Merge pull request #11756 from zhaoxiangchun/feature/zxc-cloudmon-3.7

feat(cloudmon): 3.7 add cloudmon component

build/docker/Dockerfile.cloudmon

@@ -0,0 +1,8 @@
+FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2
+
+MAINTAINER "Zexi Li <lizexi@yunionyun.com>"
+
+RUN mkdir -p /etc/yunion/data/
+
+ADD   ./_output/alpine-build/bin/cloudmon \
+      /opt/yunion/bin/

cmd/cloudmon/main.go

@@ -0,0 +1,136 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/version"
+
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+)
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClientSession(options *options.CloudMonOptions) (*mcclient.ClientSession, error) {
+	if len(options.AuthURL) == 0 {
+		return nil, errors.Error("empty auth_url")
+	}
+	if len(options.AdminUser) == 0 {
+		return nil, errors.Error("empty admin_user")
+	}
+	if len(options.AdminPassword) == 0 {
+		return nil, errors.Error("empty admin_password")
+	}
+	if len(options.AdminProject) == 0 {
+		return nil, errors.Error("empty admin_project")
+	}
+
+	client := mcclient.NewClient(
+		options.AuthURL,
+		options.Timeout,
+		options.Debug,
+		options.Insecure,
+		options.CertFile,
+		options.KeyFile,
+	)
+
+	token, err := client.AuthenticateWithSource(
+		options.AdminUser,
+		options.AdminPassword,
+		options.AdminDomain,
+		options.AdminProject,
+		options.AdminProjectDomain,
+		mcclient.AuthSourceAPI)
+	if err != nil {
+		return nil, err
+	}
+
+	session := client.NewSession(
+		context.Background(),
+		options.Region,
+		"",
+		options.EndpointType,
+		token,
+		options.ApiVersion)
+
+	return session, nil
+}
+
+func main() {
+	parser, err := options.GetArgumentParser()
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	err = parser.ParseArgs2(os.Args[1:], false, false)
+
+	opts := parser.Options().(*options.CloudMonOptions)
+
+	if opts.Help {
+		fmt.Println(parser.HelpString())
+		os.Exit(0)
+	}
+
+	if opts.Version {
+		fmt.Println(version.GetJsonString())
+		os.Exit(0)
+	}
+
+	if len(opts.Config) == 0 {
+		for _, p := range []string{"./etc", "/etc/yunion"} {
+			confTmp := path.Join(p, "cloudmon.conf")
+			if _, err := os.Stat(confTmp); err == nil {
+				opts.Config = confTmp
+				break
+			}
+		}
+	}
+
+	if len(opts.Config) > 0 {
+		err := parser.ParseFile(opts.Config)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+	}
+
+	if err != nil {
+		showErrorAndExit(err)
+	}
+
+	parser.SetDefault()
+
+	subcmd := parser.GetSubcommand()
+	subparser := subcmd.GetSubParser()
+	if err != nil {
+		if subparser != nil {
+			fmt.Print(subparser.Usage())
+		} else {
+			fmt.Print(parser.Usage())
+		}
+		showErrorAndExit(err)
+	}
+
+	suboptions := subparser.Options()
+	if opts.SUBCOMMAND == "help" {
+		err = subcmd.Invoke(suboptions)
+	} else {
+		var session *mcclient.ClientSession
+		session, err = newClientSession(opts)
+		if err != nil {
+			showErrorAndExit(err)
+		}
+		err = subcmd.Invoke(session, suboptions)
+	}
+	if err != nil {
+		showErrorAndExit(err)
+	}
+}

go.mod

@@ -113,6 +113,7 @@ require (
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/stretchr/testify v1.5.1
+	github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e
 	github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible
 	github.com/tencentyun/cos-go-sdk-v5 v0.7.24
 	github.com/tinylib/msgp v1.1.0 // indirect
@@ -147,12 +148,12 @@ require (
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
 	yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e
-	yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769
+	yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349
 	yunion.io/x/log v0.0.0-20201210064738-43181789dc74
 	yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
 	yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824
+	yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8
 	yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce
 )
 

go.sum

@@ -586,6 +586,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e h1:nt2877sKfojlHCTOBXbpWjBkuWKritFaGIfgQwbQUls=
+github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e/go.mod h1:B4+Kq1u5FlULTjFSM707Q6e/cOHFv0z/6QRoxubDIQ8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8=
 github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
 github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo=
@@ -913,8 +915,8 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ=
 yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4=
-yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
+yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349 h1:ESeezAb9LM2Dcy28DDrD2nI+h22qEf4MyAPXwuDO4uM=
+yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o=
@@ -928,7 +930,7 @@ yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c h1:/ejlt78/0xod4bdkEThUuYsQ7v
 yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824 h1:3/VcCiO+vr3+zZFLNvre41ARF2xE6NVq6Ue7JPFuJvU=
-yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8 h1:+kywPlOttXAZvbxsVqR73B5GbSVgG3Xdo4socLWRkcg=
+yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE=
 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryreport.go

@@ -0,0 +1,119 @@
+package alertrecordhistorymon
+
+import (
+	"context"
+	"fmt"
+	"strconv"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/timeutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAlertRecordHistoryFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAlertRecordHistoryFactory struct {
+}
+
+func (self *SAlertRecordHistoryFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions,
+	operatorType string) common.ICloudReport {
+	return &SAlertRecordHistoryReport{
+		common.CloudReportBase{
+			SProvider: nil,
+			Session:   session,
+			Args:      args,
+			Operator:  string(common.ALERT_RECORD),
+		},
+	}
+}
+
+func (S *SAlertRecordHistoryFactory) GetId() string {
+	return string(common.ALERT_RECORD)
+}
+
+type SAlertRecordHistoryReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAlertRecordHistoryReport) Report() error {
+	alerts, err := self.getMonitorCommonAlert()
+	if err != nil {
+		return err
+	}
+	errs := make([]error, 0)
+	recordGroup, _ := errgroup.WithContext(context.Background())
+	count := 0
+	for i, _ := range alerts {
+		tmp := alerts[i]
+		count++
+		recordGroup.Go(func() error {
+			alert_id, _ := tmp.GetString("id")
+			alertRecords, err := self.getAlertRecordsByAlertId(alert_id)
+			if err != nil {
+				return errors.Wrapf(err, "getAlertRecordsByAlertId:%s error", alert_id)
+			}
+			return self.collectMetric(alertRecords)
+		})
+		if count == 4 {
+			err := recordGroup.Wait()
+			if err != nil {
+				errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error"))
+			}
+			count = 0
+		}
+	}
+	err = recordGroup.Wait()
+	if err != nil {
+		errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error"))
+	}
+	return errors.NewAggregate(errs)
+}
+
+func (self *SAlertRecordHistoryReport) getMonitorCommonAlert() ([]jsonutils.JSONObject, error) {
+	alerts := make([]jsonutils.JSONObject, 0)
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), common.KEY_LIMIT)
+	//query.Add(jsonutils.NewBool(true), common.DETAILS)
+	query.Add(jsonutils.NewString("system"), "scope")
+
+	alerts, err := self.ListAllResource(modules.CommonAlertManager, query)
+	if err != nil {
+		return nil, errors.Wrap(err, "getMonitorCommonAlert error")
+	}
+	return alerts, nil
+}
+
+func (self *SAlertRecordHistoryReport) getAlertRecordsByAlertId(id string) ([]jsonutils.JSONObject, error) {
+	now := time.Now().UTC()
+	period64, err := strconv.ParseInt(self.Args.Interval, 10, 8)
+	if err != nil {
+		return nil, err
+	}
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), common.KEY_LIMIT)
+	query.Add(jsonutils.NewBool(true), common.DETAILS)
+	query.Add(jsonutils.NewString("system"), "scope")
+	query.Add(jsonutils.NewString(id), "alert_id")
+	query.Add(jsonutils.NewString("alerting"), "state")
+	query.Add(jsonutils.NewString(fmt.Sprintf(`created_at.between("%s","%s")`,
+		now.Add(-time.Hour*24*time.Duration(period64)).Format(timeutils.MysqlTimeFormat),
+		now.Format(timeutils.MysqlTimeFormat))),
+		"filter")
+
+	alertRecords, err := self.ListAllResource(modules.AlertRecordManager, query)
+	if err != nil {
+		return nil, err
+	}
+	return alertRecords, nil
+}

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryservice.go

@@ -0,0 +1,40 @@
+package alertrecordhistorymon
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAlertRecordHistoryReport) collectMetric(alertRecords []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	matchRecord := self.getMaxEvalMatchOfAlertRecord(alertRecords)
+	if matchRecord == nil {
+		return nil
+	}
+
+	metric, err := self.NewMetricFromJson(matchRecord)
+	if err != nil {
+		return err
+	}
+	metric.Timestamp = time.Now()
+	metric.Name = ALERT_RECORD_HISTORY_MEASUREMENT
+	dataList = append(dataList, metric)
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, ALERT_RECORD_HISTORY_DATABASE)
+}
+
+func (self *SAlertRecordHistoryReport) getMaxEvalMatchOfAlertRecord(alertRecords []jsonutils.JSONObject) jsonutils.JSONObject {
+	var matchRecord jsonutils.JSONObject
+	maxCount := int64(0)
+	for _, record := range alertRecords {
+		resNum, _ := record.Int("res_num")
+		if resNum > maxCount {
+			maxCount = resNum
+			matchRecord = record
+		}
+	}
+	return matchRecord
+}

pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryutil.go

@@ -0,0 +1,7 @@
+package alertrecordhistorymon
+
+const (
+	ALERT_RECORD_HISTORY_FIELD       = "res_num"
+	ALERT_RECORD_HISTORY_MEASUREMENT = "alert_record_history"
+	ALERT_RECORD_HISTORY_DATABASE    = "monitor"
+)

pkg/cloudmon/collectors/alertrecordhistorymon/doc.go

@@ -0,0 +1 @@
+package alertrecordhistorymon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon"

pkg/cloudmon/collectors/alertrecordhistoryservice.go

@@ -0,0 +1,15 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-alertrecord", "Report alertrecord history", reportAlertrecord)
+}
+
+func reportAlertrecord(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCustomizeCloudMetric(string(common.ALERT_RECORD), session, args)
+}

pkg/cloudmon/collectors/alimon/aliUtils.go

@@ -0,0 +1,159 @@
+package alimon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var aliMetricSpecs = map[string][]string{
+	"CPUUtilization":  {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"InternetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
+	"IntranetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
+	"InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
+	"IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
+	"DiskReadBPS":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"DiskWriteBPS":    {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"DiskReadIOPS":    {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"DiskWriteIOPS":   {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}
+var aliRdsMetricSpecs = map[string][]string{
+	"CpuUsage":                {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"MemoryUsage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"MySQL_NetworkInNew":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
+	"MySQL_NetworkOutNew":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
+	"DiskUsage":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"SQLServer_NetworkInNew":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
+	"SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
+	"ConnectionUsage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE},
+}
+var aliRedisMetricSpecs = map[string][]string{
+	"CpuUsage":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"MemoryUsage":    {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"IntranetIn":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"IntranetOut":    {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"UsedQPS":        {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"Keys":           {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"ExpiredKeys":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"UsedMemory":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+var aliOSSMetricSpecs = map[string][]string{
+	"InternetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET},
+	"InternetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET},
+	"IntranetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET},
+	"IntranetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET},
+	"GetObjectE2eLatency":  {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
+	"PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST},
+	"GetObjectCount":       {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
+	"PostObjectCount":      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST},
+	"ServerErrorCount":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
+}
+var aliElbMetricSpecs = map[string][]string{
+	"InstanceTrafficRX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
+	"InstanceTrafficTX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
+	"InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
+	"InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
+	"InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
+	"InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
+}

pkg/cloudmon/collectors/alimon/alicloudreport.go

@@ -0,0 +1,82 @@
+package alimon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAliCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAliCloudReportFactory struct {
+}
+
+func (self *SAliCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SAliCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SAliCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_ALIYUN
+}
+
+type SAliCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAliCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/alimon/aliservice.go

@@ -0,0 +1,301 @@
+package alimon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/aliyun"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAliCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	redisDeployType := make(map[string]string)
+	for _, server := range servers {
+		local_category, err := server.GetString("local_category")
+		if err != nil {
+			return err
+		}
+		switch local_category {
+		case "single":
+			redisDeployType["single"] = "Standard"
+		case "master":
+			redisDeployType["master"] = "Standard"
+		case "cluster":
+			redisDeployType["cluster"] = "Sharding"
+		case "rwsplit":
+			redisDeployType["rwsplit"] = "Splitrw"
+		}
+
+	}
+	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
+		for _, pre := range redisDeployType {
+			rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "")
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			if len(rtnArray) > 0 {
+				for _, rtnMetric := range rtnArray {
+					for _, server := range servers {
+						external_id, _ := server.GetString("external_id")
+						if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+							serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+							node_id, _ := rtnMetric.GetString("nodeId")
+							serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id})
+							if err != nil {
+								return err
+							}
+							dataList = append(dataList, serverMetric)
+						}
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliRdsMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+
+				}
+
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliOSSMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					name, _ := server.GetString("name")
+					if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name {
+						serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName,
+							influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*aliyun.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliElbMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SAliCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(UNIT_AVERAGE)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	metric.Name = measurement
+	return metric, nil
+}
+
+func (self *SAliCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(metricName)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	metric.Name = measurement
+	return metric, nil
+}

pkg/cloudmon/collectors/alimon/doc.go

@@ -0,0 +1 @@
+package alimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon"

pkg/cloudmon/collectors/apsaramon/apsaracloudreport.go

@@ -0,0 +1,82 @@
+package apsaramon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SApsaraCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SApsaraCloudReportFactory struct {
+}
+
+func (self *SApsaraCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SApsaraCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SApsaraCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_APSARA
+}
+
+type SApsaraCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SApsaraCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/apsaramon/apsaraservice.go

@@ -0,0 +1,301 @@
+package apsaramon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/apsara"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SApsaraCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	redisDeployType := make(map[string]string)
+	for _, server := range servers {
+		local_category, err := server.GetString("local_category")
+		if err != nil {
+			return err
+		}
+		switch local_category {
+		case "single":
+			redisDeployType["single"] = "Standard"
+		case "master":
+			redisDeployType["master"] = "Standard"
+		case "cluster":
+			redisDeployType["cluster"] = "Sharding"
+		case "rwsplit":
+			redisDeployType["rwsplit"] = "Splitrw"
+		}
+
+	}
+	for metricName, influxDbSpecs := range aliRedisMetricSpecs {
+		for _, pre := range redisDeployType {
+			rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "")
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			if len(rtnArray) > 0 {
+				for _, rtnMetric := range rtnArray {
+					for _, server := range servers {
+						external_id, _ := server.GetString("external_id")
+						if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+							serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+							node_id, _ := rtnMetric.GetString("nodeId")
+							serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id})
+							if err != nil {
+								return err
+							}
+							dataList = append(dataList, serverMetric)
+						}
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliRdsMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+
+				}
+
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliOSSMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					name, _ := server.GetString("name")
+					if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name {
+						serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName,
+							influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	aliReg := region.(*apsara.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range aliElbMetricSpecs {
+		rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "")
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtnArray) > 0 {
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id {
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SApsaraCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(UNIT_AVERAGE)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	metric.Name = measurement
+	return metric, nil
+}
+
+func (self *SApsaraCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject,
+	metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	timestamp, _ := rtnMetric.Get("timestamp")
+	metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0)
+	fieldValue, err := rtnMetric.Float(metricName)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	metric.Name = measurement
+	return metric, nil
+}

pkg/cloudmon/collectors/apsaramon/apsarautils.go

@@ -0,0 +1,159 @@
+package apsaramon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var aliMetricSpecs = map[string][]string{
+	"CPUUtilization":  {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"InternetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
+	"IntranetInRate":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
+	"InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
+	"IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
+	"DiskReadBPS":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"DiskWriteBPS":    {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"DiskReadIOPS":    {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"DiskWriteIOPS":   {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}
+var aliRdsMetricSpecs = map[string][]string{
+	"CpuUsage":                {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"MemoryUsage":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"MySQL_NetworkInNew":      {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL},
+	"MySQL_NetworkOutNew":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL},
+	"DiskUsage":               {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"SQLServer_NetworkInNew":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER},
+	"SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER},
+	"ConnectionUsage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE},
+}
+var aliRedisMetricSpecs = map[string][]string{
+	"CpuUsage":       {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"MemoryUsage":    {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"IntranetIn":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"IntranetOut":    {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"UsedQPS":        {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"Keys":           {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"ExpiredKeys":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"UsedMemory":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+var aliOSSMetricSpecs = map[string][]string{
+	"InternetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET},
+	"InternetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET},
+	"IntranetSend":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET},
+	"IntranetRecv":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET},
+	"GetObjectE2eLatency":  {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
+	"PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST},
+	"GetObjectCount":       {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
+	"PostObjectCount":      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST},
+	"ServerErrorCount":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
+}
+var aliElbMetricSpecs = map[string][]string{
+	"InstanceTrafficRX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
+	"InstanceTrafficTX":     {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
+	"InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
+	"InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
+	"InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
+	"InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
+}

pkg/cloudmon/collectors/apsaramon/doc.go

@@ -0,0 +1 @@
+package apsaramon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"

pkg/cloudmon/collectors/awsmon/awscloudreport.go

@@ -0,0 +1,71 @@
+package awsmon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAwsCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAwsCloudReportFactory struct {
+}
+
+func (self *SAwsCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SAwsCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SAwsCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_AWS
+}
+
+type SAwsCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAwsCloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self)
+			//err = self.collectRegionMetricOfHost(region, servers)
+			//case "redis":
+			//	err = self.collectRegionMetricOfRedis(region, servers)
+			//case "rds":
+			//	err = self.collectRegionMetricOfRds(region, servers)
+			//case "oss":
+			//	err = self.collectRegionMetricOfOss(region, servers)
+			//case "elb":
+			//	err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/awsmon/awsservice.go

@@ -0,0 +1,111 @@
+package awsmon
+
+import (
+	"strconv"
+	"strings"
+
+	aws_sdk "github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/service/cloudwatch"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/aws"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAwsCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	var err error
+	switch self.Operator {
+	case string(common.SERVER):
+		err = self.collectRegionMetricOfHost(region, servers)
+	}
+	return err
+}
+
+func (self *SAwsCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	awsRegion := region.(*aws.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range awsMetricSpecs {
+		for i, server := range servers {
+			external_id, _ := servers[i].GetString("external_id")
+			rtnArray, err := awsRegion.GetMonitorData(metricName, "AWS/EC2", external_id, since, until)
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			if len(rtnArray.Datapoints) > 0 {
+				for _, dataPoint := range rtnArray.Datapoints {
+					metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, metric)
+					serverMetric, err := self.collectMetricFromThisServerForAws(server, dataPoint, influxDbSpecs)
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, serverMetric)
+				}
+
+			}
+		}
+		err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+	return nil
+}
+
+func (self *SAwsCloudReport) collectMetricFromThisServerForAws(server jsonutils.JSONObject, datapoint *cloudwatch.Datapoint,
+	influxDbSpecs []string) (metric influxdb.SMetricData, err error) {
+	metric, err = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return metric, err
+	}
+	metric.Timestamp = aws_sdk.TimeValue(datapoint.Timestamp)
+	fieldValue := aws_sdk.Float64Value(datapoint.Average)
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	// 300:AWS metric collection period is 5m
+	if influxDbSpecs[1] == UNIT_COUNT {
+		fieldValue = (fieldValue / float64(300))
+	}
+	if influxDbSpecs[1] == UNIT_MEM {
+		fieldValue = (fieldValue / float64(300) * 8)
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{Key: common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "=")})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err != nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{Key: pairsKey,
+			Value: strconv.FormatFloat(fieldValue, 'E', -1, 64)})
+	} else {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{Key: "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10)},
+			influxdb.SKeyValue{Key: pairsKey,
+				Value: strconv.FormatFloat(fieldValue, 'E', -1, 64)})
+	}
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	metric.Name = measurement
+	return metric, nil
+}

pkg/cloudmon/collectors/awsmon/awsutils.go

@@ -0,0 +1,125 @@
+package awsmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_BPS_EBS   = INFLUXDB_FIELD_DISK_READ_BPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_DISK_WRITE_BPS_EBS  = INFLUXDB_FIELD_DISK_WRITE_BPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_DISK_READ_IOPS_EBS  = INFLUXDB_FIELD_DISK_READ_IOPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS_EBS = INFLUXDB_FIELD_DISK_WRITE_IOPS + ",disk_type=ebs"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var awsMetricSpecs = map[string][]string{
+	"CPUUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"DiskReadOps":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"EBSReadOps":     {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_READ_IOPS_EBS},
+	"DiskWriteOps":   {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+	"EBSWriteOps":    {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_WRITE_IOPS_EBS},
+	"DiskReadBytes":  {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"EBSReadBytes":   {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS_EBS},
+	"DiskWriteBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"EBSWriteBytes":  {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS_EBS},
+	"NetworkIn":      {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"NetworkOut":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+}

pkg/cloudmon/collectors/awsmon/doc.go

@@ -0,0 +1 @@
+package awsmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon"

pkg/cloudmon/collectors/azuremon/azurereport.go

@@ -0,0 +1,71 @@
+package azuremon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SAzureCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SAzureCloudReportFactory struct {
+}
+
+func (self *SAzureCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SAzureCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SAzureCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_AZURE
+}
+
+type SAzureCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SAzureCloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self)
+			//err = self.collectRegionMetricOfHost(region, servers)
+			//case "redis":
+			//	err = self.collectRegionMetricOfRedis(region, servers)
+			//case "rds":
+			//	err = self.collectRegionMetricOfRds(region, servers)
+			//case "oss":
+			//	err = self.collectRegionMetricOfOss(region, servers)
+			//case "elb":
+			//	err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/azuremon/azureservice.go

@@ -0,0 +1,128 @@
+package azuremon
+
+import (
+	"strconv"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/azure"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SAzureCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	return self.collectRegionMetricOfHost(region, servers)
+}
+
+func (self *SAzureCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	azureReg := region.(*azure.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, err := server.GetString("external_id")
+		if err != nil {
+			continue
+		}
+		metricNameArr := make([]string, 0)
+		for metricName, _ := range azureMetricSpecs {
+			metricNameArr = append(metricNameArr, metricName)
+		}
+		metricNames := strings.Join(metricNameArr, ",")
+		rtnMetrics, err := azureReg.GetMonitorData(metricNames, "Microsoft.Compute/virtualMachines", external_id, since, until)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if rtnMetrics == nil || rtnMetrics.Value == nil {
+			continue
+		}
+		for metricName, influxDbSpecs := range azureMetricSpecs {
+			for _, value := range *rtnMetrics.Value {
+				if value.Name.LocalizedValue != nil {
+					if metricName == *(value.Name.LocalizedValue) {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						if value.Timeseries != nil {
+							for _, timeserie := range *value.Timeseries {
+								serverMetric, err := self.collectMetricFromThisServer(server, timeserie, influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric...)
+							}
+						}
+					}
+				}
+			}
+		}
+		err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+		if err != nil {
+			log.Errorln(err)
+		}
+		dataList = dataList[:0]
+	}
+	return nil
+}
+
+func (self *SAzureCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	rtnMetric azure.TimeSeriesElement, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	for _, data := range *rtnMetric.Data {
+		metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		if err != nil {
+			return nil, err
+		}
+		if data.TimeStamp != nil {
+			metric.Timestamp = *data.TimeStamp
+			if data.Average != nil {
+				//根据条件拼装metric的tag和metirc信息
+				fieldValue := *data.Average
+				influxDbSpec := influxDbSpecs[2]
+				measurement := common.SubstringBefore(influxDbSpec, ".")
+				metric.Name = measurement
+				var pairsKey string
+				if strings.Contains(influxDbSpec, ",") {
+					pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+				} else {
+					pairsKey = common.SubstringAfter(influxDbSpec, ".")
+				}
+				if influxDbSpecs[1] == UNIT_MEM {
+					fieldValue = fieldValue * 8 / PERIOD
+				}
+				tag := common.SubstringAfter(influxDbSpec, ",")
+				if tag != "" && strings.Contains(influxDbSpec, "=") {
+					metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+						Key:   common.SubstringBefore(tag, "="),
+						Value: common.SubstringAfter(tag, "="),
+					})
+				}
+				cpu_cout, err := server.Get("vcpu_count")
+				if err == nil {
+					metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+						Key:   "cpu_count",
+						Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+					})
+				}
+				metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+					Key:   pairsKey,
+					Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+				})
+				self.AddMetricTag(&metric, common.OtherVmTags)
+				datas = append(datas, metric)
+			}
+		}
+	}
+
+	return datas, nil
+}

pkg/cloudmon/collectors/azuremon/azureutils.go

@@ -0,0 +1,117 @@
+package azuremon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var azureMetricSpecs = map[string][]string{
+	"Percentage CPU":            {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"Network In Total":          {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"Network Out Total":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+	"Disk Read Bytes":           {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"Disk Write Bytes":          {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"Disk Read Operations/Sec":  {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"Disk Write Operations/Sec": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_WRITE_BPS},
+}

pkg/cloudmon/collectors/azuremon/doc.go

@@ -0,0 +1 @@
+package azuremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon"

pkg/cloudmon/collectors/bucket.go

@@ -0,0 +1,74 @@
+package collectors
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BucketStatsOptions struct {
+	Debug bool `help:"debug"`
+}
+
+func bucketStatsCollect(s *mcclient.ClientSession, args *BucketStatsOptions) error {
+	metrics := make([]influxdb.SMetricData, 0)
+	listAll(s, modules.Buckets.List, nil,
+		func(data jsonutils.JSONObject) error {
+			m, err := collectBucket(s, data)
+			if err != nil {
+				return err
+			}
+			metrics = append(metrics, m)
+			return nil
+		},
+	)
+	return sendMetrics(s, metrics, args.Debug)
+}
+
+func collectBucket(s *mcclient.ClientSession, bucket jsonutils.JSONObject) (influxdb.SMetricData, error) {
+	metric := influxdb.SMetricData{}
+	bucketId, _ := bucket.GetString("id")
+	if len(bucketId) == 0 {
+		return metric, errors.Error("empty bucket id")
+	}
+	params := jsonutils.NewDict()
+	params.Set("stats_only", jsonutils.JSONTrue)
+	result, err := modules.Buckets.PerformAction(s, bucketId, "sync", params)
+	if err != nil {
+		return metric, errors.Wrap(err, "PerformAction")
+	}
+	return jsonToMetric(result.(*jsonutils.JSONDict), "bucket",
+		[]string{
+			"name",
+			"id",
+			"account",
+			"account_id",
+			"manager",
+			"manager_id",
+			"manager_domain",
+			"manager_domain_id",
+			"manager_project",
+			"manager_project_id",
+			"brand",
+			"provider",
+			"region_id",
+			"region_ext_id",
+			"tenant",
+			"tenant_id",
+			"domain_id",
+			"project_domain",
+		},
+		[]string{
+			"object_cnt",
+			"size_bytes",
+		},
+	)
+}
+
+func init() {
+	shellutils.R(&BucketStatsOptions{}, "buckets", "Bucket stats", bucketStatsCollect)
+}

pkg/cloudmon/collectors/cloudaccountmon/cloudaccountreport.go

@@ -0,0 +1,44 @@
+package cloudaccountmon
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SCloudAccountFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SCloudAccountFactory struct {
+}
+
+func (self *SCloudAccountFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions,
+	operatorType string) common.ICloudReport {
+	return &SCloudAccountReport{
+		common.CloudReportBase{
+			SProvider: nil,
+			Session:   session,
+			Args:      args,
+			Operator:  ClOUDACCOUNT_ID,
+		},
+	}
+}
+
+func (S SCloudAccountFactory) GetId() string {
+	return ClOUDACCOUNT_ID
+}
+
+type SCloudAccountReport struct {
+	common.CloudReportBase
+}
+
+func (self *SCloudAccountReport) Report() error {
+	accounts, err := self.GetAllCloudAccount(&modules.Cloudaccounts)
+	if err != nil {
+		return err
+	}
+	return self.collectMetric(accounts)
+}

pkg/cloudmon/collectors/cloudaccountmon/cloudaccountutil.go

@@ -0,0 +1,9 @@
+package cloudaccountmon
+
+const (
+	ClOUDACCOUNT_ID = "cloudaccount"
+
+	CLOUDACCOUNT_FIELD       = "balance"
+	CLOUDACCOUNT_MEASUREMENT = "cloudaccount_balance"
+	CLOUDACCOUNT_DATABASE    = "meter_db"
+)

pkg/cloudmon/collectors/cloudaccountmon/cloudaccoutservice.go

@@ -0,0 +1,24 @@
+package cloudaccountmon
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SCloudAccountReport) collectMetric(accounts []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	for _, account := range accounts {
+		metric, err := self.NewMetricFromJson(account)
+		if err != nil {
+			return err
+		}
+		metric.Timestamp = time.Now()
+		metric.Name = CLOUDACCOUNT_MEASUREMENT
+		dataList = append(dataList, metric)
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, CLOUDACCOUNT_DATABASE)
+}

pkg/cloudmon/collectors/cloudaccountmon/doc.go

@@ -0,0 +1 @@
+package cloudaccountmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon"

pkg/cloudmon/collectors/cloudaccountservice.go

@@ -0,0 +1,15 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-cloudaccount", "Report cloud account", reportCloudAccount)
+}
+
+func reportCloudAccount(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCustomizeCloudMetric(string(common.CLOUDACCOUNT), session, args)
+}

pkg/cloudmon/collectors/cloudmoninit.go

@@ -0,0 +1,25 @@
+package collectors
+
+import (
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon"
+	_ "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon"
+	_ "yunion.io/x/onecloud/pkg/multicloud/aliyun/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/apsara/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/aws/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/azure/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/esxi/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/google/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/huawei/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/qcloud/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/zstack/provider"
+)

pkg/cloudmon/collectors/common/basemon.go

@@ -0,0 +1,228 @@
+package common
+
+import (
+	"fmt"
+	"net/http"
+	"net/url"
+
+	"golang.org/x/net/http/httpproxy"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+type CloudReportBase struct {
+	SProvider *SProvider
+	Session   *mcclient.ClientSession
+	Args      *ReportOptions
+	Operator  string
+}
+
+func (self *CloudReportBase) Report() error {
+	return fmt.Errorf("No Implment the method")
+}
+
+func (self *CloudReportBase) GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+
+	query.Add(jsonutils.NewStringArray([]string{"running", "ready"}), "status")
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	query.Add(jsonutils.NewString(self.SProvider.Provider), "provider")
+	query.Add(jsonutils.NewString(self.SProvider.Id), "manager")
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) GetAllHostOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("running"), "status")
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	query.Add(jsonutils.NewString(self.SProvider.Provider), "provider")
+	query.Add(jsonutils.NewString(self.SProvider.Id), "manager")
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) GetAllCloudAccount(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewBool(true), DETAILS)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	query.Add(jsonutils.NewString(fmt.Sprintf("brand.in(%s,%s,%s)", compute.CLOUD_PROVIDER_ALIYUN,
+		compute.CLOUD_PROVIDER_QCLOUD, compute.CLOUD_PROVIDER_HUAWEI)), "filter")
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) GetAllStorage(manager modulebase.Manager) ([]jsonutils.JSONObject, error) {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewBool(true), DETAILS)
+	query.Add(jsonutils.NewString("system"), KEY_SCOPE)
+	return self.ListAllResources(manager, query)
+}
+
+func (self *CloudReportBase) ListAllResource(manager modulebase.Manager,
+	query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) {
+	resources := make([]jsonutils.JSONObject, 0)
+	offsetIndex := 0
+	for {
+		query.(*jsonutils.JSONDict).Add(jsonutils.NewInt(int64(offsetIndex)), "offset")
+		resList, err := manager.List(self.Session, query)
+		if err != nil {
+			return nil, err
+		}
+		resources = append(resources, resList.Data...)
+		offsetIndex = offsetIndex + len(resList.Data)
+		if offsetIndex >= resList.Total {
+			break
+		}
+	}
+	return resources, nil
+}
+
+func (self *CloudReportBase) InitProviderInstance() (cloudprovider.ICloudProvider, error) {
+	//secret进行Descrypt
+	secretDe, _ := utils.DescryptAESBase64(self.SProvider.Id, self.SProvider.Secret)
+	var proxyFunc httputils.TransportProxyFunc
+	{
+		cfg := &httpproxy.Config{
+			HTTPProxy:  self.SProvider.ProxySetting.HTTPProxy,
+			HTTPSProxy: self.SProvider.ProxySetting.HTTPSProxy,
+			NoProxy:    self.SProvider.ProxySetting.NoProxy,
+		}
+		cfgProxyFunc := cfg.ProxyFunc()
+		proxyFunc = func(req *http.Request) (*url.URL, error) {
+			return cfgProxyFunc(req.URL)
+		}
+	}
+	cloudAccout, err := self.getCloudAccount(&modules.Cloudaccounts)
+	if err != nil {
+		return nil, errors.Wrap(err, "getCloudAccount error")
+	}
+	endpoints := cloudprovider.SApsaraEndpoints{}
+	options, err := cloudAccout.Get("options")
+	if err == nil {
+		err := options.Unmarshal(&endpoints)
+		if err != nil {
+			log.Errorf("Unmarshal SApsaraEndpoints err:%v", err)
+		}
+	} else {
+		log.Errorf("get cloudAccout options err:%v", err)
+	}
+	cfg := cloudprovider.ProviderConfig{
+		Id:               self.SProvider.Id,
+		Name:             self.SProvider.Name,
+		URL:              self.SProvider.AccessUrl,
+		Account:          self.SProvider.Account,
+		Secret:           secretDe,
+		Vendor:           self.SProvider.Provider,
+		ProxyFunc:        proxyFunc,
+		SApsaraEndpoints: endpoints,
+	}
+	return cloudprovider.GetProvider(cfg)
+}
+
+func (self *CloudReportBase) getCloudAccount(manager modulebase.Manager) (jsonutils.JSONObject, error) {
+	return self.GetResourceById(self.SProvider.CloudaccountId, manager)
+}
+
+func (self *CloudReportBase) GetResourceById(id string, manager modulebase.Manager) (jsonutils.JSONObject, error) {
+	return manager.Get(self.Session, id, jsonutils.NewDict())
+}
+
+func (self *CloudReportBase) GetAllRegionOfServers(servers []jsonutils.JSONObject,
+	providerInstance cloudprovider.ICloudProvider) ([]cloudprovider.
+	ICloudRegion, map[string][]jsonutils.JSONObject, error) {
+	extranleIdMap := make(map[string]string)
+	regionServerList := make([]cloudprovider.ICloudRegion, 0)
+	regionServerMap := make(map[string][]jsonutils.JSONObject)
+	for i, server := range servers {
+		region_external_id, err := server.GetString("region_external_id")
+		if err != nil {
+			return nil, nil, err
+		}
+		if _, ok := extranleIdMap[region_external_id]; !ok {
+			extranleIdMap[region_external_id] = ""
+			region, err := providerInstance.GetIRegionById(region_external_id)
+			if err != nil {
+				return nil, nil, err
+			}
+			regionServerList = append(regionServerList, region)
+			regionServers := make([]jsonutils.JSONObject, 0)
+			regionServers = append(regionServers, servers[i])
+			regionServerMap[region_external_id] = regionServers
+		} else {
+			regionservers := regionServerMap[region_external_id]
+			regionservers = append(regionservers, server)
+			regionServerMap[region_external_id] = regionservers
+		}
+	}
+	return regionServerList, regionServerMap, nil
+}
+
+func (self *CloudReportBase) AddMetricTag(metric *influxdb.SMetricData, tags map[string]string) {
+	for key, value := range tags {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   key,
+			Value: value,
+		})
+	}
+}
+
+func (self *CloudReportBase) NewMetricFromJson(server jsonutils.JSONObject) (influxdb.SMetricData, error) {
+	switch MonType(self.Operator) {
+	case SERVER:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", ServerTags, make([]string, 0))
+	case HOST:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", HostTags, make([]string, 0))
+	case REDIS:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", RedisTags, make([]string, 0))
+	case RDS:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", RdsTags, make([]string, 0))
+	case OSS:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", OssTags, make([]string, 0))
+	case ELB:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", ElbTags, make([]string, 0))
+	case CLOUDACCOUNT:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", CloudAccountTags, CloudAccountFields)
+	case STORAGE:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", StorageTags, make([]string, 0))
+	case ALERT_RECORD:
+		return JsonToMetric(server.(*jsonutils.JSONDict), "", AlertRecordHistoryTags, AlertRecordHistoryFields)
+	}
+	return influxdb.SMetricData{}, fmt.Errorf("no found report operator")
+}
+
+func (self *CloudReportBase) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	panic("NO implement CollectRegionMetricOfServer")
+}
+
+func (self *CloudReportBase) ListAllResources(manager modulebase.Manager,
+	query *jsonutils.JSONDict) ([]jsonutils.JSONObject, error) {
+	offsetIndex := 0
+	resources := make([]jsonutils.JSONObject, 0)
+	for {
+		query.Add(jsonutils.NewInt(int64(offsetIndex)), "offset")
+		resList, err := manager.List(self.Session, query)
+		if err != nil {
+			return nil, err
+		}
+		resources = append(resources, resList.Data...)
+		offsetIndex = offsetIndex + len(resList.Data)
+		if offsetIndex >= resList.Total {
+			break
+		}
+	}
+	return resources, nil
+}

pkg/cloudmon/collectors/common/cloudreport.go

@@ -0,0 +1,86 @@
+package common
+
+import (
+	"fmt"
+
+	"github.com/aws/aws-sdk-go/aws/request"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+)
+
+func init() {
+	cloudReportTable = make(map[string]ICloudReportFactory)
+}
+
+type ICloudReportFactory interface {
+	NewCloudReport(provider *SProvider, session *mcclient.ClientSession, args *ReportOptions,
+		operatorType string) ICloudReport
+	GetId() string
+}
+
+type ICloudReport interface {
+	Report() error
+	GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error)
+	InitProviderInstance() (cloudprovider.ICloudProvider, error)
+	GetAllRegionOfServers(servers []jsonutils.JSONObject, providerInstance cloudprovider.ICloudProvider) (
+		[]cloudprovider.ICloudRegion, map[string][]jsonutils.JSONObject, error)
+	CollectRegionMetric(region cloudprovider.ICloudRegion,
+		servers []jsonutils.JSONObject) error
+}
+
+type SProvider struct {
+	compute.CloudproviderDetails
+	//Id        string `json:"id"
+	//Name      string `json:"name"`
+	//Provider  string `json:"provider"`
+	//Account   string `json:"account"`
+	//Secret    string `json:"secret"`
+	//AccessUrl string `json:"access_url"`
+}
+
+var cloudReportTable map[string]ICloudReportFactory
+
+func RegisterFactory(factory ICloudReportFactory) {
+	cloudReportTable[factory.GetId()] = factory
+}
+
+func GetCloudReportFactory(provider string) (ICloudReportFactory, error) {
+	factory, ok := cloudReportTable[provider]
+	if ok {
+		return factory, nil
+	}
+	log.Errorf("Provider %s not registerd", provider)
+	return nil, fmt.Errorf("No such cloudReport %s", provider)
+}
+
+func (s *SProvider) Validate() error {
+	invalidParams := request.ErrInvalidParams{Context: "SProvider"}
+	if s.Id == "" {
+		invalidParams.Add(request.NewErrParamRequired("Id"))
+	}
+	if s.Name == "" {
+		invalidParams.Add(request.NewErrParamRequired("Name"))
+	}
+	if s.Account == "" {
+		invalidParams.Add(request.NewErrParamRequired("Account"))
+	}
+	//if s.AccessUrl == "" {
+	//	invalidParams.Add(request.NewErrParamRequired("AccountUrl"))
+	//}
+	if s.Provider == "" {
+		invalidParams.Add(request.NewErrParamRequired("Provider"))
+	}
+	if s.Secret == "" {
+		invalidParams.Add(request.NewErrParamRequired("Secret"))
+	}
+	if invalidParams.Len() > 0 {
+		return invalidParams
+	}
+	return nil
+}

pkg/cloudmon/collectors/common/doc.go

@@ -0,0 +1 @@
+package common // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common"

pkg/cloudmon/collectors/common/multiCloudMonUtils.go

@@ -0,0 +1,492 @@
+package common
+
+import (
+	"context"
+	"strconv"
+	"strings"
+	"time"
+
+	"golang.org/x/sync/errgroup"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/pkg/utils"
+
+	o "yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+type MonType string
+
+const (
+	SERVER       MonType = "server"
+	HOST         MonType = "host"
+	REDIS        MonType = "redis"
+	RDS          MonType = "rds"
+	OSS          MonType = "oss"
+	ELB          MonType = "elb"
+	CLOUDACCOUNT MonType = "cloudaccount"
+	STORAGE      MonType = "storage"
+	ALERT_RECORD MonType = "alertRecord"
+)
+
+const (
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+	DETAILS    = "details"
+	KEY_SCOPE  = "scope"
+)
+
+const (
+	TYPE_VIRTUALMACHINE = "VirtualMachine"
+	TYPE_HOSTSYSTEM     = "HostSystem"
+)
+
+var OtherVmTags = map[string]string{
+	"source":   "cloudmon",
+	"res_type": "guest",
+	"is_vm":    "true",
+}
+
+var OtherHostTag = map[string]string{
+	"source":   "cloudmon",
+	"res_type": "host",
+	"is_vm":    "false",
+}
+
+type ReportOptions struct {
+	Debug     bool     `help:"debug"`
+	Batch     int      `help:"batch"`
+	Count     int      `help:"count" json:"count"`
+	Interval  string   `help:"interval""`
+	SinceTime string   `help:"sinceTime"`
+	EndTime   string   `help:"endTime"`
+	Provider  []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara" json:"provider,omitempty"`
+}
+
+var InstanceProviders = "Aliyun,Azure,Aws,Qcloud,VMWare,Huawei,Openstack,Ucloud,ZStack"
+
+//server的key-value对应保存时的Tags和Pairs
+//var ServerTags = []string{"host", "host_id", "vm_id", "vm_ip", "vm_name", "zone", "zone_id", "zone_ext_id",
+//	"hypervisor", "os_type", "status", "region", "region_id", "region_ext_id", "tenant", "tenant_id", "brand", "name"}
+var ServerTags = map[string]string{
+	"host":             "host",
+	"host_id":          "host_id",
+	"id":               "vm_id",
+	"ips":              "vm_ip",
+	"name":             "vm_name",
+	"zone":             "zone",
+	"zone_id":          "zone_id",
+	"zone_ext_id":      "zone_ext_id",
+	"os_type":          "os_type",
+	"status":           "status",
+	"cloudregion":      "cloudregion",
+	"cloudregion_id":   "cloudregion_id",
+	"region_ext_id":    "region_ext_id",
+	"tenant":           "tenant",
+	"tenant_id":        "tenant_id",
+	"brand":            "brand",
+	"scaling_group_id": "vm_scaling_group_id",
+	"domain_id":        "domain_id",
+	"project_domain":   "project_domain",
+}
+var HostTags = map[string]string{
+	"id":             "host_id",
+	"ips":            "host_ip",
+	"name":           "host",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var RdsTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "rds_id",
+	"ips":            "rds_ip",
+	"name":           "rds_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var RedisTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "redis_id",
+	"ips":            "redis_ip",
+	"name":           "redis_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var OssTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "oss_id",
+	"ips":            "oss_ip",
+	"name":           "oss_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+var ElbTags = map[string]string{
+	"host":           "host",
+	"host_id":        "host_id",
+	"id":             "elb_id",
+	"ips":            "elb_ip",
+	"name":           "elb_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"os_type":        "os_type",
+	"status":         "status",
+	"region":         "region",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"tenant":         "tenant",
+	"tenant_id":      "tenant_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+
+var CloudAccountTags = map[string]string{
+	"id":             "cloudaccount_id",
+	"name":           "cloudaccount_name",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+
+var StorageTags = map[string]string{
+	"id":             "storage_id",
+	"name":           "storage_name",
+	"zone":           "zone",
+	"zone_id":        "zone_id",
+	"zone_ext_id":    "zone_ext_id",
+	"status":         "status",
+	"cloudregion":    "cloudregion",
+	"cloudregion_id": "cloudregion_id",
+	"region_ext_id":  "region_ext_id",
+	"brand":          "brand",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+}
+
+var AlertRecordHistoryTags = map[string]string{
+	"alert_name":     "alert_name",
+	"alert_id":       "alert_id",
+	"domain_id":      "domain_id",
+	"project_domain": "project_domain",
+	"tenant_id":      "tenant_id",
+	"tenant":         "tenant",
+	"res_type":       "res_type",
+}
+
+var CloudAccountFields = []string{"balance"}
+
+var AlertRecordHistoryFields = []string{"res_num"}
+
+var ServerPairs = []string{"vcpu_count", "vmem_size", "disk"}
+
+var AddTags = map[string]string{
+	"source": "cloudmon",
+}
+
+//get substring from str before separator
+func SubstringBefore(str, separator string) string {
+	if str != "" {
+		if separator == "" {
+			return ""
+		} else {
+			if pos := strings.Index(str, separator); pos == -1 {
+				return str
+			} else {
+				return str[0:pos]
+			}
+		}
+	} else {
+		return str
+	}
+}
+
+//get substring from str after separator
+func SubstringAfter(str, separator string) string {
+	if str != "" {
+		if separator == "" {
+			return ""
+		} else {
+			if pos := strings.Index(str, separator); pos == -1 {
+				return ""
+			} else {
+				return str[pos+len(separator):]
+			}
+		}
+	} else {
+		return str
+	}
+}
+
+//get a substring from str between[open,close)
+func SubstringBetween(str, open, close string) string {
+	if str != "" && open != "" && close != "" {
+		if start := strings.Index(str, open); start != -1 {
+			if end := strings.Index(str[start+len(open):], close); end != -1 {
+				return str[start+len(open) : start+len(open)+end]
+			}
+		}
+		return ""
+	} else {
+		return ""
+	}
+}
+
+func ParseTimeStr(startTime, endTime string) (since, util time.Time, err error) {
+	since, err = timeutils.ParseTimeStr(startTime)
+	if err != nil {
+		return since, util, err
+	}
+	util, err = timeutils.ParseTimeStr(endTime)
+	if err != nil {
+		return since, util, err
+	}
+	return since, util, nil
+}
+
+func TimeRangeFromArgs(args *ReportOptions) (since, until time.Time, err error) {
+	if args.SinceTime != "" && args.EndTime != "" {
+		since, until, err = ParseTimeStr(args.SinceTime, args.EndTime)
+		if err != nil {
+			return since, until, err
+		}
+	} else {
+		period64, err := strconv.ParseInt(args.Interval, 10, 8)
+		if err != nil {
+			return since, until, err
+		}
+		since = time.Now().Add(-time.Minute * time.Duration(period64))
+		until = time.Now()
+	}
+	return since, until, nil
+}
+
+//组装server相关capability
+func FillVMCapacity(server *jsonutils.JSONDict) (influxdb.SMetricData, error) {
+	metric, err := JsonToMetric(server, "vm_capacity", ServerTags, ServerPairs)
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	hypevisor, _ := server.GetString("hypervisor")
+	metric.Timestamp = time.Now()
+	metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+		Key:   "res_type",
+		Value: "guest",
+	}, influxdb.SKeyValue{
+		Key:   "is_vm",
+		Value: "true",
+	}, influxdb.SKeyValue{
+		Key:   "platform",
+		Value: hypevisor,
+	})
+	return metric, nil
+}
+
+func GetMeasurement(action string, influxDbSpec string) (measurement string) {
+	// VirtualMachine -> VMware类型的虚拟机
+	if action == TYPE_VIRTUALMACHINE {
+		measurement = SubstringBefore(influxDbSpec, ".")
+	}
+	if action == TYPE_HOSTSYSTEM {
+		measurement = SubstringBetween(influxDbSpec, "vm_", ".")
+		if strings.Contains(influxDbSpec, "vm_netio") {
+			measurement = "net"
+		}
+	}
+	return measurement
+}
+
+func JsonToMetric(obj *jsonutils.JSONDict, name string, tags map[string]string, metrics []string) (influxdb.SMetricData, error) {
+	metric := influxdb.SMetricData{Name: name}
+	objMap, err := obj.GetMap()
+	if err != nil {
+		return metric, errors.Wrap(err, "obj.GetMap")
+	}
+	tagPairs := make([]influxdb.SKeyValue, 0)
+	metricPairs := make([]influxdb.SKeyValue, 0)
+	for k, v := range objMap {
+		val, _ := v.GetString()
+		if strings.Contains(k, "ip") {
+			if strings.Contains(val, ",") {
+				val = strings.ReplaceAll(val, ",", "|")
+			}
+		}
+		if tag, ok := tags[k]; ok {
+			tagPairs = append(tagPairs, influxdb.SKeyValue{
+				Key:   tag,
+				Value: val,
+			})
+		} else if utils.IsInStringArray(k, metrics) {
+			metricPairs = append(metricPairs, influxdb.SKeyValue{
+				Key: k, Value: val,
+			})
+		}
+	}
+	metric.Tags = tagPairs
+	metric.Metrics = metricPairs
+	return metric, nil
+}
+
+func SendMetrics(s *mcclient.ClientSession, metrics []influxdb.SMetricData, debug bool, database string) error {
+	urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType)
+	if err != nil {
+		return errors.Wrap(err, "GetServiceURLs")
+	}
+	if len(database) == 0 {
+		database = o.Options.InfluxDatabase
+	}
+	return influxdb.SendMetrics(urls, database, metrics, debug)
+}
+
+func ReportCloudMetricOfoperatorType(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error {
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("0"), KEY_LIMIT)
+	query.Add(jsonutils.NewString("true"), KEY_ADMIN)
+	//query.Add(jsonutils.NewString("true"), KEY_USABLE)
+	if len(args.Provider) > 0 {
+		for _, val := range args.Provider {
+			query.Add(jsonutils.NewString(val), "provider")
+		}
+	}
+	cloudProviderList, err := (&modules.Cloudproviders).List(session, query)
+	if err != nil {
+		return errors.Wrap(err, "cloudProviders get list error")
+	}
+	providerGroup, _ := errgroup.WithContext(context.Background())
+	tmpCount := 0
+	if args.Count == 0 {
+		args.Count = 1
+	}
+	for i := 0; i < len(cloudProviderList.Data); i++ {
+		provider := cloudProviderList.Data[i]
+		status, err := provider.GetString("status")
+		if err != nil {
+			return errors.Wrap(err, "provider get status error")
+		}
+		if status == "connected" {
+			providerStruct := SProvider{}
+			err := provider.Unmarshal(&providerStruct)
+			if err != nil {
+				return errors.Wrap(err, "provider.Unmarshal")
+			}
+			err = (&providerStruct).Validate()
+			if err != nil {
+				return errors.Wrap(err, "provider Invalidate")
+			}
+			providerStr := providerStruct.Provider
+			cloudReportFactory, err := GetCloudReportFactory(providerStr)
+			if err != nil {
+				log.Errorln(errors.Wrap(err, "GetCloudReportFactory"))
+				continue
+			}
+			tmpCount++
+			providerGroup.Go(func() error {
+				err = cloudReportFactory.NewCloudReport(&providerStruct, session, args, operatorType).Report()
+				if err != nil {
+					log.Errorln(errors.Wrap(err, "cloudReport Report method"))
+				}
+				return nil
+			})
+			if tmpCount == args.Count {
+				providerGroup.Wait()
+				tmpCount = 0
+			}
+		}
+	}
+	providerGroup.Wait()
+	return nil
+}
+
+func newGroupFunc(func()) {
+
+}
+
+func ReportCustomizeCloudMetric(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error {
+	cloudReportFactory, err := GetCloudReportFactory(operatorType)
+	if err != nil {
+		return errors.Wrap(err, "GetCloudReportFactory")
+	}
+	err = cloudReportFactory.NewCloudReport(nil, session, args, operatorType).Report()
+	if err != nil {
+		return errors.Wrap(err, "cloudReport Report method")
+	}
+	return nil
+}
+
+func CollectRegionMetricAsync(asynCount int, region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject, report ICloudReport) error {
+	metricGroup, _ := errgroup.WithContext(context.Background())
+	count := 0
+	if asynCount == 0 {
+		asynCount = 10
+	}
+	for i, _ := range servers {
+		server := servers[i]
+		metricGroup.Go(func() error {
+			return report.CollectRegionMetric(region, []jsonutils.JSONObject{server})
+		})
+		count++
+		if count == asynCount {
+			err := metricGroup.Wait()
+			if err != nil {
+				return err
+			}
+			count = 0
+		}
+	}
+	return metricGroup.Wait()
+}

pkg/cloudmon/collectors/doc.go

@@ -0,0 +1 @@
+package collectors // import "yunion.io/x/onecloud/pkg/cloudmon/collectors"

pkg/cloudmon/collectors/ecloudmon/doc.go

@@ -0,0 +1 @@
+package ecloudmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/ecloudmon"

pkg/cloudmon/collectors/ecloudmon/ecloudreport.go

@@ -0,0 +1,62 @@
+package ecloudmon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SECloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SECloudReportFactory struct {
+}
+
+func (self *SECloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SECloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SECloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_ECLOUD
+}
+
+type SECloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SECloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/ecloudmon/ecloudservice.go

@@ -0,0 +1,113 @@
+package ecloudmon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/ecloud"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+const (
+	SERVER_PRODUCT_TYPE = "vm"
+)
+
+func (self *SECloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	ecloudReg := region.(*ecloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, err := server.GetString("external_id")
+		if err != nil {
+			continue
+		}
+		name, _ := server.GetString("name")
+		external_id = strings.Split(external_id, "/")[1]
+		metrics := make([]ecloud.Metric, 0)
+		for metricName, _ := range ecloudMetricSpecs {
+			metrics = append(metrics, ecloud.Metric{Name: metricName})
+		}
+		data, err := ecloudReg.DescribeMetricList(SERVER_PRODUCT_TYPE, metrics, external_id, since, until)
+		if err != nil {
+			return errors.Wrap(err, "region DescribeMetricList err:")
+		}
+		for _, entity := range data.Entitys {
+			if influxDbSpecs, ok := ecloudMetricSpecs[entity.MetricName]; ok {
+				metricData, err := self.collectMetricFromThisServer(server, entity, influxDbSpecs)
+				if err != nil {
+					log.Errorf("server:%s collectMetric:%s err", name, entity.MetricName)
+					continue
+				}
+				dataList = append(dataList, metricData...)
+			}
+
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SECloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	entity ecloud.Entity, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	for _, point := range entity.Datapoints {
+		metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		if err != nil {
+			return nil, err
+		}
+		if len(point) != 2 {
+			log.Errorf("invalid point:%v", point)
+			continue
+		}
+		influxDbSpec := influxDbSpecs[2]
+
+		measurement := common.SubstringBefore(influxDbSpec, ".")
+		metric.Name = measurement
+		pointTime, err := strconv.ParseInt(point[1], 10, 64)
+		if err != nil {
+			return nil, errors.Wrap(err, "point parseInt err")
+		}
+		metric.Timestamp = time.Unix(pointTime, 0)
+		pointVal, err := strconv.ParseFloat(point[0], 64)
+		if err != nil {
+			return nil, errors.Wrap(err, "point parseInt err")
+		}
+		if influxDbSpecs[1] == UNIT_BYTEPS {
+			pointVal = pointVal * 8
+		}
+		if influxDbSpecs[1] == UNIT_KBYTEPS {
+			pointVal = pointVal * 8 * 1024
+		}
+		var pairsKey string
+		if strings.Contains(influxDbSpec, ",") {
+			pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+		} else {
+			pairsKey = common.SubstringAfter(influxDbSpec, ".")
+		}
+
+		tag := common.SubstringAfter(influxDbSpec, ",")
+		if tag != "" && strings.Contains(influxDbSpec, "=") {
+			metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+				Key:   common.SubstringBefore(tag, "="),
+				Value: common.SubstringAfter(tag, "="),
+			})
+		}
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   pairsKey,
+			Value: strconv.FormatFloat(pointVal, 'f', -1, 64),
+		})
+		self.AddMetricTag(&metric, common.AddTags)
+		datas = append(datas, metric)
+	}
+	return datas, nil
+}

pkg/cloudmon/collectors/ecloudmon/ecloudutils.go

@@ -0,0 +1,42 @@
+package ecloudmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_CPS           = "cps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_KBYTEPS       = "KBps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE       = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE       = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS   = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS  = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS  = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX      = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_TX      = "vm_netio.bps_sent"
+
+	KEY_LIMIT = "limit"
+	KEY_ADMIN = "admin"
+)
+
+var ecloudProTypeMetric = map[string]string{}
+
+//multiCloud查询指标列表组装
+var ecloudMetricSpecs = map[string][]string{
+	"cpu_util":                        {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"memory.util":                     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
+	"disk.device.read.bytes.rate":     {DEFAULT_STATISTICS, UNIT_KBYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"disk.device.write.bytes.rate":    {DEFAULT_STATISTICS, UNIT_KBYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"disk.device.read.requests.rate":  {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"disk.device.write.requests.rate": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+	"network.incoming.bytes":          {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX},
+	"network.outgoing.bytes":          {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX},
+}

pkg/cloudmon/collectors/elbservice.go

@@ -0,0 +1,16 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-elb", "Report Elb", reportElb)
+}
+
+//入口函数[aliyun、huawei]
+func reportElb(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.ELB), session, args)
+}

pkg/cloudmon/collectors/gcpmon/doc.go

@@ -0,0 +1 @@
+package gcpmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon"

pkg/cloudmon/collectors/gcpmon/gcpreport.go

@@ -0,0 +1,70 @@
+package gcpmon
+
+import (
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SGoogleCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SGoogleCloudReportFactory struct {
+}
+
+func (self *SGoogleCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SGoogleCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SGoogleCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_GOOGLE
+}
+
+type SGoogleCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SGoogleCloudReport) Report() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+			//case "redis":
+			//	err = self.collectRegionMetricOfRedis(region, servers)
+			//case "rds":
+			//	err = self.collectRegionMetricOfRds(region, servers)
+			//case "oss":
+			//	err = self.collectRegionMetricOfOss(region, servers)
+			//case "elb":
+			//	err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/gcpmon/gcpservice.go

@@ -0,0 +1,143 @@
+package gcpmon
+
+import (
+	"strconv"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/util/timeutils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/google"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SGoogleCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	googleReg := region.(*google.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, err := server.GetString("external_id")
+		if err != nil {
+			continue
+		}
+		serverName, _ := server.GetString("name")
+		external_id = strings.Split(external_id, "/")[1]
+		for metricName, influxDbSpecs := range gcpMetricSpecs {
+			rtn, err := googleReg.GetMonitorData(external_id, serverName, metricName, since, until)
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+			if err != nil {
+				return err
+			}
+			dataList = append(dataList, metric)
+			for _, resp := range rtn.Value() {
+				if !resp.(*jsonutils.JSONDict).Contains("points") {
+					continue
+				}
+				serverMetric, err := self.collectMetricFromThisServer(server, resp, influxDbSpecs)
+				if err != nil {
+					return err
+				}
+				dataList = append(dataList, serverMetric...)
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SGoogleCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	rtnMetric jsonutils.JSONObject, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	points_, _ := rtnMetric.Get("points")
+	for _, point := range points_.(*jsonutils.JSONArray).Value() {
+		metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		if err != nil {
+			return nil, err
+		}
+		if time, err := point.Get("interval", "startTime"); err == nil {
+			influxDbSpec := influxDbSpecs[2]
+
+			measurement := common.SubstringBefore(influxDbSpec, ".")
+			metric.Name = measurement
+
+			timestamp, err := timeutils.ParseTimeStr(time.(*jsonutils.JSONString).Value())
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			metric.Timestamp = timestamp
+
+			var pairsKey string
+			if strings.Contains(influxDbSpec, ",") {
+				pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+			} else {
+				pairsKey = common.SubstringAfter(influxDbSpec, ".")
+			}
+
+			tag := common.SubstringAfter(influxDbSpec, ",")
+			if tag != "" && strings.Contains(influxDbSpec, "=") {
+				metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+					Key:   common.SubstringBefore(tag, "="),
+					Value: common.SubstringAfter(tag, "="),
+				})
+			}
+
+			cpu_cout, err := server.Get("vcpu_count")
+			if err == nil {
+				metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+					Key:   "cpu_count",
+					Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+				})
+			}
+
+			valueTypeObj, _ := rtnMetric.Get("valueType")
+			valueType := strings.ToLower(valueTypeObj.(*jsonutils.JSONString).Value())
+			value := getMetricValue(point, valueType, influxDbSpecs[1])
+			metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+				Key:   pairsKey,
+				Value: value,
+			})
+			self.AddMetricTag(&metric, common.AddTags)
+			datas = append(datas, metric)
+		}
+	}
+	return datas, nil
+}
+
+func getMetricValue(point jsonutils.JSONObject, valueType string,
+	influxDbSpec string) string {
+	value, _ := point.Get("value", valueType+"Value")
+
+	switch valueType {
+	case "int64":
+		if val, ok := value.(*jsonutils.JSONString); ok {
+			fieldValue, _ := strconv.ParseInt(val.Value(), 10, 64)
+			if influxDbSpec == UNIT_MEM {
+				fieldValue = fieldValue * 8 / PERIOD
+			}
+			return strconv.FormatInt(fieldValue, 10)
+		}
+		fieldValue := value.(*jsonutils.JSONInt).Value()
+		if influxDbSpec == UNIT_MEM {
+			fieldValue = fieldValue * 8 / PERIOD
+		}
+		return strconv.FormatInt(fieldValue, 10)
+	case "double":
+		fieldValue := value.(*jsonutils.JSONFloat).Value()
+		if influxDbSpec == UNIT_MEM {
+			fieldValue = fieldValue * 8 / PERIOD
+		}
+		return strconv.FormatFloat(fieldValue, 'f', 3, 64)
+	}
+	return ""
+}

pkg/cloudmon/collectors/gcpmon/gcputils.go

@@ -0,0 +1,117 @@
+package gcpmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var gcpMetricSpecs = map[string][]string{
+	"compute.googleapis.com/instance/cpu/utilization":              {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"compute.googleapis.com/instance/network/received_bytes_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"compute.googleapis.com/instance/network/sent_bytes_count":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+	"compute.googleapis.com/instance/disk/read_bytes_count":        {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"compute.googleapis.com/instance/disk/write_bytes_count":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"compute.googleapis.com/instance/disk/read_ops_count":          {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"compute.googleapis.com/instance/disk/write_ops_count":         {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}

pkg/cloudmon/collectors/hostservice.go

@@ -0,0 +1,16 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-host", "Report Host", reportHost)
+}
+
+//入口函数[aliyun、huawei]
+func reportHost(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.HOST), session, args)
+}

pkg/cloudmon/collectors/huaweimon/doc.go

@@ -0,0 +1 @@
+package huaweimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon"

pkg/cloudmon/collectors/huaweimon/huaweicloudreport.go

@@ -0,0 +1,85 @@
+package huaweimon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SHwCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SHwCloudReportFactory struct {
+}
+
+func (self *SHwCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SHwCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SHwCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_HUAWEI
+}
+
+type SHwCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SHwCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/huaweimon/hwservice.go

@@ -0,0 +1,327 @@
+package huaweimon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/huawei"
+	hw_moudules "yunion.io/x/onecloud/pkg/multicloud/huawei/client/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SHwCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		instanceId, _ := server.GetString("external_id")
+		metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+		if err != nil {
+			return err
+		}
+		dataList = append(dataList, metric)
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.ECS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "instance_id", Value: instanceId})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		instanceId, _ := server.GetString("external_id")
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiRedisMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.DCS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "dcs_instance_id", Value: instanceId})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			return err
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiRedisMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		instanceId, _ := server.GetString("external_id")
+		engine, _ := server.GetString("engine")
+		metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+		if err != nil {
+			return err
+		}
+		dataList = append(dataList, metric)
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiRdsMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.RDS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			switch engine {
+			case "MySQL":
+				hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "rds_cluster_id", Value: instanceId})
+			case "PostgreSQL":
+				hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "postgresql_cluster_id", Value: instanceId})
+			case "SQLServer":
+				hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "rds_cluster_sqlserver_id", Value: instanceId})
+			}
+			metas = append(metas, hwMeta)
+		}
+		index := 0
+		tmp := 0
+		for {
+			if index > len(metas) {
+				break
+			}
+			tmp = index + 10
+			if tmp > len(metas) {
+				tmp = len(metas)
+			}
+			metricDatas, err := hwReg.GetMetricsData(metas[index:tmp], since, until)
+			if err != nil {
+				return err
+			}
+			if len(metricDatas) > 0 {
+				for _, metricData := range metricDatas {
+					for metricName, influxDbSpecs := range huaweiRdsMetricSpecs {
+						if metricData.MetricName == metricName {
+							if len(metricData.Datapoints) > 0 {
+								for _, datapoint := range metricData.Datapoints {
+									serverMetric, err := self.collectMetricFromThisServer(server, datapoint, influxDbSpecs)
+									if err != nil {
+										return err
+									}
+									dataList = append(dataList, serverMetric)
+								}
+							}
+						}
+					}
+
+				}
+			}
+			index += 10
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		name, _ := server.GetString("name")
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiOSSMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.OBS"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "bucket_name", Value: name})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			return err
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiOSSMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	hwReg := region.(*huawei.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		external_id, _ := server.GetString("external_id")
+		metas := make([]hw_moudules.SMetricMeta, 0)
+		for metricName := range huaweiOSSMetricSpecs {
+			hwMeta := hw_moudules.SMetricMeta{}
+			hwMeta.MetricName = metricName
+			hwMeta.Namespace = "SYS.ELB"
+			hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0)
+			hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "lb_instance_id", Value: external_id})
+			metas = append(metas, hwMeta)
+		}
+
+		metricDatas, err := hwReg.GetMetricsData(metas, since, until)
+		if err != nil {
+			return err
+		}
+		if len(metricDatas) > 0 {
+			for _, metricData := range metricDatas {
+				for metricName, influxDbSpecs := range huaweiOSSMetricSpecs {
+					if metricData.MetricName == metricName {
+						if len(metricData.Datapoints) > 0 {
+							for _, datapoint := range metricData.Datapoints {
+								serverMetric, err := self.collectMetricFromThisServer(server, datapoint,
+									influxDbSpecs)
+								if err != nil {
+									return err
+								}
+								dataList = append(dataList, serverMetric)
+							}
+						}
+					}
+				}
+
+			}
+		}
+
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SHwCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, datapoint hw_moudules.SDatapoint,
+	influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(server)
+	//metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	if err != nil {
+		return influxdb.SMetricData{}, err
+	}
+	metric.Timestamp = time.Unix(datapoint.Timestamp/1000, 0)
+	fieldValue := datapoint.Average
+	//根据条件拼装metric的tag和metirc信息
+	influxDbSpec := influxDbSpecs[2]
+	measurement := common.SubstringBefore(influxDbSpec, ".")
+	var pairsKey string
+	if strings.Contains(influxDbSpec, ",") {
+		pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpec, ".")
+	}
+	if influxDbSpecs[1] == UNIT_BYTEPS {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpec, ",")
+	if tag != "" && strings.Contains(influxDbSpec, "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	cpu_cout, err := server.Get("vcpu_count")
+	if err == nil {
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   "cpu_count",
+			Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	metric.Name = measurement
+	self.AddMetricTag(&metric, common.OtherVmTags)
+	return metric, nil
+}

pkg/cloudmon/collectors/huaweimon/hwutils.go

@@ -0,0 +1,162 @@
+package huaweimon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+)
+
+var huaweiMetricSpecs = map[string][]string{
+	"cpu_util":                              {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},          //CPU使用率，该指标用于统计测量对象的CPU使用率，以百分为单位。
+	"network_incoming_bytes_aggregate_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET}, //带外网络流入速率，该指标用于在虚拟化层统计每秒流入测量对象的网络流量，以字节/秒为单位。
+	"network_outgoing_bytes_aggregate_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET}, //带外网络流出速率，该指标用于在虚拟化层统计每秒流出测量对象的网络流量，以字节/秒为单位。
+	"disk_read_bytes_rate":                  {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS},       //磁盘读速率，该指标用于统计每秒从测量对象读出数据量，以字节/秒为单位。
+	"disk_write_bytes_rate":                 {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS},      //磁盘写速率，该指标用于统计每秒写到测量对象的数据量，以字节/秒为单位。
+	"disk_read_requests_rate":               {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS},         //该指标用于统计每秒从测量对象读取数据的请求次数，以请求/秒为单位。
+	"disk_write_requests_rate":              {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS},        //该指标用于统计每秒从测量对象写数据的请求次数，以请求/秒为单位
+}
+
+var huaweiRdsMetricSpecs = map[string][]string{
+	"rds001_cpu_util":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"rds002_mem_util":             {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"rds004_bytes_in":             {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_RX},
+	"rds005_bytes_out":            {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_TX},
+	"rds039_disk_util":            {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"rds049_disk_read_throughput": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_DISK_READ_BPS},
+
+	"rds050_disk_write_throughput": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_DISK_WRITE_BPS},
+
+	"rds006_conn_count": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_COUNT},
+
+	"rds008_qps": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_QPS},
+	"rds009_tps": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_TPS},
+
+	"rds013_innodb_reads":  {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_REDA_BPS},
+	"rds014_innodb_writes": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS},
+}
+
+var huaweiRedisMetricSpecs = map[string][]string{
+	"cpu_usage":                 {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"memory_usage":              {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"instantaneous_input_kbps":  {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"instantaneous_output_kbps": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"connected_clients":         {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"instantaneous_ops":         {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"keys":                      {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"expires":                   {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"used_memory_dataset":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+
+var huaweiOSSMetricSpecs = map[string][]string{
+	"download_bytes":     {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX},
+	"upload_bytes":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX},
+	"first_byte_latency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET},
+	"get_request_count":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET},
+	"request_count_4xx":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_4XX},
+	"request_count_5xx":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX},
+}
+
+var huaweiElbMetricSpecs = map[string][]string{
+	"m7_in_Bps":      {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_ELB_NET_BPS_RX},
+	"m8_out_Bps":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_ELB_NET_BPS_TX},
+	"mb_l7_qps":      {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_REQ_RATE},
+	"mc_l7_http_2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX},
+	"md_l7_http_3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX},
+	"me_l7_http_4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX},
+	"mf_l7_http_5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX},
+}

pkg/cloudmon/collectors/ossservice.go

@@ -0,0 +1,14 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-oss", "Report Oss", reportOss)
+}
+func reportOss(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.OSS), session, args)
+}

pkg/cloudmon/collectors/pinger.go

@@ -0,0 +1,212 @@
+package collectors
+
+import (
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/netutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+	"yunion.io/x/onecloud/pkg/util/sysutils"
+)
+
+type PingProbeOptions struct {
+	Debug         bool `help:"debug"`
+	ProbeCount    int  `help:"probe count, default is 3" default:"3"`
+	TimeoutSecond int  `help:"probe timeout in second, default is 1 second" default:"1"`
+}
+
+func pingProbeCoolector(s *mcclient.ClientSession, args *PingProbeOptions) error {
+	isRoot := sysutils.IsRootPermission()
+	if !isRoot {
+		return errors.Error("require root permissions")
+	}
+	metrics := make([]influxdb.SMetricData, 0)
+	params := jsonutils.NewDict()
+	params.Add(jsonutils.NewString(api.CLOUD_ENV_ON_PREMISE), "cloud_env")
+	params.Add(jsonutils.NewString(string(rbacutils.ScopeSystem)), "scope")
+	params.Add(jsonutils.JSONTrue, "is_classic")
+	listAll(s, modules.Networks.List, params, func(data jsonutils.JSONObject) error {
+		m, err := pingProbeNetwork(s, data, args)
+		if err != nil {
+			return err
+		}
+		if len(m) > 0 {
+			metrics = append(metrics, m...)
+		}
+		return nil
+	})
+	return sendMetrics(s, metrics, args.Debug)
+}
+
+type sNetwork struct {
+	Id           string
+	Name         string
+	GuestGateway string
+	GuestIpStart string
+	GuestIpEnd   string
+	Region       string
+	RegionId     string
+	ServerType   string
+	Vpc          string
+	VpcId        string
+	Wire         string
+	WireId       string
+	Zone         string
+	ZoneId       string
+	CloudEnv     string
+}
+
+func getNetworkAddrMap(s *mcclient.ClientSession, netId string) (map[string]api.SNetworkUsedAddress, error) {
+	addrListJson, err := modules.Networks.GetSpecific(s, netId, "addresses", nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetSpecific addresses")
+	}
+	addrList := make([]api.SNetworkUsedAddress, 0)
+	err = addrListJson.Unmarshal(&addrList, "addresses")
+	if err != nil {
+		return nil, errors.Wrap(err, "Unmarshal addreses")
+	}
+	addrMap := make(map[string]api.SNetworkUsedAddress)
+	for i := range addrList {
+		addrMap[addrList[i].IpAddr] = addrList[i]
+	}
+	return addrMap, nil
+}
+
+func pingProbeNetwork(s *mcclient.ClientSession, data jsonutils.JSONObject, args *PingProbeOptions) ([]influxdb.SMetricData, error) {
+	metrics := make([]influxdb.SMetricData, 0)
+	net := &sNetwork{}
+	err := data.Unmarshal(&net)
+	if err != nil {
+		log.Errorf("Unmarshal network %s: %s", data, err)
+		return nil, errors.Wrap(err, "Unmarshal network")
+	}
+	if net.CloudEnv != api.CLOUD_ENV_ON_PREMISE {
+		log.Errorf("not an onpremise network: %s", data)
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "not onpremise network")
+	}
+	if net.GuestGateway == "" {
+		log.Errorf("no valid gateway %s", data)
+		return nil, errors.Wrap(errors.ErrInvalidStatus, "unreachable network, empty gateway")
+	}
+	addrStart, err := netutils.NewIPV4Addr(net.GuestIpStart)
+	if err != nil {
+		log.Errorf("unmarshal start address %s: %s", net.GuestIpStart, err)
+		return nil, errors.Wrapf(err, "NewIPV4Addr %s", net.GuestIpStart)
+	}
+	addrEnd, err := netutils.NewIPV4Addr(net.GuestIpEnd)
+	if err != nil {
+		log.Errorf("unmarshal end address %s: %s", net.GuestIpEnd, err)
+		return nil, errors.Wrapf(err, "NewIPV4Addr %s", net.GuestIpEnd)
+	}
+	log.Infof("ping address %s - %s", addrStart, addrEnd)
+	pingAddrs := make([]string, 0)
+	for addr := addrStart; addr <= addrEnd; addr = addr.StepUp() {
+		addrStr := addr.String()
+		pingAddrs = append(pingAddrs, addrStr)
+	}
+	pingResults, err := Ping(pingAddrs, args.ProbeCount, time.Second*time.Duration(args.TimeoutSecond), args.Debug)
+	if err != nil {
+		return nil, errors.Wrap(err, "Ping")
+	}
+
+	addrMap, err := getNetworkAddrMap(s, net.Id)
+	if err != nil {
+		return nil, errors.Wrap(err, "getNetworkAddrMap")
+	}
+
+	now := time.Now().UTC()
+	for addr := addrStart; addr <= addrEnd; addr = addr.StepUp() {
+		addrStr := addr.String()
+		pingResult := pingResults[addrStr]
+		netAddr, allocated := addrMap[addrStr]
+		if allocated {
+			if netAddr.OwnerType == api.RESERVEDIP_RESOURCE_TYPES {
+				loss := pingResult.Loss()
+				status := api.RESERVEDIP_STATUS_OFFLINE
+				if loss < 100 {
+					status = api.RESERVEDIP_STATUS_ONLINE
+				}
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewString(status), "status")
+				_, err := modules.ReservedIPs.Update(s, netAddr.OwnerId, params)
+				if err != nil {
+					log.Errorf("update reserved ip %s status fail: %s", addrStr, err)
+				}
+			} else {
+				// send metrics
+				metric := influxdb.SMetricData{}
+				metric.Name = "ping"
+				metric.Timestamp = now
+				metric.Tags = []influxdb.SKeyValue{
+					{
+						Key:   "ip_addr",
+						Value: addrStr,
+					},
+					{
+						Key:   "owner_type",
+						Value: netAddr.OwnerType,
+					},
+					{
+						Key:   "owner_id",
+						Value: netAddr.OwnerId,
+					},
+					{
+						Key:   "owner",
+						Value: netAddr.Owner,
+					},
+				}
+				loss := pingResult.Loss()
+				max, avg, min := pingResult.Rtt()
+				metric.Metrics = []influxdb.SKeyValue{
+					{
+						Key:   "loss",
+						Value: strconv.FormatInt(int64(loss), 10),
+					},
+					{
+						Key:   "rtt_ms_avg",
+						Value: strconv.FormatInt(int64(avg/time.Millisecond), 10),
+					},
+					{
+						Key:   "rtt_ms_max",
+						Value: strconv.FormatInt(int64(max/time.Millisecond), 10),
+					},
+					{
+						Key:   "rtt_ms_min",
+						Value: strconv.FormatInt(int64(min/time.Millisecond), 10),
+					},
+				}
+				metrics = append(metrics, metric)
+			}
+		} else {
+			loss := pingResult.Loss()
+			if loss < 100 {
+				// reserve ip
+				log.Debugf("Free address %s is responding ping, reserve the address", addrStr)
+				params := jsonutils.NewDict()
+				params.Add(jsonutils.NewStringArray([]string{addrStr}), "ips")
+				params.Add(jsonutils.NewString("ping detect online free IP"), "notes")
+				params.Add(jsonutils.NewString(api.RESERVEDIP_STATUS_ONLINE), "status")
+				_, err := modules.Networks.PerformAction(s, net.Id, "reserve-ip", params)
+				if err != nil {
+					log.Errorf("failed to reserve ip %s: %s", addrStr, err)
+				}
+			}
+		}
+		log.Debugf("%s %s allocated %v", addrStr, netAddr, allocated)
+	}
+	return metrics, nil
+}
+
+func init() {
+	shellutils.R(&PingProbeOptions{}, "ping-probe", "Ping probe IPv4 address", pingProbeCoolector)
+}

pkg/cloudmon/collectors/pingutils.go

@@ -0,0 +1,88 @@
+package collectors
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/tatsushid/go-fastping"
+
+	"yunion.io/x/pkg/errors"
+)
+
+type SPingResult struct {
+	addr  string
+	rtt   []time.Duration
+	count int
+}
+
+func NewPingResult(addr string, count int) *SPingResult {
+	return &SPingResult{
+		addr:  addr,
+		rtt:   make([]time.Duration, 0),
+		count: count,
+	}
+}
+
+func (pr *SPingResult) Add(rtt time.Duration) {
+	pr.rtt = append(pr.rtt, rtt)
+}
+
+func (pr SPingResult) Rtt() (min time.Duration, avg time.Duration, max time.Duration) {
+	sum := time.Duration(0)
+	max = time.Duration(-1)
+	min = time.Duration(-1)
+	for _, d := range pr.rtt {
+		sum += d
+		if max < 0 || max < d {
+			max = d
+		}
+		if min < 0 || min > d {
+			min = d
+		}
+	}
+	if len(pr.rtt) > 0 {
+		avg = sum / time.Duration(len(pr.rtt))
+	}
+	return
+}
+
+func (pr SPingResult) Loss() int {
+	return 100 - len(pr.rtt)*100/pr.count
+}
+
+func (pr SPingResult) String() string {
+	min, avg, max := pr.Rtt()
+	return fmt.Sprintf("%d packets transmitted, %d received, %d%% packet loss, rtt min/avg/max = %d/%d/%d ms", pr.count, len(pr.rtt), pr.Loss(), min/time.Millisecond, avg/time.Millisecond, max/time.Millisecond)
+}
+
+func Ping(addrList []string, count int, timeout time.Duration, debug bool) (map[string]*SPingResult, error) {
+	p := fastping.NewPinger()
+	p.MaxRTT = timeout
+	p.Size = 64
+	p.Debug = debug
+	result := make(map[string]*SPingResult)
+	for _, addr := range addrList {
+		result[addr] = NewPingResult(addr, count)
+		p.AddIP(addr)
+	}
+	p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
+		result[addr.String()].Add(rtt)
+	}
+	p.OnIdle = func() {
+	}
+	p.RunLoop()
+	defer p.Stop()
+	ticker := time.NewTicker(time.Duration(count) * timeout)
+	defer ticker.Stop()
+	select {
+	case <-p.Done():
+		if err := p.Err(); err != nil {
+			return nil, errors.Wrap(err, "ping error")
+		}
+		break
+	case <-ticker.C:
+		break
+	}
+	return result, nil
+}

pkg/cloudmon/collectors/pingutils_test.go

@@ -0,0 +1,23 @@
+package collectors
+
+import (
+	"testing"
+	"time"
+)
+
+func TestPing(t *testing.T) {
+	result, err := Ping([]string{
+		"114.114.114.114",
+		"118.187.65.237",
+		"10.168.26.254",
+		"10.168.26.26",
+		"192.30.253.113",
+	}, 5, time.Second, true)
+	if err != nil {
+		// ignore error
+		t.Logf("ping error %s", err)
+	}
+	for k, v := range result {
+		t.Logf("%s: %s", k, v.String())
+	}
+}

pkg/cloudmon/collectors/qcmon/doc.go

@@ -0,0 +1 @@
+package qcmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon"

pkg/cloudmon/collectors/qcmon/qcloudreport.go

@@ -0,0 +1,85 @@
+package qcmon
+
+import (
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SQCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SQCloudReportFactory struct {
+}
+
+func (self *SQCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SQCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SQCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_QCLOUD
+}
+
+type SQCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SQCloudReport) Report() error {
+	var servers []jsonutils.JSONObject
+	var err error
+	switch self.Operator {
+	case "redis":
+		servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache)
+	case "rds":
+		servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance)
+	case "oss":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Buckets)
+	case "elb":
+		servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers)
+	default:
+		servers, err = self.GetAllserverOfThisProvider(&modules.Servers)
+	}
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		servers := regionServerMap[region.GetGlobalId()]
+		switch self.Operator {
+		case "server":
+			err = self.collectRegionMetricOfHost(region, servers)
+		case "redis":
+			err = self.collectRegionMetricOfRedis(region, servers)
+		case "rds":
+			err = self.collectRegionMetricOfRds(region, servers)
+		case "oss":
+			//err = self.collectRegionMetricOfOss(region, servers)
+		case "elb":
+			//err = self.collectRegionMetricOfElb(region, servers)
+		}
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/qcmon/qcloudservice.go

@@ -0,0 +1,224 @@
+package qcmon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/qcloud"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SQCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	var err error
+	switch self.Operator {
+	case string(common.SERVER):
+		err = self.collectRegionMetricOfHost(region, servers)
+	}
+	return err
+}
+
+func (self *SQCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	tecentReg := region.(*qcloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	dimensions := make([]qcloud.SQcMetricDimension, 0)
+	for _, server := range servers {
+		external_id, _ := server.GetString("external_id")
+		dimensions = append(dimensions, qcloud.SQcMetricDimension{
+			Name:  "InstanceId",
+			Value: external_id,
+		})
+	}
+	//dimensions := []qcloud.SQcMetricDimension{qcloud.SQcMetricDimension{Name: "InstanceId", Value: external_id}}
+	for metricName, influxDbSpecs := range tecentMetricSpecs {
+		for index, tmp := 0, 0; index < len(dimensions); index += 10 {
+			tmp = index + 10
+			if tmp > len(dimensions) {
+				tmp = len(dimensions)
+			}
+			rtnArray, err := tecentReg.GetMonitorData(metricName, "QCE/CVM", since, until,
+				dimensions[index:tmp])
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if external_id == rtnMetric.Dimensions[0].Value {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						if len(rtnMetric.Timestamps) == 0 {
+							break
+						}
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric...)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SQCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	tecentReg := region.(*qcloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	dimensions := make([]qcloud.SQcMetricDimension, 0)
+	for _, server := range servers {
+		external_id, _ := server.GetString("external_id")
+		dimensions = append(dimensions, qcloud.SQcMetricDimension{
+			Name:  "instanceid",
+			Value: external_id,
+		})
+	}
+	//dimensions := []qcloud.SQcMetricDimension{qcloud.SQcMetricDimension{Name: "InstanceId", Value: external_id}}
+	for metricName, influxDbSpecs := range tecentRedisMetricSpecs {
+		for index, tmp := 0, 0; index < len(dimensions); index += 10 {
+			tmp = index + 10
+			if tmp > len(dimensions) {
+				tmp = len(dimensions)
+			}
+			rtnArray, err := tecentReg.GetMonitorData(metricName, "QCE/REDIS", since, until,
+				dimensions[index:tmp])
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if external_id == rtnMetric.Dimensions[0].Value {
+						if len(rtnMetric.Timestamps) == 0 {
+							break
+						}
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							serverName, _ := server.Get("name")
+							log.Errorf("redis:%s,collectMetricFromThisServer err:%v", serverName, err)
+							continue
+						}
+						dataList = append(dataList, serverMetric...)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SQCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	tecentReg := region.(*qcloud.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	dimensions := make([]qcloud.SQcMetricDimension, 0)
+	for _, server := range servers {
+		external_id, _ := server.GetString("external_id")
+		dimensions = append(dimensions, qcloud.SQcMetricDimension{
+			Name:  "InstanceId",
+			Value: external_id,
+		})
+	}
+	//dimensions := []qcloud.SQcMetricDimension{qcloud.SQcMetricDimension{Name: "InstanceId", Value: external_id}}
+	for metricName, influxDbSpecs := range tecentRdsMetricSpecs {
+		for index, tmp := 0, 0; index < len(dimensions); index += 10 {
+			tmp = index + 10
+			if tmp > len(dimensions) {
+				tmp = len(dimensions)
+			}
+			rtnArray, err := tecentReg.GetMonitorData(metricName, "QCE/CDB", since, until,
+				dimensions[index:tmp])
+			if err != nil {
+				log.Errorln(err)
+				continue
+			}
+			for _, rtnMetric := range rtnArray {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if external_id == rtnMetric.Dimensions[0].Value {
+						if len(rtnMetric.Timestamps) == 0 {
+							break
+						}
+						serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs)
+						if err != nil {
+							serverName, _ := server.Get("name")
+							log.Errorf("redis:%s,collectMetricFromThisServer err:%v", serverName, err)
+							continue
+						}
+						dataList = append(dataList, serverMetric...)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SQCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric qcloud.SDataPoint, influxDbSpecs []string) ([]influxdb.SMetricData, error) {
+	datas := make([]influxdb.SMetricData, 0)
+	for index, timestamp := range rtnMetric.Timestamps {
+		metric, err := self.NewMetricFromJson(server)
+		if err != nil {
+			return nil, err
+		}
+		//根据条件拼装metric的tag和metirc信息
+		influxDbSpec := influxDbSpecs[2]
+		measurement := common.SubstringBefore(influxDbSpec, ".")
+		metric.Name = measurement
+		var pairsKey string
+		if strings.Contains(influxDbSpec, ",") {
+			pairsKey = common.SubstringBetween(influxDbSpec, ".", ",")
+		} else {
+			pairsKey = common.SubstringAfter(influxDbSpec, ".")
+		}
+		tag := common.SubstringAfter(influxDbSpec, ",")
+		if tag != "" && strings.Contains(influxDbSpec, "=") {
+			metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+				Key:   common.SubstringBefore(tag, "="),
+				Value: common.SubstringAfter(tag, "="),
+			})
+		}
+		cpu_cout, err := server.Get("vcpu_count")
+		if err == nil {
+			metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+				Key:   "cpu_count",
+				Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10),
+			})
+		}
+		metric.Timestamp = time.Unix(int64(timestamp), 0)
+		fieldValue := rtnMetric.Values[index]
+		if influxDbSpecs[1] == UNIT_MBPS {
+			fieldValue = fieldValue * 1000 * 1000
+		}
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   pairsKey,
+			Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+		})
+		self.AddMetricTag(&metric, common.OtherVmTags)
+		datas = append(datas, metric)
+	}
+	return datas, nil
+}

pkg/cloudmon/collectors/qcmon/qcutils.go

@@ -0,0 +1,143 @@
+package qcmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_INTRANET  = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_sent"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_INTRANET  = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_active"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_percent"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+var tecentMetricSpecs = map[string][]string{
+	"CPUUsage":      {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"MemUsage":      {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
+	"lanOuttraffic": {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET},
+	"WanOuttraffic": {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET},
+	"lanIntraffic":  {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET},
+	"WanIntraffic":  {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET},
+}
+
+var tecentRedisMetricSpecs = map[string][]string{
+	"CpuUsMin":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE},
+	"StorageUsMin":     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE},
+	"InFlowMin":        {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX},
+	"OutFlowMin":       {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX},
+	"ConnectionsUsMin": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE},
+	"QpsMin":           {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES},
+	"KeysMin":          {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS},
+	"ExpiredKeysMin":   {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS},
+	"StorageMin":       {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE},
+}
+
+var tecentRdsMetricSpecs = map[string][]string{
+	"CPUUseRate":        {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE},
+	"MemoryUseRate":     {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE},
+	"BytesSent":         {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_INTRANET},
+	"BytesReceived":     {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_INTRANET},
+	"VolumeRate":        {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE},
+	"ThreadsConnected":  {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_RDS_CONN_COUNT},
+	"ConnectionUseRate": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE},
+	"QPS":               {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_RDS_QPS},
+	"TPS":               {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_RDS_TPS},
+	"InnodbDataRead":    {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_REDA_BPS},
+	"InnodbDataWritten": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS},
+}

pkg/cloudmon/collectors/rdsservice.go

@@ -0,0 +1,15 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-rds", "Report Rds", reporRds)
+}
+
+func reporRds(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.RDS), session, args)
+}

pkg/cloudmon/collectors/redisservice.go

@@ -0,0 +1,15 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-redis", "Report Redis", reportRedis)
+}
+
+func reportRedis(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.REDIS), session, args)
+}

pkg/cloudmon/collectors/serverservice.go

@@ -0,0 +1,16 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-server", "Report Server", reportServer)
+}
+
+//入口函数[aliyun、huawei]
+func reportServer(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCloudMetricOfoperatorType(string(common.SERVER), session, args)
+}

pkg/cloudmon/collectors/storagemon/doc.go

@@ -0,0 +1 @@
+package storagemon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon"

pkg/cloudmon/collectors/storagemon/storagereport.go

@@ -0,0 +1,44 @@
+package storagemon
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SCloudAccountFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SCloudAccountFactory struct {
+}
+
+func (self *SCloudAccountFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions,
+	operatorType string) common.ICloudReport {
+	return &SStorageReport{
+		common.CloudReportBase{
+			SProvider: nil,
+			Session:   session,
+			Args:      args,
+			Operator:  string(common.STORAGE),
+		},
+	}
+}
+
+func (S SCloudAccountFactory) GetId() string {
+	return string(common.STORAGE)
+}
+
+type SStorageReport struct {
+	common.CloudReportBase
+}
+
+func (self *SStorageReport) Report() error {
+	accounts, err := self.GetAllStorage(&modules.Storages)
+	if err != nil {
+		return err
+	}
+	return self.collectMetric(accounts)
+}

pkg/cloudmon/collectors/storagemon/storageservice.go

@@ -0,0 +1,50 @@
+package storagemon
+
+import (
+	"strconv"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SStorageReport) collectMetric(storages []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	for _, storage := range storages {
+		metric, err := self.collectMetricFromStorage(storage)
+		if err != nil {
+			return err
+		}
+		metric.Timestamp = time.Now()
+		metric.Name = STORAGE_MEASUREMENT
+		dataList = append(dataList, metric)
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SStorageReport) collectMetricFromStorage(storage jsonutils.JSONObject) (influxdb.SMetricData, error) {
+	metric, err := self.NewMetricFromJson(storage)
+	if err != nil {
+		return metric, errors.Wrap(err, "collectMetricFromStorage NewMetricFromJson err")
+	}
+	capacity, _ := storage.Float("capacity")
+	actUsedCapacity, _ := storage.Float("actual_capacity_used")
+	var actFreeCapacity = float64(0)
+	var capacityUsage = float64(0)
+	if capacity != 0 {
+		actFreeCapacity = capacity - actUsedCapacity
+		capacityUsage = actUsedCapacity / capacity * 100
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   STORAGE_FIELD_USAGE,
+		Value: strconv.FormatFloat(capacityUsage, 'f', 2, 64),
+	}, influxdb.SKeyValue{
+		Key:   STORAGE_FIELD_FREE,
+		Value: strconv.FormatFloat(actFreeCapacity, 'f', -1, 64),
+	})
+	return metric, nil
+
+}

pkg/cloudmon/collectors/storagemon/storageutil.go

@@ -0,0 +1,10 @@
+package storagemon
+
+const (
+	STORAGE_ID = "storage"
+
+	CLOUDACCOUNT_FIELD  = "balance"
+	STORAGE_MEASUREMENT = "storage"
+	STORAGE_FIELD_USAGE = "usage_active"
+	STORAGE_FIELD_FREE  = "free"
+)

pkg/cloudmon/collectors/storageservice.go

@@ -0,0 +1,15 @@
+package collectors
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-storage", "Report Storage", reportStorage)
+}
+
+func reportStorage(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	return common.ReportCustomizeCloudMetric(string(common.STORAGE), session, args)
+}

pkg/cloudmon/collectors/usageservice.go

@@ -0,0 +1,150 @@
+package collectors
+
+import (
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+var config map[string]string = map[string]string{
+	"hypervisors":           "host-type",
+	"compute_engine_brands": "provider",
+}
+var measureMent string = "usage"
+
+func init() {
+	shellutils.R(&common.ReportOptions{}, "report-usage", "Report Usage", reportUsage)
+}
+
+func reportUsage(session *mcclient.ClientSession, args *common.ReportOptions) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	nowTime := time.Now()
+	//镜像使用量信息
+	imageUsageFields, err := getImageUsageFields(session)
+	if err != nil {
+		return err
+	}
+	//查询到的Usage信息统一放置在metric中
+	imageUsageFieldsDict := imageUsageFields.(*jsonutils.JSONDict)
+	capabilitesQuery := jsonutils.NewDict()
+	capabilitesQuery.Add(jsonutils.NewString("system"), "scope")
+	capabilites, err := modules.Capabilities.List(session, capabilitesQuery)
+	if err != nil {
+		return err
+	}
+	//通过capabilities中的信息遍历hypevisors和brands
+	for i := 0; i < len(capabilites.Data); i++ {
+		capabilitesObj := capabilites.Data[i]
+		capDict, ok := capabilitesObj.(*jsonutils.JSONDict)
+		if !ok {
+			return errors.ErrClient
+		}
+		for _, capKey := range capDict.SortedKeys() {
+			if _, ok := config[capKey]; ok {
+				hypeOrBrandObj, _ := capDict.Get(capKey)
+				if hypeOrBrandObj != nil {
+					hypeOrBrandArr, _ := hypeOrBrandObj.(*jsonutils.JSONArray)
+					for i := 0; i < len(hypeOrBrandArr.Value()); i++ {
+						hypeOrBrand := hypeOrBrandArr.Value()[i].(*jsonutils.JSONString)
+						dataList, err = packMetricList(session, dataList, imageUsageFieldsDict, config[capKey],
+							hypeOrBrand.String(), nowTime)
+						if err != nil {
+							return err
+						}
+					}
+				}
+			}
+		}
+	}
+	//查询host-type==""的情况，对应onecloud-控制面板-全部 要展示的内容
+	dataList, err = packMetricList(session, dataList, imageUsageFieldsDict, "host-type", "", nowTime)
+	//写入influDb
+	return sendMetrics(session, dataList, args.Debug)
+}
+
+//根据capabilities中的hypevisors和brands中的对应属性，组装Metric
+func packMetricList(session *mcclient.ClientSession, dataList []influxdb.SMetricData,
+	imageUsageFieldsDict *jsonutils.JSONDict, paramKey string,
+	paramValue string, nowTime time.Time) (rtnList []influxdb.SMetricData, err error) {
+	//query，sql信息，查询主机compute的使用信息
+	query := jsonutils.NewDict()
+	query.Add(jsonutils.NewString("system"), "scope")
+	if paramValue != "" {
+		if paramKey == "host-type" {
+			if paramValue == "kvm" {
+				query.Add(jsonutils.NewString("hypervisor"), paramKey)
+			}
+			query.Add(jsonutils.NewString(paramValue), paramKey)
+		}
+		query.Add(jsonutils.NewString(paramValue), paramKey)
+	}
+	metric := &influxdb.SMetricData{Name: measureMent, Timestamp: nowTime}
+	//查询到的镜像的使用信息放到SMetricData中的metric
+	metric, _ = jsonTometricData(imageUsageFieldsDict, metric, "metric")
+	//compute主机使用量
+	respObj, err := modules.Usages.GetGeneralUsage(session, query)
+	if err != nil {
+		return nil, err
+	}
+	respObjDict := respObj.(*jsonutils.JSONDict)
+	//查询到的主机的使用信息放到SMetricData中的metric
+	metric, _ = jsonTometricData(respObjDict, metric, "metric")
+	if paramValue != "" {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key: paramKey, Value: paramValue,
+		})
+	} else {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key: paramKey, Value: "all",
+		})
+	}
+	dataList = append(dataList, *metric)
+	return dataList, nil
+}
+
+//获得镜像使用量
+func getImageUsageFields(session *mcclient.ClientSession) (jsonutils.JSONObject, error) {
+	respObj, e := (&modules.ImageUsages).GetUsage(session, nil)
+	if e != nil {
+		return nil, e
+	}
+	respDict, ok := respObj.(*jsonutils.JSONDict)
+	if !ok {
+		return nil, jsonutils.ErrInvalidJsonDict
+	}
+	return respDict, nil
+}
+
+//将JSONDict的信息放置到SMetricData中
+func jsonTometricData(obj *jsonutils.JSONDict, metric *influxdb.SMetricData,
+	metricDataType string) (*influxdb.SMetricData, error) {
+
+	objMap, err := obj.GetMap()
+	if err != nil {
+		return nil, errors.Wrap(err, "obj.GetMap")
+	}
+	tagPairs := make([]influxdb.SKeyValue, 0)
+	metricPairs := make([]influxdb.SKeyValue, 0)
+	for k, v := range objMap {
+		val, _ := v.GetString()
+		if metricDataType == "tag" {
+			tagPairs = append(tagPairs, influxdb.SKeyValue{
+				Key: k, Value: val,
+			})
+		} else if metricDataType == "metric" {
+			metricPairs = append(metricPairs, influxdb.SKeyValue{
+				Key: k, Value: val,
+			})
+		}
+	}
+	metric.Tags = append(metric.Tags, tagPairs...)
+	metric.Metrics = append(metric.Metrics, metricPairs...)
+	return metric, nil
+}

pkg/cloudmon/collectors/utils.go

@@ -0,0 +1,80 @@
+package collectors
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	o "yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func jsonToMetric(obj *jsonutils.JSONDict, name string, tags []string, metrics []string) (influxdb.SMetricData, error) {
+	metric := influxdb.SMetricData{Name: name}
+	objMap, err := obj.GetMap()
+	if err != nil {
+		return metric, errors.Wrap(err, "obj.GetMap")
+	}
+	tagPairs := make([]influxdb.SKeyValue, 0)
+	metricPairs := make([]influxdb.SKeyValue, 0)
+	for k, v := range objMap {
+		val, _ := v.GetString()
+		if utils.IsInStringArray(k, tags) {
+			tagPairs = append(tagPairs, influxdb.SKeyValue{
+				Key: k, Value: val,
+			})
+		} else if utils.IsInStringArray(k, metrics) {
+			metricPairs = append(metricPairs, influxdb.SKeyValue{
+				Key: k, Value: val,
+			})
+		}
+	}
+	metric.Tags = tagPairs
+	metric.Metrics = metricPairs
+	return metric, nil
+}
+
+func sendMetrics(s *mcclient.ClientSession, metrics []influxdb.SMetricData, debug bool) error {
+	urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType)
+	if err != nil {
+		return errors.Wrap(err, "GetServiceURLs")
+	}
+	return influxdb.SendMetrics(urls, o.Options.InfluxDatabase, metrics, debug)
+}
+
+type TListFunc func(*mcclient.ClientSession, jsonutils.JSONObject) (*modulebase.ListResult, error)
+type TProcessFunc func(jsonutils.JSONObject) error
+
+func listAll(s *mcclient.ClientSession, listFunc TListFunc, kwargs jsonutils.JSONObject, processFunc TProcessFunc) error {
+	type sListParams struct {
+		Limit  int
+		Offset int
+	}
+	offset := 0
+	total := -1
+	for total < 0 || offset < total {
+		params := jsonutils.Marshal(sListParams{
+			Limit:  100,
+			Offset: offset,
+		})
+		if kwargs != nil {
+			params.(*jsonutils.JSONDict).Update(kwargs)
+		}
+		result, err := listFunc(s, jsonutils.Marshal(params))
+		if err != nil {
+			return err
+		}
+		total = result.Total
+		for i := range result.Data {
+			offset += 1
+			err = processFunc(result.Data[i])
+			if err != nil {
+				log.Errorf("fail to processData %s: %s", result.Data[i], err)
+			}
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/vmwaremon/doc.go

@@ -0,0 +1 @@
+package vmwaremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon"

pkg/cloudmon/collectors/vmwaremon/vmwarereport.go

@@ -0,0 +1,86 @@
+package vmwaremon
+
+import (
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SEsxiCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SEsxiCloudReportFactory struct {
+}
+
+func (self *SEsxiCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SEsxiCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SEsxiCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_VMWARE
+}
+
+type SEsxiCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SEsxiCloudReport) Report() error {
+	var err error
+	switch self.Operator {
+	case string(common.SERVER):
+		//servers, err := self.getAllserverOfThisProvider(&modules.Servers)
+		servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+		if err != nil {
+			return err
+		}
+		err = self.CollectServerMetricByProvider(servers)
+	case string(common.HOST):
+		hosts, err := self.GetAllHostOfThisProvider(&modules.Hosts)
+		if err != nil {
+			return err
+		}
+		//err = common.CollectRegionMetricAsync(40, nil, hosts, self)
+		err = self.CollectRegionHostMetricAsync(hosts)
+	}
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (self *SEsxiCloudReport) CollectServerMetricByProvider(servers []jsonutils.JSONObject) error {
+	return self.collectRegionMetricOfServerBatch("", servers)
+
+}
+
+func (self *SEsxiCloudReport) CollectRegionHostMetricAsync(servers []jsonutils.JSONObject) error {
+	log.Errorf("cloudproviderid: %s,%s count:%d", self.SProvider.Id, self.getMonType(), len(servers))
+	if self.Args.Batch == 0 {
+		self.Args.Batch = 100
+	}
+	for i := 0; i < len(servers); i += self.Args.Batch {
+		tmp := i + self.Args.Batch
+		if tmp > len(servers) {
+			tmp = len(servers)
+		}
+		err := self.collectRegionMetricOfServerBatch("", servers[i:tmp])
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/vmwaremon/vmwareservice.go

@@ -0,0 +1,337 @@
+package vmwaremon
+
+import (
+	"context"
+	"net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/vmware/govmomi/performance"
+	"github.com/vmware/govmomi/vim25/types"
+	"golang.org/x/sync/errgroup"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/esxi"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SEsxiCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	var err error
+	switch self.Operator {
+	case string(common.SERVER):
+		err = self.collectRegionMetricOfServer(servers)
+	case string(common.HOST):
+		err = self.collectRegionMetricOfHost(servers)
+	}
+	return err
+}
+
+func (self *SEsxiCloudReport) getMonType() string {
+	switch self.Operator {
+	case string(common.SERVER):
+		return common.TYPE_VIRTUALMACHINE
+	case string(common.HOST):
+		return common.TYPE_HOSTSYSTEM
+	}
+	return ""
+}
+
+func (self *SEsxiCloudReport) collectRegionMetricOfServerBatch(hostExtId string, servers []jsonutils.JSONObject) error {
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+
+	client, err := self.newEsxiClient()
+	if err != nil {
+		return err
+	}
+	metrics := make([]string, 0)
+	for metric, _ := range esxiMetricSpecs {
+		metrics = append(metrics, metric)
+	}
+	now := time.Now()
+	perfEntityMetrics, err := client.GetMonitorDataList(hostExtId, servers, self.getMonType(), metrics, since,
+		until)
+	if err != nil {
+		return errors.Wrap(err, "SEsxiCloudReport GetMonitorDataList error")
+	}
+
+	log.Infof("get %s metriclist cost: %f s", self.getMonType(), time.Now().Sub(now).Seconds())
+
+	writeGroup, _ := errgroup.WithContext(context.Background())
+	for i, _ := range servers {
+		tmpSer := servers[i]
+		writeGroup.Go(func() error {
+			extId, _ := tmpSer.GetString("external_id")
+			dataList := make([]influxdb.SMetricData, 0)
+			if entityMetric, ok := perfEntityMetrics[extId]; ok {
+				if self.Operator == string(common.SERVER) {
+					metric, err := common.FillVMCapacity(tmpSer.(*jsonutils.JSONDict))
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, metric)
+				}
+				serverMetric := self.collectMetricFromThisServer_(tmpSer, self.getMonType(), entityMetric)
+				dataList = append(dataList, serverMetric...)
+				writStartTime := time.Now()
+				common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+				log.Errorf("influxdb write cost:%f s", time.Now().Sub(writStartTime).Seconds())
+			}
+			return nil
+		})
+	}
+	err = writeGroup.Wait()
+	log.Infof("collect %s num:%d", self.getMonType(), len(servers))
+	return nil
+}
+
+func (self *SEsxiCloudReport) collectRegionMetricOfServer(servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+
+	client, err := self.newEsxiClient()
+	if err != nil {
+		return err
+	}
+	for _, server := range servers {
+		perfEntityMetrics, metricIdNamedTable, err := client.GetMonitorData(server, common.TYPE_VIRTUALMACHINE,
+			esxiMetricSpecsSync, since, until)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		for _, perfEntityMetric := range perfEntityMetrics {
+			perfMetricSeries := perfEntityMetric.Value
+			perfSampleInfos := perfEntityMetric.SampleInfo
+			for _, perfMetricSerie := range perfMetricSeries {
+				if perfMetricIntSerie, ok := perfMetricSerie.(*types.PerfMetricIntSeries); ok {
+					metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+					if err != nil {
+						return err
+					}
+					dataList = append(dataList, metric)
+					serverMetric := self.collectMetricFromThisServer(server, common.TYPE_VIRTUALMACHINE, perfMetricIntSerie,
+						perfSampleInfos, metricIdNamedTable)
+					dataList = append(dataList, serverMetric...)
+				}
+			}
+		}
+		err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+	return nil
+}
+
+func (self *SEsxiCloudReport) collectRegionMetricOfHost(hosts []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	client, err := self.newEsxiClient()
+	if err != nil {
+		return err
+	}
+	for _, host := range hosts {
+		perfEntityMetrics, metricIdNamedTable, err := client.GetMonitorData(host, common.TYPE_HOSTSYSTEM, esxiMetricSpecsSync,
+			since, until)
+		if err != nil {
+			continue
+		}
+		for _, perfEntityMetric := range perfEntityMetrics {
+			perfMetricSeries := perfEntityMetric.Value
+			perfSampleInfos := perfEntityMetric.SampleInfo
+			for _, perfMetricSerie := range perfMetricSeries {
+				if perfMetricIntSerie, ok := perfMetricSerie.(*types.PerfMetricIntSeries); ok {
+					serverMetric := self.collectMetricFromThisServer(host, common.TYPE_HOSTSYSTEM, perfMetricIntSerie,
+						perfSampleInfos, metricIdNamedTable)
+					dataList = append(dataList, serverMetric...)
+				}
+			}
+		}
+		err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+	return nil
+}
+
+func (self *SEsxiCloudReport) collectMetricFromThisServer_(server jsonutils.JSONObject, monType string,
+	entityMetric performance.EntityMetric) []influxdb.SMetricData {
+	datas := make([]influxdb.SMetricData, 0)
+	for _, metricSeries := range entityMetric.Value {
+		if _, ok := esxiMetricSpecs[metricSeries.Name]; !ok {
+			continue
+		}
+		for i, value := range metricSeries.Value {
+			metric := influxdb.SMetricData{}
+			if monType == common.TYPE_HOSTSYSTEM {
+				metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.HostTags, make([]string, 0))
+			} else {
+				metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+			}
+
+			if len(entityMetric.SampleInfo) > 0 {
+				metric.Timestamp = entityMetric.SampleInfo[i].Timestamp
+			}
+			influxDbSpecs := esxiMetricSpecs[metricSeries.Name]
+			metric.Name = common.GetMeasurement(monType, influxDbSpecs[2])
+			var pairsKey string
+			if strings.Contains(influxDbSpecs[2], ",") {
+				pairsKey = common.SubstringBetween(influxDbSpecs[2], ".", ",")
+			} else {
+				pairsKey = common.SubstringAfter(influxDbSpecs[2], ".")
+			}
+			tag := common.SubstringAfter(influxDbSpecs[2], ",")
+			if tag != "" && strings.Contains(influxDbSpecs[2], "=") {
+				metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+					Key:   common.SubstringBefore(tag, "="),
+					Value: common.SubstringAfter(tag, "="),
+				})
+			}
+			value = formateValue(value, influxDbSpecs)
+			metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+				Key:   pairsKey,
+				Value: strconv.FormatInt(value, 10),
+			})
+			if monType == common.TYPE_HOSTSYSTEM {
+				self.AddMetricTag(&metric, common.OtherHostTag)
+			} else {
+				self.AddMetricTag(&metric, common.OtherVmTags)
+			}
+			datas = append(datas, metric)
+		}
+	}
+	return datas
+}
+
+func (self *SEsxiCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, monType string,
+	perfMetricIntSerie *types.PerfMetricIntSeries, perfSampleInfos []types.PerfSampleInfo,
+	metricIdNamedTable map[int32]string) []influxdb.SMetricData {
+	datas := make([]influxdb.SMetricData, 0)
+	for i, value := range perfMetricIntSerie.Value {
+		metric := influxdb.SMetricData{}
+		if monType == common.TYPE_HOSTSYSTEM {
+			metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.HostTags, make([]string, 0))
+		} else {
+			metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+		}
+		counterId := perfMetricIntSerie.Id.CounterId
+		instance := perfMetricIntSerie.Id.Instance
+		if instance != "" {
+			metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+				Key:   "perf_instance",
+				Value: instance,
+			})
+		}
+		if len(perfSampleInfos) > 0 {
+			metric.Timestamp = perfSampleInfos[i].Timestamp
+		}
+
+		influxDbSpecs := esxiMetricSpecsSync[metricIdNamedTable[counterId]]
+		metric.Name = common.GetMeasurement(monType, influxDbSpecs[2])
+		var pairsKey string
+		if strings.Contains(influxDbSpecs[2], ",") {
+			pairsKey = common.SubstringBetween(influxDbSpecs[2], ".", ",")
+		} else {
+			pairsKey = common.SubstringAfter(influxDbSpecs[2], ".")
+		}
+		tag := common.SubstringAfter(influxDbSpecs[2], ",")
+		if tag != "" && strings.Contains(influxDbSpecs[2], "=") {
+			metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+				Key:   common.SubstringBefore(tag, "="),
+				Value: common.SubstringAfter(tag, "="),
+			})
+		}
+		value = formateValue(value, influxDbSpecs)
+		metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+			Key:   pairsKey,
+			Value: strconv.FormatInt(value, 10),
+		})
+		if monType == common.TYPE_HOSTSYSTEM {
+			self.AddMetricTag(&metric, common.OtherHostTag)
+		} else {
+			self.AddMetricTag(&metric, common.OtherVmTags)
+		}
+		datas = append(datas, metric)
+	}
+	return datas
+}
+
+func formateValue(value int64, influxDbSpecs []string) int64 {
+	if influxDbSpecs[1] == UNIT_KBPS && strings.Contains(influxDbSpecs[2], "bps") {
+		value = value * 1000
+	}
+	if influxDbSpecs[1] == UNIT_KBPS && strings.Contains(influxDbSpecs[2], "bytes") {
+		value = value * 1000
+	}
+	if influxDbSpecs[1] == UNIT_PERCENT && strings.Contains(influxDbSpecs[2], "usage_active") {
+		value = value / 100
+	}
+	if influxDbSpecs[1] == UNIT_PERCENT && strings.Contains(influxDbSpecs[2], "used_percent") {
+		value = value / 100
+	}
+	return value
+}
+
+func (self *SEsxiCloudReport) newEsxiClient() (*esxi.SESXiClient, error) {
+	parts, err := url.Parse(self.SProvider.AccessUrl)
+	if err != nil {
+		return nil, err
+	}
+	host, port, err := parseHostPort(parts.Host, 443)
+	if err != nil {
+		return nil, err
+	}
+
+	secretDe, _ := utils.DescryptAESBase64(self.SProvider.Id, self.SProvider.Secret)
+	esxiCfg := esxi.NewESXiClientConfig(host, port, self.SProvider.Account, secretDe)
+	proCfg := cloudprovider.ProviderConfig{
+		Id:      self.SProvider.Id,
+		Name:    self.SProvider.Name,
+		Account: self.SProvider.Account,
+		Secret:  secretDe,
+		URL:     self.SProvider.AccessUrl,
+		Vendor:  self.SProvider.Provider,
+	}
+	esxiCfg.CloudproviderConfig(proCfg)
+	client, err := esxi.NewESXiClient(esxiCfg)
+	if err != nil {
+		return nil, err
+	}
+	return client, nil
+}
+
+func parseHostPort(host string, defPort int) (string, int, error) {
+	colonPos := strings.IndexByte(host, ':')
+	if colonPos > 0 {
+		h := host[:colonPos]
+		p, err := strconv.Atoi(host[colonPos+1:])
+		if err != nil {
+			log.Errorf("Invalid host %s", host)
+			return "", 0, err
+		}
+		if p == 0 {
+			p = defPort
+		}
+		return h, p, nil
+	} else {
+		return host, defPort, nil
+	}
+}

pkg/cloudmon/collectors/vmwaremon/vmwareutils.go

@@ -0,0 +1,132 @@
+package vmwaremon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_KBPS          = "kiloBytesPerSecond"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE              = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE              = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS          = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_READ_BPS_VIRTUAL  = INFLUXDB_FIELD_DISK_READ_BPS + ",disk_type=virtual"
+	INFLUXDB_FIELD_DISK_WRITE_BPS         = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS_VIRTUAL = INFLUXDB_FIELD_DISK_WRITE_BPS + ",disk_type=virtual"
+	INFLUXDB_FIELD_DISK_READ_IOPS         = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS        = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX             = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET    = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET    = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX             = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET    = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET    = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC          = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC           = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG              = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG               = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+)
+
+//multiCloud查询指标列表组装
+var esxiMetricSpecs = map[string][]string{
+	"cpu.usage.average":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"mem.usage.average":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
+	"net.received.average":      {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_RX},
+	"net.transmitted.average":   {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_TX},
+	"disk.read.average":         {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"disk.write.average":        {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"virtualDisk.read.average":  {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS_VIRTUAL},
+	"virtualDisk.write.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS_VIRTUAL},
+}
+
+var esxiMetricSpecsSync = map[string][]string{
+	"cpu_usage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"mem_usage":         {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE},
+	"net_received":      {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_RX},
+	"net_transmitted":   {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_TX},
+	"disk_read":         {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS},
+	"disk_write":        {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"virtualDisk_read":  {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS_VIRTUAL},
+	"virtualDisk_write": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS_VIRTUAL},
+}

pkg/cloudmon/collectors/zstackmon/doc.go

@@ -0,0 +1 @@
+package zstackmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon"

pkg/cloudmon/collectors/zstackmon/zstackreport.go

@@ -0,0 +1,94 @@
+package zstackmon
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+func init() {
+	factory := SZStackCloudReportFactory{}
+	common.RegisterFactory(&factory)
+}
+
+type SZStackCloudReportFactory struct {
+}
+
+func (self *SZStackCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession,
+	args *common.ReportOptions, operatorType string) common.ICloudReport {
+	return &SZStackCloudReport{
+		common.CloudReportBase{
+			SProvider: provider,
+			Session:   session,
+			Args:      args,
+			Operator:  operatorType,
+		},
+	}
+}
+
+func (self *SZStackCloudReportFactory) GetId() string {
+	return compute.CLOUD_PROVIDER_ZSTACK
+}
+
+type SZStackCloudReport struct {
+	common.CloudReportBase
+}
+
+func (self *SZStackCloudReport) Report() error {
+	switch self.Operator {
+	case string(common.SERVER):
+		return self.getServerMetrics()
+	case string(common.HOST):
+		return self.getHoseMetrics()
+	}
+	return nil
+}
+
+func (self *SZStackCloudReport) getServerMetrics() error {
+	servers, err := self.GetAllserverOfThisProvider(&modules.Servers)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		err = common.CollectRegionMetricAsync(self.Args.Batch, region, regionServerMap[region.GetGlobalId()], self)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+	}
+	return nil
+}
+
+func (self *SZStackCloudReport) getHoseMetrics() error {
+	hosts, err := self.GetAllHostOfThisProvider(&modules.Hosts)
+	if err != nil {
+		return err
+	}
+	providerInstance, err := self.InitProviderInstance()
+	if err != nil {
+		return err
+	}
+	regionList, regionServerMap, err := self.GetAllRegionOfServers(hosts, providerInstance)
+	if err != nil {
+		return err
+	}
+	for _, region := range regionList {
+		err = common.CollectRegionMetricAsync(self.Args.Batch, region, regionServerMap[region.GetGlobalId()], self)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+	}
+	return nil
+}

pkg/cloudmon/collectors/zstackmon/zstackservice.go

@@ -0,0 +1,135 @@
+package zstackmon
+
+import (
+	"strconv"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudmon/collectors/common"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud/zstack"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func (self *SZStackCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	var err error
+	switch self.Operator {
+	case string(common.SERVER):
+		err = self.collectRegionMetricOfServer(region, servers)
+	case string(common.HOST):
+		err = self.collectRegionMetricOfHost(region, servers)
+	}
+	return err
+}
+
+func (self *SZStackCloudReport) collectRegionMetricOfServer(region cloudprovider.ICloudRegion,
+	servers []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	zstackReg := region.(*zstack.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range zstackMetricSpecs {
+		rtn, err := zstackReg.GetMonitorData(metricName, NAMESPACE_VM, since, until)
+		if err != nil {
+			log.Errorln(err)
+			continue
+		}
+		if len(rtn.DataPoints) > 0 {
+			for _, dataPoint := range rtn.DataPoints {
+				for _, server := range servers {
+					external_id, _ := server.GetString("external_id")
+					if dataPoint.Labels.VMUuid == external_id {
+						metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict))
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, metric)
+						serverMetric, err := self.collectMetricFromThisServer(server, common.TYPE_VIRTUALMACHINE, dataPoint, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SZStackCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion,
+	hosts []jsonutils.JSONObject) error {
+	dataList := make([]influxdb.SMetricData, 0)
+	zstackReg := region.(*zstack.SRegion)
+	since, until, err := common.TimeRangeFromArgs(self.Args)
+	if err != nil {
+		return err
+	}
+	for metricName, influxDbSpecs := range zstackMetricSpecs {
+		rtn, err := zstackReg.GetMonitorData(metricName, NAMESPACE_HOST, since, until)
+		if err != nil {
+			return err
+		}
+		if len(rtn.DataPoints) > 0 {
+			for _, dataPoint := range rtn.DataPoints {
+				for _, host := range hosts {
+					external_id, _ := host.GetString("external_id")
+					if dataPoint.Labels.HostUuid == external_id {
+						serverMetric, err := self.collectMetricFromThisServer(host, common.TYPE_HOSTSYSTEM, dataPoint, influxDbSpecs)
+						if err != nil {
+							return err
+						}
+						dataList = append(dataList, serverMetric)
+					}
+				}
+			}
+		}
+	}
+	return common.SendMetrics(self.Session, dataList, self.Args.Debug, "")
+}
+
+func (self *SZStackCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject,
+	monType string, dataPoint zstack.DataPoint, influxDbSpecs []string) (influxdb.SMetricData, error) {
+	metric := influxdb.SMetricData{}
+	if monType == common.TYPE_HOSTSYSTEM {
+		metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.HostTags, make([]string, 0))
+	} else {
+		metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0))
+	}
+	fieldValue := dataPoint.Value
+	metric.Timestamp = time.Unix(dataPoint.TimeStemp, 0)
+	//根据条件拼装metric的tag和metirc信息
+	metric.Name = common.GetMeasurement(monType, influxDbSpecs[2])
+	var pairsKey string
+	if strings.Contains(influxDbSpecs[2], ",") {
+		pairsKey = common.SubstringBetween(influxDbSpecs[2], ".", ",")
+	} else {
+		pairsKey = common.SubstringAfter(influxDbSpecs[2], ".")
+	}
+	if influxDbSpecs[1] == UNIT_MEM {
+		fieldValue = fieldValue * 8
+	}
+	tag := common.SubstringAfter(influxDbSpecs[2], ",")
+	if tag != "" && strings.Contains(influxDbSpecs[2], "=") {
+		metric.Tags = append(metric.Tags, influxdb.SKeyValue{
+			Key:   common.SubstringBefore(tag, "="),
+			Value: common.SubstringAfter(tag, "="),
+		})
+	}
+	metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{
+		Key:   pairsKey,
+		Value: strconv.FormatFloat(fieldValue, 'E', -1, 64),
+	})
+	if monType == common.TYPE_HOSTSYSTEM {
+		self.AddMetricTag(&metric, common.OtherHostTag)
+	} else {
+		self.AddMetricTag(&metric, common.OtherVmTags)
+	}
+	return metric, nil
+}

pkg/cloudmon/collectors/zstackmon/zstackutils.go

@@ -0,0 +1,123 @@
+package zstackmon
+
+const (
+	PERIOD             = 60
+	UNIT_AVERAGE       = "Average"
+	DEFAULT_STATISTICS = "Average,Minimum,Maximum"
+	UNIT_PERCENT       = "Percent"
+	UNIT_BPS           = "bps"
+	UNIT_MBPS          = "Mbps"
+	UNIT_BYTEPS        = "Bps"
+	UNIT_CPS           = "cps"
+	UNIT_COUNT         = "count"
+	UNIT_MEM           = "byte"
+	UNIT_MSEC          = "ms"
+	UNIT_COUNT_SEC     = "count/s"
+
+	TYPE_VIRTUALMACHINE = "VirtualMachine"
+	TYPE_HOSTSYSTEM     = "HostSystem"
+
+	//ESC监控指标
+	INFLUXDB_FIELD_CPU_USAGE           = "vm_cpu.usage_active"
+	INFLUXDB_FIELD_MEM_USAGE           = "vm_mem.used_percent"
+	INFLUXDB_FIELD_DISK_READ_BPS       = "vm_diskio.read_bps"
+	INFLUXDB_FIELD_DISK_WRITE_BPS      = "vm_diskio.write_bps"
+	INFLUXDB_FIELD_DISK_READ_IOPS      = "vm_diskio.read_iops"
+	INFLUXDB_FIELD_DISK_WRITE_IOPS     = "vm_diskio.write_iops"
+	INFLUXDB_FIELD_NET_BPS_RX          = "vm_netio.bps_recv"
+	INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_NET_BPS_TX          = "vm_netio.bps_sent"
+	INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_WANOUTTRAFFIC       = "vm_eipio.bps_out"
+	INFLUXDB_FIELD_WANINTRAFFIC        = "vm_eipio.bps_in"
+	INFLUXDB_FIELD_WANOUTPKG           = "vm_eipio.pps_out"
+	INFLUXDB_FIELD_WANINPKG            = "vm_eipio.pps_in"
+
+	//RDS监控指标
+	INFLUXDB_FIELD_RDS_CPU_USAGE            = "rds_cpu.usage_active"
+	INFLUXDB_FIELD_RDS_MEM_USAGE            = "rds_mem.used_percent"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX           = "rds_netio.bps_recv"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX           = "rds_netio.bps_send"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL     = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql"
+	INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver"
+	INFLUXDB_FIELD_RDS_DISK_USAGE           = "rds_disk.used_percent"
+	INFLUXDB_FIELD_RDS_DISK_READ_BPS        = "rds_diskio.read_bps"
+	INFLUXDB_FIELD_RDS_DISK_WRITE_BPS       = "rds_diskio.write_bps"
+	INFLUXDB_FIELD_RDS_CONN_COUNT           = "rds_conn.used_count"
+	INFLUXDB_FIELD_RDS_CONN_USAGE           = "rds_conn.used_percent"
+
+	INFLUXDB_FIELD_RDS_QPS              = "rds_qps.query_qps"
+	INFLUXDB_FIELD_RDS_TPS              = "rds_tps.trans_qps"
+	INFLUXDB_FIELD_RDS_INNODB_REDA_BPS  = "rds_innodb.read_bps"
+	INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps"
+
+	//REDIS监控指标
+	INFLUXDB_FIELD_REDIS_CPU_USAGE      = "dcs_cpu.usage_percent"
+	INFLUXDB_FIELD_REDIS_MEM_USAGE      = "dcs_mem.used_percent"
+	INFLUXDB_FIELD_REDIS_NET_BPS_RX     = "dcs_netio.bps_recv"
+	INFLUXDB_FIELD_REDIS_NET_BPS_TX     = "dcs_netio.bps_sent"
+	INFLUXDB_FIFLD_REDIS_CONN_USAGE     = "dcs_conn.used_conn"
+	INFLUXDB_FIFLD_REDIS_OPT_SES        = "dcs_instantopt.opt_sec"
+	INFLUXDB_FIFLD_REDIS_CACHE_KEYS     = "dcs_cachekeys.key_count"
+	INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire"
+	INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte"
+
+	//对象存储OSS监控指标
+	INFLUXDB_FIELD_OSS_NET_BPS_RX          = "oss_netio.bps_recv"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX          = "oss_netio.bps_sent"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet"
+	INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet"
+	INFLUXDB_FIELD_OSS_LATECY              = "oss_latency.req_late"
+	INFLUXDB_FIELD_OSS_LATECY_GET          = INFLUXDB_FIELD_OSS_LATECY + ",request=get"
+	INFLUXDB_FIELD_OSS_LATECY_POST         = INFLUXDB_FIELD_OSS_LATECY + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT           = "oss_req.req_count"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_GET       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_POST      = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_5XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_OSS_REQ_COUNT_4XX       = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx"
+
+	//负载均衡监控指标
+	INFLUXDB_FIELD_ELB_NET_BPS_RX     = "haproxy.bin"
+	INFLUXDB_FIELD_ELB_NET_BPS_TX     = "haproxy.bout"
+	INFLUXDB_FIELD_ELB_REQ_RATE       = "haproxy.req_rate,request=http"
+	INFLUXDB_FIELD_ELB_CONN_RATE      = "haproxy.conn_rate,request=tcp"
+	INFLUXDB_FIELD_ELB_DREQ_COUNT     = "haproxy.dreq,request=http"
+	INFLUXDB_FIELD_ELB_DCONN_COUNT    = "haproxy.dcon,request=tcp"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT     = "haproxy.hrsp_Nxx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx"
+	INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx"
+	INFLUXDB_FIELD_ELB_CHC_STATUS     = "haproxy.check_status"
+	INFLUXDB_FIELD_ELB_CHC_CODE       = "haproxy.check_code"
+	INFLUXDB_FIELD_ELB_LAST_CHC       = "haproxy.last_chk"
+
+	KEY_VMS   = "vms"
+	KEY_CPUS  = "cpus"
+	KEY_MEMS  = "mems"
+	KEY_DISKS = "disks"
+
+	KEY_LIMIT  = "limit"
+	KEY_ADMIN  = "admin"
+	KEY_USABLE = "usable"
+
+	NAMESPACE_VM   = "ZStack/VM"
+	NAMESPACE_HOST = "ZStack/Host"
+)
+
+//multiCloud查询指标列表组装
+var zstackMetricSpecs = map[string][]string{
+	"CPUAverageUsedUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE},
+	"DiskReadBytes":             {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS},
+	"DiskWriteBytes":            {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS},
+	"NetworkInBytes":            {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX},
+	"NetworkOutBytes":           {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX},
+	"DiskReadOps":               {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_IOPS},
+	"DiskWriteOps":              {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_IOPS},
+}

pkg/cloudmon/options/doc.go

@@ -0,0 +1 @@
+package options // import "yunion.io/x/onecloud/pkg/cloudmon/options"

pkg/cloudmon/options/options.go

@@ -0,0 +1,75 @@
+package options
+
+import (
+	"fmt"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type CloudMonOptions struct {
+	structarg.BaseOptions
+
+	Region       string `help:"Region name"`
+	EndpointType string `default:"internalURL" help:"Defaults to internalURL" choices:"publicURL|internalURL|adminURL"`
+	ApiVersion   string `help:"override default modules service api version"`
+
+	Debug    bool   `help:"Show debug information"`
+	Timeout  int    `default:"600" help:"Number of seconds to wait for a response"`
+	Insecure bool   `default:"true" help:"Allow skip server cert verification if URL is https" short-token:"k"`
+	CertFile string `help:"certificate file"`
+	KeyFile  string `help:"private key file"`
+
+	AuthURL            string `help:"Keystone auth URL" alias:"auth-uri"`
+	AdminUser          string `help:"Admin username"`
+	AdminDomain        string `help:"Admin user domain"`
+	AdminPassword      string `help:"Admin password" alias:"admin-passwd"`
+	AdminProject       string `help:"Admin project" default:"system" alias:"admin-tenant-name"`
+	AdminProjectDomain string `help:"Domain of Admin project"`
+
+	SessionEndpointType string `help:"Client session end point type"`
+
+	InfluxDatabase string `help:"influxdb database name, default telegraf" default:"telegraf"`
+
+	SUBCOMMAND string `help:"climc subcommand" subcommand:"true"`
+}
+
+func GetArgumentParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&Options,
+		"cloudmon",
+		`Command-line interface to collect cloud monitoring data.`,
+		`See "cloudmon help COMMAND" for help on a specific command.`)
+	if e != nil {
+		return nil, e
+	}
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, errors.Error("No subcommand argument")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"Sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help information of any subcommand", func(suboptions *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(suboptions.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+var (
+	Options CloudMonOptions
+)

pkg/hostman/guestfs/kvmpart/kvmpart.go

@@ -267,7 +267,7 @@ func (p *SKVMGuestDiskPartition) Umount() error {
 				log.Warningf("blockdev --flushbufs %s error: %v", p.partDev, err)
 			}
 			if err := os.Remove(p.mountPath); err != nil {
-				log.Warningf("remove mount path %s error: %v", p.mountPath)
+				log.Warningf("remove mount path %s error: %v", p.mountPath, err)
 			}
 			log.Infof("umount %s successfully", p.partDev)
 			return nil

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -93,11 +93,12 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 	return ret, nil
 }
 
-func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) {
+func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty,
+	err error) {
 	// There will be some occasional unknown panic, so temporarily capture panic here.
 	defer func() {
 		if r := recover(); r != nil {
-			log.Errorf("DeployGuestFs: %s, stack:\n %s", debug.Stack())
+			log.Errorf("DeployGuestFs: %s, stack:\n %s", req.String(), debug.Stack())
 			msg := "panic: "
 			if str, ok := r.(fmt.Stringer); ok {
 				msg += str.String()

pkg/monitor/suggestsysdrivers/doc.go

@@ -1 +0,0 @@
-package suggestsysdrivers // import "yunion.io/x/onecloud/pkg/monitor/suggestsysdrivers"

pkg/monitor/suggestsysdrivers/influxdbbasedriver.go

@@ -1,389 +0,0 @@
-package suggestsysdrivers
-
-import (
-	"context"
-	"fmt"
-	"strings"
-	"time"
-
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/log"
-	"yunion.io/x/pkg/errors"
-	"yunion.io/x/pkg/utils"
-
-	"yunion.io/x/onecloud/pkg/apis/monitor"
-	"yunion.io/x/onecloud/pkg/httperrors"
-	"yunion.io/x/onecloud/pkg/mcclient"
-	"yunion.io/x/onecloud/pkg/mcclient/auth"
-	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
-	"yunion.io/x/onecloud/pkg/mcclient/modules"
-	"yunion.io/x/onecloud/pkg/monitor/alerting"
-	"yunion.io/x/onecloud/pkg/monitor/alerting/conditions"
-	merrors "yunion.io/x/onecloud/pkg/monitor/errors"
-	"yunion.io/x/onecloud/pkg/monitor/models"
-	"yunion.io/x/onecloud/pkg/monitor/validators"
-)
-
-type InfluxdbBaseDriver struct {
-	*baseDriver
-}
-
-func NewInfluxdbBaseDriver(driverType monitor.SuggestDriverType, resourceType monitor.MonitorResourceType,
-	action monitor.SuggestDriverAction, suggest monitor.MonitorSuggest, rule monitor.SuggestSysRuleCreateInput) *InfluxdbBaseDriver {
-	return &InfluxdbBaseDriver{
-		baseDriver: newBaseDriver(
-			driverType,
-			resourceType,
-			action,
-			suggest,
-			rule,
-		)}
-}
-
-func (drv *InfluxdbBaseDriver) ValidateSetting(input *monitor.SSuggestSysAlertSetting) error {
-	if input.ScaleRule == nil || len(*input.ScaleRule) == 0 {
-		return merrors.NewArgIsEmptyErr("ScaleRule")
-	}
-	for _, scale := range *input.ScaleRule {
-		if scale.Database == "" {
-			return merrors.NewArgIsEmptyErr("database")
-		}
-		if scale.Measurement == "" {
-			return merrors.NewArgIsEmptyErr("measurement")
-		}
-		if scale.Field == "" {
-			return merrors.NewArgIsEmptyErr("field")
-		}
-		if !utils.IsInStringArray(getQueryEvalType(scale), validators.EvaluatorDefaultTypes) {
-			return httperrors.NewInputParameterError("the evalType is illegal")
-		}
-		if scale.Threshold == 0 {
-			return merrors.NewArgIsEmptyErr("threshold")
-		}
-	}
-	return nil
-}
-
-func getQueryEvalType(scale monitor.Scale) string {
-	typ := ""
-	switch scale.EvalType {
-	case ">=", ">":
-		typ = "gt"
-	case "<=", "<":
-		typ = "lt"
-	}
-	return typ
-}
-
-func (drv *InfluxdbBaseDriver) DoSuggestSysRule(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
-	doSuggestSysRule(ctx, userCred, isStart, drv)
-}
-
-func (drv *InfluxdbBaseDriver) Run(rule *models.SSuggestSysRule, instance *monitor.SSuggestSysAlertSetting) {
-	Run(drv, rule, instance)
-}
-
-func (drv *InfluxdbBaseDriver) GetLatestAlerts(rule *models.SSuggestSysRule, instance *monitor.SSuggestSysAlertSetting) ([]jsonutils.JSONObject, error) {
-	//scaleEvalMatchs := make([]*monitor.EvalMatch, 0)
-	ret := make([]jsonutils.JSONObject, 0)
-	firing, evalMatchMap, err := drv.getScaleEvalResult(*instance.ScaleRule)
-	if err != nil {
-		return ret, errors.Wrap(err, "rule getScaleEvalResult happen error")
-	}
-	if firing {
-		log.Errorf("evalMatchMapLen:%d", len(evalMatchMap))
-		serverArr, err := drv.getResourcesByEvalMatchsMap(evalMatchMap, instance)
-		if err != nil {
-			return ret, errors.Wrap(err, "rule  getResource error")
-		}
-		return serverArr, nil
-	}
-	return ret, nil
-}
-
-func (drv *InfluxdbBaseDriver) getScaleEvalResult(scales []monitor.Scale) (bool, map[string][]*monitor.EvalMatch,
-	error) {
-	firing := false
-	scaleEvalMatchs := make(map[string][]*monitor.EvalMatch, 0)
-	for index, scale := range scales {
-		condition := monitor.AlertCondition{
-			Type:      "suggest_query",
-			Query:     drv.newAlertQuery(scale),
-			Evaluator: monitor.Condition{Type: getQueryEvalType(scale), Params: []float64{scale.Threshold}},
-			Reducer:   monitor.Condition{Type: "avg"},
-			Operator:  scale.Operator,
-		}
-		factory := alerting.GetConditionFactories()[condition.Type]
-		queryCondition, err := factory(&condition, index)
-		if err != nil {
-			return firing, scaleEvalMatchs, errors.Wrapf(err, "construct query condition %s",
-				jsonutils.Marshal(condition))
-		}
-		duration, _ := time.ParseDuration(condition.Query.From)
-		queryCon := queryCondition.(*conditions.SuggestQueryCondition)
-		queryCon.Reducer = conditions.NewSuggestRuleReducer(queryCon.Reducer.GetType(), duration)
-		//evalContext := alerting.NewEvalContext(context.Background(), auth.AdminCredential(), nil)
-		evalContext := alerting.EvalContext{
-			Ctx:       context.Background(),
-			UserCred:  auth.AdminCredential(),
-			IsDebug:   true,
-			IsTestRun: true,
-		}
-		conditionResult, err := queryCondition.Eval(&evalContext)
-		if err != nil {
-			return firing, scaleEvalMatchs, errors.Wrap(err, "condition eval error")
-		}
-		if index == 0 {
-			firing = conditionResult.Firing
-		}
-
-		// calculating Firing based on operator
-		if conditionResult.Operator == "or" {
-			firing = firing || conditionResult.Firing
-		} else {
-			firing = firing && conditionResult.Firing
-		}
-		if firing {
-			evalMatchs := conditionResult.EvalMatches
-			if conditionResult.Operator == "and" {
-				if index != 0 {
-					evalMatchs = getAndEvalMatches(scaleEvalMatchs, evalMatchs)
-					if len(evalMatchs) == 0 {
-						return false, scaleEvalMatchs, nil
-					}
-				}
-			} else {
-				if index != 0 {
-					getOrEvalMatches(scaleEvalMatchs, evalMatchs)
-				}
-			}
-			key := fmt.Sprintf("%s--%d", scale.Field, index)
-			scaleEvalMatchs[key] = evalMatchs
-		}
-	}
-	return firing, scaleEvalMatchs, nil
-}
-
-func (drv *InfluxdbBaseDriver) getResourcesByEvalMatchsMap(evalMatchsMap map[string][]*monitor.EvalMatch,
-	instance *monitor.SSuggestSysAlertSetting) ([]jsonutils.JSONObject, error) {
-	matchLength := 0
-	var maxEvalMatch []*monitor.EvalMatch
-	for _, evalMatchs := range evalMatchsMap {
-		if len(evalMatchs) > matchLength {
-			matchLength = len(evalMatchs)
-			maxEvalMatch = evalMatchs
-		}
-	}
-	resArr := jsonutils.NewArray()
-	for _, evalMatch := range maxEvalMatch {
-		res, mappingId, err := drv.getResourceFromEvalMatch(evalMatch)
-		if err != nil {
-			log.Errorln(err)
-			continue
-		}
-		suggestSysAlert, err := getSuggestSysAlertFromJson(res, drv)
-		if err != nil {
-			return nil, errors.Wrap(err, "Scale getSuggestSysAlertFromJson error")
-		}
-		suggestSysAlert.Action = string(monitor.SCALE_DOWN_DRIVER_ACTION)
-		suggestSysAlert.MonitorConfig = jsonutils.Marshal(instance)
-		suggestSysAlert.Problem = drv.describeEvalResultTojson(evalMatchsMap, mappingId)
-		resArr.Add(jsonutils.Marshal(suggestSysAlert))
-	}
-	return resArr.GetArray()
-}
-
-func (drv *InfluxdbBaseDriver) getResourceFromEvalMatch(evalMatch *monitor.EvalMatch) (jsonutils.JSONObject, string, error) {
-	var server jsonutils.JSONObject
-	mappingId := ""
-	id, err := drv.getMetricId(evalMatch)
-	if err != nil {
-		return server, mappingId, errors.Wrap(err, "InfluxdbBaseDriver getMetricId err")
-	}
-	serverobj, err := drv.getResourceById(id)
-	if err != nil {
-		return server, mappingId, errors.Wrapf(err, "InfluxdbBaseDriver getResourceById:%s err", id)
-	}
-	server = serverobj
-	mappingId = id
-	return server, mappingId, nil
-}
-
-func (drv *InfluxdbBaseDriver) getMetricId(evalMatch *monitor.EvalMatch) (string, error) {
-	var id string
-	switch drv.GetResourceType() {
-	case monitor.SCALE_MONTITOR_RES_TYPE:
-		id = evalMatch.Tags[monitor.METRIC_VM_ID]
-	case monitor.REDIS_UNREASONABLE_MONITOR_RES_TYPE:
-		id = evalMatch.Tags[monitor.METRIC_REDIS_ID]
-	case monitor.RDS_UNREASONABLE_MONITOR_RES_TYPE:
-		id = evalMatch.Tags[monitor.METRIC_RDS_ID]
-	case monitor.OSS_UNREASONABLE_MONITOR_RES_TYPE:
-		id = evalMatch.Tags[monitor.METRIC_OSS_ID]
-	}
-	if len(id) == 0 {
-		return id, fmt.Errorf("no find resourceId by the driver type:%s", string(drv.GetResourceType()))
-	}
-	return id, nil
-}
-
-func (drv *InfluxdbBaseDriver) getResourceById(id string) (jsonutils.JSONObject, error) {
-	switch drv.GetResourceType() {
-	case monitor.SCALE_MONTITOR_RES_TYPE:
-		return getResource(id, &modules.Servers)
-	case monitor.REDIS_UNREASONABLE_MONITOR_RES_TYPE:
-		return getResource(id, &modules.ElasticCache)
-	case monitor.RDS_UNREASONABLE_MONITOR_RES_TYPE:
-		return getResource(id, &modules.DBInstance)
-	case monitor.OSS_UNREASONABLE_MONITOR_RES_TYPE:
-		return getResource(id, &modules.Buckets)
-	}
-	return nil, fmt.Errorf("unsupporttd to get resource by the driver type:%s", string(drv.GetResourceType()))
-}
-
-func (drv *InfluxdbBaseDriver) describeEvalResultTojson(evalMatchsMap map[string][]*monitor.EvalMatch,
-	mappingId string) jsonutils.JSONObject {
-	problem := jsonutils.NewDict()
-loopEvalMap:
-	for _, evalMatchs := range evalMatchsMap {
-		for _, evalMatch := range evalMatchs {
-			metricId, _ := drv.getMetricId(evalMatch)
-			if metricId == mappingId {
-				problem.Add(jsonutils.NewFloat64(*evalMatch.Value), evalMatch.Metric)
-				break loopEvalMap
-			}
-		}
-	}
-	return problem
-}
-
-func getResource(id string, manager modulebase.Manager) (jsonutils.JSONObject, error) {
-	session := auth.GetAdminSession(context.Background(), "", "")
-	query := jsonutils.NewDict()
-	query.Add(jsonutils.NewString("0"), "limit")
-	query.Add(jsonutils.NewString("system"), "scope")
-	server, err := manager.GetById(session, id, query)
-	if err != nil {
-		return nil, err
-	}
-	return server, nil
-}
-
-func getMetricIdTag(tags map[string]string) map[string]string {
-	idTags := make(map[string]string, 0)
-	for key, val := range tags {
-		if strings.HasSuffix(key, "_id") {
-			idTags[key] = val
-		}
-	}
-	return idTags
-}
-
-func getAndEvalMatches(scaleEvalMatchs map[string][]*monitor.EvalMatch, andscaleEvalMatchs []*monitor.EvalMatch) []*monitor.EvalMatch {
-	for key, evalMatchs := range scaleEvalMatchs {
-		andscaleEvalMatchs = getAndEvalMatches_(evalMatchs, andscaleEvalMatchs)
-		if len(andscaleEvalMatchs) == 0 {
-			return andscaleEvalMatchs
-		}
-		scaleEvalMatchs[key] = getAndEvalMatches_(andscaleEvalMatchs, evalMatchs)
-	}
-	return andscaleEvalMatchs
-}
-
-func getOrEvalMatches(scaleEvalMatchs map[string][]*monitor.EvalMatch, orscaleEvalMatchs []*monitor.EvalMatch) {
-	for key, evalMatchs := range scaleEvalMatchs {
-		matches_ := getOrEvalMatches_(evalMatchs, orscaleEvalMatchs)
-		scaleEvalMatchs[key] = append(evalMatchs, matches_...)
-	}
-}
-
-//by first  param to scale other param's length
-func getAndEvalMatches_(scaleEvalMatchs, andscaleEvalMatchs []*monitor.EvalMatch) []*monitor.EvalMatch {
-	resEvalMatchs := make([]*monitor.EvalMatch, 0)
-	for _, evalMatch := range scaleEvalMatchs {
-		idTags := getMetricIdTag(evalMatch.Tags)
-	twoLoop:
-		for _, andEvalMatch := range andscaleEvalMatchs {
-			andIdTags := getMetricIdTag(andEvalMatch.Tags)
-			//all the idTags must be equals
-			for key, val := range idTags {
-				if andVal, ok := andIdTags[key]; ok {
-					if val != andVal {
-						continue twoLoop
-					}
-				}
-				continue twoLoop
-			}
-			resEvalMatchs = append(resEvalMatchs, andEvalMatch)
-		}
-	}
-	return resEvalMatchs
-}
-
-func getOrEvalMatches_(scaleEvalMatchs, orscaleEvalMatchs []*monitor.EvalMatch) []*monitor.EvalMatch {
-	resEvalMatchs := make([]*monitor.EvalMatch, 0)
-	containsEvalMatchMap := make(map[int]string)
-	for _, evalMatch := range scaleEvalMatchs {
-		idTags := getMetricIdTag(evalMatch.Tags)
-	twoLoop:
-		for i, orEvalMatch := range orscaleEvalMatchs {
-			orIdTags := getMetricIdTag(orEvalMatch.Tags)
-			//all the idTags must be equals
-			for key, val := range idTags {
-				if orVal, ok := orIdTags[key]; ok {
-					if val != orVal {
-						continue twoLoop
-					}
-				}
-				continue twoLoop
-			}
-			containsEvalMatchMap[i] = ""
-		}
-	}
-	for i, andEvalMatch := range orscaleEvalMatchs {
-		if _, ok := containsEvalMatchMap[i]; ok {
-			continue
-		}
-		resEvalMatchs = append(resEvalMatchs, andEvalMatch)
-	}
-	return resEvalMatchs
-}
-
-func (drv *InfluxdbBaseDriver) newAlertQuery(scale monitor.Scale) monitor.AlertQuery {
-	suggestSysRules, _ := models.SuggestSysRuleManager.GetRules(drv.GetType())
-	datasource, _ := models.DataSourceManager.GetDefaultSource()
-	return monitor.AlertQuery{
-		Model:        newMetricQuery(scale),
-		DataSourceId: datasource.Id,
-		From:         suggestSysRules[0].TimeFrom,
-		To:           "now",
-	}
-}
-
-func newMetricQuery(scale monitor.Scale) monitor.MetricQuery {
-	sels := make([]monitor.MetricQuerySelect, 0)
-	sels = append(sels, monitor.NewMetricQuerySelect(monitor.MetricQueryPart{Type: "field", Params: []string{scale.Field}}))
-	return monitor.MetricQuery{
-		Database:    scale.Database,
-		Measurement: scale.Measurement,
-		Selects:     sels,
-		GroupBy: []monitor.MetricQueryPart{
-			{
-				Type:   "field",
-				Params: []string{"*"},
-			},
-		},
-	}
-}
-
-func (drv *InfluxdbBaseDriver) StartResolveTask(ctx context.Context, userCred mcclient.TokenCredential,
-	suggestSysAlert *models.SSuggestSysAlert,
-	params *jsonutils.JSONDict) error {
-	log.Println("InfluxdbBaseDriver StartResolveTask do nothing")
-	return nil
-}
-
-func (drv *InfluxdbBaseDriver) Resolve(data *models.SSuggestSysAlert) error {
-	log.Println("InfluxdbBaseDriver Resolve do nothing")
-	return nil
-}

pkg/vpcagent/models/modelset.go

@@ -94,7 +94,7 @@ func (ms Vpcs) joinRouteTables(subEntries RouteTables) bool {
 		}
 		subEntry.Vpc = m
 		if m.RouteTable != nil {
-			log.Warningf("vpc %s has more than 1 route table available, skipping %s(%s)",
+			log.Warningf("vpc %s(%s) has more than 1 route table available, skipping %s(%s)",
 				m.Name, m.Id, subEntry.Name, subEntry.Id)
 			correct = false
 			continue

vendor/github.com/mitchellh/copystructure/README.md

@@ -1,21 +1,21 @@
-# copystructure
-
-copystructure is a Go library for deep copying values in Go.
-
-This allows you to copy Go values that may contain reference values
-such as maps, slices, or pointers, and copy their data as well instead
-of just their references.
-
-## Installation
-
-Standard `go get`:
-
-```
-$ go get github.com/mitchellh/copystructure
-```
-
-## Usage & Example
-
-For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/copystructure).
-
-The `Copy` function has examples associated with it there.
+# copystructure
+
+copystructure is a Go library for deep copying values in Go.
+
+This allows you to copy Go values that may contain reference values
+such as maps, slices, or pointers, and copy their data as well instead
+of just their references.
+
+## Installation
+
+Standard `go get`:
+
+```
+$ go get github.com/mitchellh/copystructure
+```
+
+## Usage & Example
+
+For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/copystructure).
+
+The `Copy` function has examples associated with it there.

vendor/github.com/tatsushid/go-fastping/.gitignore

@@ -0,0 +1,27 @@
+# Compiled Object files, Static and Dynamic libs (Shared Objects)
+*.o
+*.a
+*.so
+
+# Folders
+_obj
+_test
+
+# Architecture specific extensions/prefixes
+*.[568vq]
+[568vq].out
+
+*.cgo1.go
+*.cgo2.c
+_cgo_defun.c
+_cgo_gotypes.go
+_cgo_export.*
+
+_testmain.go
+
+*.exe
+
+cover.out
+
+ping
+!ping/

vendor/github.com/tatsushid/go-fastping/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2013 Tatsushi Demachi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

vendor/github.com/tatsushid/go-fastping/README.md

@@ -0,0 +1,54 @@
+go-fastping
+===========
+
+go-fastping is a Go language ICMP ping library, inspired by the `AnyEvent::FastPing`
+Perl module, for quickly sending ICMP ECHO REQUEST packets. Original Perl module
+is available at http://search.cpan.org/~mlehmann/AnyEvent-FastPing-2.01/
+
+All original functions haven't been implemented yet.
+
+[![GoDoc](https://godoc.org/github.com/tatsushid/go-fastping?status.svg)](https://godoc.org/github.com/tatsushid/go-fastping)
+
+## Installation
+
+Install and update with `go get -u github.com/tatsushid/go-fastping`
+
+## Examples
+
+Import this package and write
+
+```go
+p := fastping.NewPinger()
+ra, err := net.ResolveIPAddr("ip4:icmp", os.Args[1])
+if err != nil {
+	fmt.Println(err)
+	os.Exit(1)
+}
+p.AddIPAddr(ra)
+p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
+	fmt.Printf("IP Addr: %s receive, RTT: %v\n", addr.String(), rtt)
+}
+p.OnIdle = func() {
+	fmt.Println("finish")
+}
+err = p.Run()
+if err != nil {
+	fmt.Println(err)
+}
+```
+
+The example sends an ICMP packet and waits for a response. If it receives a
+response, it calls the "receive" callback. After that, once MaxRTT time has
+passed, it calls the "idle" callback. For more details,
+refer [to the godoc][godoc], and if you need more examples,
+please see "cmd/ping/ping.go".
+
+## Caution
+This package implements ICMP ping using both raw socket and UDP. If your program
+uses this package in raw socket mode, it needs to be run as a root user.
+
+## License
+go-fastping is under MIT License. See the [LICENSE][license] file for details.
+
+[godoc]: http://godoc.org/github.com/tatsushid/go-fastping
+[license]: https://github.com/tatsushid/go-fastping/blob/master/LICENSE

vendor/github.com/tatsushid/go-fastping/fastping.go

@@ -0,0 +1,685 @@
+// Package fastping is an ICMP ping library inspired by AnyEvent::FastPing Perl
+// module to send ICMP ECHO REQUEST packets quickly. Original Perl module is
+// available at
+// http://search.cpan.org/~mlehmann/AnyEvent-FastPing-2.01/
+//
+// It hasn't been fully implemented original functions yet.
+//
+// Here is an example:
+//
+//	p := fastping.NewPinger()
+//	ra, err := net.ResolveIPAddr("ip4:icmp", os.Args[1])
+//	if err != nil {
+//		fmt.Println(err)
+//		os.Exit(1)
+//	}
+//	p.AddIPAddr(ra)
+//	p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) {
+//		fmt.Printf("IP Addr: %s receive, RTT: %v\n", addr.String(), rtt)
+//	}
+//	p.OnIdle = func() {
+//		fmt.Println("finish")
+//	}
+//	err = p.Run()
+//	if err != nil {
+//		fmt.Println(err)
+//	}
+//
+// It sends an ICMP packet and wait a response. If it receives a response,
+// it calls "receive" callback. After that, MaxRTT time passed, it calls
+// "idle" callback. If you need more example, please see "cmd/ping/ping.go".
+//
+// This library needs to run as a superuser for sending ICMP packets when
+// privileged raw ICMP endpoints is used so in such a case, to run go test
+// for the package, please run like a following
+//
+//	sudo go test
+//
+package fastping
+
+import (
+	"errors"
+	"fmt"
+	"log"
+	"math/rand"
+	"net"
+	"sync"
+	"syscall"
+	"time"
+
+	"golang.org/x/net/icmp"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+const (
+	TimeSliceLength  = 8
+	ProtocolICMP     = 1
+	ProtocolIPv6ICMP = 58
+)
+
+var (
+	ipv4Proto = map[string]string{"ip": "ip4:icmp", "udp": "udp4"}
+	ipv6Proto = map[string]string{"ip": "ip6:ipv6-icmp", "udp": "udp6"}
+)
+
+func byteSliceOfSize(n int) []byte {
+	b := make([]byte, n)
+	for i := 0; i < len(b); i++ {
+		b[i] = 1
+	}
+
+	return b
+}
+
+func timeToBytes(t time.Time) []byte {
+	nsec := t.UnixNano()
+	b := make([]byte, 8)
+	for i := uint8(0); i < 8; i++ {
+		b[i] = byte((nsec >> ((7 - i) * 8)) & 0xff)
+	}
+	return b
+}
+
+func bytesToTime(b []byte) time.Time {
+	var nsec int64
+	for i := uint8(0); i < 8; i++ {
+		nsec += int64(b[i]) << ((7 - i) * 8)
+	}
+	return time.Unix(nsec/1000000000, nsec%1000000000)
+}
+
+func isIPv4(ip net.IP) bool {
+	return len(ip.To4()) == net.IPv4len
+}
+
+func isIPv6(ip net.IP) bool {
+	return len(ip) == net.IPv6len
+}
+
+func ipv4Payload(b []byte) []byte {
+	if len(b) < ipv4.HeaderLen {
+		return b
+	}
+	hdrlen := int(b[0]&0x0f) << 2
+	return b[hdrlen:]
+}
+
+type packet struct {
+	bytes []byte
+	addr  net.Addr
+}
+
+type context struct {
+	stop chan bool
+	done chan bool
+	err  error
+}
+
+func newContext() *context {
+	return &context{
+		stop: make(chan bool),
+		done: make(chan bool),
+	}
+}
+
+// Pinger represents ICMP packet sender/receiver
+type Pinger struct {
+	id  int
+	seq int
+	// key string is IPAddr.String()
+	addrs   map[string]*net.IPAddr
+	network string
+	source  string
+	source6 string
+	hasIPv4 bool
+	hasIPv6 bool
+	ctx     *context
+	mu      sync.Mutex
+
+	// Size in bytes of the payload to send
+	Size int
+	// Number of (nano,milli)seconds of an idle timeout. Once it passed,
+	// the library calls an idle callback function. It is also used for an
+	// interval time of RunLoop() method
+	MaxRTT time.Duration
+	// OnRecv is called with a response packet's source address and its
+	// elapsed time when Pinger receives a response packet.
+	OnRecv func(*net.IPAddr, time.Duration)
+	// OnIdle is called when MaxRTT time passed
+	OnIdle func()
+	// If Debug is true, it prints debug messages to stdout.
+	Debug bool
+}
+
+// NewPinger returns a new Pinger struct pointer
+func NewPinger() *Pinger {
+	rand.Seed(time.Now().UnixNano())
+	return &Pinger{
+		id:      rand.Intn(0xffff),
+		seq:     rand.Intn(0xffff),
+		addrs:   make(map[string]*net.IPAddr),
+		network: "ip",
+		source:  "",
+		source6: "",
+		hasIPv4: false,
+		hasIPv6: false,
+		Size:    TimeSliceLength,
+		MaxRTT:  time.Second,
+		OnRecv:  nil,
+		OnIdle:  nil,
+		Debug:   false,
+	}
+}
+
+// Network sets a network endpoints for ICMP ping and returns the previous
+// setting. network arg should be "ip" or "udp" string or if others are
+// specified, it returns an error. If this function isn't called, Pinger
+// uses "ip" as default.
+func (p *Pinger) Network(network string) (string, error) {
+	origNet := p.network
+	switch network {
+	case "ip":
+		fallthrough
+	case "udp":
+		p.network = network
+	default:
+		return origNet, errors.New(network + " can't be used as ICMP endpoint")
+	}
+	return origNet, nil
+}
+
+// Source sets ipv4/ipv6 source IP for sending ICMP packets and returns the previous
+// setting. Empty value indicates to use system default one (for both ipv4 and ipv6).
+func (p *Pinger) Source(source string) (string, error) {
+	// using ipv4 previous value for new empty one
+	origSource := p.source
+	if "" == source {
+		p.mu.Lock()
+		p.source = ""
+		p.source6 = ""
+		p.mu.Unlock()
+		return origSource, nil
+	}
+
+	addr := net.ParseIP(source)
+	if addr == nil {
+		return origSource, errors.New(source + " is not a valid textual representation of an IPv4/IPv6 address")
+	}
+
+	if isIPv4(addr) {
+		p.mu.Lock()
+		p.source = source
+		p.mu.Unlock()
+	} else if isIPv6(addr) {
+		origSource = p.source6
+		p.mu.Lock()
+		p.source6 = source
+		p.mu.Unlock()
+	} else {
+		return origSource, errors.New(source + " is not a valid textual representation of an IPv4/IPv6 address")
+	}
+
+	return origSource, nil
+}
+
+// AddIP adds an IP address to Pinger. ipaddr arg should be a string like
+// "192.0.2.1".
+func (p *Pinger) AddIP(ipaddr string) error {
+	addr := net.ParseIP(ipaddr)
+	if addr == nil {
+		return fmt.Errorf("%s is not a valid textual representation of an IP address", ipaddr)
+	}
+	p.mu.Lock()
+	p.addrs[addr.String()] = &net.IPAddr{IP: addr}
+	if isIPv4(addr) {
+		p.hasIPv4 = true
+	} else if isIPv6(addr) {
+		p.hasIPv6 = true
+	}
+	p.mu.Unlock()
+	return nil
+}
+
+// AddIPAddr adds an IP address to Pinger. ip arg should be a net.IPAddr
+// pointer.
+func (p *Pinger) AddIPAddr(ip *net.IPAddr) {
+	p.mu.Lock()
+	p.addrs[ip.String()] = ip
+	if isIPv4(ip.IP) {
+		p.hasIPv4 = true
+	} else if isIPv6(ip.IP) {
+		p.hasIPv6 = true
+	}
+	p.mu.Unlock()
+}
+
+// RemoveIP removes an IP address from Pinger. ipaddr arg should be a string
+// like "192.0.2.1".
+func (p *Pinger) RemoveIP(ipaddr string) error {
+	addr := net.ParseIP(ipaddr)
+	if addr == nil {
+		return fmt.Errorf("%s is not a valid textual representation of an IP address", ipaddr)
+	}
+	p.mu.Lock()
+	delete(p.addrs, addr.String())
+	p.mu.Unlock()
+	return nil
+}
+
+// RemoveIPAddr removes an IP address from Pinger. ip arg should be a net.IPAddr
+// pointer.
+func (p *Pinger) RemoveIPAddr(ip *net.IPAddr) {
+	p.mu.Lock()
+	delete(p.addrs, ip.String())
+	p.mu.Unlock()
+}
+
+// AddHandler adds event handler to Pinger. event arg should be "receive" or
+// "idle" string.
+//
+// **CAUTION** This function is deprecated. Please use OnRecv and OnIdle field
+// of Pinger struct to set following handlers.
+//
+// "receive" handler should be
+//
+//	func(addr *net.IPAddr, rtt time.Duration)
+//
+// type function. The handler is called with a response packet's source address
+// and its elapsed time when Pinger receives a response packet.
+//
+// "idle" handler should be
+//
+//	func()
+//
+// type function. The handler is called when MaxRTT time passed. For more
+// detail, please see Run() and RunLoop().
+func (p *Pinger) AddHandler(event string, handler interface{}) error {
+	switch event {
+	case "receive":
+		if hdl, ok := handler.(func(*net.IPAddr, time.Duration)); ok {
+			p.mu.Lock()
+			p.OnRecv = hdl
+			p.mu.Unlock()
+			return nil
+		}
+		return errors.New("receive event handler should be `func(*net.IPAddr, time.Duration)`")
+	case "idle":
+		if hdl, ok := handler.(func()); ok {
+			p.mu.Lock()
+			p.OnIdle = hdl
+			p.mu.Unlock()
+			return nil
+		}
+		return errors.New("idle event handler should be `func()`")
+	}
+	return errors.New("No such event: " + event)
+}
+
+// Run invokes a single send/receive procedure. It sends packets to all hosts
+// which have already been added by AddIP() etc. and wait those responses. When
+// it receives a response, it calls "receive" handler registered by AddHander().
+// After MaxRTT seconds, it calls "idle" handler and returns to caller with
+// an error value. It means it blocks until MaxRTT seconds passed. For the
+// purpose of sending/receiving packets over and over, use RunLoop().
+func (p *Pinger) Run() error {
+	p.mu.Lock()
+	p.ctx = newContext()
+	p.mu.Unlock()
+	p.run(true)
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.ctx.err
+}
+
+// RunLoop invokes send/receive procedure repeatedly. It sends packets to all
+// hosts which have already been added by AddIP() etc. and wait those responses.
+// When it receives a response, it calls "receive" handler registered by
+// AddHander(). After MaxRTT seconds, it calls "idle" handler, resend packets
+// and wait those response. MaxRTT works as an interval time.
+//
+// This is a non-blocking method so immediately returns. If you want to monitor
+// and stop sending packets, use Done() and Stop() methods. For example,
+//
+//	p.RunLoop()
+//	ticker := time.NewTicker(time.Millisecond * 250)
+//	select {
+//	case <-p.Done():
+//		if err := p.Err(); err != nil {
+//			log.Fatalf("Ping failed: %v", err)
+//		}
+//	case <-ticker.C:
+//		break
+//	}
+//	ticker.Stop()
+//	p.Stop()
+//
+// For more details, please see "cmd/ping/ping.go".
+func (p *Pinger) RunLoop() {
+	p.mu.Lock()
+	p.ctx = newContext()
+	p.mu.Unlock()
+	go p.run(false)
+}
+
+// Done returns a channel that is closed when RunLoop() is stopped by an error
+// or Stop(). It must be called after RunLoop() call. If not, it causes panic.
+func (p *Pinger) Done() <-chan bool {
+	return p.ctx.done
+}
+
+// Stop stops RunLoop(). It must be called after RunLoop(). If not, it causes
+// panic.
+func (p *Pinger) Stop() {
+	p.debugln("Stop(): close(p.ctx.stop)")
+	close(p.ctx.stop)
+	p.debugln("Stop(): <-p.ctx.done")
+	<-p.ctx.done
+}
+
+// Err returns an error that is set by RunLoop(). It must be called after
+// RunLoop(). If not, it causes panic.
+func (p *Pinger) Err() error {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	return p.ctx.err
+}
+
+func (p *Pinger) listen(netProto string, source string) *icmp.PacketConn {
+	conn, err := icmp.ListenPacket(netProto, source)
+	if err != nil {
+		p.mu.Lock()
+		p.ctx.err = err
+		p.mu.Unlock()
+		p.debugln("Run(): close(p.ctx.done)")
+		close(p.ctx.done)
+		return nil
+	}
+	return conn
+}
+
+func (p *Pinger) run(once bool) {
+	p.debugln("Run(): Start")
+	var conn, conn6 *icmp.PacketConn
+	if p.hasIPv4 {
+		if conn = p.listen(ipv4Proto[p.network], p.source); conn == nil {
+			return
+		}
+		defer conn.Close()
+	}
+
+	if p.hasIPv6 {
+		if conn6 = p.listen(ipv6Proto[p.network], p.source6); conn6 == nil {
+			return
+		}
+		defer conn6.Close()
+	}
+
+	recv := make(chan *packet, 1)
+	recvCtx := newContext()
+	wg := new(sync.WaitGroup)
+
+	p.debugln("Run(): call recvICMP()")
+	if conn != nil {
+		wg.Add(1)
+		go p.recvICMP(conn, recv, recvCtx, wg)
+	}
+	if conn6 != nil {
+		wg.Add(1)
+		go p.recvICMP(conn6, recv, recvCtx, wg)
+	}
+
+	p.debugln("Run(): call sendICMP()")
+	queue, err := p.sendICMP(conn, conn6)
+
+	ticker := time.NewTicker(p.MaxRTT)
+
+mainloop:
+	for {
+		select {
+		case <-p.ctx.stop:
+			p.debugln("Run(): <-p.ctx.stop")
+			break mainloop
+		case <-recvCtx.done:
+			p.debugln("Run(): <-recvCtx.done")
+			p.mu.Lock()
+			err = recvCtx.err
+			p.mu.Unlock()
+			break mainloop
+		case <-ticker.C:
+			p.mu.Lock()
+			handler := p.OnIdle
+			p.mu.Unlock()
+			if handler != nil {
+				handler()
+			}
+			if once || err != nil {
+				break mainloop
+			}
+			p.debugln("Run(): call sendICMP()")
+			queue, err = p.sendICMP(conn, conn6)
+		case r := <-recv:
+			p.debugln("Run(): <-recv")
+			p.procRecv(r, queue)
+		}
+	}
+
+	ticker.Stop()
+
+	p.debugln("Run(): close(recvCtx.stop)")
+	close(recvCtx.stop)
+	p.debugln("Run(): wait recvICMP()")
+	wg.Wait()
+
+	p.mu.Lock()
+	p.ctx.err = err
+	p.mu.Unlock()
+
+	p.debugln("Run(): close(p.ctx.done)")
+	close(p.ctx.done)
+	p.debugln("Run(): End")
+}
+
+func (p *Pinger) sendICMP(conn, conn6 *icmp.PacketConn) (map[string]*net.IPAddr, error) {
+	p.debugln("sendICMP(): Start")
+	p.mu.Lock()
+	p.id = rand.Intn(0xffff)
+	p.seq = rand.Intn(0xffff)
+	p.mu.Unlock()
+	queue := make(map[string]*net.IPAddr)
+	wg := new(sync.WaitGroup)
+	for key, addr := range p.addrs {
+		var typ icmp.Type
+		var cn *icmp.PacketConn
+		if isIPv4(addr.IP) {
+			typ = ipv4.ICMPTypeEcho
+			cn = conn
+		} else if isIPv6(addr.IP) {
+			typ = ipv6.ICMPTypeEchoRequest
+			cn = conn6
+		} else {
+			continue
+		}
+		if cn == nil {
+			continue
+		}
+
+		t := timeToBytes(time.Now())
+
+		if p.Size-TimeSliceLength != 0 {
+			t = append(t, byteSliceOfSize(p.Size-TimeSliceLength)...)
+		}
+
+		p.mu.Lock()
+		bytes, err := (&icmp.Message{
+			Type: typ, Code: 0,
+			Body: &icmp.Echo{
+				ID: p.id, Seq: p.seq,
+				Data: t,
+			},
+		}).Marshal(nil)
+		p.mu.Unlock()
+		if err != nil {
+			wg.Wait()
+			return queue, err
+		}
+
+		queue[key] = addr
+		var dst net.Addr = addr
+		if p.network == "udp" {
+			dst = &net.UDPAddr{IP: addr.IP, Zone: addr.Zone}
+		}
+
+		p.debugln("sendICMP(): Invoke goroutine")
+		wg.Add(1)
+		go func(conn *icmp.PacketConn, ra net.Addr, b []byte) {
+			for {
+				if _, err := conn.WriteTo(bytes, ra); err != nil {
+					if neterr, ok := err.(*net.OpError); ok {
+						if neterr.Err == syscall.ENOBUFS {
+							continue
+						}
+					}
+				}
+				break
+			}
+			p.debugln("sendICMP(): WriteTo End")
+			wg.Done()
+		}(cn, dst, bytes)
+	}
+	wg.Wait()
+	p.debugln("sendICMP(): End")
+	return queue, nil
+}
+
+func (p *Pinger) recvICMP(conn *icmp.PacketConn, recv chan<- *packet, ctx *context, wg *sync.WaitGroup) {
+	p.debugln("recvICMP(): Start")
+	for {
+		select {
+		case <-ctx.stop:
+			p.debugln("recvICMP(): <-ctx.stop")
+			wg.Done()
+			p.debugln("recvICMP(): wg.Done()")
+			return
+		default:
+		}
+
+		bytes := make([]byte, 512)
+		conn.SetReadDeadline(time.Now().Add(time.Millisecond * 100))
+		p.debugln("recvICMP(): ReadFrom Start")
+		_, ra, err := conn.ReadFrom(bytes)
+		p.debugln("recvICMP(): ReadFrom End")
+		if err != nil {
+			if neterr, ok := err.(*net.OpError); ok {
+				if neterr.Timeout() {
+					p.debugln("recvICMP(): Read Timeout")
+					continue
+				} else {
+					p.debugln("recvICMP(): OpError happen", err)
+					p.mu.Lock()
+					ctx.err = err
+					p.mu.Unlock()
+					p.debugln("recvICMP(): close(ctx.done)")
+					close(ctx.done)
+					p.debugln("recvICMP(): wg.Done()")
+					wg.Done()
+					return
+				}
+			}
+		}
+		p.debugln("recvICMP(): p.recv <- packet")
+
+		select {
+		case recv <- &packet{bytes: bytes, addr: ra}:
+		case <-ctx.stop:
+			p.debugln("recvICMP(): <-ctx.stop")
+			wg.Done()
+			p.debugln("recvICMP(): wg.Done()")
+			return
+		}
+	}
+}
+
+func (p *Pinger) procRecv(recv *packet, queue map[string]*net.IPAddr) {
+	var ipaddr *net.IPAddr
+	switch adr := recv.addr.(type) {
+	case *net.IPAddr:
+		ipaddr = adr
+	case *net.UDPAddr:
+		ipaddr = &net.IPAddr{IP: adr.IP, Zone: adr.Zone}
+	default:
+		return
+	}
+
+	addr := ipaddr.String()
+	p.mu.Lock()
+	if _, ok := p.addrs[addr]; !ok {
+		p.mu.Unlock()
+		return
+	}
+	p.mu.Unlock()
+
+	var bytes []byte
+	var proto int
+	if isIPv4(ipaddr.IP) {
+		if p.network == "ip" {
+			bytes = ipv4Payload(recv.bytes)
+		} else {
+			bytes = recv.bytes
+		}
+		proto = ProtocolICMP
+	} else if isIPv6(ipaddr.IP) {
+		bytes = recv.bytes
+		proto = ProtocolIPv6ICMP
+	} else {
+		return
+	}
+
+	var m *icmp.Message
+	var err error
+	if m, err = icmp.ParseMessage(proto, bytes); err != nil {
+		return
+	}
+
+	if m.Type != ipv4.ICMPTypeEchoReply && m.Type != ipv6.ICMPTypeEchoReply {
+		return
+	}
+
+	var rtt time.Duration
+	switch pkt := m.Body.(type) {
+	case *icmp.Echo:
+		p.mu.Lock()
+		if pkt.ID == p.id && pkt.Seq == p.seq {
+			rtt = time.Since(bytesToTime(pkt.Data[:TimeSliceLength]))
+		}
+		p.mu.Unlock()
+	default:
+		return
+	}
+
+	if _, ok := queue[addr]; ok {
+		delete(queue, addr)
+		p.mu.Lock()
+		handler := p.OnRecv
+		p.mu.Unlock()
+		if handler != nil {
+			handler(ipaddr, rtt)
+		}
+	}
+}
+
+func (p *Pinger) debugln(args ...interface{}) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.Debug {
+		log.Println(args...)
+	}
+}
+
+func (p *Pinger) debugf(format string, args ...interface{}) {
+	p.mu.Lock()
+	defer p.mu.Unlock()
+	if p.Debug {
+		log.Printf(format, args...)
+	}
+}

vendor/golang.org/x/net/icmp/dstunreach.go

@@ -0,0 +1,59 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"golang.org/x/net/internal/iana"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+// A DstUnreach represents an ICMP destination unreachable message
+// body.
+type DstUnreach struct {
+	Data       []byte      // data, known as original datagram field
+	Extensions []Extension // extensions
+}
+
+// Len implements the Len method of MessageBody interface.
+func (p *DstUnreach) Len(proto int) int {
+	if p == nil {
+		return 0
+	}
+	l, _ := multipartMessageBodyDataLen(proto, true, p.Data, p.Extensions)
+	return l
+}
+
+// Marshal implements the Marshal method of MessageBody interface.
+func (p *DstUnreach) Marshal(proto int) ([]byte, error) {
+	var typ Type
+	switch proto {
+	case iana.ProtocolICMP:
+		typ = ipv4.ICMPTypeDestinationUnreachable
+	case iana.ProtocolIPv6ICMP:
+		typ = ipv6.ICMPTypeDestinationUnreachable
+	default:
+		return nil, errInvalidProtocol
+	}
+	if !validExtensions(typ, p.Extensions) {
+		return nil, errInvalidExtension
+	}
+	return marshalMultipartMessageBody(proto, true, p.Data, p.Extensions)
+}
+
+// parseDstUnreach parses b as an ICMP destination unreachable message
+// body.
+func parseDstUnreach(proto int, typ Type, b []byte) (MessageBody, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	p := &DstUnreach{}
+	var err error
+	p.Data, p.Extensions, err = parseMultipartMessageBody(proto, typ, b)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}

vendor/golang.org/x/net/icmp/echo.go

@@ -0,0 +1,173 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"encoding/binary"
+
+	"golang.org/x/net/internal/iana"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+// An Echo represents an ICMP echo request or reply message body.
+type Echo struct {
+	ID   int    // identifier
+	Seq  int    // sequence number
+	Data []byte // data
+}
+
+// Len implements the Len method of MessageBody interface.
+func (p *Echo) Len(proto int) int {
+	if p == nil {
+		return 0
+	}
+	return 4 + len(p.Data)
+}
+
+// Marshal implements the Marshal method of MessageBody interface.
+func (p *Echo) Marshal(proto int) ([]byte, error) {
+	b := make([]byte, 4+len(p.Data))
+	binary.BigEndian.PutUint16(b[:2], uint16(p.ID))
+	binary.BigEndian.PutUint16(b[2:4], uint16(p.Seq))
+	copy(b[4:], p.Data)
+	return b, nil
+}
+
+// parseEcho parses b as an ICMP echo request or reply message body.
+func parseEcho(proto int, _ Type, b []byte) (MessageBody, error) {
+	bodyLen := len(b)
+	if bodyLen < 4 {
+		return nil, errMessageTooShort
+	}
+	p := &Echo{ID: int(binary.BigEndian.Uint16(b[:2])), Seq: int(binary.BigEndian.Uint16(b[2:4]))}
+	if bodyLen > 4 {
+		p.Data = make([]byte, bodyLen-4)
+		copy(p.Data, b[4:])
+	}
+	return p, nil
+}
+
+// An ExtendedEchoRequest represents an ICMP extended echo request
+// message body.
+type ExtendedEchoRequest struct {
+	ID         int         // identifier
+	Seq        int         // sequence number
+	Local      bool        // must be true when identifying by name or index
+	Extensions []Extension // extensions
+}
+
+// Len implements the Len method of MessageBody interface.
+func (p *ExtendedEchoRequest) Len(proto int) int {
+	if p == nil {
+		return 0
+	}
+	l, _ := multipartMessageBodyDataLen(proto, false, nil, p.Extensions)
+	return l
+}
+
+// Marshal implements the Marshal method of MessageBody interface.
+func (p *ExtendedEchoRequest) Marshal(proto int) ([]byte, error) {
+	var typ Type
+	switch proto {
+	case iana.ProtocolICMP:
+		typ = ipv4.ICMPTypeExtendedEchoRequest
+	case iana.ProtocolIPv6ICMP:
+		typ = ipv6.ICMPTypeExtendedEchoRequest
+	default:
+		return nil, errInvalidProtocol
+	}
+	if !validExtensions(typ, p.Extensions) {
+		return nil, errInvalidExtension
+	}
+	b, err := marshalMultipartMessageBody(proto, false, nil, p.Extensions)
+	if err != nil {
+		return nil, err
+	}
+	binary.BigEndian.PutUint16(b[:2], uint16(p.ID))
+	b[2] = byte(p.Seq)
+	if p.Local {
+		b[3] |= 0x01
+	}
+	return b, nil
+}
+
+// parseExtendedEchoRequest parses b as an ICMP extended echo request
+// message body.
+func parseExtendedEchoRequest(proto int, typ Type, b []byte) (MessageBody, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	p := &ExtendedEchoRequest{ID: int(binary.BigEndian.Uint16(b[:2])), Seq: int(b[2])}
+	if b[3]&0x01 != 0 {
+		p.Local = true
+	}
+	var err error
+	_, p.Extensions, err = parseMultipartMessageBody(proto, typ, b)
+	if err != nil {
+		return nil, err
+	}
+	return p, nil
+}
+
+// An ExtendedEchoReply represents an ICMP extended echo reply message
+// body.
+type ExtendedEchoReply struct {
+	ID     int  // identifier
+	Seq    int  // sequence number
+	State  int  // 3-bit state working together with Message.Code
+	Active bool // probed interface is active
+	IPv4   bool // probed interface runs IPv4
+	IPv6   bool // probed interface runs IPv6
+}
+
+// Len implements the Len method of MessageBody interface.
+func (p *ExtendedEchoReply) Len(proto int) int {
+	if p == nil {
+		return 0
+	}
+	return 4
+}
+
+// Marshal implements the Marshal method of MessageBody interface.
+func (p *ExtendedEchoReply) Marshal(proto int) ([]byte, error) {
+	b := make([]byte, 4)
+	binary.BigEndian.PutUint16(b[:2], uint16(p.ID))
+	b[2] = byte(p.Seq)
+	b[3] = byte(p.State<<5) & 0xe0
+	if p.Active {
+		b[3] |= 0x04
+	}
+	if p.IPv4 {
+		b[3] |= 0x02
+	}
+	if p.IPv6 {
+		b[3] |= 0x01
+	}
+	return b, nil
+}
+
+// parseExtendedEchoReply parses b as an ICMP extended echo reply
+// message body.
+func parseExtendedEchoReply(proto int, _ Type, b []byte) (MessageBody, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	p := &ExtendedEchoReply{
+		ID:    int(binary.BigEndian.Uint16(b[:2])),
+		Seq:   int(b[2]),
+		State: int(b[3]) >> 5,
+	}
+	if b[3]&0x04 != 0 {
+		p.Active = true
+	}
+	if b[3]&0x02 != 0 {
+		p.IPv4 = true
+	}
+	if b[3]&0x01 != 0 {
+		p.IPv6 = true
+	}
+	return p, nil
+}

vendor/golang.org/x/net/icmp/endpoint.go

@@ -0,0 +1,113 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"net"
+	"runtime"
+	"time"
+
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+var _ net.PacketConn = &PacketConn{}
+
+// A PacketConn represents a packet network endpoint that uses either
+// ICMPv4 or ICMPv6.
+type PacketConn struct {
+	c  net.PacketConn
+	p4 *ipv4.PacketConn
+	p6 *ipv6.PacketConn
+}
+
+func (c *PacketConn) ok() bool { return c != nil && c.c != nil }
+
+// IPv4PacketConn returns the ipv4.PacketConn of c.
+// It returns nil when c is not created as the endpoint for ICMPv4.
+func (c *PacketConn) IPv4PacketConn() *ipv4.PacketConn {
+	if !c.ok() {
+		return nil
+	}
+	return c.p4
+}
+
+// IPv6PacketConn returns the ipv6.PacketConn of c.
+// It returns nil when c is not created as the endpoint for ICMPv6.
+func (c *PacketConn) IPv6PacketConn() *ipv6.PacketConn {
+	if !c.ok() {
+		return nil
+	}
+	return c.p6
+}
+
+// ReadFrom reads an ICMP message from the connection.
+func (c *PacketConn) ReadFrom(b []byte) (int, net.Addr, error) {
+	if !c.ok() {
+		return 0, nil, errInvalidConn
+	}
+	// Please be informed that ipv4.NewPacketConn enables
+	// IP_STRIPHDR option by default on Darwin.
+	// See golang.org/issue/9395 for further information.
+	if (runtime.GOOS == "darwin" || runtime.GOOS == "ios") && c.p4 != nil {
+		n, _, peer, err := c.p4.ReadFrom(b)
+		return n, peer, err
+	}
+	return c.c.ReadFrom(b)
+}
+
+// WriteTo writes the ICMP message b to dst.
+// The provided dst must be net.UDPAddr when c is a non-privileged
+// datagram-oriented ICMP endpoint.
+// Otherwise it must be net.IPAddr.
+func (c *PacketConn) WriteTo(b []byte, dst net.Addr) (int, error) {
+	if !c.ok() {
+		return 0, errInvalidConn
+	}
+	return c.c.WriteTo(b, dst)
+}
+
+// Close closes the endpoint.
+func (c *PacketConn) Close() error {
+	if !c.ok() {
+		return errInvalidConn
+	}
+	return c.c.Close()
+}
+
+// LocalAddr returns the local network address.
+func (c *PacketConn) LocalAddr() net.Addr {
+	if !c.ok() {
+		return nil
+	}
+	return c.c.LocalAddr()
+}
+
+// SetDeadline sets the read and write deadlines associated with the
+// endpoint.
+func (c *PacketConn) SetDeadline(t time.Time) error {
+	if !c.ok() {
+		return errInvalidConn
+	}
+	return c.c.SetDeadline(t)
+}
+
+// SetReadDeadline sets the read deadline associated with the
+// endpoint.
+func (c *PacketConn) SetReadDeadline(t time.Time) error {
+	if !c.ok() {
+		return errInvalidConn
+	}
+	return c.c.SetReadDeadline(t)
+}
+
+// SetWriteDeadline sets the write deadline associated with the
+// endpoint.
+func (c *PacketConn) SetWriteDeadline(t time.Time) error {
+	if !c.ok() {
+		return errInvalidConn
+	}
+	return c.c.SetWriteDeadline(t)
+}

vendor/golang.org/x/net/icmp/extension.go

@@ -0,0 +1,170 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"encoding/binary"
+
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+// An Extension represents an ICMP extension.
+type Extension interface {
+	// Len returns the length of ICMP extension.
+	// The provided proto must be either the ICMPv4 or ICMPv6
+	// protocol number.
+	Len(proto int) int
+
+	// Marshal returns the binary encoding of ICMP extension.
+	// The provided proto must be either the ICMPv4 or ICMPv6
+	// protocol number.
+	Marshal(proto int) ([]byte, error)
+}
+
+const extensionVersion = 2
+
+func validExtensionHeader(b []byte) bool {
+	v := int(b[0]&0xf0) >> 4
+	s := binary.BigEndian.Uint16(b[2:4])
+	if s != 0 {
+		s = checksum(b)
+	}
+	if v != extensionVersion || s != 0 {
+		return false
+	}
+	return true
+}
+
+// parseExtensions parses b as a list of ICMP extensions.
+// The length attribute l must be the length attribute field in
+// received icmp messages.
+//
+// It will return a list of ICMP extensions and an adjusted length
+// attribute that represents the length of the padded original
+// datagram field. Otherwise, it returns an error.
+func parseExtensions(typ Type, b []byte, l int) ([]Extension, int, error) {
+	// Still a lot of non-RFC 4884 compliant implementations are
+	// out there. Set the length attribute l to 128 when it looks
+	// inappropriate for backwards compatibility.
+	//
+	// A minimal extension at least requires 8 octets; 4 octets
+	// for an extension header, and 4 octets for a single object
+	// header.
+	//
+	// See RFC 4884 for further information.
+	switch typ {
+	case ipv4.ICMPTypeExtendedEchoRequest, ipv6.ICMPTypeExtendedEchoRequest:
+		if len(b) < 8 || !validExtensionHeader(b) {
+			return nil, -1, errNoExtension
+		}
+		l = 0
+	default:
+		if 128 > l || l+8 > len(b) {
+			l = 128
+		}
+		if l+8 > len(b) {
+			return nil, -1, errNoExtension
+		}
+		if !validExtensionHeader(b[l:]) {
+			if l == 128 {
+				return nil, -1, errNoExtension
+			}
+			l = 128
+			if !validExtensionHeader(b[l:]) {
+				return nil, -1, errNoExtension
+			}
+		}
+	}
+	var exts []Extension
+	for b = b[l+4:]; len(b) >= 4; {
+		ol := int(binary.BigEndian.Uint16(b[:2]))
+		if 4 > ol || ol > len(b) {
+			break
+		}
+		switch b[2] {
+		case classMPLSLabelStack:
+			ext, err := parseMPLSLabelStack(b[:ol])
+			if err != nil {
+				return nil, -1, err
+			}
+			exts = append(exts, ext)
+		case classInterfaceInfo:
+			ext, err := parseInterfaceInfo(b[:ol])
+			if err != nil {
+				return nil, -1, err
+			}
+			exts = append(exts, ext)
+		case classInterfaceIdent:
+			ext, err := parseInterfaceIdent(b[:ol])
+			if err != nil {
+				return nil, -1, err
+			}
+			exts = append(exts, ext)
+		default:
+			ext := &RawExtension{Data: make([]byte, ol)}
+			copy(ext.Data, b[:ol])
+			exts = append(exts, ext)
+		}
+		b = b[ol:]
+	}
+	return exts, l, nil
+}
+
+func validExtensions(typ Type, exts []Extension) bool {
+	switch typ {
+	case ipv4.ICMPTypeDestinationUnreachable, ipv4.ICMPTypeTimeExceeded, ipv4.ICMPTypeParameterProblem,
+		ipv6.ICMPTypeDestinationUnreachable, ipv6.ICMPTypeTimeExceeded:
+		for i := range exts {
+			switch exts[i].(type) {
+			case *MPLSLabelStack, *InterfaceInfo, *RawExtension:
+			default:
+				return false
+			}
+		}
+		return true
+	case ipv4.ICMPTypeExtendedEchoRequest, ipv6.ICMPTypeExtendedEchoRequest:
+		var n int
+		for i := range exts {
+			switch exts[i].(type) {
+			case *InterfaceIdent:
+				n++
+			case *RawExtension:
+			default:
+				return false
+			}
+		}
+		// Not a single InterfaceIdent object or a combo of
+		// RawExtension and InterfaceIdent objects is not
+		// allowed.
+		if n == 1 && len(exts) > 1 {
+			return false
+		}
+		return true
+	default:
+		return false
+	}
+}
+
+// A RawExtension represents a raw extension.
+//
+// A raw extension is excluded from message processing and can be used
+// to construct applications such as protocol conformance testing.
+type RawExtension struct {
+	Data []byte // data
+}
+
+// Len implements the Len method of Extension interface.
+func (p *RawExtension) Len(proto int) int {
+	if p == nil {
+		return 0
+	}
+	return len(p.Data)
+}
+
+// Marshal implements the Marshal method of Extension interface.
+func (p *RawExtension) Marshal(proto int) ([]byte, error) {
+	return p.Data, nil
+}

vendor/golang.org/x/net/icmp/helper_posix.go

@@ -0,0 +1,75 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris windows
+
+package icmp
+
+import (
+	"net"
+	"strconv"
+	"syscall"
+)
+
+func sockaddr(family int, address string) (syscall.Sockaddr, error) {
+	switch family {
+	case syscall.AF_INET:
+		a, err := net.ResolveIPAddr("ip4", address)
+		if err != nil {
+			return nil, err
+		}
+		if len(a.IP) == 0 {
+			a.IP = net.IPv4zero
+		}
+		if a.IP = a.IP.To4(); a.IP == nil {
+			return nil, net.InvalidAddrError("non-ipv4 address")
+		}
+		sa := &syscall.SockaddrInet4{}
+		copy(sa.Addr[:], a.IP)
+		return sa, nil
+	case syscall.AF_INET6:
+		a, err := net.ResolveIPAddr("ip6", address)
+		if err != nil {
+			return nil, err
+		}
+		if len(a.IP) == 0 {
+			a.IP = net.IPv6unspecified
+		}
+		if a.IP.Equal(net.IPv4zero) {
+			a.IP = net.IPv6unspecified
+		}
+		if a.IP = a.IP.To16(); a.IP == nil || a.IP.To4() != nil {
+			return nil, net.InvalidAddrError("non-ipv6 address")
+		}
+		sa := &syscall.SockaddrInet6{ZoneId: zoneToUint32(a.Zone)}
+		copy(sa.Addr[:], a.IP)
+		return sa, nil
+	default:
+		return nil, net.InvalidAddrError("unexpected family")
+	}
+}
+
+func zoneToUint32(zone string) uint32 {
+	if zone == "" {
+		return 0
+	}
+	if ifi, err := net.InterfaceByName(zone); err == nil {
+		return uint32(ifi.Index)
+	}
+	n, err := strconv.Atoi(zone)
+	if err != nil {
+		return 0
+	}
+	return uint32(n)
+}
+
+func last(s string, b byte) int {
+	i := len(s)
+	for i--; i >= 0; i-- {
+		if s[i] == b {
+			break
+		}
+	}
+	return i
+}

vendor/golang.org/x/net/icmp/interface.go

@@ -0,0 +1,322 @@
+// Copyright 2015 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"encoding/binary"
+	"net"
+	"strings"
+
+	"golang.org/x/net/internal/iana"
+)
+
+const (
+	classInterfaceInfo = 2
+)
+
+const (
+	attrMTU = 1 << iota
+	attrName
+	attrIPAddr
+	attrIfIndex
+)
+
+// An InterfaceInfo represents interface and next-hop identification.
+type InterfaceInfo struct {
+	Class     int // extension object class number
+	Type      int // extension object sub-type
+	Interface *net.Interface
+	Addr      *net.IPAddr
+}
+
+func (ifi *InterfaceInfo) nameLen() int {
+	if len(ifi.Interface.Name) > 63 {
+		return 64
+	}
+	l := 1 + len(ifi.Interface.Name)
+	return (l + 3) &^ 3
+}
+
+func (ifi *InterfaceInfo) attrsAndLen(proto int) (attrs, l int) {
+	l = 4
+	if ifi.Interface != nil && ifi.Interface.Index > 0 {
+		attrs |= attrIfIndex
+		l += 4
+		if len(ifi.Interface.Name) > 0 {
+			attrs |= attrName
+			l += ifi.nameLen()
+		}
+		if ifi.Interface.MTU > 0 {
+			attrs |= attrMTU
+			l += 4
+		}
+	}
+	if ifi.Addr != nil {
+		switch proto {
+		case iana.ProtocolICMP:
+			if ifi.Addr.IP.To4() != nil {
+				attrs |= attrIPAddr
+				l += 4 + net.IPv4len
+			}
+		case iana.ProtocolIPv6ICMP:
+			if ifi.Addr.IP.To16() != nil && ifi.Addr.IP.To4() == nil {
+				attrs |= attrIPAddr
+				l += 4 + net.IPv6len
+			}
+		}
+	}
+	return
+}
+
+// Len implements the Len method of Extension interface.
+func (ifi *InterfaceInfo) Len(proto int) int {
+	_, l := ifi.attrsAndLen(proto)
+	return l
+}
+
+// Marshal implements the Marshal method of Extension interface.
+func (ifi *InterfaceInfo) Marshal(proto int) ([]byte, error) {
+	attrs, l := ifi.attrsAndLen(proto)
+	b := make([]byte, l)
+	if err := ifi.marshal(proto, b, attrs, l); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (ifi *InterfaceInfo) marshal(proto int, b []byte, attrs, l int) error {
+	binary.BigEndian.PutUint16(b[:2], uint16(l))
+	b[2], b[3] = classInterfaceInfo, byte(ifi.Type)
+	for b = b[4:]; len(b) > 0 && attrs != 0; {
+		switch {
+		case attrs&attrIfIndex != 0:
+			b = ifi.marshalIfIndex(proto, b)
+			attrs &^= attrIfIndex
+		case attrs&attrIPAddr != 0:
+			b = ifi.marshalIPAddr(proto, b)
+			attrs &^= attrIPAddr
+		case attrs&attrName != 0:
+			b = ifi.marshalName(proto, b)
+			attrs &^= attrName
+		case attrs&attrMTU != 0:
+			b = ifi.marshalMTU(proto, b)
+			attrs &^= attrMTU
+		}
+	}
+	return nil
+}
+
+func (ifi *InterfaceInfo) marshalIfIndex(proto int, b []byte) []byte {
+	binary.BigEndian.PutUint32(b[:4], uint32(ifi.Interface.Index))
+	return b[4:]
+}
+
+func (ifi *InterfaceInfo) parseIfIndex(b []byte) ([]byte, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	ifi.Interface.Index = int(binary.BigEndian.Uint32(b[:4]))
+	return b[4:], nil
+}
+
+func (ifi *InterfaceInfo) marshalIPAddr(proto int, b []byte) []byte {
+	switch proto {
+	case iana.ProtocolICMP:
+		binary.BigEndian.PutUint16(b[:2], uint16(iana.AddrFamilyIPv4))
+		copy(b[4:4+net.IPv4len], ifi.Addr.IP.To4())
+		b = b[4+net.IPv4len:]
+	case iana.ProtocolIPv6ICMP:
+		binary.BigEndian.PutUint16(b[:2], uint16(iana.AddrFamilyIPv6))
+		copy(b[4:4+net.IPv6len], ifi.Addr.IP.To16())
+		b = b[4+net.IPv6len:]
+	}
+	return b
+}
+
+func (ifi *InterfaceInfo) parseIPAddr(b []byte) ([]byte, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	afi := int(binary.BigEndian.Uint16(b[:2]))
+	b = b[4:]
+	switch afi {
+	case iana.AddrFamilyIPv4:
+		if len(b) < net.IPv4len {
+			return nil, errMessageTooShort
+		}
+		ifi.Addr.IP = make(net.IP, net.IPv4len)
+		copy(ifi.Addr.IP, b[:net.IPv4len])
+		b = b[net.IPv4len:]
+	case iana.AddrFamilyIPv6:
+		if len(b) < net.IPv6len {
+			return nil, errMessageTooShort
+		}
+		ifi.Addr.IP = make(net.IP, net.IPv6len)
+		copy(ifi.Addr.IP, b[:net.IPv6len])
+		b = b[net.IPv6len:]
+	}
+	return b, nil
+}
+
+func (ifi *InterfaceInfo) marshalName(proto int, b []byte) []byte {
+	l := byte(ifi.nameLen())
+	b[0] = l
+	copy(b[1:], []byte(ifi.Interface.Name))
+	return b[l:]
+}
+
+func (ifi *InterfaceInfo) parseName(b []byte) ([]byte, error) {
+	if 4 > len(b) || len(b) < int(b[0]) {
+		return nil, errMessageTooShort
+	}
+	l := int(b[0])
+	if l%4 != 0 || 4 > l || l > 64 {
+		return nil, errInvalidExtension
+	}
+	var name [63]byte
+	copy(name[:], b[1:l])
+	ifi.Interface.Name = strings.Trim(string(name[:]), "\000")
+	return b[l:], nil
+}
+
+func (ifi *InterfaceInfo) marshalMTU(proto int, b []byte) []byte {
+	binary.BigEndian.PutUint32(b[:4], uint32(ifi.Interface.MTU))
+	return b[4:]
+}
+
+func (ifi *InterfaceInfo) parseMTU(b []byte) ([]byte, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	ifi.Interface.MTU = int(binary.BigEndian.Uint32(b[:4]))
+	return b[4:], nil
+}
+
+func parseInterfaceInfo(b []byte) (Extension, error) {
+	ifi := &InterfaceInfo{
+		Class: int(b[2]),
+		Type:  int(b[3]),
+	}
+	if ifi.Type&(attrIfIndex|attrName|attrMTU) != 0 {
+		ifi.Interface = &net.Interface{}
+	}
+	if ifi.Type&attrIPAddr != 0 {
+		ifi.Addr = &net.IPAddr{}
+	}
+	attrs := ifi.Type & (attrIfIndex | attrIPAddr | attrName | attrMTU)
+	for b = b[4:]; len(b) > 0 && attrs != 0; {
+		var err error
+		switch {
+		case attrs&attrIfIndex != 0:
+			b, err = ifi.parseIfIndex(b)
+			attrs &^= attrIfIndex
+		case attrs&attrIPAddr != 0:
+			b, err = ifi.parseIPAddr(b)
+			attrs &^= attrIPAddr
+		case attrs&attrName != 0:
+			b, err = ifi.parseName(b)
+			attrs &^= attrName
+		case attrs&attrMTU != 0:
+			b, err = ifi.parseMTU(b)
+			attrs &^= attrMTU
+		}
+		if err != nil {
+			return nil, err
+		}
+	}
+	if ifi.Interface != nil && ifi.Interface.Name != "" && ifi.Addr != nil && ifi.Addr.IP.To16() != nil && ifi.Addr.IP.To4() == nil {
+		ifi.Addr.Zone = ifi.Interface.Name
+	}
+	return ifi, nil
+}
+
+const (
+	classInterfaceIdent    = 3
+	typeInterfaceByName    = 1
+	typeInterfaceByIndex   = 2
+	typeInterfaceByAddress = 3
+)
+
+// An InterfaceIdent represents interface identification.
+type InterfaceIdent struct {
+	Class int    // extension object class number
+	Type  int    // extension object sub-type
+	Name  string // interface name
+	Index int    // interface index
+	AFI   int    // address family identifier; see address family numbers in IANA registry
+	Addr  []byte // address
+}
+
+// Len implements the Len method of Extension interface.
+func (ifi *InterfaceIdent) Len(_ int) int {
+	switch ifi.Type {
+	case typeInterfaceByName:
+		l := len(ifi.Name)
+		if l > 255 {
+			l = 255
+		}
+		return 4 + (l+3)&^3
+	case typeInterfaceByIndex:
+		return 4 + 4
+	case typeInterfaceByAddress:
+		return 4 + 4 + (len(ifi.Addr)+3)&^3
+	default:
+		return 4
+	}
+}
+
+// Marshal implements the Marshal method of Extension interface.
+func (ifi *InterfaceIdent) Marshal(proto int) ([]byte, error) {
+	b := make([]byte, ifi.Len(proto))
+	if err := ifi.marshal(proto, b); err != nil {
+		return nil, err
+	}
+	return b, nil
+}
+
+func (ifi *InterfaceIdent) marshal(proto int, b []byte) error {
+	l := ifi.Len(proto)
+	binary.BigEndian.PutUint16(b[:2], uint16(l))
+	b[2], b[3] = classInterfaceIdent, byte(ifi.Type)
+	switch ifi.Type {
+	case typeInterfaceByName:
+		copy(b[4:], ifi.Name)
+	case typeInterfaceByIndex:
+		binary.BigEndian.PutUint32(b[4:4+4], uint32(ifi.Index))
+	case typeInterfaceByAddress:
+		binary.BigEndian.PutUint16(b[4:4+2], uint16(ifi.AFI))
+		b[4+2] = byte(len(ifi.Addr))
+		copy(b[4+4:], ifi.Addr)
+	}
+	return nil
+}
+
+func parseInterfaceIdent(b []byte) (Extension, error) {
+	ifi := &InterfaceIdent{
+		Class: int(b[2]),
+		Type:  int(b[3]),
+	}
+	switch ifi.Type {
+	case typeInterfaceByName:
+		ifi.Name = strings.Trim(string(b[4:]), "\x00")
+	case typeInterfaceByIndex:
+		if len(b[4:]) < 4 {
+			return nil, errInvalidExtension
+		}
+		ifi.Index = int(binary.BigEndian.Uint32(b[4 : 4+4]))
+	case typeInterfaceByAddress:
+		if len(b[4:]) < 4 {
+			return nil, errInvalidExtension
+		}
+		ifi.AFI = int(binary.BigEndian.Uint16(b[4 : 4+2]))
+		l := int(b[4+2])
+		if len(b[4+4:]) < l {
+			return nil, errInvalidExtension
+		}
+		ifi.Addr = make([]byte, l)
+		copy(ifi.Addr, b[4+4:])
+	}
+	return ifi, nil
+}

vendor/golang.org/x/net/icmp/ipv4.go

@@ -0,0 +1,69 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"encoding/binary"
+	"net"
+	"runtime"
+
+	"golang.org/x/net/internal/socket"
+	"golang.org/x/net/ipv4"
+)
+
+// freebsdVersion is set in sys_freebsd.go.
+// See http://www.freebsd.org/doc/en/books/porters-handbook/freebsd-versions.html.
+var freebsdVersion uint32
+
+// ParseIPv4Header returns the IPv4 header of the IPv4 packet that
+// triggered an ICMP error message.
+// This is found in the Data field of the ICMP error message body.
+//
+// The provided b must be in the format used by a raw ICMP socket on
+// the local system.
+// This may differ from the wire format, and the format used by a raw
+// IP socket, depending on the system.
+//
+// To parse an IPv6 header, use ipv6.ParseHeader.
+func ParseIPv4Header(b []byte) (*ipv4.Header, error) {
+	if len(b) < ipv4.HeaderLen {
+		return nil, errHeaderTooShort
+	}
+	hdrlen := int(b[0]&0x0f) << 2
+	if hdrlen > len(b) {
+		return nil, errBufferTooShort
+	}
+	h := &ipv4.Header{
+		Version:  int(b[0] >> 4),
+		Len:      hdrlen,
+		TOS:      int(b[1]),
+		ID:       int(binary.BigEndian.Uint16(b[4:6])),
+		FragOff:  int(binary.BigEndian.Uint16(b[6:8])),
+		TTL:      int(b[8]),
+		Protocol: int(b[9]),
+		Checksum: int(binary.BigEndian.Uint16(b[10:12])),
+		Src:      net.IPv4(b[12], b[13], b[14], b[15]),
+		Dst:      net.IPv4(b[16], b[17], b[18], b[19]),
+	}
+	switch runtime.GOOS {
+	case "darwin", "ios":
+		h.TotalLen = int(socket.NativeEndian.Uint16(b[2:4]))
+	case "freebsd":
+		if freebsdVersion >= 1000000 {
+			h.TotalLen = int(binary.BigEndian.Uint16(b[2:4]))
+		} else {
+			h.TotalLen = int(socket.NativeEndian.Uint16(b[2:4]))
+		}
+	default:
+		h.TotalLen = int(binary.BigEndian.Uint16(b[2:4]))
+	}
+	h.Flags = ipv4.HeaderFlags(h.FragOff&0xe000) >> 13
+	h.FragOff = h.FragOff & 0x1fff
+	if hdrlen-ipv4.HeaderLen > 0 {
+		h.Options = make([]byte, hdrlen-ipv4.HeaderLen)
+		copy(h.Options, b[ipv4.HeaderLen:])
+	}
+	return h, nil
+}

vendor/golang.org/x/net/icmp/ipv6.go

@@ -0,0 +1,23 @@
+// Copyright 2013 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package icmp
+
+import (
+	"net"
+
+	"golang.org/x/net/internal/iana"
+)
+
+const ipv6PseudoHeaderLen = 2*net.IPv6len + 8
+
+// IPv6PseudoHeader returns an IPv6 pseudo header for checksum
+// calculation.
+func IPv6PseudoHeader(src, dst net.IP) []byte {
+	b := make([]byte, ipv6PseudoHeaderLen)
+	copy(b, src.To16())
+	copy(b[net.IPv6len:], dst.To16())
+	b[len(b)-1] = byte(iana.ProtocolIPv6ICMP)
+	return b
+}

vendor/golang.org/x/net/icmp/listen_posix.go

@@ -0,0 +1,103 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris windows
+
+package icmp
+
+import (
+	"net"
+	"os"
+	"runtime"
+	"syscall"
+
+	"golang.org/x/net/internal/iana"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+const sysIP_STRIPHDR = 0x17 // for now only darwin supports this option
+
+// ListenPacket listens for incoming ICMP packets addressed to
+// address. See net.Dial for the syntax of address.
+//
+// For non-privileged datagram-oriented ICMP endpoints, network must
+// be "udp4" or "udp6". The endpoint allows to read, write a few
+// limited ICMP messages such as echo request and echo reply.
+// Currently only Darwin and Linux support this.
+//
+// Examples:
+//	ListenPacket("udp4", "192.168.0.1")
+//	ListenPacket("udp4", "0.0.0.0")
+//	ListenPacket("udp6", "fe80::1%en0")
+//	ListenPacket("udp6", "::")
+//
+// For privileged raw ICMP endpoints, network must be "ip4" or "ip6"
+// followed by a colon and an ICMP protocol number or name.
+//
+// Examples:
+//	ListenPacket("ip4:icmp", "192.168.0.1")
+//	ListenPacket("ip4:1", "0.0.0.0")
+//	ListenPacket("ip6:ipv6-icmp", "fe80::1%en0")
+//	ListenPacket("ip6:58", "::")
+func ListenPacket(network, address string) (*PacketConn, error) {
+	var family, proto int
+	switch network {
+	case "udp4":
+		family, proto = syscall.AF_INET, iana.ProtocolICMP
+	case "udp6":
+		family, proto = syscall.AF_INET6, iana.ProtocolIPv6ICMP
+	default:
+		i := last(network, ':')
+		if i < 0 {
+			i = len(network)
+		}
+		switch network[:i] {
+		case "ip4":
+			proto = iana.ProtocolICMP
+		case "ip6":
+			proto = iana.ProtocolIPv6ICMP
+		}
+	}
+	var cerr error
+	var c net.PacketConn
+	switch family {
+	case syscall.AF_INET, syscall.AF_INET6:
+		s, err := syscall.Socket(family, syscall.SOCK_DGRAM, proto)
+		if err != nil {
+			return nil, os.NewSyscallError("socket", err)
+		}
+		if (runtime.GOOS == "darwin" || runtime.GOOS == "ios") && family == syscall.AF_INET {
+			if err := syscall.SetsockoptInt(s, iana.ProtocolIP, sysIP_STRIPHDR, 1); err != nil {
+				syscall.Close(s)
+				return nil, os.NewSyscallError("setsockopt", err)
+			}
+		}
+		sa, err := sockaddr(family, address)
+		if err != nil {
+			syscall.Close(s)
+			return nil, err
+		}
+		if err := syscall.Bind(s, sa); err != nil {
+			syscall.Close(s)
+			return nil, os.NewSyscallError("bind", err)
+		}
+		f := os.NewFile(uintptr(s), "datagram-oriented icmp")
+		c, cerr = net.FilePacketConn(f)
+		f.Close()
+	default:
+		c, cerr = net.ListenPacket(network, address)
+	}
+	if cerr != nil {
+		return nil, cerr
+	}
+	switch proto {
+	case iana.ProtocolICMP:
+		return &PacketConn{c: c, p4: ipv4.NewPacketConn(c)}, nil
+	case iana.ProtocolIPv6ICMP:
+		return &PacketConn{c: c, p6: ipv6.NewPacketConn(c)}, nil
+	default:
+		return &PacketConn{c: c}, nil
+	}
+}

vendor/golang.org/x/net/icmp/listen_stub.go

@@ -0,0 +1,33 @@
+// Copyright 2014 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows
+
+package icmp
+
+// ListenPacket listens for incoming ICMP packets addressed to
+// address. See net.Dial for the syntax of address.
+//
+// For non-privileged datagram-oriented ICMP endpoints, network must
+// be "udp4" or "udp6". The endpoint allows to read, write a few
+// limited ICMP messages such as echo request and echo reply.
+// Currently only Darwin and Linux support this.
+//
+// Examples:
+//	ListenPacket("udp4", "192.168.0.1")
+//	ListenPacket("udp4", "0.0.0.0")
+//	ListenPacket("udp6", "fe80::1%en0")
+//	ListenPacket("udp6", "::")
+//
+// For privileged raw ICMP endpoints, network must be "ip4" or "ip6"
+// followed by a colon and an ICMP protocol number or name.
+//
+// Examples:
+//	ListenPacket("ip4:icmp", "192.168.0.1")
+//	ListenPacket("ip4:1", "0.0.0.0")
+//	ListenPacket("ip6:ipv6-icmp", "fe80::1%en0")
+//	ListenPacket("ip6:58", "::")
+func ListenPacket(network, address string) (*PacketConn, error) {
+	return nil, errNotImplemented
+}

vendor/golang.org/x/net/icmp/message.go

@@ -0,0 +1,162 @@
+// Copyright 2012 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package icmp provides basic functions for the manipulation of
+// messages used in the Internet Control Message Protocols,
+// ICMPv4 and ICMPv6.
+//
+// ICMPv4 and ICMPv6 are defined in RFC 792 and RFC 4443.
+// Multi-part message support for ICMP is defined in RFC 4884.
+// ICMP extensions for MPLS are defined in RFC 4950.
+// ICMP extensions for interface and next-hop identification are
+// defined in RFC 5837.
+// PROBE: A utility for probing interfaces is defined in RFC 8335.
+package icmp // import "golang.org/x/net/icmp"
+
+import (
+	"encoding/binary"
+	"errors"
+	"net"
+	"runtime"
+
+	"golang.org/x/net/internal/iana"
+	"golang.org/x/net/ipv4"
+	"golang.org/x/net/ipv6"
+)
+
+// BUG(mikio): This package is not implemented on JS, NaCl and Plan 9.
+
+var (
+	errInvalidConn      = errors.New("invalid connection")
+	errInvalidProtocol  = errors.New("invalid protocol")
+	errMessageTooShort  = errors.New("message too short")
+	errHeaderTooShort   = errors.New("header too short")
+	errBufferTooShort   = errors.New("buffer too short")
+	errInvalidBody      = errors.New("invalid body")
+	errNoExtension      = errors.New("no extension")
+	errInvalidExtension = errors.New("invalid extension")
+	errNotImplemented   = errors.New("not implemented on " + runtime.GOOS + "/" + runtime.GOARCH)
+)
+
+func checksum(b []byte) uint16 {
+	csumcv := len(b) - 1 // checksum coverage
+	s := uint32(0)
+	for i := 0; i < csumcv; i += 2 {
+		s += uint32(b[i+1])<<8 | uint32(b[i])
+	}
+	if csumcv&1 == 0 {
+		s += uint32(b[csumcv])
+	}
+	s = s>>16 + s&0xffff
+	s = s + s>>16
+	return ^uint16(s)
+}
+
+// A Type represents an ICMP message type.
+type Type interface {
+	Protocol() int
+}
+
+// A Message represents an ICMP message.
+type Message struct {
+	Type     Type        // type, either ipv4.ICMPType or ipv6.ICMPType
+	Code     int         // code
+	Checksum int         // checksum
+	Body     MessageBody // body
+}
+
+// Marshal returns the binary encoding of the ICMP message m.
+//
+// For an ICMPv4 message, the returned message always contains the
+// calculated checksum field.
+//
+// For an ICMPv6 message, the returned message contains the calculated
+// checksum field when psh is not nil, otherwise the kernel will
+// compute the checksum field during the message transmission.
+// When psh is not nil, it must be the pseudo header for IPv6.
+func (m *Message) Marshal(psh []byte) ([]byte, error) {
+	var mtype byte
+	switch typ := m.Type.(type) {
+	case ipv4.ICMPType:
+		mtype = byte(typ)
+	case ipv6.ICMPType:
+		mtype = byte(typ)
+	default:
+		return nil, errInvalidProtocol
+	}
+	b := []byte{mtype, byte(m.Code), 0, 0}
+	proto := m.Type.Protocol()
+	if proto == iana.ProtocolIPv6ICMP && psh != nil {
+		b = append(psh, b...)
+	}
+	if m.Body != nil && m.Body.Len(proto) != 0 {
+		mb, err := m.Body.Marshal(proto)
+		if err != nil {
+			return nil, err
+		}
+		b = append(b, mb...)
+	}
+	if proto == iana.ProtocolIPv6ICMP {
+		if psh == nil { // cannot calculate checksum here
+			return b, nil
+		}
+		off, l := 2*net.IPv6len, len(b)-len(psh)
+		binary.BigEndian.PutUint32(b[off:off+4], uint32(l))
+	}
+	s := checksum(b)
+	// Place checksum back in header; using ^= avoids the
+	// assumption the checksum bytes are zero.
+	b[len(psh)+2] ^= byte(s)
+	b[len(psh)+3] ^= byte(s >> 8)
+	return b[len(psh):], nil
+}
+
+var parseFns = map[Type]func(int, Type, []byte) (MessageBody, error){
+	ipv4.ICMPTypeDestinationUnreachable: parseDstUnreach,
+	ipv4.ICMPTypeTimeExceeded:           parseTimeExceeded,
+	ipv4.ICMPTypeParameterProblem:       parseParamProb,
+
+	ipv4.ICMPTypeEcho:                parseEcho,
+	ipv4.ICMPTypeEchoReply:           parseEcho,
+	ipv4.ICMPTypeExtendedEchoRequest: parseExtendedEchoRequest,
+	ipv4.ICMPTypeExtendedEchoReply:   parseExtendedEchoReply,
+
+	ipv6.ICMPTypeDestinationUnreachable: parseDstUnreach,
+	ipv6.ICMPTypePacketTooBig:           parsePacketTooBig,
+	ipv6.ICMPTypeTimeExceeded:           parseTimeExceeded,
+	ipv6.ICMPTypeParameterProblem:       parseParamProb,
+
+	ipv6.ICMPTypeEchoRequest:         parseEcho,
+	ipv6.ICMPTypeEchoReply:           parseEcho,
+	ipv6.ICMPTypeExtendedEchoRequest: parseExtendedEchoRequest,
+	ipv6.ICMPTypeExtendedEchoReply:   parseExtendedEchoReply,
+}
+
+// ParseMessage parses b as an ICMP message.
+// The provided proto must be either the ICMPv4 or ICMPv6 protocol
+// number.
+func ParseMessage(proto int, b []byte) (*Message, error) {
+	if len(b) < 4 {
+		return nil, errMessageTooShort
+	}
+	var err error
+	m := &Message{Code: int(b[1]), Checksum: int(binary.BigEndian.Uint16(b[2:4]))}
+	switch proto {
+	case iana.ProtocolICMP:
+		m.Type = ipv4.ICMPType(b[0])
+	case iana.ProtocolIPv6ICMP:
+		m.Type = ipv6.ICMPType(b[0])
+	default:
+		return nil, errInvalidProtocol
+	}
+	if fn, ok := parseFns[m.Type]; !ok {
+		m.Body, err = parseRawBody(proto, b[4:])
+	} else {
+		m.Body, err = fn(proto, m.Type, b[4:])
+	}
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}