diff --git a/build/docker/Dockerfile.cloudmon b/build/docker/Dockerfile.cloudmon new file mode 100644 index 0000000000..a870d09d3b --- /dev/null +++ b/build/docker/Dockerfile.cloudmon @@ -0,0 +1,8 @@ +FROM registry.cn-beijing.aliyuncs.com/yunionio/onecloud-base:v0.2 + +MAINTAINER "Zexi Li " + +RUN mkdir -p /etc/yunion/data/ + +ADD ./_output/alpine-build/bin/cloudmon \ + /opt/yunion/bin/ \ No newline at end of file diff --git a/cmd/cloudmon/main.go b/cmd/cloudmon/main.go new file mode 100644 index 0000000000..eea37d01b5 --- /dev/null +++ b/cmd/cloudmon/main.go @@ -0,0 +1,136 @@ +package main + +import ( + "context" + "fmt" + "os" + "path" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/version" + + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors" + "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/mcclient" +) + +func showErrorAndExit(e error) { + fmt.Fprintf(os.Stderr, "%s", e) + fmt.Fprintln(os.Stderr) + os.Exit(1) +} + +func newClientSession(options *options.CloudMonOptions) (*mcclient.ClientSession, error) { + if len(options.AuthURL) == 0 { + return nil, errors.Error("empty auth_url") + } + if len(options.AdminUser) == 0 { + return nil, errors.Error("empty admin_user") + } + if len(options.AdminPassword) == 0 { + return nil, errors.Error("empty admin_password") + } + if len(options.AdminProject) == 0 { + return nil, errors.Error("empty admin_project") + } + + client := mcclient.NewClient( + options.AuthURL, + options.Timeout, + options.Debug, + options.Insecure, + options.CertFile, + options.KeyFile, + ) + + token, err := client.AuthenticateWithSource( + options.AdminUser, + options.AdminPassword, + options.AdminDomain, + options.AdminProject, + options.AdminProjectDomain, + mcclient.AuthSourceAPI) + if err != nil { + return nil, err + } + + session := client.NewSession( + context.Background(), + options.Region, + "", + options.EndpointType, + token, + options.ApiVersion) + + return session, nil +} + +func main() { + parser, err := options.GetArgumentParser() + if err != nil { + showErrorAndExit(err) + } + + err = parser.ParseArgs2(os.Args[1:], false, false) + + opts := parser.Options().(*options.CloudMonOptions) + + if opts.Help { + fmt.Println(parser.HelpString()) + os.Exit(0) + } + + if opts.Version { + fmt.Println(version.GetJsonString()) + os.Exit(0) + } + + if len(opts.Config) == 0 { + for _, p := range []string{"./etc", "/etc/yunion"} { + confTmp := path.Join(p, "cloudmon.conf") + if _, err := os.Stat(confTmp); err == nil { + opts.Config = confTmp + break + } + } + } + + if len(opts.Config) > 0 { + err := parser.ParseFile(opts.Config) + if err != nil { + showErrorAndExit(err) + } + } + + if err != nil { + showErrorAndExit(err) + } + + parser.SetDefault() + + subcmd := parser.GetSubcommand() + subparser := subcmd.GetSubParser() + if err != nil { + if subparser != nil { + fmt.Print(subparser.Usage()) + } else { + fmt.Print(parser.Usage()) + } + showErrorAndExit(err) + } + + suboptions := subparser.Options() + if opts.SUBCOMMAND == "help" { + err = subcmd.Invoke(suboptions) + } else { + var session *mcclient.ClientSession + session, err = newClientSession(opts) + if err != nil { + showErrorAndExit(err) + } + err = subcmd.Invoke(session, suboptions) + } + if err != nil { + showErrorAndExit(err) + } +} diff --git a/go.mod b/go.mod index fe230198c7..ec02ebe2d1 100644 --- a/go.mod +++ b/go.mod @@ -113,6 +113,7 @@ require ( github.com/smartystreets/goconvey v1.6.4 github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.5.1 + github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible github.com/tencentyun/cos-go-sdk-v5 v0.7.24 github.com/tinylib/msgp v1.1.0 // indirect @@ -147,12 +148,12 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e - yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 + yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824 + yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce ) diff --git a/go.sum b/go.sum index 525660fbea..641b983c1d 100644 --- a/go.sum +++ b/go.sum @@ -586,6 +586,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M= +github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e h1:nt2877sKfojlHCTOBXbpWjBkuWKritFaGIfgQwbQUls= +github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e/go.mod h1:B4+Kq1u5FlULTjFSM707Q6e/cOHFv0z/6QRoxubDIQ8= github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible h1:QIMoFqKCmNp4HPLiTR+couZbHsIZfoOllncHYvtqse8= github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= github.com/tencentyun/cos-go-sdk-v5 v0.7.24 h1:ZsZij764lOaPsj7mEAlyxXvslGt6/m312Tzqj/zeRpo= @@ -913,8 +915,8 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e h1:uGmKmmHm4zeLXhc/MEbAFtEDtGtyEYhC0OaBRGFi+fQ= yunion.io/x/executor v0.0.0-20210310041834-3fcd1574f68e/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= -yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 h1:LIQ4hhLGQuQK+XxlV+8JrKBuL37WUT+5ZTVxBwHOTD4= -yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= +yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349 h1:ESeezAb9LM2Dcy28DDrD2nI+h22qEf4MyAPXwuDO4uM= +yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE= yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o= @@ -928,7 +930,7 @@ yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c h1:/ejlt78/0xod4bdkEThUuYsQ7v yunion.io/x/pkg v0.0.0-20210721081124-55078288ca4c/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824 h1:3/VcCiO+vr3+zZFLNvre41ARF2xE6NVq6Ue7JPFuJvU= -yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8 h1:+kywPlOttXAZvbxsVqR73B5GbSVgG3Xdo4socLWRkcg= +yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryreport.go b/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryreport.go new file mode 100644 index 0000000000..faaba279b8 --- /dev/null +++ b/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryreport.go @@ -0,0 +1,119 @@ +package alertrecordhistorymon + +import ( + "context" + "fmt" + "strconv" + "time" + + "golang.org/x/sync/errgroup" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/timeutils" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SAlertRecordHistoryFactory{} + common.RegisterFactory(&factory) +} + +type SAlertRecordHistoryFactory struct { +} + +func (self *SAlertRecordHistoryFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, + operatorType string) common.ICloudReport { + return &SAlertRecordHistoryReport{ + common.CloudReportBase{ + SProvider: nil, + Session: session, + Args: args, + Operator: string(common.ALERT_RECORD), + }, + } +} + +func (S *SAlertRecordHistoryFactory) GetId() string { + return string(common.ALERT_RECORD) +} + +type SAlertRecordHistoryReport struct { + common.CloudReportBase +} + +func (self *SAlertRecordHistoryReport) Report() error { + alerts, err := self.getMonitorCommonAlert() + if err != nil { + return err + } + errs := make([]error, 0) + recordGroup, _ := errgroup.WithContext(context.Background()) + count := 0 + for i, _ := range alerts { + tmp := alerts[i] + count++ + recordGroup.Go(func() error { + alert_id, _ := tmp.GetString("id") + alertRecords, err := self.getAlertRecordsByAlertId(alert_id) + if err != nil { + return errors.Wrapf(err, "getAlertRecordsByAlertId:%s error", alert_id) + } + return self.collectMetric(alertRecords) + }) + if count == 4 { + err := recordGroup.Wait() + if err != nil { + errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error")) + } + count = 0 + } + } + err = recordGroup.Wait() + if err != nil { + errs = append(errs, errors.Wrap(err, "alertRecordHistoryReport collectMetric error")) + } + return errors.NewAggregate(errs) +} + +func (self *SAlertRecordHistoryReport) getMonitorCommonAlert() ([]jsonutils.JSONObject, error) { + alerts := make([]jsonutils.JSONObject, 0) + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("0"), common.KEY_LIMIT) + //query.Add(jsonutils.NewBool(true), common.DETAILS) + query.Add(jsonutils.NewString("system"), "scope") + + alerts, err := self.ListAllResource(modules.CommonAlertManager, query) + if err != nil { + return nil, errors.Wrap(err, "getMonitorCommonAlert error") + } + return alerts, nil +} + +func (self *SAlertRecordHistoryReport) getAlertRecordsByAlertId(id string) ([]jsonutils.JSONObject, error) { + now := time.Now().UTC() + period64, err := strconv.ParseInt(self.Args.Interval, 10, 8) + if err != nil { + return nil, err + } + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("0"), common.KEY_LIMIT) + query.Add(jsonutils.NewBool(true), common.DETAILS) + query.Add(jsonutils.NewString("system"), "scope") + query.Add(jsonutils.NewString(id), "alert_id") + query.Add(jsonutils.NewString("alerting"), "state") + query.Add(jsonutils.NewString(fmt.Sprintf(`created_at.between("%s","%s")`, + now.Add(-time.Hour*24*time.Duration(period64)).Format(timeutils.MysqlTimeFormat), + now.Format(timeutils.MysqlTimeFormat))), + "filter") + + alertRecords, err := self.ListAllResource(modules.AlertRecordManager, query) + if err != nil { + return nil, err + } + return alertRecords, nil +} diff --git a/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryservice.go b/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryservice.go new file mode 100644 index 0000000000..8c9b50810b --- /dev/null +++ b/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryservice.go @@ -0,0 +1,40 @@ +package alertrecordhistorymon + +import ( + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SAlertRecordHistoryReport) collectMetric(alertRecords []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + matchRecord := self.getMaxEvalMatchOfAlertRecord(alertRecords) + if matchRecord == nil { + return nil + } + + metric, err := self.NewMetricFromJson(matchRecord) + if err != nil { + return err + } + metric.Timestamp = time.Now() + metric.Name = ALERT_RECORD_HISTORY_MEASUREMENT + dataList = append(dataList, metric) + return common.SendMetrics(self.Session, dataList, self.Args.Debug, ALERT_RECORD_HISTORY_DATABASE) +} + +func (self *SAlertRecordHistoryReport) getMaxEvalMatchOfAlertRecord(alertRecords []jsonutils.JSONObject) jsonutils.JSONObject { + var matchRecord jsonutils.JSONObject + maxCount := int64(0) + for _, record := range alertRecords { + resNum, _ := record.Int("res_num") + if resNum > maxCount { + maxCount = resNum + matchRecord = record + } + } + return matchRecord +} diff --git a/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryutil.go b/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryutil.go new file mode 100644 index 0000000000..a99bd3bba9 --- /dev/null +++ b/pkg/cloudmon/collectors/alertrecordhistorymon/alertrecordhistoryutil.go @@ -0,0 +1,7 @@ +package alertrecordhistorymon + +const ( + ALERT_RECORD_HISTORY_FIELD = "res_num" + ALERT_RECORD_HISTORY_MEASUREMENT = "alert_record_history" + ALERT_RECORD_HISTORY_DATABASE = "monitor" +) diff --git a/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go b/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go new file mode 100644 index 0000000000..9e40053e99 --- /dev/null +++ b/pkg/cloudmon/collectors/alertrecordhistorymon/doc.go @@ -0,0 +1 @@ +package alertrecordhistorymon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon" diff --git a/pkg/cloudmon/collectors/alertrecordhistoryservice.go b/pkg/cloudmon/collectors/alertrecordhistoryservice.go new file mode 100644 index 0000000000..f89df07930 --- /dev/null +++ b/pkg/cloudmon/collectors/alertrecordhistoryservice.go @@ -0,0 +1,15 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-alertrecord", "Report alertrecord history", reportAlertrecord) +} + +func reportAlertrecord(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCustomizeCloudMetric(string(common.ALERT_RECORD), session, args) +} diff --git a/pkg/cloudmon/collectors/alimon/aliUtils.go b/pkg/cloudmon/collectors/alimon/aliUtils.go new file mode 100644 index 0000000000..4f2347f8a0 --- /dev/null +++ b/pkg/cloudmon/collectors/alimon/aliUtils.go @@ -0,0 +1,159 @@ +package alimon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_sent" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +//multiCloud查询指标列表组装 +var aliMetricSpecs = map[string][]string{ + "CPUUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "InternetInRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET}, + "IntranetInRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET}, + "InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET}, + "IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET}, + "DiskReadBPS": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS}, + "DiskWriteBPS": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "DiskReadIOPS": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS}, + "DiskWriteIOPS": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS}, +} +var aliRdsMetricSpecs = map[string][]string{ + "CpuUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE}, + "MemoryUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE}, + "MySQL_NetworkInNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL}, + "MySQL_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL}, + "DiskUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE}, + "SQLServer_NetworkInNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER}, + "SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER}, + "ConnectionUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE}, +} +var aliRedisMetricSpecs = map[string][]string{ + "CpuUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE}, + "MemoryUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE}, + "IntranetIn": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX}, + "IntranetOut": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX}, + "UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE}, + "UsedQPS": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES}, + "Keys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS}, + "ExpiredKeys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS}, + "UsedMemory": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE}, +} +var aliOSSMetricSpecs = map[string][]string{ + "InternetSend": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET}, + "InternetRecv": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET}, + "IntranetSend": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET}, + "IntranetRecv": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET}, + "GetObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET}, + "PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST}, + "GetObjectCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET}, + "PostObjectCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST}, + "ServerErrorCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX}, +} +var aliElbMetricSpecs = map[string][]string{ + "InstanceTrafficRX": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX}, + "InstanceTrafficTX": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX}, + "InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX}, + "InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX}, + "InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX}, + "InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX}, +} diff --git a/pkg/cloudmon/collectors/alimon/alicloudreport.go b/pkg/cloudmon/collectors/alimon/alicloudreport.go new file mode 100644 index 0000000000..6ac824c7d2 --- /dev/null +++ b/pkg/cloudmon/collectors/alimon/alicloudreport.go @@ -0,0 +1,82 @@ +package alimon + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SAliCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SAliCloudReportFactory struct { +} + +func (self *SAliCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SAliCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SAliCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_ALIYUN +} + +type SAliCloudReport struct { + common.CloudReportBase +} + +func (self *SAliCloudReport) Report() error { + var servers []jsonutils.JSONObject + var err error + switch self.Operator { + case "redis": + servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache) + case "rds": + servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance) + case "oss": + servers, err = self.GetAllserverOfThisProvider(&modules.Buckets) + case "elb": + servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers) + default: + servers, err = self.GetAllserverOfThisProvider(&modules.Servers) + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = self.collectRegionMetricOfHost(region, servers) + case "redis": + err = self.collectRegionMetricOfRedis(region, servers) + case "rds": + err = self.collectRegionMetricOfRds(region, servers) + case "oss": + err = self.collectRegionMetricOfOss(region, servers) + case "elb": + err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/alimon/aliservice.go b/pkg/cloudmon/collectors/alimon/aliservice.go new file mode 100644 index 0000000000..060d13bc0a --- /dev/null +++ b/pkg/cloudmon/collectors/alimon/aliservice.go @@ -0,0 +1,301 @@ +package alimon + +import ( + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/aliyun" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SAliCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*aliyun.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SAliCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*aliyun.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + redisDeployType := make(map[string]string) + for _, server := range servers { + local_category, err := server.GetString("local_category") + if err != nil { + return err + } + switch local_category { + case "single": + redisDeployType["single"] = "Standard" + case "master": + redisDeployType["master"] = "Standard" + case "cluster": + redisDeployType["cluster"] = "Sharding" + case "rwsplit": + redisDeployType["rwsplit"] = "Splitrw" + } + + } + for metricName, influxDbSpecs := range aliRedisMetricSpecs { + for _, pre := range redisDeployType { + rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + node_id, _ := rtnMetric.GetString("nodeId") + serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id}) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SAliCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*aliyun.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliRdsMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + + } + + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SAliCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*aliyun.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliOSSMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + name, _ := server.GetString("name") + if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name { + serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName, + influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SAliCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*aliyun.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliElbMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SAliCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject, + influxDbSpecs []string) (influxdb.SMetricData, error) { + metric, err := self.NewMetricFromJson(server) + //metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return influxdb.SMetricData{}, err + } + timestamp, _ := rtnMetric.Get("timestamp") + metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0) + fieldValue, err := rtnMetric.Float(UNIT_AVERAGE) + if err != nil { + return influxdb.SMetricData{}, err + } + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) { + fieldValue = fieldValue * 8 + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + self.AddMetricTag(&metric, common.OtherVmTags) + metric.Name = measurement + return metric, nil +} + +func (self *SAliCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject, + metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) { + metric, err := self.NewMetricFromJson(server) + //metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return influxdb.SMetricData{}, err + } + timestamp, _ := rtnMetric.Get("timestamp") + metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0) + fieldValue, err := rtnMetric.Float(metricName) + if err != nil { + return influxdb.SMetricData{}, err + } + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) { + fieldValue = fieldValue * 8 + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + metric.Name = measurement + return metric, nil +} diff --git a/pkg/cloudmon/collectors/alimon/doc.go b/pkg/cloudmon/collectors/alimon/doc.go new file mode 100644 index 0000000000..b37cec1ba8 --- /dev/null +++ b/pkg/cloudmon/collectors/alimon/doc.go @@ -0,0 +1 @@ +package alimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon" diff --git a/pkg/cloudmon/collectors/apsaramon/apsaracloudreport.go b/pkg/cloudmon/collectors/apsaramon/apsaracloudreport.go new file mode 100644 index 0000000000..c91de8900c --- /dev/null +++ b/pkg/cloudmon/collectors/apsaramon/apsaracloudreport.go @@ -0,0 +1,82 @@ +package apsaramon + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SApsaraCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SApsaraCloudReportFactory struct { +} + +func (self *SApsaraCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SApsaraCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SApsaraCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_APSARA +} + +type SApsaraCloudReport struct { + common.CloudReportBase +} + +func (self *SApsaraCloudReport) Report() error { + var servers []jsonutils.JSONObject + var err error + switch self.Operator { + case "redis": + servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache) + case "rds": + servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance) + case "oss": + servers, err = self.GetAllserverOfThisProvider(&modules.Buckets) + case "elb": + servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers) + default: + servers, err = self.GetAllserverOfThisProvider(&modules.Servers) + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = self.collectRegionMetricOfHost(region, servers) + case "redis": + err = self.collectRegionMetricOfRedis(region, servers) + case "rds": + err = self.collectRegionMetricOfRds(region, servers) + case "oss": + err = self.collectRegionMetricOfOss(region, servers) + case "elb": + err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/apsaramon/apsaraservice.go b/pkg/cloudmon/collectors/apsaramon/apsaraservice.go new file mode 100644 index 0000000000..c31eacd77d --- /dev/null +++ b/pkg/cloudmon/collectors/apsaramon/apsaraservice.go @@ -0,0 +1,301 @@ +package apsaramon + +import ( + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/apsara" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SApsaraCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*apsara.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_ecs_dashboard", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SApsaraCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*apsara.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + redisDeployType := make(map[string]string) + for _, server := range servers { + local_category, err := server.GetString("local_category") + if err != nil { + return err + } + switch local_category { + case "single": + redisDeployType["single"] = "Standard" + case "master": + redisDeployType["master"] = "Standard" + case "cluster": + redisDeployType["cluster"] = "Sharding" + case "rwsplit": + redisDeployType["rwsplit"] = "Splitrw" + } + + } + for metricName, influxDbSpecs := range aliRedisMetricSpecs { + for _, pre := range redisDeployType { + rtnArray, _, err := aliReg.DescribeMetricList(pre+metricName, "acs_kvstore", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + node_id, _ := rtnMetric.GetString("nodeId") + serverMetric.Tags = append(serverMetric.Tags, influxdb.SKeyValue{Key: "node_id", Value: node_id}) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SApsaraCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*apsara.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliRdsMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_rds_dashboard", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + + } + + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SApsaraCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*apsara.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliOSSMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_oss", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + name, _ := server.GetString("name") + if bucketName, _ := rtnMetric.GetString("BucketName"); bucketName == name { + serverMetric, err := self.collectOssMetricFromThisServer(server, rtnMetric, metricName, + influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SApsaraCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + aliReg := region.(*apsara.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range aliElbMetricSpecs { + rtnArray, _, err := aliReg.DescribeMetricList(metricName, "acs_slb_dashboard", since, until, "") + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray) > 0 { + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if instanceId, _ := rtnMetric.GetString("instanceId"); instanceId == external_id { + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SApsaraCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject, + influxDbSpecs []string) (influxdb.SMetricData, error) { + metric, err := self.NewMetricFromJson(server) + //metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return influxdb.SMetricData{}, err + } + timestamp, _ := rtnMetric.Get("timestamp") + metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0) + fieldValue, err := rtnMetric.Float(UNIT_AVERAGE) + if err != nil { + return influxdb.SMetricData{}, err + } + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) { + fieldValue = fieldValue * 8 + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + self.AddMetricTag(&metric, common.OtherVmTags) + metric.Name = measurement + return metric, nil +} + +func (self *SApsaraCloudReport) collectOssMetricFromThisServer(server jsonutils.JSONObject, rtnMetric jsonutils.JSONObject, + metricName string, influxDbSpecs []string) (influxdb.SMetricData, error) { + metric, err := self.NewMetricFromJson(server) + //metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return influxdb.SMetricData{}, err + } + timestamp, _ := rtnMetric.Get("timestamp") + metric.Timestamp = time.Unix(timestamp.(*jsonutils.JSONInt).Value()/1000, 0) + fieldValue, err := rtnMetric.Float(metricName) + if err != nil { + return influxdb.SMetricData{}, err + } + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + if influxDbSpecs[1] == UNIT_BYTEPS && strings.Contains(pairsKey, UNIT_BPS) { + fieldValue = fieldValue * 8 + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + metric.Name = measurement + return metric, nil +} diff --git a/pkg/cloudmon/collectors/apsaramon/apsarautils.go b/pkg/cloudmon/collectors/apsaramon/apsarautils.go new file mode 100644 index 0000000000..3b3e445e4f --- /dev/null +++ b/pkg/cloudmon/collectors/apsaramon/apsarautils.go @@ -0,0 +1,159 @@ +package apsaramon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_sent" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +//multiCloud查询指标列表组装 +var aliMetricSpecs = map[string][]string{ + "CPUUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "InternetInRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET}, + "IntranetInRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET}, + "InternetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET}, + "IntranetOutRate": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET}, + "DiskReadBPS": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS}, + "DiskWriteBPS": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "DiskReadIOPS": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS}, + "DiskWriteIOPS": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS}, +} +var aliRdsMetricSpecs = map[string][]string{ + "CpuUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE}, + "MemoryUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE}, + "MySQL_NetworkInNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL}, + "MySQL_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL}, + "DiskUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE}, + "SQLServer_NetworkInNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER}, + "SQLServer_NetworkOutNew": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER}, + "ConnectionUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE}, +} +var aliRedisMetricSpecs = map[string][]string{ + "CpuUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE}, + "MemoryUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE}, + "IntranetIn": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX}, + "IntranetOut": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX}, + "UsedConnection": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE}, + "UsedQPS": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES}, + "Keys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS}, + "ExpiredKeys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS}, + "UsedMemory": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE}, +} +var aliOSSMetricSpecs = map[string][]string{ + "InternetSend": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET}, + "InternetRecv": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET}, + "IntranetSend": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET}, + "IntranetRecv": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET}, + "GetObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET}, + "PostObjectE2eLatency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_POST}, + "GetObjectCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET}, + "PostObjectCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_POST}, + "ServerErrorCount": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX}, +} +var aliElbMetricSpecs = map[string][]string{ + "InstanceTrafficRX": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_RX}, + "InstanceTrafficTX": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_ELB_NET_BPS_TX}, + "InstanceStatusCode2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX}, + "InstanceStatusCode3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX}, + "InstanceStatusCode4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX}, + "InstanceStatusCode5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX}, +} diff --git a/pkg/cloudmon/collectors/apsaramon/doc.go b/pkg/cloudmon/collectors/apsaramon/doc.go new file mode 100644 index 0000000000..fbb1f87aeb --- /dev/null +++ b/pkg/cloudmon/collectors/apsaramon/doc.go @@ -0,0 +1 @@ +package apsaramon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon" diff --git a/pkg/cloudmon/collectors/awsmon/awscloudreport.go b/pkg/cloudmon/collectors/awsmon/awscloudreport.go new file mode 100644 index 0000000000..224cf97e15 --- /dev/null +++ b/pkg/cloudmon/collectors/awsmon/awscloudreport.go @@ -0,0 +1,71 @@ +package awsmon + +import ( + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SAwsCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SAwsCloudReportFactory struct { +} + +func (self *SAwsCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SAwsCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SAwsCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_AWS +} + +type SAwsCloudReport struct { + common.CloudReportBase +} + +func (self *SAwsCloudReport) Report() error { + servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self) + //err = self.collectRegionMetricOfHost(region, servers) + //case "redis": + // err = self.collectRegionMetricOfRedis(region, servers) + //case "rds": + // err = self.collectRegionMetricOfRds(region, servers) + //case "oss": + // err = self.collectRegionMetricOfOss(region, servers) + //case "elb": + // err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/awsmon/awsservice.go b/pkg/cloudmon/collectors/awsmon/awsservice.go new file mode 100644 index 0000000000..7419f4e94e --- /dev/null +++ b/pkg/cloudmon/collectors/awsmon/awsservice.go @@ -0,0 +1,111 @@ +package awsmon + +import ( + "strconv" + "strings" + + aws_sdk "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/cloudwatch" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/aws" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SAwsCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + var err error + switch self.Operator { + case string(common.SERVER): + err = self.collectRegionMetricOfHost(region, servers) + } + return err +} + +func (self *SAwsCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + awsRegion := region.(*aws.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range awsMetricSpecs { + for i, server := range servers { + external_id, _ := servers[i].GetString("external_id") + rtnArray, err := awsRegion.GetMonitorData(metricName, "AWS/EC2", external_id, since, until) + if err != nil { + log.Errorln(err) + continue + } + if len(rtnArray.Datapoints) > 0 { + for _, dataPoint := range rtnArray.Datapoints { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric, err := self.collectMetricFromThisServerForAws(server, dataPoint, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + + } + } + err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "") + if err != nil { + log.Errorln(err) + } + } + return nil +} + +func (self *SAwsCloudReport) collectMetricFromThisServerForAws(server jsonutils.JSONObject, datapoint *cloudwatch.Datapoint, + influxDbSpecs []string) (metric influxdb.SMetricData, err error) { + metric, err = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return metric, err + } + metric.Timestamp = aws_sdk.TimeValue(datapoint.Timestamp) + fieldValue := aws_sdk.Float64Value(datapoint.Average) + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + // 300:AWS metric collection period is 5m + if influxDbSpecs[1] == UNIT_COUNT { + fieldValue = (fieldValue / float64(300)) + } + if influxDbSpecs[1] == UNIT_MEM { + fieldValue = (fieldValue / float64(300) * 8) + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "=")}) + } + cpu_cout, err := server.Get("vcpu_count") + if err != nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64)}) + } else { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10)}, + influxdb.SKeyValue{Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64)}) + } + self.AddMetricTag(&metric, common.OtherVmTags) + metric.Name = measurement + return metric, nil +} diff --git a/pkg/cloudmon/collectors/awsmon/awsutils.go b/pkg/cloudmon/collectors/awsmon/awsutils.go new file mode 100644 index 0000000000..505d87e8da --- /dev/null +++ b/pkg/cloudmon/collectors/awsmon/awsutils.go @@ -0,0 +1,125 @@ +package awsmon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_BPS_EBS = INFLUXDB_FIELD_DISK_READ_BPS + ",disk_type=ebs" + INFLUXDB_FIELD_DISK_WRITE_BPS_EBS = INFLUXDB_FIELD_DISK_WRITE_BPS + ",disk_type=ebs" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_DISK_READ_IOPS_EBS = INFLUXDB_FIELD_DISK_READ_IOPS + ",disk_type=ebs" + INFLUXDB_FIELD_DISK_WRITE_IOPS_EBS = INFLUXDB_FIELD_DISK_WRITE_IOPS + ",disk_type=ebs" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_send" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +//multiCloud查询指标列表组装 +var awsMetricSpecs = map[string][]string{ + "CPUUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "DiskReadOps": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_READ_IOPS}, + "EBSReadOps": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_READ_IOPS_EBS}, + "DiskWriteOps": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_WRITE_IOPS}, + "EBSWriteOps": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_DISK_WRITE_IOPS_EBS}, + "DiskReadBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS}, + "EBSReadBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS_EBS}, + "DiskWriteBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "EBSWriteBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS_EBS}, + "NetworkIn": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX}, + "NetworkOut": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX}, +} diff --git a/pkg/cloudmon/collectors/awsmon/doc.go b/pkg/cloudmon/collectors/awsmon/doc.go new file mode 100644 index 0000000000..69a475d946 --- /dev/null +++ b/pkg/cloudmon/collectors/awsmon/doc.go @@ -0,0 +1 @@ +package awsmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon" diff --git a/pkg/cloudmon/collectors/azuremon/azurereport.go b/pkg/cloudmon/collectors/azuremon/azurereport.go new file mode 100644 index 0000000000..93c62b8e7f --- /dev/null +++ b/pkg/cloudmon/collectors/azuremon/azurereport.go @@ -0,0 +1,71 @@ +package azuremon + +import ( + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SAzureCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SAzureCloudReportFactory struct { +} + +func (self *SAzureCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SAzureCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SAzureCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_AZURE +} + +type SAzureCloudReport struct { + common.CloudReportBase +} + +func (self *SAzureCloudReport) Report() error { + servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = common.CollectRegionMetricAsync(self.Args.Batch, region, servers, self) + //err = self.collectRegionMetricOfHost(region, servers) + //case "redis": + // err = self.collectRegionMetricOfRedis(region, servers) + //case "rds": + // err = self.collectRegionMetricOfRds(region, servers) + //case "oss": + // err = self.collectRegionMetricOfOss(region, servers) + //case "elb": + // err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/azuremon/azureservice.go b/pkg/cloudmon/collectors/azuremon/azureservice.go new file mode 100644 index 0000000000..c2ad02cde6 --- /dev/null +++ b/pkg/cloudmon/collectors/azuremon/azureservice.go @@ -0,0 +1,128 @@ +package azuremon + +import ( + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/azure" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SAzureCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + return self.collectRegionMetricOfHost(region, servers) +} + +func (self *SAzureCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + azureReg := region.(*azure.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + external_id, err := server.GetString("external_id") + if err != nil { + continue + } + metricNameArr := make([]string, 0) + for metricName, _ := range azureMetricSpecs { + metricNameArr = append(metricNameArr, metricName) + } + metricNames := strings.Join(metricNameArr, ",") + rtnMetrics, err := azureReg.GetMonitorData(metricNames, "Microsoft.Compute/virtualMachines", external_id, since, until) + if err != nil { + log.Errorln(err) + continue + } + if rtnMetrics == nil || rtnMetrics.Value == nil { + continue + } + for metricName, influxDbSpecs := range azureMetricSpecs { + for _, value := range *rtnMetrics.Value { + if value.Name.LocalizedValue != nil { + if metricName == *(value.Name.LocalizedValue) { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + if value.Timeseries != nil { + for _, timeserie := range *value.Timeseries { + serverMetric, err := self.collectMetricFromThisServer(server, timeserie, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric...) + } + } + } + } + } + } + err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "") + if err != nil { + log.Errorln(err) + } + dataList = dataList[:0] + } + return nil +} + +func (self *SAzureCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, + rtnMetric azure.TimeSeriesElement, influxDbSpecs []string) ([]influxdb.SMetricData, error) { + datas := make([]influxdb.SMetricData, 0) + for _, data := range *rtnMetric.Data { + metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return nil, err + } + if data.TimeStamp != nil { + metric.Timestamp = *data.TimeStamp + if data.Average != nil { + //根据条件拼装metric的tag和metirc信息 + fieldValue := *data.Average + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + metric.Name = measurement + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + if influxDbSpecs[1] == UNIT_MEM { + fieldValue = fieldValue * 8 / PERIOD + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + self.AddMetricTag(&metric, common.OtherVmTags) + datas = append(datas, metric) + } + } + } + + return datas, nil +} diff --git a/pkg/cloudmon/collectors/azuremon/azureutils.go b/pkg/cloudmon/collectors/azuremon/azureutils.go new file mode 100644 index 0000000000..e562ecbbaa --- /dev/null +++ b/pkg/cloudmon/collectors/azuremon/azureutils.go @@ -0,0 +1,117 @@ +package azuremon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_send" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +//multiCloud查询指标列表组装 +var azureMetricSpecs = map[string][]string{ + "Percentage CPU": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "Network In Total": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX}, + "Network Out Total": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX}, + "Disk Read Bytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS}, + "Disk Write Bytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "Disk Read Operations/Sec": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_READ_IOPS}, + "Disk Write Operations/Sec": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_DISK_WRITE_BPS}, +} diff --git a/pkg/cloudmon/collectors/azuremon/doc.go b/pkg/cloudmon/collectors/azuremon/doc.go new file mode 100644 index 0000000000..37c41c3e3f --- /dev/null +++ b/pkg/cloudmon/collectors/azuremon/doc.go @@ -0,0 +1 @@ +package azuremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon" diff --git a/pkg/cloudmon/collectors/bucket.go b/pkg/cloudmon/collectors/bucket.go new file mode 100644 index 0000000000..b195c5271e --- /dev/null +++ b/pkg/cloudmon/collectors/bucket.go @@ -0,0 +1,74 @@ +package collectors + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/influxdb" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +type BucketStatsOptions struct { + Debug bool `help:"debug"` +} + +func bucketStatsCollect(s *mcclient.ClientSession, args *BucketStatsOptions) error { + metrics := make([]influxdb.SMetricData, 0) + listAll(s, modules.Buckets.List, nil, + func(data jsonutils.JSONObject) error { + m, err := collectBucket(s, data) + if err != nil { + return err + } + metrics = append(metrics, m) + return nil + }, + ) + return sendMetrics(s, metrics, args.Debug) +} + +func collectBucket(s *mcclient.ClientSession, bucket jsonutils.JSONObject) (influxdb.SMetricData, error) { + metric := influxdb.SMetricData{} + bucketId, _ := bucket.GetString("id") + if len(bucketId) == 0 { + return metric, errors.Error("empty bucket id") + } + params := jsonutils.NewDict() + params.Set("stats_only", jsonutils.JSONTrue) + result, err := modules.Buckets.PerformAction(s, bucketId, "sync", params) + if err != nil { + return metric, errors.Wrap(err, "PerformAction") + } + return jsonToMetric(result.(*jsonutils.JSONDict), "bucket", + []string{ + "name", + "id", + "account", + "account_id", + "manager", + "manager_id", + "manager_domain", + "manager_domain_id", + "manager_project", + "manager_project_id", + "brand", + "provider", + "region_id", + "region_ext_id", + "tenant", + "tenant_id", + "domain_id", + "project_domain", + }, + []string{ + "object_cnt", + "size_bytes", + }, + ) +} + +func init() { + shellutils.R(&BucketStatsOptions{}, "buckets", "Bucket stats", bucketStatsCollect) +} diff --git a/pkg/cloudmon/collectors/cloudaccountmon/cloudaccountreport.go b/pkg/cloudmon/collectors/cloudaccountmon/cloudaccountreport.go new file mode 100644 index 0000000000..7b8e465ffd --- /dev/null +++ b/pkg/cloudmon/collectors/cloudaccountmon/cloudaccountreport.go @@ -0,0 +1,44 @@ +package cloudaccountmon + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SCloudAccountFactory{} + common.RegisterFactory(&factory) +} + +type SCloudAccountFactory struct { +} + +func (self *SCloudAccountFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, + operatorType string) common.ICloudReport { + return &SCloudAccountReport{ + common.CloudReportBase{ + SProvider: nil, + Session: session, + Args: args, + Operator: ClOUDACCOUNT_ID, + }, + } +} + +func (S SCloudAccountFactory) GetId() string { + return ClOUDACCOUNT_ID +} + +type SCloudAccountReport struct { + common.CloudReportBase +} + +func (self *SCloudAccountReport) Report() error { + accounts, err := self.GetAllCloudAccount(&modules.Cloudaccounts) + if err != nil { + return err + } + return self.collectMetric(accounts) +} diff --git a/pkg/cloudmon/collectors/cloudaccountmon/cloudaccountutil.go b/pkg/cloudmon/collectors/cloudaccountmon/cloudaccountutil.go new file mode 100644 index 0000000000..4571eb90e7 --- /dev/null +++ b/pkg/cloudmon/collectors/cloudaccountmon/cloudaccountutil.go @@ -0,0 +1,9 @@ +package cloudaccountmon + +const ( + ClOUDACCOUNT_ID = "cloudaccount" + + CLOUDACCOUNT_FIELD = "balance" + CLOUDACCOUNT_MEASUREMENT = "cloudaccount_balance" + CLOUDACCOUNT_DATABASE = "meter_db" +) diff --git a/pkg/cloudmon/collectors/cloudaccountmon/cloudaccoutservice.go b/pkg/cloudmon/collectors/cloudaccountmon/cloudaccoutservice.go new file mode 100644 index 0000000000..d380ff9a0a --- /dev/null +++ b/pkg/cloudmon/collectors/cloudaccountmon/cloudaccoutservice.go @@ -0,0 +1,24 @@ +package cloudaccountmon + +import ( + "time" + + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SCloudAccountReport) collectMetric(accounts []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + for _, account := range accounts { + metric, err := self.NewMetricFromJson(account) + if err != nil { + return err + } + metric.Timestamp = time.Now() + metric.Name = CLOUDACCOUNT_MEASUREMENT + dataList = append(dataList, metric) + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, CLOUDACCOUNT_DATABASE) +} diff --git a/pkg/cloudmon/collectors/cloudaccountmon/doc.go b/pkg/cloudmon/collectors/cloudaccountmon/doc.go new file mode 100644 index 0000000000..5f90225102 --- /dev/null +++ b/pkg/cloudmon/collectors/cloudaccountmon/doc.go @@ -0,0 +1 @@ +package cloudaccountmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon" diff --git a/pkg/cloudmon/collectors/cloudaccountservice.go b/pkg/cloudmon/collectors/cloudaccountservice.go new file mode 100644 index 0000000000..aaac6de709 --- /dev/null +++ b/pkg/cloudmon/collectors/cloudaccountservice.go @@ -0,0 +1,15 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-cloudaccount", "Report cloud account", reportCloudAccount) +} + +func reportCloudAccount(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCustomizeCloudMetric(string(common.CLOUDACCOUNT), session, args) +} diff --git a/pkg/cloudmon/collectors/cloudmoninit.go b/pkg/cloudmon/collectors/cloudmoninit.go new file mode 100644 index 0000000000..0568d07e5b --- /dev/null +++ b/pkg/cloudmon/collectors/cloudmoninit.go @@ -0,0 +1,25 @@ +package collectors + +import ( + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/alertrecordhistorymon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/alimon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/apsaramon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/awsmon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/azuremon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/cloudaccountmon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon" + _ "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon" + _ "yunion.io/x/onecloud/pkg/multicloud/aliyun/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/apsara/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/aws/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/azure/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/esxi/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/google/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/huawei/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/qcloud/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/zstack/provider" +) diff --git a/pkg/cloudmon/collectors/common/basemon.go b/pkg/cloudmon/collectors/common/basemon.go new file mode 100644 index 0000000000..a4fb7e7744 --- /dev/null +++ b/pkg/cloudmon/collectors/common/basemon.go @@ -0,0 +1,228 @@ +package common + +import ( + "fmt" + "net/http" + "net/url" + + "golang.org/x/net/http/httpproxy" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/httputils" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +type CloudReportBase struct { + SProvider *SProvider + Session *mcclient.ClientSession + Args *ReportOptions + Operator string +} + +func (self *CloudReportBase) Report() error { + return fmt.Errorf("No Implment the method") +} + +func (self *CloudReportBase) GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) { + query := jsonutils.NewDict() + + query.Add(jsonutils.NewStringArray([]string{"running", "ready"}), "status") + query.Add(jsonutils.NewString("0"), KEY_LIMIT) + query.Add(jsonutils.NewString("true"), KEY_ADMIN) + query.Add(jsonutils.NewString(self.SProvider.Provider), "provider") + query.Add(jsonutils.NewString(self.SProvider.Id), "manager") + return self.ListAllResources(manager, query) +} + +func (self *CloudReportBase) GetAllHostOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) { + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("running"), "status") + query.Add(jsonutils.NewString("0"), KEY_LIMIT) + query.Add(jsonutils.NewString("true"), KEY_ADMIN) + query.Add(jsonutils.NewString(self.SProvider.Provider), "provider") + query.Add(jsonutils.NewString(self.SProvider.Id), "manager") + return self.ListAllResources(manager, query) +} + +func (self *CloudReportBase) GetAllCloudAccount(manager modulebase.Manager) ([]jsonutils.JSONObject, error) { + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("0"), KEY_LIMIT) + query.Add(jsonutils.NewBool(true), DETAILS) + query.Add(jsonutils.NewString("true"), KEY_ADMIN) + query.Add(jsonutils.NewString(fmt.Sprintf("brand.in(%s,%s,%s)", compute.CLOUD_PROVIDER_ALIYUN, + compute.CLOUD_PROVIDER_QCLOUD, compute.CLOUD_PROVIDER_HUAWEI)), "filter") + return self.ListAllResources(manager, query) +} + +func (self *CloudReportBase) GetAllStorage(manager modulebase.Manager) ([]jsonutils.JSONObject, error) { + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("0"), KEY_LIMIT) + query.Add(jsonutils.NewBool(true), DETAILS) + query.Add(jsonutils.NewString("system"), KEY_SCOPE) + return self.ListAllResources(manager, query) +} + +func (self *CloudReportBase) ListAllResource(manager modulebase.Manager, + query jsonutils.JSONObject) ([]jsonutils.JSONObject, error) { + resources := make([]jsonutils.JSONObject, 0) + offsetIndex := 0 + for { + query.(*jsonutils.JSONDict).Add(jsonutils.NewInt(int64(offsetIndex)), "offset") + resList, err := manager.List(self.Session, query) + if err != nil { + return nil, err + } + resources = append(resources, resList.Data...) + offsetIndex = offsetIndex + len(resList.Data) + if offsetIndex >= resList.Total { + break + } + } + return resources, nil +} + +func (self *CloudReportBase) InitProviderInstance() (cloudprovider.ICloudProvider, error) { + //secret进行Descrypt + secretDe, _ := utils.DescryptAESBase64(self.SProvider.Id, self.SProvider.Secret) + var proxyFunc httputils.TransportProxyFunc + { + cfg := &httpproxy.Config{ + HTTPProxy: self.SProvider.ProxySetting.HTTPProxy, + HTTPSProxy: self.SProvider.ProxySetting.HTTPSProxy, + NoProxy: self.SProvider.ProxySetting.NoProxy, + } + cfgProxyFunc := cfg.ProxyFunc() + proxyFunc = func(req *http.Request) (*url.URL, error) { + return cfgProxyFunc(req.URL) + } + } + cloudAccout, err := self.getCloudAccount(&modules.Cloudaccounts) + if err != nil { + return nil, errors.Wrap(err, "getCloudAccount error") + } + endpoints := cloudprovider.SApsaraEndpoints{} + options, err := cloudAccout.Get("options") + if err == nil { + err := options.Unmarshal(&endpoints) + if err != nil { + log.Errorf("Unmarshal SApsaraEndpoints err:%v", err) + } + } else { + log.Errorf("get cloudAccout options err:%v", err) + } + cfg := cloudprovider.ProviderConfig{ + Id: self.SProvider.Id, + Name: self.SProvider.Name, + URL: self.SProvider.AccessUrl, + Account: self.SProvider.Account, + Secret: secretDe, + Vendor: self.SProvider.Provider, + ProxyFunc: proxyFunc, + SApsaraEndpoints: endpoints, + } + return cloudprovider.GetProvider(cfg) +} + +func (self *CloudReportBase) getCloudAccount(manager modulebase.Manager) (jsonutils.JSONObject, error) { + return self.GetResourceById(self.SProvider.CloudaccountId, manager) +} + +func (self *CloudReportBase) GetResourceById(id string, manager modulebase.Manager) (jsonutils.JSONObject, error) { + return manager.Get(self.Session, id, jsonutils.NewDict()) +} + +func (self *CloudReportBase) GetAllRegionOfServers(servers []jsonutils.JSONObject, + providerInstance cloudprovider.ICloudProvider) ([]cloudprovider. + ICloudRegion, map[string][]jsonutils.JSONObject, error) { + extranleIdMap := make(map[string]string) + regionServerList := make([]cloudprovider.ICloudRegion, 0) + regionServerMap := make(map[string][]jsonutils.JSONObject) + for i, server := range servers { + region_external_id, err := server.GetString("region_external_id") + if err != nil { + return nil, nil, err + } + if _, ok := extranleIdMap[region_external_id]; !ok { + extranleIdMap[region_external_id] = "" + region, err := providerInstance.GetIRegionById(region_external_id) + if err != nil { + return nil, nil, err + } + regionServerList = append(regionServerList, region) + regionServers := make([]jsonutils.JSONObject, 0) + regionServers = append(regionServers, servers[i]) + regionServerMap[region_external_id] = regionServers + } else { + regionservers := regionServerMap[region_external_id] + regionservers = append(regionservers, server) + regionServerMap[region_external_id] = regionservers + } + } + return regionServerList, regionServerMap, nil +} + +func (self *CloudReportBase) AddMetricTag(metric *influxdb.SMetricData, tags map[string]string) { + for key, value := range tags { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: key, + Value: value, + }) + } +} + +func (self *CloudReportBase) NewMetricFromJson(server jsonutils.JSONObject) (influxdb.SMetricData, error) { + switch MonType(self.Operator) { + case SERVER: + return JsonToMetric(server.(*jsonutils.JSONDict), "", ServerTags, make([]string, 0)) + case HOST: + return JsonToMetric(server.(*jsonutils.JSONDict), "", HostTags, make([]string, 0)) + case REDIS: + return JsonToMetric(server.(*jsonutils.JSONDict), "", RedisTags, make([]string, 0)) + case RDS: + return JsonToMetric(server.(*jsonutils.JSONDict), "", RdsTags, make([]string, 0)) + case OSS: + return JsonToMetric(server.(*jsonutils.JSONDict), "", OssTags, make([]string, 0)) + case ELB: + return JsonToMetric(server.(*jsonutils.JSONDict), "", ElbTags, make([]string, 0)) + case CLOUDACCOUNT: + return JsonToMetric(server.(*jsonutils.JSONDict), "", CloudAccountTags, CloudAccountFields) + case STORAGE: + return JsonToMetric(server.(*jsonutils.JSONDict), "", StorageTags, make([]string, 0)) + case ALERT_RECORD: + return JsonToMetric(server.(*jsonutils.JSONDict), "", AlertRecordHistoryTags, AlertRecordHistoryFields) + } + return influxdb.SMetricData{}, fmt.Errorf("no found report operator") +} + +func (self *CloudReportBase) CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + panic("NO implement CollectRegionMetricOfServer") +} + +func (self *CloudReportBase) ListAllResources(manager modulebase.Manager, + query *jsonutils.JSONDict) ([]jsonutils.JSONObject, error) { + offsetIndex := 0 + resources := make([]jsonutils.JSONObject, 0) + for { + query.Add(jsonutils.NewInt(int64(offsetIndex)), "offset") + resList, err := manager.List(self.Session, query) + if err != nil { + return nil, err + } + resources = append(resources, resList.Data...) + offsetIndex = offsetIndex + len(resList.Data) + if offsetIndex >= resList.Total { + break + } + } + return resources, nil +} diff --git a/pkg/cloudmon/collectors/common/cloudreport.go b/pkg/cloudmon/collectors/common/cloudreport.go new file mode 100644 index 0000000000..ea2e6ec5af --- /dev/null +++ b/pkg/cloudmon/collectors/common/cloudreport.go @@ -0,0 +1,86 @@ +package common + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws/request" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" +) + +func init() { + cloudReportTable = make(map[string]ICloudReportFactory) +} + +type ICloudReportFactory interface { + NewCloudReport(provider *SProvider, session *mcclient.ClientSession, args *ReportOptions, + operatorType string) ICloudReport + GetId() string +} + +type ICloudReport interface { + Report() error + GetAllserverOfThisProvider(manager modulebase.Manager) ([]jsonutils.JSONObject, error) + InitProviderInstance() (cloudprovider.ICloudProvider, error) + GetAllRegionOfServers(servers []jsonutils.JSONObject, providerInstance cloudprovider.ICloudProvider) ( + []cloudprovider.ICloudRegion, map[string][]jsonutils.JSONObject, error) + CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error +} + +type SProvider struct { + compute.CloudproviderDetails + //Id string `json:"id" + //Name string `json:"name"` + //Provider string `json:"provider"` + //Account string `json:"account"` + //Secret string `json:"secret"` + //AccessUrl string `json:"access_url"` +} + +var cloudReportTable map[string]ICloudReportFactory + +func RegisterFactory(factory ICloudReportFactory) { + cloudReportTable[factory.GetId()] = factory +} + +func GetCloudReportFactory(provider string) (ICloudReportFactory, error) { + factory, ok := cloudReportTable[provider] + if ok { + return factory, nil + } + log.Errorf("Provider %s not registerd", provider) + return nil, fmt.Errorf("No such cloudReport %s", provider) +} + +func (s *SProvider) Validate() error { + invalidParams := request.ErrInvalidParams{Context: "SProvider"} + if s.Id == "" { + invalidParams.Add(request.NewErrParamRequired("Id")) + } + if s.Name == "" { + invalidParams.Add(request.NewErrParamRequired("Name")) + } + if s.Account == "" { + invalidParams.Add(request.NewErrParamRequired("Account")) + } + //if s.AccessUrl == "" { + // invalidParams.Add(request.NewErrParamRequired("AccountUrl")) + //} + if s.Provider == "" { + invalidParams.Add(request.NewErrParamRequired("Provider")) + } + if s.Secret == "" { + invalidParams.Add(request.NewErrParamRequired("Secret")) + } + if invalidParams.Len() > 0 { + return invalidParams + } + return nil +} diff --git a/pkg/cloudmon/collectors/common/doc.go b/pkg/cloudmon/collectors/common/doc.go new file mode 100644 index 0000000000..ee93a28f61 --- /dev/null +++ b/pkg/cloudmon/collectors/common/doc.go @@ -0,0 +1 @@ +package common // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" diff --git a/pkg/cloudmon/collectors/common/multiCloudMonUtils.go b/pkg/cloudmon/collectors/common/multiCloudMonUtils.go new file mode 100644 index 0000000000..566ea8141f --- /dev/null +++ b/pkg/cloudmon/collectors/common/multiCloudMonUtils.go @@ -0,0 +1,492 @@ +package common + +import ( + "context" + "strconv" + "strings" + "time" + + "golang.org/x/sync/errgroup" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/pkg/utils" + + o "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +type MonType string + +const ( + SERVER MonType = "server" + HOST MonType = "host" + REDIS MonType = "redis" + RDS MonType = "rds" + OSS MonType = "oss" + ELB MonType = "elb" + CLOUDACCOUNT MonType = "cloudaccount" + STORAGE MonType = "storage" + ALERT_RECORD MonType = "alertRecord" +) + +const ( + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" + DETAILS = "details" + KEY_SCOPE = "scope" +) + +const ( + TYPE_VIRTUALMACHINE = "VirtualMachine" + TYPE_HOSTSYSTEM = "HostSystem" +) + +var OtherVmTags = map[string]string{ + "source": "cloudmon", + "res_type": "guest", + "is_vm": "true", +} + +var OtherHostTag = map[string]string{ + "source": "cloudmon", + "res_type": "host", + "is_vm": "false", +} + +type ReportOptions struct { + Debug bool `help:"debug"` + Batch int `help:"batch"` + Count int `help:"count" json:"count"` + Interval string `help:"interval""` + SinceTime string `help:"sinceTime"` + EndTime string `help:"endTime"` + Provider []string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|ZStack|Google|Apsara" json:"provider,omitempty"` +} + +var InstanceProviders = "Aliyun,Azure,Aws,Qcloud,VMWare,Huawei,Openstack,Ucloud,ZStack" + +//server的key-value对应保存时的Tags和Pairs +//var ServerTags = []string{"host", "host_id", "vm_id", "vm_ip", "vm_name", "zone", "zone_id", "zone_ext_id", +// "hypervisor", "os_type", "status", "region", "region_id", "region_ext_id", "tenant", "tenant_id", "brand", "name"} +var ServerTags = map[string]string{ + "host": "host", + "host_id": "host_id", + "id": "vm_id", + "ips": "vm_ip", + "name": "vm_name", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "os_type": "os_type", + "status": "status", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "region_ext_id": "region_ext_id", + "tenant": "tenant", + "tenant_id": "tenant_id", + "brand": "brand", + "scaling_group_id": "vm_scaling_group_id", + "domain_id": "domain_id", + "project_domain": "project_domain", +} +var HostTags = map[string]string{ + "id": "host_id", + "ips": "host_ip", + "name": "host", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "os_type": "os_type", + "status": "status", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "region_ext_id": "region_ext_id", + "tenant": "tenant", + "tenant_id": "tenant_id", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} +var RdsTags = map[string]string{ + "host": "host", + "host_id": "host_id", + "id": "rds_id", + "ips": "rds_ip", + "name": "rds_name", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "os_type": "os_type", + "status": "status", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "region_ext_id": "region_ext_id", + "tenant": "tenant", + "tenant_id": "tenant_id", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} +var RedisTags = map[string]string{ + "host": "host", + "host_id": "host_id", + "id": "redis_id", + "ips": "redis_ip", + "name": "redis_name", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "os_type": "os_type", + "status": "status", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "region_ext_id": "region_ext_id", + "tenant": "tenant", + "tenant_id": "tenant_id", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} +var OssTags = map[string]string{ + "host": "host", + "host_id": "host_id", + "id": "oss_id", + "ips": "oss_ip", + "name": "oss_name", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "os_type": "os_type", + "status": "status", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "region_ext_id": "region_ext_id", + "tenant": "tenant", + "tenant_id": "tenant_id", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} +var ElbTags = map[string]string{ + "host": "host", + "host_id": "host_id", + "id": "elb_id", + "ips": "elb_ip", + "name": "elb_name", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "os_type": "os_type", + "status": "status", + "region": "region", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "tenant": "tenant", + "tenant_id": "tenant_id", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} + +var CloudAccountTags = map[string]string{ + "id": "cloudaccount_id", + "name": "cloudaccount_name", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} + +var StorageTags = map[string]string{ + "id": "storage_id", + "name": "storage_name", + "zone": "zone", + "zone_id": "zone_id", + "zone_ext_id": "zone_ext_id", + "status": "status", + "cloudregion": "cloudregion", + "cloudregion_id": "cloudregion_id", + "region_ext_id": "region_ext_id", + "brand": "brand", + "domain_id": "domain_id", + "project_domain": "project_domain", +} + +var AlertRecordHistoryTags = map[string]string{ + "alert_name": "alert_name", + "alert_id": "alert_id", + "domain_id": "domain_id", + "project_domain": "project_domain", + "tenant_id": "tenant_id", + "tenant": "tenant", + "res_type": "res_type", +} + +var CloudAccountFields = []string{"balance"} + +var AlertRecordHistoryFields = []string{"res_num"} + +var ServerPairs = []string{"vcpu_count", "vmem_size", "disk"} + +var AddTags = map[string]string{ + "source": "cloudmon", +} + +//get substring from str before separator +func SubstringBefore(str, separator string) string { + if str != "" { + if separator == "" { + return "" + } else { + if pos := strings.Index(str, separator); pos == -1 { + return str + } else { + return str[0:pos] + } + } + } else { + return str + } +} + +//get substring from str after separator +func SubstringAfter(str, separator string) string { + if str != "" { + if separator == "" { + return "" + } else { + if pos := strings.Index(str, separator); pos == -1 { + return "" + } else { + return str[pos+len(separator):] + } + } + } else { + return str + } +} + +//get a substring from str between[open,close) +func SubstringBetween(str, open, close string) string { + if str != "" && open != "" && close != "" { + if start := strings.Index(str, open); start != -1 { + if end := strings.Index(str[start+len(open):], close); end != -1 { + return str[start+len(open) : start+len(open)+end] + } + } + return "" + } else { + return "" + } +} + +func ParseTimeStr(startTime, endTime string) (since, util time.Time, err error) { + since, err = timeutils.ParseTimeStr(startTime) + if err != nil { + return since, util, err + } + util, err = timeutils.ParseTimeStr(endTime) + if err != nil { + return since, util, err + } + return since, util, nil +} + +func TimeRangeFromArgs(args *ReportOptions) (since, until time.Time, err error) { + if args.SinceTime != "" && args.EndTime != "" { + since, until, err = ParseTimeStr(args.SinceTime, args.EndTime) + if err != nil { + return since, until, err + } + } else { + period64, err := strconv.ParseInt(args.Interval, 10, 8) + if err != nil { + return since, until, err + } + since = time.Now().Add(-time.Minute * time.Duration(period64)) + until = time.Now() + } + return since, until, nil +} + +//组装server相关capability +func FillVMCapacity(server *jsonutils.JSONDict) (influxdb.SMetricData, error) { + metric, err := JsonToMetric(server, "vm_capacity", ServerTags, ServerPairs) + if err != nil { + return influxdb.SMetricData{}, err + } + hypevisor, _ := server.GetString("hypervisor") + metric.Timestamp = time.Now() + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: "res_type", + Value: "guest", + }, influxdb.SKeyValue{ + Key: "is_vm", + Value: "true", + }, influxdb.SKeyValue{ + Key: "platform", + Value: hypevisor, + }) + return metric, nil +} + +func GetMeasurement(action string, influxDbSpec string) (measurement string) { + // VirtualMachine -> VMware类型的虚拟机 + if action == TYPE_VIRTUALMACHINE { + measurement = SubstringBefore(influxDbSpec, ".") + } + if action == TYPE_HOSTSYSTEM { + measurement = SubstringBetween(influxDbSpec, "vm_", ".") + if strings.Contains(influxDbSpec, "vm_netio") { + measurement = "net" + } + } + return measurement +} + +func JsonToMetric(obj *jsonutils.JSONDict, name string, tags map[string]string, metrics []string) (influxdb.SMetricData, error) { + metric := influxdb.SMetricData{Name: name} + objMap, err := obj.GetMap() + if err != nil { + return metric, errors.Wrap(err, "obj.GetMap") + } + tagPairs := make([]influxdb.SKeyValue, 0) + metricPairs := make([]influxdb.SKeyValue, 0) + for k, v := range objMap { + val, _ := v.GetString() + if strings.Contains(k, "ip") { + if strings.Contains(val, ",") { + val = strings.ReplaceAll(val, ",", "|") + } + } + if tag, ok := tags[k]; ok { + tagPairs = append(tagPairs, influxdb.SKeyValue{ + Key: tag, + Value: val, + }) + } else if utils.IsInStringArray(k, metrics) { + metricPairs = append(metricPairs, influxdb.SKeyValue{ + Key: k, Value: val, + }) + } + } + metric.Tags = tagPairs + metric.Metrics = metricPairs + return metric, nil +} + +func SendMetrics(s *mcclient.ClientSession, metrics []influxdb.SMetricData, debug bool, database string) error { + urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType) + if err != nil { + return errors.Wrap(err, "GetServiceURLs") + } + if len(database) == 0 { + database = o.Options.InfluxDatabase + } + return influxdb.SendMetrics(urls, database, metrics, debug) +} + +func ReportCloudMetricOfoperatorType(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error { + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("0"), KEY_LIMIT) + query.Add(jsonutils.NewString("true"), KEY_ADMIN) + //query.Add(jsonutils.NewString("true"), KEY_USABLE) + if len(args.Provider) > 0 { + for _, val := range args.Provider { + query.Add(jsonutils.NewString(val), "provider") + } + } + cloudProviderList, err := (&modules.Cloudproviders).List(session, query) + if err != nil { + return errors.Wrap(err, "cloudProviders get list error") + } + providerGroup, _ := errgroup.WithContext(context.Background()) + tmpCount := 0 + if args.Count == 0 { + args.Count = 1 + } + for i := 0; i < len(cloudProviderList.Data); i++ { + provider := cloudProviderList.Data[i] + status, err := provider.GetString("status") + if err != nil { + return errors.Wrap(err, "provider get status error") + } + if status == "connected" { + providerStruct := SProvider{} + err := provider.Unmarshal(&providerStruct) + if err != nil { + return errors.Wrap(err, "provider.Unmarshal") + } + err = (&providerStruct).Validate() + if err != nil { + return errors.Wrap(err, "provider Invalidate") + } + providerStr := providerStruct.Provider + cloudReportFactory, err := GetCloudReportFactory(providerStr) + if err != nil { + log.Errorln(errors.Wrap(err, "GetCloudReportFactory")) + continue + } + tmpCount++ + providerGroup.Go(func() error { + err = cloudReportFactory.NewCloudReport(&providerStruct, session, args, operatorType).Report() + if err != nil { + log.Errorln(errors.Wrap(err, "cloudReport Report method")) + } + return nil + }) + if tmpCount == args.Count { + providerGroup.Wait() + tmpCount = 0 + } + } + } + providerGroup.Wait() + return nil +} + +func newGroupFunc(func()) { + +} + +func ReportCustomizeCloudMetric(operatorType string, session *mcclient.ClientSession, args *ReportOptions) error { + cloudReportFactory, err := GetCloudReportFactory(operatorType) + if err != nil { + return errors.Wrap(err, "GetCloudReportFactory") + } + err = cloudReportFactory.NewCloudReport(nil, session, args, operatorType).Report() + if err != nil { + return errors.Wrap(err, "cloudReport Report method") + } + return nil +} + +func CollectRegionMetricAsync(asynCount int, region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject, report ICloudReport) error { + metricGroup, _ := errgroup.WithContext(context.Background()) + count := 0 + if asynCount == 0 { + asynCount = 10 + } + for i, _ := range servers { + server := servers[i] + metricGroup.Go(func() error { + return report.CollectRegionMetric(region, []jsonutils.JSONObject{server}) + }) + count++ + if count == asynCount { + err := metricGroup.Wait() + if err != nil { + return err + } + count = 0 + } + } + return metricGroup.Wait() +} diff --git a/pkg/cloudmon/collectors/doc.go b/pkg/cloudmon/collectors/doc.go new file mode 100644 index 0000000000..bcc8136e9f --- /dev/null +++ b/pkg/cloudmon/collectors/doc.go @@ -0,0 +1 @@ +package collectors // import "yunion.io/x/onecloud/pkg/cloudmon/collectors" diff --git a/pkg/cloudmon/collectors/ecloudmon/doc.go b/pkg/cloudmon/collectors/ecloudmon/doc.go new file mode 100644 index 0000000000..52dca82637 --- /dev/null +++ b/pkg/cloudmon/collectors/ecloudmon/doc.go @@ -0,0 +1 @@ +package ecloudmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/ecloudmon" diff --git a/pkg/cloudmon/collectors/ecloudmon/ecloudreport.go b/pkg/cloudmon/collectors/ecloudmon/ecloudreport.go new file mode 100644 index 0000000000..da19adcb79 --- /dev/null +++ b/pkg/cloudmon/collectors/ecloudmon/ecloudreport.go @@ -0,0 +1,62 @@ +package ecloudmon + +import ( + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SECloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SECloudReportFactory struct { +} + +func (self *SECloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SECloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SECloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_ECLOUD +} + +type SECloudReport struct { + common.CloudReportBase +} + +func (self *SECloudReport) Report() error { + servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = self.collectRegionMetricOfHost(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/ecloudmon/ecloudservice.go b/pkg/cloudmon/collectors/ecloudmon/ecloudservice.go new file mode 100644 index 0000000000..42cac508fb --- /dev/null +++ b/pkg/cloudmon/collectors/ecloudmon/ecloudservice.go @@ -0,0 +1,113 @@ +package ecloudmon + +import ( + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/ecloud" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +const ( + SERVER_PRODUCT_TYPE = "vm" +) + +func (self *SECloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + ecloudReg := region.(*ecloud.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + external_id, err := server.GetString("external_id") + if err != nil { + continue + } + name, _ := server.GetString("name") + external_id = strings.Split(external_id, "/")[1] + metrics := make([]ecloud.Metric, 0) + for metricName, _ := range ecloudMetricSpecs { + metrics = append(metrics, ecloud.Metric{Name: metricName}) + } + data, err := ecloudReg.DescribeMetricList(SERVER_PRODUCT_TYPE, metrics, external_id, since, until) + if err != nil { + return errors.Wrap(err, "region DescribeMetricList err:") + } + for _, entity := range data.Entitys { + if influxDbSpecs, ok := ecloudMetricSpecs[entity.MetricName]; ok { + metricData, err := self.collectMetricFromThisServer(server, entity, influxDbSpecs) + if err != nil { + log.Errorf("server:%s collectMetric:%s err", name, entity.MetricName) + continue + } + dataList = append(dataList, metricData...) + } + + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SECloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, + entity ecloud.Entity, influxDbSpecs []string) ([]influxdb.SMetricData, error) { + datas := make([]influxdb.SMetricData, 0) + for _, point := range entity.Datapoints { + metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return nil, err + } + if len(point) != 2 { + log.Errorf("invalid point:%v", point) + continue + } + influxDbSpec := influxDbSpecs[2] + + measurement := common.SubstringBefore(influxDbSpec, ".") + metric.Name = measurement + pointTime, err := strconv.ParseInt(point[1], 10, 64) + if err != nil { + return nil, errors.Wrap(err, "point parseInt err") + } + metric.Timestamp = time.Unix(pointTime, 0) + pointVal, err := strconv.ParseFloat(point[0], 64) + if err != nil { + return nil, errors.Wrap(err, "point parseInt err") + } + if influxDbSpecs[1] == UNIT_BYTEPS { + pointVal = pointVal * 8 + } + if influxDbSpecs[1] == UNIT_KBYTEPS { + pointVal = pointVal * 8 * 1024 + } + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(pointVal, 'f', -1, 64), + }) + self.AddMetricTag(&metric, common.AddTags) + datas = append(datas, metric) + } + return datas, nil +} diff --git a/pkg/cloudmon/collectors/ecloudmon/ecloudutils.go b/pkg/cloudmon/collectors/ecloudmon/ecloudutils.go new file mode 100644 index 0000000000..bde79abb92 --- /dev/null +++ b/pkg/cloudmon/collectors/ecloudmon/ecloudutils.go @@ -0,0 +1,42 @@ +package ecloudmon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_CPS = "cps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_KBYTEPS = "KBps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" +) + +var ecloudProTypeMetric = map[string]string{} + +//multiCloud查询指标列表组装 +var ecloudMetricSpecs = map[string][]string{ + "cpu_util": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "memory.util": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE}, + "disk.device.read.bytes.rate": {DEFAULT_STATISTICS, UNIT_KBYTEPS, INFLUXDB_FIELD_DISK_READ_BPS}, + "disk.device.write.bytes.rate": {DEFAULT_STATISTICS, UNIT_KBYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "disk.device.read.requests.rate": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS}, + "disk.device.write.requests.rate": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS}, + "network.incoming.bytes": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX}, + "network.outgoing.bytes": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX}, +} diff --git a/pkg/cloudmon/collectors/elbservice.go b/pkg/cloudmon/collectors/elbservice.go new file mode 100644 index 0000000000..fcd73ed57f --- /dev/null +++ b/pkg/cloudmon/collectors/elbservice.go @@ -0,0 +1,16 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-elb", "Report Elb", reportElb) +} + +//入口函数[aliyun、huawei] +func reportElb(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCloudMetricOfoperatorType(string(common.ELB), session, args) +} diff --git a/pkg/cloudmon/collectors/gcpmon/doc.go b/pkg/cloudmon/collectors/gcpmon/doc.go new file mode 100644 index 0000000000..ce3e544c34 --- /dev/null +++ b/pkg/cloudmon/collectors/gcpmon/doc.go @@ -0,0 +1 @@ +package gcpmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/gcpmon" diff --git a/pkg/cloudmon/collectors/gcpmon/gcpreport.go b/pkg/cloudmon/collectors/gcpmon/gcpreport.go new file mode 100644 index 0000000000..ad9cda0d81 --- /dev/null +++ b/pkg/cloudmon/collectors/gcpmon/gcpreport.go @@ -0,0 +1,70 @@ +package gcpmon + +import ( + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SGoogleCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SGoogleCloudReportFactory struct { +} + +func (self *SGoogleCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SGoogleCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SGoogleCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_GOOGLE +} + +type SGoogleCloudReport struct { + common.CloudReportBase +} + +func (self *SGoogleCloudReport) Report() error { + servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = self.collectRegionMetricOfHost(region, servers) + //case "redis": + // err = self.collectRegionMetricOfRedis(region, servers) + //case "rds": + // err = self.collectRegionMetricOfRds(region, servers) + //case "oss": + // err = self.collectRegionMetricOfOss(region, servers) + //case "elb": + // err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/gcpmon/gcpservice.go b/pkg/cloudmon/collectors/gcpmon/gcpservice.go new file mode 100644 index 0000000000..d0076f3761 --- /dev/null +++ b/pkg/cloudmon/collectors/gcpmon/gcpservice.go @@ -0,0 +1,143 @@ +package gcpmon + +import ( + "strconv" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/util/timeutils" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/google" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SGoogleCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + googleReg := region.(*google.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + external_id, err := server.GetString("external_id") + if err != nil { + continue + } + serverName, _ := server.GetString("name") + external_id = strings.Split(external_id, "/")[1] + for metricName, influxDbSpecs := range gcpMetricSpecs { + rtn, err := googleReg.GetMonitorData(external_id, serverName, metricName, since, until) + if err != nil { + log.Errorln(err) + continue + } + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + for _, resp := range rtn.Value() { + if !resp.(*jsonutils.JSONDict).Contains("points") { + continue + } + serverMetric, err := self.collectMetricFromThisServer(server, resp, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric...) + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SGoogleCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, + rtnMetric jsonutils.JSONObject, influxDbSpecs []string) ([]influxdb.SMetricData, error) { + datas := make([]influxdb.SMetricData, 0) + points_, _ := rtnMetric.Get("points") + for _, point := range points_.(*jsonutils.JSONArray).Value() { + metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return nil, err + } + if time, err := point.Get("interval", "startTime"); err == nil { + influxDbSpec := influxDbSpecs[2] + + measurement := common.SubstringBefore(influxDbSpec, ".") + metric.Name = measurement + + timestamp, err := timeutils.ParseTimeStr(time.(*jsonutils.JSONString).Value()) + if err != nil { + log.Errorln(err) + continue + } + metric.Timestamp = timestamp + + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + + valueTypeObj, _ := rtnMetric.Get("valueType") + valueType := strings.ToLower(valueTypeObj.(*jsonutils.JSONString).Value()) + value := getMetricValue(point, valueType, influxDbSpecs[1]) + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: value, + }) + self.AddMetricTag(&metric, common.AddTags) + datas = append(datas, metric) + } + } + return datas, nil +} + +func getMetricValue(point jsonutils.JSONObject, valueType string, + influxDbSpec string) string { + value, _ := point.Get("value", valueType+"Value") + + switch valueType { + case "int64": + if val, ok := value.(*jsonutils.JSONString); ok { + fieldValue, _ := strconv.ParseInt(val.Value(), 10, 64) + if influxDbSpec == UNIT_MEM { + fieldValue = fieldValue * 8 / PERIOD + } + return strconv.FormatInt(fieldValue, 10) + } + fieldValue := value.(*jsonutils.JSONInt).Value() + if influxDbSpec == UNIT_MEM { + fieldValue = fieldValue * 8 / PERIOD + } + return strconv.FormatInt(fieldValue, 10) + case "double": + fieldValue := value.(*jsonutils.JSONFloat).Value() + if influxDbSpec == UNIT_MEM { + fieldValue = fieldValue * 8 / PERIOD + } + return strconv.FormatFloat(fieldValue, 'f', 3, 64) + } + return "" +} diff --git a/pkg/cloudmon/collectors/gcpmon/gcputils.go b/pkg/cloudmon/collectors/gcpmon/gcputils.go new file mode 100644 index 0000000000..c8807ce7b1 --- /dev/null +++ b/pkg/cloudmon/collectors/gcpmon/gcputils.go @@ -0,0 +1,117 @@ +package gcpmon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_send" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +//multiCloud查询指标列表组装 +var gcpMetricSpecs = map[string][]string{ + "compute.googleapis.com/instance/cpu/utilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "compute.googleapis.com/instance/network/received_bytes_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX}, + "compute.googleapis.com/instance/network/sent_bytes_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX}, + "compute.googleapis.com/instance/disk/read_bytes_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS}, + "compute.googleapis.com/instance/disk/write_bytes_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "compute.googleapis.com/instance/disk/read_ops_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_IOPS}, + "compute.googleapis.com/instance/disk/write_ops_count": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_IOPS}, +} diff --git a/pkg/cloudmon/collectors/hostservice.go b/pkg/cloudmon/collectors/hostservice.go new file mode 100644 index 0000000000..c142e332ad --- /dev/null +++ b/pkg/cloudmon/collectors/hostservice.go @@ -0,0 +1,16 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-host", "Report Host", reportHost) +} + +//入口函数[aliyun、huawei] +func reportHost(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCloudMetricOfoperatorType(string(common.HOST), session, args) +} diff --git a/pkg/cloudmon/collectors/huaweimon/doc.go b/pkg/cloudmon/collectors/huaweimon/doc.go new file mode 100644 index 0000000000..0d2f51138b --- /dev/null +++ b/pkg/cloudmon/collectors/huaweimon/doc.go @@ -0,0 +1 @@ +package huaweimon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/huaweimon" diff --git a/pkg/cloudmon/collectors/huaweimon/huaweicloudreport.go b/pkg/cloudmon/collectors/huaweimon/huaweicloudreport.go new file mode 100644 index 0000000000..40139a20b6 --- /dev/null +++ b/pkg/cloudmon/collectors/huaweimon/huaweicloudreport.go @@ -0,0 +1,85 @@ +package huaweimon + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SHwCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SHwCloudReportFactory struct { +} + +func (self *SHwCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SHwCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SHwCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_HUAWEI +} + +type SHwCloudReport struct { + common.CloudReportBase +} + +func (self *SHwCloudReport) Report() error { + var servers []jsonutils.JSONObject + var err error + switch self.Operator { + case "redis": + servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache) + case "rds": + servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance) + case "oss": + servers, err = self.GetAllserverOfThisProvider(&modules.Buckets) + case "elb": + servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers) + default: + servers, err = self.GetAllserverOfThisProvider(&modules.Servers) + } + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = self.collectRegionMetricOfHost(region, servers) + case "redis": + err = self.collectRegionMetricOfRedis(region, servers) + case "rds": + err = self.collectRegionMetricOfRds(region, servers) + case "oss": + err = self.collectRegionMetricOfOss(region, servers) + case "elb": + err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/huaweimon/hwservice.go b/pkg/cloudmon/collectors/huaweimon/hwservice.go new file mode 100644 index 0000000000..f6ff634a3a --- /dev/null +++ b/pkg/cloudmon/collectors/huaweimon/hwservice.go @@ -0,0 +1,327 @@ +package huaweimon + +import ( + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/huawei" + hw_moudules "yunion.io/x/onecloud/pkg/multicloud/huawei/client/modules" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SHwCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + hwReg := region.(*huawei.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + instanceId, _ := server.GetString("external_id") + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + metas := make([]hw_moudules.SMetricMeta, 0) + for metricName := range huaweiMetricSpecs { + hwMeta := hw_moudules.SMetricMeta{} + hwMeta.MetricName = metricName + hwMeta.Namespace = "SYS.ECS" + hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0) + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "instance_id", Value: instanceId}) + metas = append(metas, hwMeta) + } + + metricDatas, err := hwReg.GetMetricsData(metas, since, until) + if err != nil { + log.Errorln(err) + continue + } + if len(metricDatas) > 0 { + for _, metricData := range metricDatas { + for metricName, influxDbSpecs := range huaweiMetricSpecs { + if metricData.MetricName == metricName { + if len(metricData.Datapoints) > 0 { + for _, datapoint := range metricData.Datapoints { + serverMetric, err := self.collectMetricFromThisServer(server, datapoint, + influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + + } + } + + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SHwCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + + hwReg := region.(*huawei.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + instanceId, _ := server.GetString("external_id") + metas := make([]hw_moudules.SMetricMeta, 0) + for metricName := range huaweiRedisMetricSpecs { + hwMeta := hw_moudules.SMetricMeta{} + hwMeta.MetricName = metricName + hwMeta.Namespace = "SYS.DCS" + hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0) + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "dcs_instance_id", Value: instanceId}) + metas = append(metas, hwMeta) + } + + metricDatas, err := hwReg.GetMetricsData(metas, since, until) + if err != nil { + return err + } + if len(metricDatas) > 0 { + for _, metricData := range metricDatas { + for metricName, influxDbSpecs := range huaweiRedisMetricSpecs { + if metricData.MetricName == metricName { + if len(metricData.Datapoints) > 0 { + for _, datapoint := range metricData.Datapoints { + serverMetric, err := self.collectMetricFromThisServer(server, datapoint, + influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + + } + } + + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SHwCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + + hwReg := region.(*huawei.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + instanceId, _ := server.GetString("external_id") + engine, _ := server.GetString("engine") + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + metas := make([]hw_moudules.SMetricMeta, 0) + for metricName := range huaweiRdsMetricSpecs { + hwMeta := hw_moudules.SMetricMeta{} + hwMeta.MetricName = metricName + hwMeta.Namespace = "SYS.RDS" + hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0) + switch engine { + case "MySQL": + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "rds_cluster_id", Value: instanceId}) + case "PostgreSQL": + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "postgresql_cluster_id", Value: instanceId}) + case "SQLServer": + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "rds_cluster_sqlserver_id", Value: instanceId}) + } + metas = append(metas, hwMeta) + } + index := 0 + tmp := 0 + for { + if index > len(metas) { + break + } + tmp = index + 10 + if tmp > len(metas) { + tmp = len(metas) + } + metricDatas, err := hwReg.GetMetricsData(metas[index:tmp], since, until) + if err != nil { + return err + } + if len(metricDatas) > 0 { + for _, metricData := range metricDatas { + for metricName, influxDbSpecs := range huaweiRdsMetricSpecs { + if metricData.MetricName == metricName { + if len(metricData.Datapoints) > 0 { + for _, datapoint := range metricData.Datapoints { + serverMetric, err := self.collectMetricFromThisServer(server, datapoint, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + + } + } + index += 10 + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SHwCloudReport) collectRegionMetricOfOss(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + hwReg := region.(*huawei.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + name, _ := server.GetString("name") + metas := make([]hw_moudules.SMetricMeta, 0) + for metricName := range huaweiOSSMetricSpecs { + hwMeta := hw_moudules.SMetricMeta{} + hwMeta.MetricName = metricName + hwMeta.Namespace = "SYS.OBS" + hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0) + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "bucket_name", Value: name}) + metas = append(metas, hwMeta) + } + + metricDatas, err := hwReg.GetMetricsData(metas, since, until) + if err != nil { + return err + } + if len(metricDatas) > 0 { + for _, metricData := range metricDatas { + for metricName, influxDbSpecs := range huaweiOSSMetricSpecs { + if metricData.MetricName == metricName { + if len(metricData.Datapoints) > 0 { + for _, datapoint := range metricData.Datapoints { + serverMetric, err := self.collectMetricFromThisServer(server, datapoint, + influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + + } + } + + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SHwCloudReport) collectRegionMetricOfElb(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + hwReg := region.(*huawei.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for _, server := range servers { + external_id, _ := server.GetString("external_id") + metas := make([]hw_moudules.SMetricMeta, 0) + for metricName := range huaweiOSSMetricSpecs { + hwMeta := hw_moudules.SMetricMeta{} + hwMeta.MetricName = metricName + hwMeta.Namespace = "SYS.ELB" + hwMeta.Dimensions = make([]hw_moudules.SMetricDimension, 0) + hwMeta.Dimensions = append(hwMeta.Dimensions, hw_moudules.SMetricDimension{Name: "lb_instance_id", Value: external_id}) + metas = append(metas, hwMeta) + } + + metricDatas, err := hwReg.GetMetricsData(metas, since, until) + if err != nil { + return err + } + if len(metricDatas) > 0 { + for _, metricData := range metricDatas { + for metricName, influxDbSpecs := range huaweiOSSMetricSpecs { + if metricData.MetricName == metricName { + if len(metricData.Datapoints) > 0 { + for _, datapoint := range metricData.Datapoints { + serverMetric, err := self.collectMetricFromThisServer(server, datapoint, + influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + + } + } + + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SHwCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, datapoint hw_moudules.SDatapoint, + influxDbSpecs []string) (influxdb.SMetricData, error) { + metric, err := self.NewMetricFromJson(server) + //metric, err := common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + if err != nil { + return influxdb.SMetricData{}, err + } + metric.Timestamp = time.Unix(datapoint.Timestamp/1000, 0) + fieldValue := datapoint.Average + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + if influxDbSpecs[1] == UNIT_BYTEPS { + fieldValue = fieldValue * 8 + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + metric.Name = measurement + self.AddMetricTag(&metric, common.OtherVmTags) + return metric, nil +} diff --git a/pkg/cloudmon/collectors/huaweimon/hwutils.go b/pkg/cloudmon/collectors/huaweimon/hwutils.go new file mode 100644 index 0000000000..be92a9cd1c --- /dev/null +++ b/pkg/cloudmon/collectors/huaweimon/hwutils.go @@ -0,0 +1,162 @@ +package huaweimon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_sent" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" +) + +var huaweiMetricSpecs = map[string][]string{ + "cpu_util": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, //CPU使用率，该指标用于统计测量对象的CPU使用率，以百分为单位。 + "network_incoming_bytes_aggregate_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET}, //带外网络流入速率，该指标用于在虚拟化层统计每秒流入测量对象的网络流量，以字节/秒为单位。 + "network_outgoing_bytes_aggregate_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET}, //带外网络流出速率，该指标用于在虚拟化层统计每秒流出测量对象的网络流量，以字节/秒为单位。 + "disk_read_bytes_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_READ_BPS}, //磁盘读速率，该指标用于统计每秒从测量对象读出数据量，以字节/秒为单位。 + "disk_write_bytes_rate": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, //磁盘写速率，该指标用于统计每秒写到测量对象的数据量，以字节/秒为单位。 + "disk_read_requests_rate": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_READ_IOPS}, //该指标用于统计每秒从测量对象读取数据的请求次数，以请求/秒为单位。 + "disk_write_requests_rate": {DEFAULT_STATISTICS, UNIT_CPS, INFLUXDB_FIELD_DISK_WRITE_IOPS}, //该指标用于统计每秒从测量对象写数据的请求次数，以请求/秒为单位 +} + +var huaweiRdsMetricSpecs = map[string][]string{ + "rds001_cpu_util": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE}, + "rds002_mem_util": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE}, + "rds004_bytes_in": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_RX}, + "rds005_bytes_out": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_TX}, + "rds039_disk_util": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE}, + "rds049_disk_read_throughput": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_DISK_READ_BPS}, + + "rds050_disk_write_throughput": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_DISK_WRITE_BPS}, + + "rds006_conn_count": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_COUNT}, + + "rds008_qps": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_QPS}, + "rds009_tps": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_TPS}, + + "rds013_innodb_reads": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_REDA_BPS}, + "rds014_innodb_writes": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS}, +} + +var huaweiRedisMetricSpecs = map[string][]string{ + "cpu_usage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE}, + "memory_usage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE}, + "instantaneous_input_kbps": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX}, + "instantaneous_output_kbps": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX}, + "connected_clients": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE}, + "instantaneous_ops": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES}, + "keys": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS}, + "expires": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS}, + "used_memory_dataset": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE}, +} + +var huaweiOSSMetricSpecs = map[string][]string{ + "download_bytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_TX}, + "upload_bytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_OSS_NET_BPS_RX}, + "first_byte_latency": {DEFAULT_STATISTICS, UNIT_MSEC, INFLUXDB_FIELD_OSS_LATECY_GET}, + "get_request_count": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_GET}, + "request_count_4xx": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_4XX}, + "request_count_5xx": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_OSS_REQ_COUNT_5XX}, +} + +var huaweiElbMetricSpecs = map[string][]string{ + "m7_in_Bps": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_ELB_NET_BPS_RX}, + "m8_out_Bps": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_ELB_NET_BPS_TX}, + "mb_l7_qps": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_REQ_RATE}, + "mc_l7_http_2xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX}, + "md_l7_http_3xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX}, + "me_l7_http_4xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX}, + "mf_l7_http_5xx": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX}, +} diff --git a/pkg/cloudmon/collectors/ossservice.go b/pkg/cloudmon/collectors/ossservice.go new file mode 100644 index 0000000000..d0c05fa03a --- /dev/null +++ b/pkg/cloudmon/collectors/ossservice.go @@ -0,0 +1,14 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-oss", "Report Oss", reportOss) +} +func reportOss(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCloudMetricOfoperatorType(string(common.OSS), session, args) +} diff --git a/pkg/cloudmon/collectors/pinger.go b/pkg/cloudmon/collectors/pinger.go new file mode 100644 index 0000000000..ea77a937c9 --- /dev/null +++ b/pkg/cloudmon/collectors/pinger.go @@ -0,0 +1,212 @@ +package collectors + +import ( + "strconv" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/netutils" + + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/influxdb" + "yunion.io/x/onecloud/pkg/util/rbacutils" + "yunion.io/x/onecloud/pkg/util/shellutils" + "yunion.io/x/onecloud/pkg/util/sysutils" +) + +type PingProbeOptions struct { + Debug bool `help:"debug"` + ProbeCount int `help:"probe count, default is 3" default:"3"` + TimeoutSecond int `help:"probe timeout in second, default is 1 second" default:"1"` +} + +func pingProbeCoolector(s *mcclient.ClientSession, args *PingProbeOptions) error { + isRoot := sysutils.IsRootPermission() + if !isRoot { + return errors.Error("require root permissions") + } + metrics := make([]influxdb.SMetricData, 0) + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(api.CLOUD_ENV_ON_PREMISE), "cloud_env") + params.Add(jsonutils.NewString(string(rbacutils.ScopeSystem)), "scope") + params.Add(jsonutils.JSONTrue, "is_classic") + listAll(s, modules.Networks.List, params, func(data jsonutils.JSONObject) error { + m, err := pingProbeNetwork(s, data, args) + if err != nil { + return err + } + if len(m) > 0 { + metrics = append(metrics, m...) + } + return nil + }) + return sendMetrics(s, metrics, args.Debug) +} + +type sNetwork struct { + Id string + Name string + GuestGateway string + GuestIpStart string + GuestIpEnd string + Region string + RegionId string + ServerType string + Vpc string + VpcId string + Wire string + WireId string + Zone string + ZoneId string + CloudEnv string +} + +func getNetworkAddrMap(s *mcclient.ClientSession, netId string) (map[string]api.SNetworkUsedAddress, error) { + addrListJson, err := modules.Networks.GetSpecific(s, netId, "addresses", nil) + if err != nil { + return nil, errors.Wrap(err, "GetSpecific addresses") + } + addrList := make([]api.SNetworkUsedAddress, 0) + err = addrListJson.Unmarshal(&addrList, "addresses") + if err != nil { + return nil, errors.Wrap(err, "Unmarshal addreses") + } + addrMap := make(map[string]api.SNetworkUsedAddress) + for i := range addrList { + addrMap[addrList[i].IpAddr] = addrList[i] + } + return addrMap, nil +} + +func pingProbeNetwork(s *mcclient.ClientSession, data jsonutils.JSONObject, args *PingProbeOptions) ([]influxdb.SMetricData, error) { + metrics := make([]influxdb.SMetricData, 0) + net := &sNetwork{} + err := data.Unmarshal(&net) + if err != nil { + log.Errorf("Unmarshal network %s: %s", data, err) + return nil, errors.Wrap(err, "Unmarshal network") + } + if net.CloudEnv != api.CLOUD_ENV_ON_PREMISE { + log.Errorf("not an onpremise network: %s", data) + return nil, errors.Wrap(errors.ErrInvalidStatus, "not onpremise network") + } + if net.GuestGateway == "" { + log.Errorf("no valid gateway %s", data) + return nil, errors.Wrap(errors.ErrInvalidStatus, "unreachable network, empty gateway") + } + addrStart, err := netutils.NewIPV4Addr(net.GuestIpStart) + if err != nil { + log.Errorf("unmarshal start address %s: %s", net.GuestIpStart, err) + return nil, errors.Wrapf(err, "NewIPV4Addr %s", net.GuestIpStart) + } + addrEnd, err := netutils.NewIPV4Addr(net.GuestIpEnd) + if err != nil { + log.Errorf("unmarshal end address %s: %s", net.GuestIpEnd, err) + return nil, errors.Wrapf(err, "NewIPV4Addr %s", net.GuestIpEnd) + } + log.Infof("ping address %s - %s", addrStart, addrEnd) + pingAddrs := make([]string, 0) + for addr := addrStart; addr <= addrEnd; addr = addr.StepUp() { + addrStr := addr.String() + pingAddrs = append(pingAddrs, addrStr) + } + pingResults, err := Ping(pingAddrs, args.ProbeCount, time.Second*time.Duration(args.TimeoutSecond), args.Debug) + if err != nil { + return nil, errors.Wrap(err, "Ping") + } + + addrMap, err := getNetworkAddrMap(s, net.Id) + if err != nil { + return nil, errors.Wrap(err, "getNetworkAddrMap") + } + + now := time.Now().UTC() + for addr := addrStart; addr <= addrEnd; addr = addr.StepUp() { + addrStr := addr.String() + pingResult := pingResults[addrStr] + netAddr, allocated := addrMap[addrStr] + if allocated { + if netAddr.OwnerType == api.RESERVEDIP_RESOURCE_TYPES { + loss := pingResult.Loss() + status := api.RESERVEDIP_STATUS_OFFLINE + if loss < 100 { + status = api.RESERVEDIP_STATUS_ONLINE + } + params := jsonutils.NewDict() + params.Add(jsonutils.NewString(status), "status") + _, err := modules.ReservedIPs.Update(s, netAddr.OwnerId, params) + if err != nil { + log.Errorf("update reserved ip %s status fail: %s", addrStr, err) + } + } else { + // send metrics + metric := influxdb.SMetricData{} + metric.Name = "ping" + metric.Timestamp = now + metric.Tags = []influxdb.SKeyValue{ + { + Key: "ip_addr", + Value: addrStr, + }, + { + Key: "owner_type", + Value: netAddr.OwnerType, + }, + { + Key: "owner_id", + Value: netAddr.OwnerId, + }, + { + Key: "owner", + Value: netAddr.Owner, + }, + } + loss := pingResult.Loss() + max, avg, min := pingResult.Rtt() + metric.Metrics = []influxdb.SKeyValue{ + { + Key: "loss", + Value: strconv.FormatInt(int64(loss), 10), + }, + { + Key: "rtt_ms_avg", + Value: strconv.FormatInt(int64(avg/time.Millisecond), 10), + }, + { + Key: "rtt_ms_max", + Value: strconv.FormatInt(int64(max/time.Millisecond), 10), + }, + { + Key: "rtt_ms_min", + Value: strconv.FormatInt(int64(min/time.Millisecond), 10), + }, + } + metrics = append(metrics, metric) + } + } else { + loss := pingResult.Loss() + if loss < 100 { + // reserve ip + log.Debugf("Free address %s is responding ping, reserve the address", addrStr) + params := jsonutils.NewDict() + params.Add(jsonutils.NewStringArray([]string{addrStr}), "ips") + params.Add(jsonutils.NewString("ping detect online free IP"), "notes") + params.Add(jsonutils.NewString(api.RESERVEDIP_STATUS_ONLINE), "status") + _, err := modules.Networks.PerformAction(s, net.Id, "reserve-ip", params) + if err != nil { + log.Errorf("failed to reserve ip %s: %s", addrStr, err) + } + } + } + log.Debugf("%s %s allocated %v", addrStr, netAddr, allocated) + } + return metrics, nil +} + +func init() { + shellutils.R(&PingProbeOptions{}, "ping-probe", "Ping probe IPv4 address", pingProbeCoolector) +} diff --git a/pkg/cloudmon/collectors/pingutils.go b/pkg/cloudmon/collectors/pingutils.go new file mode 100644 index 0000000000..d0f2e7a43c --- /dev/null +++ b/pkg/cloudmon/collectors/pingutils.go @@ -0,0 +1,88 @@ +package collectors + +import ( + "fmt" + "net" + "time" + + "github.com/tatsushid/go-fastping" + + "yunion.io/x/pkg/errors" +) + +type SPingResult struct { + addr string + rtt []time.Duration + count int +} + +func NewPingResult(addr string, count int) *SPingResult { + return &SPingResult{ + addr: addr, + rtt: make([]time.Duration, 0), + count: count, + } +} + +func (pr *SPingResult) Add(rtt time.Duration) { + pr.rtt = append(pr.rtt, rtt) +} + +func (pr SPingResult) Rtt() (min time.Duration, avg time.Duration, max time.Duration) { + sum := time.Duration(0) + max = time.Duration(-1) + min = time.Duration(-1) + for _, d := range pr.rtt { + sum += d + if max < 0 || max < d { + max = d + } + if min < 0 || min > d { + min = d + } + } + if len(pr.rtt) > 0 { + avg = sum / time.Duration(len(pr.rtt)) + } + return +} + +func (pr SPingResult) Loss() int { + return 100 - len(pr.rtt)*100/pr.count +} + +func (pr SPingResult) String() string { + min, avg, max := pr.Rtt() + return fmt.Sprintf("%d packets transmitted, %d received, %d%% packet loss, rtt min/avg/max = %d/%d/%d ms", pr.count, len(pr.rtt), pr.Loss(), min/time.Millisecond, avg/time.Millisecond, max/time.Millisecond) +} + +func Ping(addrList []string, count int, timeout time.Duration, debug bool) (map[string]*SPingResult, error) { + p := fastping.NewPinger() + p.MaxRTT = timeout + p.Size = 64 + p.Debug = debug + result := make(map[string]*SPingResult) + for _, addr := range addrList { + result[addr] = NewPingResult(addr, count) + p.AddIP(addr) + } + p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) { + result[addr.String()].Add(rtt) + } + p.OnIdle = func() { + } + p.RunLoop() + defer p.Stop() + ticker := time.NewTicker(time.Duration(count) * timeout) + defer ticker.Stop() + select { + case <-p.Done(): + if err := p.Err(); err != nil { + return nil, errors.Wrap(err, "ping error") + } + break + case <-ticker.C: + break + } + return result, nil +} diff --git a/pkg/cloudmon/collectors/pingutils_test.go b/pkg/cloudmon/collectors/pingutils_test.go new file mode 100644 index 0000000000..bc376aae22 --- /dev/null +++ b/pkg/cloudmon/collectors/pingutils_test.go @@ -0,0 +1,23 @@ +package collectors + +import ( + "testing" + "time" +) + +func TestPing(t *testing.T) { + result, err := Ping([]string{ + "114.114.114.114", + "118.187.65.237", + "10.168.26.254", + "10.168.26.26", + "192.30.253.113", + }, 5, time.Second, true) + if err != nil { + // ignore error + t.Logf("ping error %s", err) + } + for k, v := range result { + t.Logf("%s: %s", k, v.String()) + } +} diff --git a/pkg/cloudmon/collectors/qcmon/doc.go b/pkg/cloudmon/collectors/qcmon/doc.go new file mode 100644 index 0000000000..c8ce0c74d8 --- /dev/null +++ b/pkg/cloudmon/collectors/qcmon/doc.go @@ -0,0 +1 @@ +package qcmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/qcmon" diff --git a/pkg/cloudmon/collectors/qcmon/qcloudreport.go b/pkg/cloudmon/collectors/qcmon/qcloudreport.go new file mode 100644 index 0000000000..0e0d896211 --- /dev/null +++ b/pkg/cloudmon/collectors/qcmon/qcloudreport.go @@ -0,0 +1,85 @@ +package qcmon + +import ( + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SQCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SQCloudReportFactory struct { +} + +func (self *SQCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SQCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SQCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_QCLOUD +} + +type SQCloudReport struct { + common.CloudReportBase +} + +func (self *SQCloudReport) Report() error { + var servers []jsonutils.JSONObject + var err error + switch self.Operator { + case "redis": + servers, err = self.GetAllserverOfThisProvider(&modules.ElasticCache) + case "rds": + servers, err = self.GetAllserverOfThisProvider(&modules.DBInstance) + case "oss": + servers, err = self.GetAllserverOfThisProvider(&modules.Buckets) + case "elb": + servers, err = self.GetAllserverOfThisProvider(&modules.Loadbalancers) + default: + servers, err = self.GetAllserverOfThisProvider(&modules.Servers) + } + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + servers := regionServerMap[region.GetGlobalId()] + switch self.Operator { + case "server": + err = self.collectRegionMetricOfHost(region, servers) + case "redis": + err = self.collectRegionMetricOfRedis(region, servers) + case "rds": + err = self.collectRegionMetricOfRds(region, servers) + case "oss": + //err = self.collectRegionMetricOfOss(region, servers) + case "elb": + //err = self.collectRegionMetricOfElb(region, servers) + } + if err != nil { + return err + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/qcmon/qcloudservice.go b/pkg/cloudmon/collectors/qcmon/qcloudservice.go new file mode 100644 index 0000000000..eb993726b6 --- /dev/null +++ b/pkg/cloudmon/collectors/qcmon/qcloudservice.go @@ -0,0 +1,224 @@ +package qcmon + +import ( + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/qcloud" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SQCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + var err error + switch self.Operator { + case string(common.SERVER): + err = self.collectRegionMetricOfHost(region, servers) + } + return err +} + +func (self *SQCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + tecentReg := region.(*qcloud.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + dimensions := make([]qcloud.SQcMetricDimension, 0) + for _, server := range servers { + external_id, _ := server.GetString("external_id") + dimensions = append(dimensions, qcloud.SQcMetricDimension{ + Name: "InstanceId", + Value: external_id, + }) + } + //dimensions := []qcloud.SQcMetricDimension{qcloud.SQcMetricDimension{Name: "InstanceId", Value: external_id}} + for metricName, influxDbSpecs := range tecentMetricSpecs { + for index, tmp := 0, 0; index < len(dimensions); index += 10 { + tmp = index + 10 + if tmp > len(dimensions) { + tmp = len(dimensions) + } + rtnArray, err := tecentReg.GetMonitorData(metricName, "QCE/CVM", since, until, + dimensions[index:tmp]) + if err != nil { + log.Errorln(err) + continue + } + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if external_id == rtnMetric.Dimensions[0].Value { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + if len(rtnMetric.Timestamps) == 0 { + break + } + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric...) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SQCloudReport) collectRegionMetricOfRedis(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + tecentReg := region.(*qcloud.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + dimensions := make([]qcloud.SQcMetricDimension, 0) + for _, server := range servers { + external_id, _ := server.GetString("external_id") + dimensions = append(dimensions, qcloud.SQcMetricDimension{ + Name: "instanceid", + Value: external_id, + }) + } + //dimensions := []qcloud.SQcMetricDimension{qcloud.SQcMetricDimension{Name: "InstanceId", Value: external_id}} + for metricName, influxDbSpecs := range tecentRedisMetricSpecs { + for index, tmp := 0, 0; index < len(dimensions); index += 10 { + tmp = index + 10 + if tmp > len(dimensions) { + tmp = len(dimensions) + } + rtnArray, err := tecentReg.GetMonitorData(metricName, "QCE/REDIS", since, until, + dimensions[index:tmp]) + if err != nil { + log.Errorln(err) + continue + } + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if external_id == rtnMetric.Dimensions[0].Value { + if len(rtnMetric.Timestamps) == 0 { + break + } + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + serverName, _ := server.Get("name") + log.Errorf("redis:%s,collectMetricFromThisServer err:%v", serverName, err) + continue + } + dataList = append(dataList, serverMetric...) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SQCloudReport) collectRegionMetricOfRds(region cloudprovider.ICloudRegion, servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + tecentReg := region.(*qcloud.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + dimensions := make([]qcloud.SQcMetricDimension, 0) + for _, server := range servers { + external_id, _ := server.GetString("external_id") + dimensions = append(dimensions, qcloud.SQcMetricDimension{ + Name: "InstanceId", + Value: external_id, + }) + } + //dimensions := []qcloud.SQcMetricDimension{qcloud.SQcMetricDimension{Name: "InstanceId", Value: external_id}} + for metricName, influxDbSpecs := range tecentRdsMetricSpecs { + for index, tmp := 0, 0; index < len(dimensions); index += 10 { + tmp = index + 10 + if tmp > len(dimensions) { + tmp = len(dimensions) + } + rtnArray, err := tecentReg.GetMonitorData(metricName, "QCE/CDB", since, until, + dimensions[index:tmp]) + if err != nil { + log.Errorln(err) + continue + } + for _, rtnMetric := range rtnArray { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if external_id == rtnMetric.Dimensions[0].Value { + if len(rtnMetric.Timestamps) == 0 { + break + } + serverMetric, err := self.collectMetricFromThisServer(server, rtnMetric, influxDbSpecs) + if err != nil { + serverName, _ := server.Get("name") + log.Errorf("redis:%s,collectMetricFromThisServer err:%v", serverName, err) + continue + } + dataList = append(dataList, serverMetric...) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SQCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, rtnMetric qcloud.SDataPoint, influxDbSpecs []string) ([]influxdb.SMetricData, error) { + datas := make([]influxdb.SMetricData, 0) + for index, timestamp := range rtnMetric.Timestamps { + metric, err := self.NewMetricFromJson(server) + if err != nil { + return nil, err + } + //根据条件拼装metric的tag和metirc信息 + influxDbSpec := influxDbSpecs[2] + measurement := common.SubstringBefore(influxDbSpec, ".") + metric.Name = measurement + var pairsKey string + if strings.Contains(influxDbSpec, ",") { + pairsKey = common.SubstringBetween(influxDbSpec, ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpec, ".") + } + tag := common.SubstringAfter(influxDbSpec, ",") + if tag != "" && strings.Contains(influxDbSpec, "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + cpu_cout, err := server.Get("vcpu_count") + if err == nil { + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: "cpu_count", + Value: strconv.FormatInt(cpu_cout.(*jsonutils.JSONInt).Value(), 10), + }) + } + metric.Timestamp = time.Unix(int64(timestamp), 0) + fieldValue := rtnMetric.Values[index] + if influxDbSpecs[1] == UNIT_MBPS { + fieldValue = fieldValue * 1000 * 1000 + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + self.AddMetricTag(&metric, common.OtherVmTags) + datas = append(datas, metric) + } + return datas, nil +} diff --git a/pkg/cloudmon/collectors/qcmon/qcutils.go b/pkg/cloudmon/collectors/qcmon/qcutils.go new file mode 100644 index 0000000000..a2aab9e0a5 --- /dev/null +++ b/pkg/cloudmon/collectors/qcmon/qcutils.go @@ -0,0 +1,143 @@ +package qcmon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_sent" + INFLUXDB_FIELD_RDS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_active" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_percent" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +var tecentMetricSpecs = map[string][]string{ + "CPUUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "MemUsage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE}, + "lanOuttraffic": {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_TX_INTRANET}, + "WanOuttraffic": {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_TX_INTERNET}, + "lanIntraffic": {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_RX_INTRANET}, + "WanIntraffic": {DEFAULT_STATISTICS, UNIT_MBPS, INFLUXDB_FIELD_NET_BPS_RX_INTERNET}, +} + +var tecentRedisMetricSpecs = map[string][]string{ + "CpuUsMin": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_CPU_USAGE}, + "StorageUsMin": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_REDIS_MEM_USAGE}, + "InFlowMin": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_RX}, + "OutFlowMin": {DEFAULT_STATISTICS, UNIT_BPS, INFLUXDB_FIELD_REDIS_NET_BPS_TX}, + "ConnectionsUsMin": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CONN_USAGE}, + "QpsMin": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_OPT_SES}, + "KeysMin": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_KEYS}, + "ExpiredKeysMin": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS}, + "StorageMin": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE}, +} + +var tecentRdsMetricSpecs = map[string][]string{ + "CPUUseRate": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CPU_USAGE}, + "MemoryUseRate": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_MEM_USAGE}, + "BytesSent": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_TX_INTRANET}, + "BytesReceived": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_NET_BPS_RX_INTRANET}, + "VolumeRate": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_DISK_USAGE}, + "ThreadsConnected": {DEFAULT_STATISTICS, UNIT_COUNT, INFLUXDB_FIELD_RDS_CONN_COUNT}, + "ConnectionUseRate": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_RDS_CONN_USAGE}, + "QPS": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_RDS_QPS}, + "TPS": {DEFAULT_STATISTICS, UNIT_COUNT_SEC, INFLUXDB_FIELD_RDS_TPS}, + "InnodbDataRead": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_REDA_BPS}, + "InnodbDataWritten": {DEFAULT_STATISTICS, UNIT_BYTEPS, INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS}, +} diff --git a/pkg/cloudmon/collectors/rdsservice.go b/pkg/cloudmon/collectors/rdsservice.go new file mode 100644 index 0000000000..b5f25e565a --- /dev/null +++ b/pkg/cloudmon/collectors/rdsservice.go @@ -0,0 +1,15 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-rds", "Report Rds", reporRds) +} + +func reporRds(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCloudMetricOfoperatorType(string(common.RDS), session, args) +} diff --git a/pkg/cloudmon/collectors/redisservice.go b/pkg/cloudmon/collectors/redisservice.go new file mode 100644 index 0000000000..1c76766b97 --- /dev/null +++ b/pkg/cloudmon/collectors/redisservice.go @@ -0,0 +1,15 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-redis", "Report Redis", reportRedis) +} + +func reportRedis(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCloudMetricOfoperatorType(string(common.REDIS), session, args) +} diff --git a/pkg/cloudmon/collectors/serverservice.go b/pkg/cloudmon/collectors/serverservice.go new file mode 100644 index 0000000000..68744eeff3 --- /dev/null +++ b/pkg/cloudmon/collectors/serverservice.go @@ -0,0 +1,16 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-server", "Report Server", reportServer) +} + +//入口函数[aliyun、huawei] +func reportServer(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCloudMetricOfoperatorType(string(common.SERVER), session, args) +} diff --git a/pkg/cloudmon/collectors/storagemon/doc.go b/pkg/cloudmon/collectors/storagemon/doc.go new file mode 100644 index 0000000000..1a6ad7e3fb --- /dev/null +++ b/pkg/cloudmon/collectors/storagemon/doc.go @@ -0,0 +1 @@ +package storagemon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/storagemon" diff --git a/pkg/cloudmon/collectors/storagemon/storagereport.go b/pkg/cloudmon/collectors/storagemon/storagereport.go new file mode 100644 index 0000000000..38a00fcc1d --- /dev/null +++ b/pkg/cloudmon/collectors/storagemon/storagereport.go @@ -0,0 +1,44 @@ +package storagemon + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SCloudAccountFactory{} + common.RegisterFactory(&factory) +} + +type SCloudAccountFactory struct { +} + +func (self *SCloudAccountFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, + operatorType string) common.ICloudReport { + return &SStorageReport{ + common.CloudReportBase{ + SProvider: nil, + Session: session, + Args: args, + Operator: string(common.STORAGE), + }, + } +} + +func (S SCloudAccountFactory) GetId() string { + return string(common.STORAGE) +} + +type SStorageReport struct { + common.CloudReportBase +} + +func (self *SStorageReport) Report() error { + accounts, err := self.GetAllStorage(&modules.Storages) + if err != nil { + return err + } + return self.collectMetric(accounts) +} diff --git a/pkg/cloudmon/collectors/storagemon/storageservice.go b/pkg/cloudmon/collectors/storagemon/storageservice.go new file mode 100644 index 0000000000..2a418db0f6 --- /dev/null +++ b/pkg/cloudmon/collectors/storagemon/storageservice.go @@ -0,0 +1,50 @@ +package storagemon + +import ( + "strconv" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SStorageReport) collectMetric(storages []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + for _, storage := range storages { + metric, err := self.collectMetricFromStorage(storage) + if err != nil { + return err + } + metric.Timestamp = time.Now() + metric.Name = STORAGE_MEASUREMENT + dataList = append(dataList, metric) + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SStorageReport) collectMetricFromStorage(storage jsonutils.JSONObject) (influxdb.SMetricData, error) { + metric, err := self.NewMetricFromJson(storage) + if err != nil { + return metric, errors.Wrap(err, "collectMetricFromStorage NewMetricFromJson err") + } + capacity, _ := storage.Float("capacity") + actUsedCapacity, _ := storage.Float("actual_capacity_used") + var actFreeCapacity = float64(0) + var capacityUsage = float64(0) + if capacity != 0 { + actFreeCapacity = capacity - actUsedCapacity + capacityUsage = actUsedCapacity / capacity * 100 + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: STORAGE_FIELD_USAGE, + Value: strconv.FormatFloat(capacityUsage, 'f', 2, 64), + }, influxdb.SKeyValue{ + Key: STORAGE_FIELD_FREE, + Value: strconv.FormatFloat(actFreeCapacity, 'f', -1, 64), + }) + return metric, nil + +} diff --git a/pkg/cloudmon/collectors/storagemon/storageutil.go b/pkg/cloudmon/collectors/storagemon/storageutil.go new file mode 100644 index 0000000000..b6b51f05d8 --- /dev/null +++ b/pkg/cloudmon/collectors/storagemon/storageutil.go @@ -0,0 +1,10 @@ +package storagemon + +const ( + STORAGE_ID = "storage" + + CLOUDACCOUNT_FIELD = "balance" + STORAGE_MEASUREMENT = "storage" + STORAGE_FIELD_USAGE = "usage_active" + STORAGE_FIELD_FREE = "free" +) diff --git a/pkg/cloudmon/collectors/storageservice.go b/pkg/cloudmon/collectors/storageservice.go new file mode 100644 index 0000000000..8f0ef47292 --- /dev/null +++ b/pkg/cloudmon/collectors/storageservice.go @@ -0,0 +1,15 @@ +package collectors + +import ( + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +func init() { + shellutils.R(&common.ReportOptions{}, "report-storage", "Report Storage", reportStorage) +} + +func reportStorage(session *mcclient.ClientSession, args *common.ReportOptions) error { + return common.ReportCustomizeCloudMetric(string(common.STORAGE), session, args) +} diff --git a/pkg/cloudmon/collectors/usageservice.go b/pkg/cloudmon/collectors/usageservice.go new file mode 100644 index 0000000000..364e8cc0d7 --- /dev/null +++ b/pkg/cloudmon/collectors/usageservice.go @@ -0,0 +1,150 @@ +package collectors + +import ( + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/util/influxdb" + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +var config map[string]string = map[string]string{ + "hypervisors": "host-type", + "compute_engine_brands": "provider", +} +var measureMent string = "usage" + +func init() { + shellutils.R(&common.ReportOptions{}, "report-usage", "Report Usage", reportUsage) +} + +func reportUsage(session *mcclient.ClientSession, args *common.ReportOptions) error { + dataList := make([]influxdb.SMetricData, 0) + nowTime := time.Now() + //镜像使用量信息 + imageUsageFields, err := getImageUsageFields(session) + if err != nil { + return err + } + //查询到的Usage信息统一放置在metric中 + imageUsageFieldsDict := imageUsageFields.(*jsonutils.JSONDict) + capabilitesQuery := jsonutils.NewDict() + capabilitesQuery.Add(jsonutils.NewString("system"), "scope") + capabilites, err := modules.Capabilities.List(session, capabilitesQuery) + if err != nil { + return err + } + //通过capabilities中的信息遍历hypevisors和brands + for i := 0; i < len(capabilites.Data); i++ { + capabilitesObj := capabilites.Data[i] + capDict, ok := capabilitesObj.(*jsonutils.JSONDict) + if !ok { + return errors.ErrClient + } + for _, capKey := range capDict.SortedKeys() { + if _, ok := config[capKey]; ok { + hypeOrBrandObj, _ := capDict.Get(capKey) + if hypeOrBrandObj != nil { + hypeOrBrandArr, _ := hypeOrBrandObj.(*jsonutils.JSONArray) + for i := 0; i < len(hypeOrBrandArr.Value()); i++ { + hypeOrBrand := hypeOrBrandArr.Value()[i].(*jsonutils.JSONString) + dataList, err = packMetricList(session, dataList, imageUsageFieldsDict, config[capKey], + hypeOrBrand.String(), nowTime) + if err != nil { + return err + } + } + } + } + } + } + //查询host-type==""的情况，对应onecloud-控制面板-全部 要展示的内容 + dataList, err = packMetricList(session, dataList, imageUsageFieldsDict, "host-type", "", nowTime) + //写入influDb + return sendMetrics(session, dataList, args.Debug) +} + +//根据capabilities中的hypevisors和brands中的对应属性，组装Metric +func packMetricList(session *mcclient.ClientSession, dataList []influxdb.SMetricData, + imageUsageFieldsDict *jsonutils.JSONDict, paramKey string, + paramValue string, nowTime time.Time) (rtnList []influxdb.SMetricData, err error) { + //query，sql信息，查询主机compute的使用信息 + query := jsonutils.NewDict() + query.Add(jsonutils.NewString("system"), "scope") + if paramValue != "" { + if paramKey == "host-type" { + if paramValue == "kvm" { + query.Add(jsonutils.NewString("hypervisor"), paramKey) + } + query.Add(jsonutils.NewString(paramValue), paramKey) + } + query.Add(jsonutils.NewString(paramValue), paramKey) + } + metric := &influxdb.SMetricData{Name: measureMent, Timestamp: nowTime} + //查询到的镜像的使用信息放到SMetricData中的metric + metric, _ = jsonTometricData(imageUsageFieldsDict, metric, "metric") + //compute主机使用量 + respObj, err := modules.Usages.GetGeneralUsage(session, query) + if err != nil { + return nil, err + } + respObjDict := respObj.(*jsonutils.JSONDict) + //查询到的主机的使用信息放到SMetricData中的metric + metric, _ = jsonTometricData(respObjDict, metric, "metric") + if paramValue != "" { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: paramKey, Value: paramValue, + }) + } else { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: paramKey, Value: "all", + }) + } + dataList = append(dataList, *metric) + return dataList, nil +} + +//获得镜像使用量 +func getImageUsageFields(session *mcclient.ClientSession) (jsonutils.JSONObject, error) { + respObj, e := (&modules.ImageUsages).GetUsage(session, nil) + if e != nil { + return nil, e + } + respDict, ok := respObj.(*jsonutils.JSONDict) + if !ok { + return nil, jsonutils.ErrInvalidJsonDict + } + return respDict, nil +} + +//将JSONDict的信息放置到SMetricData中 +func jsonTometricData(obj *jsonutils.JSONDict, metric *influxdb.SMetricData, + metricDataType string) (*influxdb.SMetricData, error) { + + objMap, err := obj.GetMap() + if err != nil { + return nil, errors.Wrap(err, "obj.GetMap") + } + tagPairs := make([]influxdb.SKeyValue, 0) + metricPairs := make([]influxdb.SKeyValue, 0) + for k, v := range objMap { + val, _ := v.GetString() + if metricDataType == "tag" { + tagPairs = append(tagPairs, influxdb.SKeyValue{ + Key: k, Value: val, + }) + } else if metricDataType == "metric" { + metricPairs = append(metricPairs, influxdb.SKeyValue{ + Key: k, Value: val, + }) + } + } + metric.Tags = append(metric.Tags, tagPairs...) + metric.Metrics = append(metric.Metrics, metricPairs...) + return metric, nil +} diff --git a/pkg/cloudmon/collectors/utils.go b/pkg/cloudmon/collectors/utils.go new file mode 100644 index 0000000000..e3fc3ac231 --- /dev/null +++ b/pkg/cloudmon/collectors/utils.go @@ -0,0 +1,80 @@ +package collectors + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + o "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func jsonToMetric(obj *jsonutils.JSONDict, name string, tags []string, metrics []string) (influxdb.SMetricData, error) { + metric := influxdb.SMetricData{Name: name} + objMap, err := obj.GetMap() + if err != nil { + return metric, errors.Wrap(err, "obj.GetMap") + } + tagPairs := make([]influxdb.SKeyValue, 0) + metricPairs := make([]influxdb.SKeyValue, 0) + for k, v := range objMap { + val, _ := v.GetString() + if utils.IsInStringArray(k, tags) { + tagPairs = append(tagPairs, influxdb.SKeyValue{ + Key: k, Value: val, + }) + } else if utils.IsInStringArray(k, metrics) { + metricPairs = append(metricPairs, influxdb.SKeyValue{ + Key: k, Value: val, + }) + } + } + metric.Tags = tagPairs + metric.Metrics = metricPairs + return metric, nil +} + +func sendMetrics(s *mcclient.ClientSession, metrics []influxdb.SMetricData, debug bool) error { + urls, err := s.GetServiceURLs("influxdb", o.Options.SessionEndpointType) + if err != nil { + return errors.Wrap(err, "GetServiceURLs") + } + return influxdb.SendMetrics(urls, o.Options.InfluxDatabase, metrics, debug) +} + +type TListFunc func(*mcclient.ClientSession, jsonutils.JSONObject) (*modulebase.ListResult, error) +type TProcessFunc func(jsonutils.JSONObject) error + +func listAll(s *mcclient.ClientSession, listFunc TListFunc, kwargs jsonutils.JSONObject, processFunc TProcessFunc) error { + type sListParams struct { + Limit int + Offset int + } + offset := 0 + total := -1 + for total < 0 || offset < total { + params := jsonutils.Marshal(sListParams{ + Limit: 100, + Offset: offset, + }) + if kwargs != nil { + params.(*jsonutils.JSONDict).Update(kwargs) + } + result, err := listFunc(s, jsonutils.Marshal(params)) + if err != nil { + return err + } + total = result.Total + for i := range result.Data { + offset += 1 + err = processFunc(result.Data[i]) + if err != nil { + log.Errorf("fail to processData %s: %s", result.Data[i], err) + } + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/vmwaremon/doc.go b/pkg/cloudmon/collectors/vmwaremon/doc.go new file mode 100644 index 0000000000..1f87d5e8f3 --- /dev/null +++ b/pkg/cloudmon/collectors/vmwaremon/doc.go @@ -0,0 +1 @@ +package vmwaremon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/vmwaremon" diff --git a/pkg/cloudmon/collectors/vmwaremon/vmwarereport.go b/pkg/cloudmon/collectors/vmwaremon/vmwarereport.go new file mode 100644 index 0000000000..0fa99b6474 --- /dev/null +++ b/pkg/cloudmon/collectors/vmwaremon/vmwarereport.go @@ -0,0 +1,86 @@ +package vmwaremon + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SEsxiCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SEsxiCloudReportFactory struct { +} + +func (self *SEsxiCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SEsxiCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SEsxiCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_VMWARE +} + +type SEsxiCloudReport struct { + common.CloudReportBase +} + +func (self *SEsxiCloudReport) Report() error { + var err error + switch self.Operator { + case string(common.SERVER): + //servers, err := self.getAllserverOfThisProvider(&modules.Servers) + servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + if err != nil { + return err + } + err = self.CollectServerMetricByProvider(servers) + case string(common.HOST): + hosts, err := self.GetAllHostOfThisProvider(&modules.Hosts) + if err != nil { + return err + } + //err = common.CollectRegionMetricAsync(40, nil, hosts, self) + err = self.CollectRegionHostMetricAsync(hosts) + } + if err != nil { + return err + } + return nil +} + +func (self *SEsxiCloudReport) CollectServerMetricByProvider(servers []jsonutils.JSONObject) error { + return self.collectRegionMetricOfServerBatch("", servers) + +} + +func (self *SEsxiCloudReport) CollectRegionHostMetricAsync(servers []jsonutils.JSONObject) error { + log.Errorf("cloudproviderid: %s,%s count:%d", self.SProvider.Id, self.getMonType(), len(servers)) + if self.Args.Batch == 0 { + self.Args.Batch = 100 + } + for i := 0; i < len(servers); i += self.Args.Batch { + tmp := i + self.Args.Batch + if tmp > len(servers) { + tmp = len(servers) + } + err := self.collectRegionMetricOfServerBatch("", servers[i:tmp]) + if err != nil { + log.Errorln(err) + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/vmwaremon/vmwareservice.go b/pkg/cloudmon/collectors/vmwaremon/vmwareservice.go new file mode 100644 index 0000000000..5bb29227af --- /dev/null +++ b/pkg/cloudmon/collectors/vmwaremon/vmwareservice.go @@ -0,0 +1,337 @@ +package vmwaremon + +import ( + "context" + "net/url" + "strconv" + "strings" + "time" + + "github.com/vmware/govmomi/performance" + "github.com/vmware/govmomi/vim25/types" + "golang.org/x/sync/errgroup" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/esxi" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SEsxiCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + var err error + switch self.Operator { + case string(common.SERVER): + err = self.collectRegionMetricOfServer(servers) + case string(common.HOST): + err = self.collectRegionMetricOfHost(servers) + } + return err +} + +func (self *SEsxiCloudReport) getMonType() string { + switch self.Operator { + case string(common.SERVER): + return common.TYPE_VIRTUALMACHINE + case string(common.HOST): + return common.TYPE_HOSTSYSTEM + } + return "" +} + +func (self *SEsxiCloudReport) collectRegionMetricOfServerBatch(hostExtId string, servers []jsonutils.JSONObject) error { + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + + client, err := self.newEsxiClient() + if err != nil { + return err + } + metrics := make([]string, 0) + for metric, _ := range esxiMetricSpecs { + metrics = append(metrics, metric) + } + now := time.Now() + perfEntityMetrics, err := client.GetMonitorDataList(hostExtId, servers, self.getMonType(), metrics, since, + until) + if err != nil { + return errors.Wrap(err, "SEsxiCloudReport GetMonitorDataList error") + } + + log.Infof("get %s metriclist cost: %f s", self.getMonType(), time.Now().Sub(now).Seconds()) + + writeGroup, _ := errgroup.WithContext(context.Background()) + for i, _ := range servers { + tmpSer := servers[i] + writeGroup.Go(func() error { + extId, _ := tmpSer.GetString("external_id") + dataList := make([]influxdb.SMetricData, 0) + if entityMetric, ok := perfEntityMetrics[extId]; ok { + if self.Operator == string(common.SERVER) { + metric, err := common.FillVMCapacity(tmpSer.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + } + serverMetric := self.collectMetricFromThisServer_(tmpSer, self.getMonType(), entityMetric) + dataList = append(dataList, serverMetric...) + writStartTime := time.Now() + common.SendMetrics(self.Session, dataList, self.Args.Debug, "") + log.Errorf("influxdb write cost:%f s", time.Now().Sub(writStartTime).Seconds()) + } + return nil + }) + } + err = writeGroup.Wait() + log.Infof("collect %s num:%d", self.getMonType(), len(servers)) + return nil +} + +func (self *SEsxiCloudReport) collectRegionMetricOfServer(servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + + client, err := self.newEsxiClient() + if err != nil { + return err + } + for _, server := range servers { + perfEntityMetrics, metricIdNamedTable, err := client.GetMonitorData(server, common.TYPE_VIRTUALMACHINE, + esxiMetricSpecsSync, since, until) + if err != nil { + log.Errorln(err) + continue + } + for _, perfEntityMetric := range perfEntityMetrics { + perfMetricSeries := perfEntityMetric.Value + perfSampleInfos := perfEntityMetric.SampleInfo + for _, perfMetricSerie := range perfMetricSeries { + if perfMetricIntSerie, ok := perfMetricSerie.(*types.PerfMetricIntSeries); ok { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric := self.collectMetricFromThisServer(server, common.TYPE_VIRTUALMACHINE, perfMetricIntSerie, + perfSampleInfos, metricIdNamedTable) + dataList = append(dataList, serverMetric...) + } + } + } + err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "") + if err != nil { + log.Errorln(err) + } + } + return nil +} + +func (self *SEsxiCloudReport) collectRegionMetricOfHost(hosts []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + client, err := self.newEsxiClient() + if err != nil { + return err + } + for _, host := range hosts { + perfEntityMetrics, metricIdNamedTable, err := client.GetMonitorData(host, common.TYPE_HOSTSYSTEM, esxiMetricSpecsSync, + since, until) + if err != nil { + continue + } + for _, perfEntityMetric := range perfEntityMetrics { + perfMetricSeries := perfEntityMetric.Value + perfSampleInfos := perfEntityMetric.SampleInfo + for _, perfMetricSerie := range perfMetricSeries { + if perfMetricIntSerie, ok := perfMetricSerie.(*types.PerfMetricIntSeries); ok { + serverMetric := self.collectMetricFromThisServer(host, common.TYPE_HOSTSYSTEM, perfMetricIntSerie, + perfSampleInfos, metricIdNamedTable) + dataList = append(dataList, serverMetric...) + } + } + } + err = common.SendMetrics(self.Session, dataList, self.Args.Debug, "") + if err != nil { + log.Errorln(err) + } + } + return nil +} + +func (self *SEsxiCloudReport) collectMetricFromThisServer_(server jsonutils.JSONObject, monType string, + entityMetric performance.EntityMetric) []influxdb.SMetricData { + datas := make([]influxdb.SMetricData, 0) + for _, metricSeries := range entityMetric.Value { + if _, ok := esxiMetricSpecs[metricSeries.Name]; !ok { + continue + } + for i, value := range metricSeries.Value { + metric := influxdb.SMetricData{} + if monType == common.TYPE_HOSTSYSTEM { + metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.HostTags, make([]string, 0)) + } else { + metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + } + + if len(entityMetric.SampleInfo) > 0 { + metric.Timestamp = entityMetric.SampleInfo[i].Timestamp + } + influxDbSpecs := esxiMetricSpecs[metricSeries.Name] + metric.Name = common.GetMeasurement(monType, influxDbSpecs[2]) + var pairsKey string + if strings.Contains(influxDbSpecs[2], ",") { + pairsKey = common.SubstringBetween(influxDbSpecs[2], ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpecs[2], ".") + } + tag := common.SubstringAfter(influxDbSpecs[2], ",") + if tag != "" && strings.Contains(influxDbSpecs[2], "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + value = formateValue(value, influxDbSpecs) + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatInt(value, 10), + }) + if monType == common.TYPE_HOSTSYSTEM { + self.AddMetricTag(&metric, common.OtherHostTag) + } else { + self.AddMetricTag(&metric, common.OtherVmTags) + } + datas = append(datas, metric) + } + } + return datas +} + +func (self *SEsxiCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, monType string, + perfMetricIntSerie *types.PerfMetricIntSeries, perfSampleInfos []types.PerfSampleInfo, + metricIdNamedTable map[int32]string) []influxdb.SMetricData { + datas := make([]influxdb.SMetricData, 0) + for i, value := range perfMetricIntSerie.Value { + metric := influxdb.SMetricData{} + if monType == common.TYPE_HOSTSYSTEM { + metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.HostTags, make([]string, 0)) + } else { + metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + } + counterId := perfMetricIntSerie.Id.CounterId + instance := perfMetricIntSerie.Id.Instance + if instance != "" { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: "perf_instance", + Value: instance, + }) + } + if len(perfSampleInfos) > 0 { + metric.Timestamp = perfSampleInfos[i].Timestamp + } + + influxDbSpecs := esxiMetricSpecsSync[metricIdNamedTable[counterId]] + metric.Name = common.GetMeasurement(monType, influxDbSpecs[2]) + var pairsKey string + if strings.Contains(influxDbSpecs[2], ",") { + pairsKey = common.SubstringBetween(influxDbSpecs[2], ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpecs[2], ".") + } + tag := common.SubstringAfter(influxDbSpecs[2], ",") + if tag != "" && strings.Contains(influxDbSpecs[2], "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + value = formateValue(value, influxDbSpecs) + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatInt(value, 10), + }) + if monType == common.TYPE_HOSTSYSTEM { + self.AddMetricTag(&metric, common.OtherHostTag) + } else { + self.AddMetricTag(&metric, common.OtherVmTags) + } + datas = append(datas, metric) + } + return datas +} + +func formateValue(value int64, influxDbSpecs []string) int64 { + if influxDbSpecs[1] == UNIT_KBPS && strings.Contains(influxDbSpecs[2], "bps") { + value = value * 1000 + } + if influxDbSpecs[1] == UNIT_KBPS && strings.Contains(influxDbSpecs[2], "bytes") { + value = value * 1000 + } + if influxDbSpecs[1] == UNIT_PERCENT && strings.Contains(influxDbSpecs[2], "usage_active") { + value = value / 100 + } + if influxDbSpecs[1] == UNIT_PERCENT && strings.Contains(influxDbSpecs[2], "used_percent") { + value = value / 100 + } + return value +} + +func (self *SEsxiCloudReport) newEsxiClient() (*esxi.SESXiClient, error) { + parts, err := url.Parse(self.SProvider.AccessUrl) + if err != nil { + return nil, err + } + host, port, err := parseHostPort(parts.Host, 443) + if err != nil { + return nil, err + } + + secretDe, _ := utils.DescryptAESBase64(self.SProvider.Id, self.SProvider.Secret) + esxiCfg := esxi.NewESXiClientConfig(host, port, self.SProvider.Account, secretDe) + proCfg := cloudprovider.ProviderConfig{ + Id: self.SProvider.Id, + Name: self.SProvider.Name, + Account: self.SProvider.Account, + Secret: secretDe, + URL: self.SProvider.AccessUrl, + Vendor: self.SProvider.Provider, + } + esxiCfg.CloudproviderConfig(proCfg) + client, err := esxi.NewESXiClient(esxiCfg) + if err != nil { + return nil, err + } + return client, nil +} + +func parseHostPort(host string, defPort int) (string, int, error) { + colonPos := strings.IndexByte(host, ':') + if colonPos > 0 { + h := host[:colonPos] + p, err := strconv.Atoi(host[colonPos+1:]) + if err != nil { + log.Errorf("Invalid host %s", host) + return "", 0, err + } + if p == 0 { + p = defPort + } + return h, p, nil + } else { + return host, defPort, nil + } +} diff --git a/pkg/cloudmon/collectors/vmwaremon/vmwareutils.go b/pkg/cloudmon/collectors/vmwaremon/vmwareutils.go new file mode 100644 index 0000000000..cecaaf225f --- /dev/null +++ b/pkg/cloudmon/collectors/vmwaremon/vmwareutils.go @@ -0,0 +1,132 @@ +package vmwaremon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_KBPS = "kiloBytesPerSecond" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_READ_BPS_VIRTUAL = INFLUXDB_FIELD_DISK_READ_BPS + ",disk_type=virtual" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS_VIRTUAL = INFLUXDB_FIELD_DISK_WRITE_BPS + ",disk_type=virtual" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_send" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" +) + +//multiCloud查询指标列表组装 +var esxiMetricSpecs = map[string][]string{ + "cpu.usage.average": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "mem.usage.average": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE}, + "net.received.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_RX}, + "net.transmitted.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_TX}, + "disk.read.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS}, + "disk.write.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "virtualDisk.read.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS_VIRTUAL}, + "virtualDisk.write.average": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS_VIRTUAL}, +} + +var esxiMetricSpecsSync = map[string][]string{ + "cpu_usage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "mem_usage": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_MEM_USAGE}, + "net_received": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_RX}, + "net_transmitted": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_NET_BPS_TX}, + "disk_read": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS}, + "disk_write": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "virtualDisk_read": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_READ_BPS_VIRTUAL}, + "virtualDisk_write": {DEFAULT_STATISTICS, UNIT_KBPS, INFLUXDB_FIELD_DISK_WRITE_BPS_VIRTUAL}, +} diff --git a/pkg/cloudmon/collectors/zstackmon/doc.go b/pkg/cloudmon/collectors/zstackmon/doc.go new file mode 100644 index 0000000000..4e68378322 --- /dev/null +++ b/pkg/cloudmon/collectors/zstackmon/doc.go @@ -0,0 +1 @@ +package zstackmon // import "yunion.io/x/onecloud/pkg/cloudmon/collectors/zstackmon" diff --git a/pkg/cloudmon/collectors/zstackmon/zstackreport.go b/pkg/cloudmon/collectors/zstackmon/zstackreport.go new file mode 100644 index 0000000000..553f0bf55a --- /dev/null +++ b/pkg/cloudmon/collectors/zstackmon/zstackreport.go @@ -0,0 +1,94 @@ +package zstackmon + +import ( + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" +) + +func init() { + factory := SZStackCloudReportFactory{} + common.RegisterFactory(&factory) +} + +type SZStackCloudReportFactory struct { +} + +func (self *SZStackCloudReportFactory) NewCloudReport(provider *common.SProvider, session *mcclient.ClientSession, + args *common.ReportOptions, operatorType string) common.ICloudReport { + return &SZStackCloudReport{ + common.CloudReportBase{ + SProvider: provider, + Session: session, + Args: args, + Operator: operatorType, + }, + } +} + +func (self *SZStackCloudReportFactory) GetId() string { + return compute.CLOUD_PROVIDER_ZSTACK +} + +type SZStackCloudReport struct { + common.CloudReportBase +} + +func (self *SZStackCloudReport) Report() error { + switch self.Operator { + case string(common.SERVER): + return self.getServerMetrics() + case string(common.HOST): + return self.getHoseMetrics() + } + return nil +} + +func (self *SZStackCloudReport) getServerMetrics() error { + servers, err := self.GetAllserverOfThisProvider(&modules.Servers) + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(servers, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + err = common.CollectRegionMetricAsync(self.Args.Batch, region, regionServerMap[region.GetGlobalId()], self) + if err != nil { + log.Errorln(err) + continue + } + } + return nil +} + +func (self *SZStackCloudReport) getHoseMetrics() error { + hosts, err := self.GetAllHostOfThisProvider(&modules.Hosts) + if err != nil { + return err + } + providerInstance, err := self.InitProviderInstance() + if err != nil { + return err + } + regionList, regionServerMap, err := self.GetAllRegionOfServers(hosts, providerInstance) + if err != nil { + return err + } + for _, region := range regionList { + err = common.CollectRegionMetricAsync(self.Args.Batch, region, regionServerMap[region.GetGlobalId()], self) + if err != nil { + log.Errorln(err) + continue + } + } + return nil +} diff --git a/pkg/cloudmon/collectors/zstackmon/zstackservice.go b/pkg/cloudmon/collectors/zstackmon/zstackservice.go new file mode 100644 index 0000000000..c1a5465550 --- /dev/null +++ b/pkg/cloudmon/collectors/zstackmon/zstackservice.go @@ -0,0 +1,135 @@ +package zstackmon + +import ( + "strconv" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + + "yunion.io/x/onecloud/pkg/cloudmon/collectors/common" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/multicloud/zstack" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func (self *SZStackCloudReport) CollectRegionMetric(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + var err error + switch self.Operator { + case string(common.SERVER): + err = self.collectRegionMetricOfServer(region, servers) + case string(common.HOST): + err = self.collectRegionMetricOfHost(region, servers) + } + return err +} + +func (self *SZStackCloudReport) collectRegionMetricOfServer(region cloudprovider.ICloudRegion, + servers []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + zstackReg := region.(*zstack.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range zstackMetricSpecs { + rtn, err := zstackReg.GetMonitorData(metricName, NAMESPACE_VM, since, until) + if err != nil { + log.Errorln(err) + continue + } + if len(rtn.DataPoints) > 0 { + for _, dataPoint := range rtn.DataPoints { + for _, server := range servers { + external_id, _ := server.GetString("external_id") + if dataPoint.Labels.VMUuid == external_id { + metric, err := common.FillVMCapacity(server.(*jsonutils.JSONDict)) + if err != nil { + return err + } + dataList = append(dataList, metric) + serverMetric, err := self.collectMetricFromThisServer(server, common.TYPE_VIRTUALMACHINE, dataPoint, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SZStackCloudReport) collectRegionMetricOfHost(region cloudprovider.ICloudRegion, + hosts []jsonutils.JSONObject) error { + dataList := make([]influxdb.SMetricData, 0) + zstackReg := region.(*zstack.SRegion) + since, until, err := common.TimeRangeFromArgs(self.Args) + if err != nil { + return err + } + for metricName, influxDbSpecs := range zstackMetricSpecs { + rtn, err := zstackReg.GetMonitorData(metricName, NAMESPACE_HOST, since, until) + if err != nil { + return err + } + if len(rtn.DataPoints) > 0 { + for _, dataPoint := range rtn.DataPoints { + for _, host := range hosts { + external_id, _ := host.GetString("external_id") + if dataPoint.Labels.HostUuid == external_id { + serverMetric, err := self.collectMetricFromThisServer(host, common.TYPE_HOSTSYSTEM, dataPoint, influxDbSpecs) + if err != nil { + return err + } + dataList = append(dataList, serverMetric) + } + } + } + } + } + return common.SendMetrics(self.Session, dataList, self.Args.Debug, "") +} + +func (self *SZStackCloudReport) collectMetricFromThisServer(server jsonutils.JSONObject, + monType string, dataPoint zstack.DataPoint, influxDbSpecs []string) (influxdb.SMetricData, error) { + metric := influxdb.SMetricData{} + if monType == common.TYPE_HOSTSYSTEM { + metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.HostTags, make([]string, 0)) + } else { + metric, _ = common.JsonToMetric(server.(*jsonutils.JSONDict), "", common.ServerTags, make([]string, 0)) + } + fieldValue := dataPoint.Value + metric.Timestamp = time.Unix(dataPoint.TimeStemp, 0) + //根据条件拼装metric的tag和metirc信息 + metric.Name = common.GetMeasurement(monType, influxDbSpecs[2]) + var pairsKey string + if strings.Contains(influxDbSpecs[2], ",") { + pairsKey = common.SubstringBetween(influxDbSpecs[2], ".", ",") + } else { + pairsKey = common.SubstringAfter(influxDbSpecs[2], ".") + } + if influxDbSpecs[1] == UNIT_MEM { + fieldValue = fieldValue * 8 + } + tag := common.SubstringAfter(influxDbSpecs[2], ",") + if tag != "" && strings.Contains(influxDbSpecs[2], "=") { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: common.SubstringBefore(tag, "="), + Value: common.SubstringAfter(tag, "="), + }) + } + metric.Metrics = append(metric.Metrics, influxdb.SKeyValue{ + Key: pairsKey, + Value: strconv.FormatFloat(fieldValue, 'E', -1, 64), + }) + if monType == common.TYPE_HOSTSYSTEM { + self.AddMetricTag(&metric, common.OtherHostTag) + } else { + self.AddMetricTag(&metric, common.OtherVmTags) + } + return metric, nil +} diff --git a/pkg/cloudmon/collectors/zstackmon/zstackutils.go b/pkg/cloudmon/collectors/zstackmon/zstackutils.go new file mode 100644 index 0000000000..f222187efc --- /dev/null +++ b/pkg/cloudmon/collectors/zstackmon/zstackutils.go @@ -0,0 +1,123 @@ +package zstackmon + +const ( + PERIOD = 60 + UNIT_AVERAGE = "Average" + DEFAULT_STATISTICS = "Average,Minimum,Maximum" + UNIT_PERCENT = "Percent" + UNIT_BPS = "bps" + UNIT_MBPS = "Mbps" + UNIT_BYTEPS = "Bps" + UNIT_CPS = "cps" + UNIT_COUNT = "count" + UNIT_MEM = "byte" + UNIT_MSEC = "ms" + UNIT_COUNT_SEC = "count/s" + + TYPE_VIRTUALMACHINE = "VirtualMachine" + TYPE_HOSTSYSTEM = "HostSystem" + + //ESC监控指标 + INFLUXDB_FIELD_CPU_USAGE = "vm_cpu.usage_active" + INFLUXDB_FIELD_MEM_USAGE = "vm_mem.used_percent" + INFLUXDB_FIELD_DISK_READ_BPS = "vm_diskio.read_bps" + INFLUXDB_FIELD_DISK_WRITE_BPS = "vm_diskio.write_bps" + INFLUXDB_FIELD_DISK_READ_IOPS = "vm_diskio.read_iops" + INFLUXDB_FIELD_DISK_WRITE_IOPS = "vm_diskio.write_iops" + INFLUXDB_FIELD_NET_BPS_RX = "vm_netio.bps_recv" + INFLUXDB_FIELD_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_NET_BPS_TX = "vm_netio.bps_sent" + INFLUXDB_FIELD_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_WANOUTTRAFFIC = "vm_eipio.bps_out" + INFLUXDB_FIELD_WANINTRAFFIC = "vm_eipio.bps_in" + INFLUXDB_FIELD_WANOUTPKG = "vm_eipio.pps_out" + INFLUXDB_FIELD_WANINPKG = "vm_eipio.pps_in" + + //RDS监控指标 + INFLUXDB_FIELD_RDS_CPU_USAGE = "rds_cpu.usage_active" + INFLUXDB_FIELD_RDS_MEM_USAGE = "rds_mem.used_percent" + INFLUXDB_FIELD_RDS_NET_BPS_RX = "rds_netio.bps_recv" + INFLUXDB_FIELD_RDS_NET_BPS_RX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_RX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_RX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_NET_BPS_TX = "rds_netio.bps_send" + INFLUXDB_FIELD_RDS_NET_BPS_TX_MYSQL = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=mysql" + INFLUXDB_FIELD_RDS_NET_BPS_TX_SQLSERVER = INFLUXDB_FIELD_RDS_NET_BPS_TX + ",server_type=sqlserver" + INFLUXDB_FIELD_RDS_DISK_USAGE = "rds_disk.used_percent" + INFLUXDB_FIELD_RDS_DISK_READ_BPS = "rds_diskio.read_bps" + INFLUXDB_FIELD_RDS_DISK_WRITE_BPS = "rds_diskio.write_bps" + INFLUXDB_FIELD_RDS_CONN_COUNT = "rds_conn.used_count" + INFLUXDB_FIELD_RDS_CONN_USAGE = "rds_conn.used_percent" + + INFLUXDB_FIELD_RDS_QPS = "rds_qps.query_qps" + INFLUXDB_FIELD_RDS_TPS = "rds_tps.trans_qps" + INFLUXDB_FIELD_RDS_INNODB_REDA_BPS = "rds_innodb.read_bps" + INFLUXDB_FIELD_RDS_INNODB_WRITE_BPS = "rds_innodb.write_bps" + + //REDIS监控指标 + INFLUXDB_FIELD_REDIS_CPU_USAGE = "dcs_cpu.usage_percent" + INFLUXDB_FIELD_REDIS_MEM_USAGE = "dcs_mem.used_percent" + INFLUXDB_FIELD_REDIS_NET_BPS_RX = "dcs_netio.bps_recv" + INFLUXDB_FIELD_REDIS_NET_BPS_TX = "dcs_netio.bps_sent" + INFLUXDB_FIFLD_REDIS_CONN_USAGE = "dcs_conn.used_conn" + INFLUXDB_FIFLD_REDIS_OPT_SES = "dcs_instantopt.opt_sec" + INFLUXDB_FIFLD_REDIS_CACHE_KEYS = "dcs_cachekeys.key_count" + INFLUXDB_FIFLD_REDIS_CACHE_EXP_KEYS = INFLUXDB_FIFLD_REDIS_CACHE_KEYS + ",exp=expire" + INFLUXDB_FIFLD_REDIS_DATA_MEM_USAGE = "dcs_datamem.used_byte" + + //对象存储OSS监控指标 + INFLUXDB_FIELD_OSS_NET_BPS_RX = "oss_netio.bps_recv" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_RX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_RX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_NET_BPS_TX = "oss_netio.bps_sent" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTERNET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=internet" + INFLUXDB_FIELD_OSS_NET_BPS_TX_INTRANET = INFLUXDB_FIELD_OSS_NET_BPS_TX + ",net_type=intranet" + INFLUXDB_FIELD_OSS_LATECY = "oss_latency.req_late" + INFLUXDB_FIELD_OSS_LATECY_GET = INFLUXDB_FIELD_OSS_LATECY + ",request=get" + INFLUXDB_FIELD_OSS_LATECY_POST = INFLUXDB_FIELD_OSS_LATECY + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT = "oss_req.req_count" + INFLUXDB_FIELD_OSS_REQ_COUNT_GET = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=get" + INFLUXDB_FIELD_OSS_REQ_COUNT_POST = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=post" + INFLUXDB_FIELD_OSS_REQ_COUNT_5XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=5xx" + INFLUXDB_FIELD_OSS_REQ_COUNT_4XX = INFLUXDB_FIELD_OSS_REQ_COUNT + ",request=4xx" + + //负载均衡监控指标 + INFLUXDB_FIELD_ELB_NET_BPS_RX = "haproxy.bin" + INFLUXDB_FIELD_ELB_NET_BPS_TX = "haproxy.bout" + INFLUXDB_FIELD_ELB_REQ_RATE = "haproxy.req_rate,request=http" + INFLUXDB_FIELD_ELB_CONN_RATE = "haproxy.conn_rate,request=tcp" + INFLUXDB_FIELD_ELB_DREQ_COUNT = "haproxy.dreq,request=http" + INFLUXDB_FIELD_ELB_DCONN_COUNT = "haproxy.dcon,request=tcp" + INFLUXDB_FIELD_ELB_HRSP_COUNT = "haproxy.hrsp_Nxx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_2XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=2xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_3XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=3xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_4XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=4xx" + INFLUXDB_FIELD_ELB_HRSP_COUNT_5XX = INFLUXDB_FIELD_ELB_HRSP_COUNT + ",request=5xx" + INFLUXDB_FIELD_ELB_CHC_STATUS = "haproxy.check_status" + INFLUXDB_FIELD_ELB_CHC_CODE = "haproxy.check_code" + INFLUXDB_FIELD_ELB_LAST_CHC = "haproxy.last_chk" + + KEY_VMS = "vms" + KEY_CPUS = "cpus" + KEY_MEMS = "mems" + KEY_DISKS = "disks" + + KEY_LIMIT = "limit" + KEY_ADMIN = "admin" + KEY_USABLE = "usable" + + NAMESPACE_VM = "ZStack/VM" + NAMESPACE_HOST = "ZStack/Host" +) + +//multiCloud查询指标列表组装 +var zstackMetricSpecs = map[string][]string{ + "CPUAverageUsedUtilization": {DEFAULT_STATISTICS, UNIT_PERCENT, INFLUXDB_FIELD_CPU_USAGE}, + "DiskReadBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_BPS}, + "DiskWriteBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_BPS}, + "NetworkInBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_RX}, + "NetworkOutBytes": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_NET_BPS_TX}, + "DiskReadOps": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_READ_IOPS}, + "DiskWriteOps": {DEFAULT_STATISTICS, UNIT_MEM, INFLUXDB_FIELD_DISK_WRITE_IOPS}, +} diff --git a/pkg/cloudmon/options/doc.go b/pkg/cloudmon/options/doc.go new file mode 100644 index 0000000000..d49590c17e --- /dev/null +++ b/pkg/cloudmon/options/doc.go @@ -0,0 +1 @@ +package options // import "yunion.io/x/onecloud/pkg/cloudmon/options" diff --git a/pkg/cloudmon/options/options.go b/pkg/cloudmon/options/options.go new file mode 100644 index 0000000000..7e8598e913 --- /dev/null +++ b/pkg/cloudmon/options/options.go @@ -0,0 +1,75 @@ +package options + +import ( + "fmt" + + "yunion.io/x/pkg/errors" + "yunion.io/x/structarg" + + "yunion.io/x/onecloud/pkg/util/shellutils" +) + +type CloudMonOptions struct { + structarg.BaseOptions + + Region string `help:"Region name"` + EndpointType string `default:"internalURL" help:"Defaults to internalURL" choices:"publicURL|internalURL|adminURL"` + ApiVersion string `help:"override default modules service api version"` + + Debug bool `help:"Show debug information"` + Timeout int `default:"600" help:"Number of seconds to wait for a response"` + Insecure bool `default:"true" help:"Allow skip server cert verification if URL is https" short-token:"k"` + CertFile string `help:"certificate file"` + KeyFile string `help:"private key file"` + + AuthURL string `help:"Keystone auth URL" alias:"auth-uri"` + AdminUser string `help:"Admin username"` + AdminDomain string `help:"Admin user domain"` + AdminPassword string `help:"Admin password" alias:"admin-passwd"` + AdminProject string `help:"Admin project" default:"system" alias:"admin-tenant-name"` + AdminProjectDomain string `help:"Domain of Admin project"` + + SessionEndpointType string `help:"Client session end point type"` + + InfluxDatabase string `help:"influxdb database name, default telegraf" default:"telegraf"` + + SUBCOMMAND string `help:"climc subcommand" subcommand:"true"` +} + +func GetArgumentParser() (*structarg.ArgumentParser, error) { + parse, e := structarg.NewArgumentParser(&Options, + "cloudmon", + `Command-line interface to collect cloud monitoring data.`, + `See "cloudmon help COMMAND" for help on a specific command.`) + if e != nil { + return nil, e + } + subcmd := parse.GetSubcommand() + if subcmd == nil { + return nil, errors.Error("No subcommand argument") + } + type HelpOptions struct { + SUBCOMMAND string `help:"Sub-command name"` + } + shellutils.R(&HelpOptions{}, "help", "Show help information of any subcommand", func(suboptions *HelpOptions) error { + helpstr, e := subcmd.SubHelpString(suboptions.SUBCOMMAND) + if e != nil { + return e + } else { + fmt.Print(helpstr) + return nil + } + }) + for _, v := range shellutils.CommandTable { + _, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback) + + if e != nil { + return nil, e + } + } + return parse, nil +} + +var ( + Options CloudMonOptions +) diff --git a/pkg/hostman/guestfs/kvmpart/kvmpart.go b/pkg/hostman/guestfs/kvmpart/kvmpart.go index 4e45f5f6ff..1d9d94c1b0 100644 --- a/pkg/hostman/guestfs/kvmpart/kvmpart.go +++ b/pkg/hostman/guestfs/kvmpart/kvmpart.go @@ -267,7 +267,7 @@ func (p *SKVMGuestDiskPartition) Umount() error { log.Warningf("blockdev --flushbufs %s error: %v", p.partDev, err) } if err := os.Remove(p.mountPath); err != nil { - log.Warningf("remove mount path %s error: %v", p.mountPath) + log.Warningf("remove mount path %s error: %v", p.mountPath, err) } log.Infof("umount %s successfully", p.partDev) return nil diff --git a/pkg/hostman/hostdeployer/deployserver/deployserver.go b/pkg/hostman/hostdeployer/deployserver/deployserver.go index d9a8cdbfc4..d9b4482e69 100644 --- a/pkg/hostman/hostdeployer/deployserver/deployserver.go +++ b/pkg/hostman/hostdeployer/deployserver/deployserver.go @@ -93,11 +93,12 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP return ret, nil } -func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) { +func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty, + err error) { // There will be some occasional unknown panic, so temporarily capture panic here. defer func() { if r := recover(); r != nil { - log.Errorf("DeployGuestFs: %s, stack:\n %s", debug.Stack()) + log.Errorf("DeployGuestFs: %s, stack:\n %s", req.String(), debug.Stack()) msg := "panic: " if str, ok := r.(fmt.Stringer); ok { msg += str.String() diff --git a/pkg/monitor/suggestsysdrivers/doc.go b/pkg/monitor/suggestsysdrivers/doc.go deleted file mode 100644 index 5e11404e71..0000000000 --- a/pkg/monitor/suggestsysdrivers/doc.go +++ /dev/null @@ -1 +0,0 @@ -package suggestsysdrivers // import "yunion.io/x/onecloud/pkg/monitor/suggestsysdrivers" diff --git a/pkg/monitor/suggestsysdrivers/influxdbbasedriver.go b/pkg/monitor/suggestsysdrivers/influxdbbasedriver.go deleted file mode 100644 index a6e1f46d61..0000000000 --- a/pkg/monitor/suggestsysdrivers/influxdbbasedriver.go +++ /dev/null @@ -1,389 +0,0 @@ -package suggestsysdrivers - -import ( - "context" - "fmt" - "strings" - "time" - - "yunion.io/x/jsonutils" - "yunion.io/x/log" - "yunion.io/x/pkg/errors" - "yunion.io/x/pkg/utils" - - "yunion.io/x/onecloud/pkg/apis/monitor" - "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/onecloud/pkg/mcclient" - "yunion.io/x/onecloud/pkg/mcclient/auth" - "yunion.io/x/onecloud/pkg/mcclient/modulebase" - "yunion.io/x/onecloud/pkg/mcclient/modules" - "yunion.io/x/onecloud/pkg/monitor/alerting" - "yunion.io/x/onecloud/pkg/monitor/alerting/conditions" - merrors "yunion.io/x/onecloud/pkg/monitor/errors" - "yunion.io/x/onecloud/pkg/monitor/models" - "yunion.io/x/onecloud/pkg/monitor/validators" -) - -type InfluxdbBaseDriver struct { - *baseDriver -} - -func NewInfluxdbBaseDriver(driverType monitor.SuggestDriverType, resourceType monitor.MonitorResourceType, - action monitor.SuggestDriverAction, suggest monitor.MonitorSuggest, rule monitor.SuggestSysRuleCreateInput) *InfluxdbBaseDriver { - return &InfluxdbBaseDriver{ - baseDriver: newBaseDriver( - driverType, - resourceType, - action, - suggest, - rule, - )} -} - -func (drv *InfluxdbBaseDriver) ValidateSetting(input *monitor.SSuggestSysAlertSetting) error { - if input.ScaleRule == nil || len(*input.ScaleRule) == 0 { - return merrors.NewArgIsEmptyErr("ScaleRule") - } - for _, scale := range *input.ScaleRule { - if scale.Database == "" { - return merrors.NewArgIsEmptyErr("database") - } - if scale.Measurement == "" { - return merrors.NewArgIsEmptyErr("measurement") - } - if scale.Field == "" { - return merrors.NewArgIsEmptyErr("field") - } - if !utils.IsInStringArray(getQueryEvalType(scale), validators.EvaluatorDefaultTypes) { - return httperrors.NewInputParameterError("the evalType is illegal") - } - if scale.Threshold == 0 { - return merrors.NewArgIsEmptyErr("threshold") - } - } - return nil -} - -func getQueryEvalType(scale monitor.Scale) string { - typ := "" - switch scale.EvalType { - case ">=", ">": - typ = "gt" - case "<=", "<": - typ = "lt" - } - return typ -} - -func (drv *InfluxdbBaseDriver) DoSuggestSysRule(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - doSuggestSysRule(ctx, userCred, isStart, drv) -} - -func (drv *InfluxdbBaseDriver) Run(rule *models.SSuggestSysRule, instance *monitor.SSuggestSysAlertSetting) { - Run(drv, rule, instance) -} - -func (drv *InfluxdbBaseDriver) GetLatestAlerts(rule *models.SSuggestSysRule, instance *monitor.SSuggestSysAlertSetting) ([]jsonutils.JSONObject, error) { - //scaleEvalMatchs := make([]*monitor.EvalMatch, 0) - ret := make([]jsonutils.JSONObject, 0) - firing, evalMatchMap, err := drv.getScaleEvalResult(*instance.ScaleRule) - if err != nil { - return ret, errors.Wrap(err, "rule getScaleEvalResult happen error") - } - if firing { - log.Errorf("evalMatchMapLen:%d", len(evalMatchMap)) - serverArr, err := drv.getResourcesByEvalMatchsMap(evalMatchMap, instance) - if err != nil { - return ret, errors.Wrap(err, "rule getResource error") - } - return serverArr, nil - } - return ret, nil -} - -func (drv *InfluxdbBaseDriver) getScaleEvalResult(scales []monitor.Scale) (bool, map[string][]*monitor.EvalMatch, - error) { - firing := false - scaleEvalMatchs := make(map[string][]*monitor.EvalMatch, 0) - for index, scale := range scales { - condition := monitor.AlertCondition{ - Type: "suggest_query", - Query: drv.newAlertQuery(scale), - Evaluator: monitor.Condition{Type: getQueryEvalType(scale), Params: []float64{scale.Threshold}}, - Reducer: monitor.Condition{Type: "avg"}, - Operator: scale.Operator, - } - factory := alerting.GetConditionFactories()[condition.Type] - queryCondition, err := factory(&condition, index) - if err != nil { - return firing, scaleEvalMatchs, errors.Wrapf(err, "construct query condition %s", - jsonutils.Marshal(condition)) - } - duration, _ := time.ParseDuration(condition.Query.From) - queryCon := queryCondition.(*conditions.SuggestQueryCondition) - queryCon.Reducer = conditions.NewSuggestRuleReducer(queryCon.Reducer.GetType(), duration) - //evalContext := alerting.NewEvalContext(context.Background(), auth.AdminCredential(), nil) - evalContext := alerting.EvalContext{ - Ctx: context.Background(), - UserCred: auth.AdminCredential(), - IsDebug: true, - IsTestRun: true, - } - conditionResult, err := queryCondition.Eval(&evalContext) - if err != nil { - return firing, scaleEvalMatchs, errors.Wrap(err, "condition eval error") - } - if index == 0 { - firing = conditionResult.Firing - } - - // calculating Firing based on operator - if conditionResult.Operator == "or" { - firing = firing || conditionResult.Firing - } else { - firing = firing && conditionResult.Firing - } - if firing { - evalMatchs := conditionResult.EvalMatches - if conditionResult.Operator == "and" { - if index != 0 { - evalMatchs = getAndEvalMatches(scaleEvalMatchs, evalMatchs) - if len(evalMatchs) == 0 { - return false, scaleEvalMatchs, nil - } - } - } else { - if index != 0 { - getOrEvalMatches(scaleEvalMatchs, evalMatchs) - } - } - key := fmt.Sprintf("%s--%d", scale.Field, index) - scaleEvalMatchs[key] = evalMatchs - } - } - return firing, scaleEvalMatchs, nil -} - -func (drv *InfluxdbBaseDriver) getResourcesByEvalMatchsMap(evalMatchsMap map[string][]*monitor.EvalMatch, - instance *monitor.SSuggestSysAlertSetting) ([]jsonutils.JSONObject, error) { - matchLength := 0 - var maxEvalMatch []*monitor.EvalMatch - for _, evalMatchs := range evalMatchsMap { - if len(evalMatchs) > matchLength { - matchLength = len(evalMatchs) - maxEvalMatch = evalMatchs - } - } - resArr := jsonutils.NewArray() - for _, evalMatch := range maxEvalMatch { - res, mappingId, err := drv.getResourceFromEvalMatch(evalMatch) - if err != nil { - log.Errorln(err) - continue - } - suggestSysAlert, err := getSuggestSysAlertFromJson(res, drv) - if err != nil { - return nil, errors.Wrap(err, "Scale getSuggestSysAlertFromJson error") - } - suggestSysAlert.Action = string(monitor.SCALE_DOWN_DRIVER_ACTION) - suggestSysAlert.MonitorConfig = jsonutils.Marshal(instance) - suggestSysAlert.Problem = drv.describeEvalResultTojson(evalMatchsMap, mappingId) - resArr.Add(jsonutils.Marshal(suggestSysAlert)) - } - return resArr.GetArray() -} - -func (drv *InfluxdbBaseDriver) getResourceFromEvalMatch(evalMatch *monitor.EvalMatch) (jsonutils.JSONObject, string, error) { - var server jsonutils.JSONObject - mappingId := "" - id, err := drv.getMetricId(evalMatch) - if err != nil { - return server, mappingId, errors.Wrap(err, "InfluxdbBaseDriver getMetricId err") - } - serverobj, err := drv.getResourceById(id) - if err != nil { - return server, mappingId, errors.Wrapf(err, "InfluxdbBaseDriver getResourceById:%s err", id) - } - server = serverobj - mappingId = id - return server, mappingId, nil -} - -func (drv *InfluxdbBaseDriver) getMetricId(evalMatch *monitor.EvalMatch) (string, error) { - var id string - switch drv.GetResourceType() { - case monitor.SCALE_MONTITOR_RES_TYPE: - id = evalMatch.Tags[monitor.METRIC_VM_ID] - case monitor.REDIS_UNREASONABLE_MONITOR_RES_TYPE: - id = evalMatch.Tags[monitor.METRIC_REDIS_ID] - case monitor.RDS_UNREASONABLE_MONITOR_RES_TYPE: - id = evalMatch.Tags[monitor.METRIC_RDS_ID] - case monitor.OSS_UNREASONABLE_MONITOR_RES_TYPE: - id = evalMatch.Tags[monitor.METRIC_OSS_ID] - } - if len(id) == 0 { - return id, fmt.Errorf("no find resourceId by the driver type:%s", string(drv.GetResourceType())) - } - return id, nil -} - -func (drv *InfluxdbBaseDriver) getResourceById(id string) (jsonutils.JSONObject, error) { - switch drv.GetResourceType() { - case monitor.SCALE_MONTITOR_RES_TYPE: - return getResource(id, &modules.Servers) - case monitor.REDIS_UNREASONABLE_MONITOR_RES_TYPE: - return getResource(id, &modules.ElasticCache) - case monitor.RDS_UNREASONABLE_MONITOR_RES_TYPE: - return getResource(id, &modules.DBInstance) - case monitor.OSS_UNREASONABLE_MONITOR_RES_TYPE: - return getResource(id, &modules.Buckets) - } - return nil, fmt.Errorf("unsupporttd to get resource by the driver type:%s", string(drv.GetResourceType())) -} - -func (drv *InfluxdbBaseDriver) describeEvalResultTojson(evalMatchsMap map[string][]*monitor.EvalMatch, - mappingId string) jsonutils.JSONObject { - problem := jsonutils.NewDict() -loopEvalMap: - for _, evalMatchs := range evalMatchsMap { - for _, evalMatch := range evalMatchs { - metricId, _ := drv.getMetricId(evalMatch) - if metricId == mappingId { - problem.Add(jsonutils.NewFloat64(*evalMatch.Value), evalMatch.Metric) - break loopEvalMap - } - } - } - return problem -} - -func getResource(id string, manager modulebase.Manager) (jsonutils.JSONObject, error) { - session := auth.GetAdminSession(context.Background(), "", "") - query := jsonutils.NewDict() - query.Add(jsonutils.NewString("0"), "limit") - query.Add(jsonutils.NewString("system"), "scope") - server, err := manager.GetById(session, id, query) - if err != nil { - return nil, err - } - return server, nil -} - -func getMetricIdTag(tags map[string]string) map[string]string { - idTags := make(map[string]string, 0) - for key, val := range tags { - if strings.HasSuffix(key, "_id") { - idTags[key] = val - } - } - return idTags -} - -func getAndEvalMatches(scaleEvalMatchs map[string][]*monitor.EvalMatch, andscaleEvalMatchs []*monitor.EvalMatch) []*monitor.EvalMatch { - for key, evalMatchs := range scaleEvalMatchs { - andscaleEvalMatchs = getAndEvalMatches_(evalMatchs, andscaleEvalMatchs) - if len(andscaleEvalMatchs) == 0 { - return andscaleEvalMatchs - } - scaleEvalMatchs[key] = getAndEvalMatches_(andscaleEvalMatchs, evalMatchs) - } - return andscaleEvalMatchs -} - -func getOrEvalMatches(scaleEvalMatchs map[string][]*monitor.EvalMatch, orscaleEvalMatchs []*monitor.EvalMatch) { - for key, evalMatchs := range scaleEvalMatchs { - matches_ := getOrEvalMatches_(evalMatchs, orscaleEvalMatchs) - scaleEvalMatchs[key] = append(evalMatchs, matches_...) - } -} - -//by first param to scale other param's length -func getAndEvalMatches_(scaleEvalMatchs, andscaleEvalMatchs []*monitor.EvalMatch) []*monitor.EvalMatch { - resEvalMatchs := make([]*monitor.EvalMatch, 0) - for _, evalMatch := range scaleEvalMatchs { - idTags := getMetricIdTag(evalMatch.Tags) - twoLoop: - for _, andEvalMatch := range andscaleEvalMatchs { - andIdTags := getMetricIdTag(andEvalMatch.Tags) - //all the idTags must be equals - for key, val := range idTags { - if andVal, ok := andIdTags[key]; ok { - if val != andVal { - continue twoLoop - } - } - continue twoLoop - } - resEvalMatchs = append(resEvalMatchs, andEvalMatch) - } - } - return resEvalMatchs -} - -func getOrEvalMatches_(scaleEvalMatchs, orscaleEvalMatchs []*monitor.EvalMatch) []*monitor.EvalMatch { - resEvalMatchs := make([]*monitor.EvalMatch, 0) - containsEvalMatchMap := make(map[int]string) - for _, evalMatch := range scaleEvalMatchs { - idTags := getMetricIdTag(evalMatch.Tags) - twoLoop: - for i, orEvalMatch := range orscaleEvalMatchs { - orIdTags := getMetricIdTag(orEvalMatch.Tags) - //all the idTags must be equals - for key, val := range idTags { - if orVal, ok := orIdTags[key]; ok { - if val != orVal { - continue twoLoop - } - } - continue twoLoop - } - containsEvalMatchMap[i] = "" - } - } - for i, andEvalMatch := range orscaleEvalMatchs { - if _, ok := containsEvalMatchMap[i]; ok { - continue - } - resEvalMatchs = append(resEvalMatchs, andEvalMatch) - } - return resEvalMatchs -} - -func (drv *InfluxdbBaseDriver) newAlertQuery(scale monitor.Scale) monitor.AlertQuery { - suggestSysRules, _ := models.SuggestSysRuleManager.GetRules(drv.GetType()) - datasource, _ := models.DataSourceManager.GetDefaultSource() - return monitor.AlertQuery{ - Model: newMetricQuery(scale), - DataSourceId: datasource.Id, - From: suggestSysRules[0].TimeFrom, - To: "now", - } -} - -func newMetricQuery(scale monitor.Scale) monitor.MetricQuery { - sels := make([]monitor.MetricQuerySelect, 0) - sels = append(sels, monitor.NewMetricQuerySelect(monitor.MetricQueryPart{Type: "field", Params: []string{scale.Field}})) - return monitor.MetricQuery{ - Database: scale.Database, - Measurement: scale.Measurement, - Selects: sels, - GroupBy: []monitor.MetricQueryPart{ - { - Type: "field", - Params: []string{"*"}, - }, - }, - } -} - -func (drv *InfluxdbBaseDriver) StartResolveTask(ctx context.Context, userCred mcclient.TokenCredential, - suggestSysAlert *models.SSuggestSysAlert, - params *jsonutils.JSONDict) error { - log.Println("InfluxdbBaseDriver StartResolveTask do nothing") - return nil -} - -func (drv *InfluxdbBaseDriver) Resolve(data *models.SSuggestSysAlert) error { - log.Println("InfluxdbBaseDriver Resolve do nothing") - return nil -} diff --git a/pkg/vpcagent/models/modelset.go b/pkg/vpcagent/models/modelset.go index 2bc38c7012..377faf4095 100644 --- a/pkg/vpcagent/models/modelset.go +++ b/pkg/vpcagent/models/modelset.go @@ -94,7 +94,7 @@ func (ms Vpcs) joinRouteTables(subEntries RouteTables) bool { } subEntry.Vpc = m if m.RouteTable != nil { - log.Warningf("vpc %s has more than 1 route table available, skipping %s(%s)", + log.Warningf("vpc %s(%s) has more than 1 route table available, skipping %s(%s)", m.Name, m.Id, subEntry.Name, subEntry.Id) correct = false continue diff --git a/vendor/github.com/mitchellh/copystructure/README.md b/vendor/github.com/mitchellh/copystructure/README.md index f0fbd2e5c9..bcb8c8d2cb 100644 --- a/vendor/github.com/mitchellh/copystructure/README.md +++ b/vendor/github.com/mitchellh/copystructure/README.md @@ -1,21 +1,21 @@ -# copystructure - -copystructure is a Go library for deep copying values in Go. - -This allows you to copy Go values that may contain reference values -such as maps, slices, or pointers, and copy their data as well instead -of just their references. - -## Installation - -Standard `go get`: - -``` -$ go get github.com/mitchellh/copystructure -``` - -## Usage & Example - -For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/copystructure). - -The `Copy` function has examples associated with it there. +# copystructure + +copystructure is a Go library for deep copying values in Go. + +This allows you to copy Go values that may contain reference values +such as maps, slices, or pointers, and copy their data as well instead +of just their references. + +## Installation + +Standard `go get`: + +``` +$ go get github.com/mitchellh/copystructure +``` + +## Usage & Example + +For usage and examples see the [Godoc](http://godoc.org/github.com/mitchellh/copystructure). + +The `Copy` function has examples associated with it there. diff --git a/vendor/github.com/tatsushid/go-fastping/.gitignore b/vendor/github.com/tatsushid/go-fastping/.gitignore new file mode 100644 index 0000000000..494eb387d3 --- /dev/null +++ b/vendor/github.com/tatsushid/go-fastping/.gitignore @@ -0,0 +1,27 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe + +cover.out + +ping +!ping/ diff --git a/vendor/github.com/tatsushid/go-fastping/LICENSE b/vendor/github.com/tatsushid/go-fastping/LICENSE new file mode 100644 index 0000000000..abb8deeeec --- /dev/null +++ b/vendor/github.com/tatsushid/go-fastping/LICENSE @@ -0,0 +1,20 @@ +The MIT License (MIT) + +Copyright (c) 2013 Tatsushi Demachi + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/vendor/github.com/tatsushid/go-fastping/README.md b/vendor/github.com/tatsushid/go-fastping/README.md new file mode 100644 index 0000000000..d77a261929 --- /dev/null +++ b/vendor/github.com/tatsushid/go-fastping/README.md @@ -0,0 +1,54 @@ +go-fastping +=========== + +go-fastping is a Go language ICMP ping library, inspired by the `AnyEvent::FastPing` +Perl module, for quickly sending ICMP ECHO REQUEST packets. Original Perl module +is available at http://search.cpan.org/~mlehmann/AnyEvent-FastPing-2.01/ + +All original functions haven't been implemented yet. + +[![GoDoc](https://godoc.org/github.com/tatsushid/go-fastping?status.svg)](https://godoc.org/github.com/tatsushid/go-fastping) + +## Installation + +Install and update with `go get -u github.com/tatsushid/go-fastping` + +## Examples + +Import this package and write + +```go +p := fastping.NewPinger() +ra, err := net.ResolveIPAddr("ip4:icmp", os.Args[1]) +if err != nil { + fmt.Println(err) + os.Exit(1) +} +p.AddIPAddr(ra) +p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) { + fmt.Printf("IP Addr: %s receive, RTT: %v\n", addr.String(), rtt) +} +p.OnIdle = func() { + fmt.Println("finish") +} +err = p.Run() +if err != nil { + fmt.Println(err) +} +``` + +The example sends an ICMP packet and waits for a response. If it receives a +response, it calls the "receive" callback. After that, once MaxRTT time has +passed, it calls the "idle" callback. For more details, +refer [to the godoc][godoc], and if you need more examples, +please see "cmd/ping/ping.go". + +## Caution +This package implements ICMP ping using both raw socket and UDP. If your program +uses this package in raw socket mode, it needs to be run as a root user. + +## License +go-fastping is under MIT License. See the [LICENSE][license] file for details. + +[godoc]: http://godoc.org/github.com/tatsushid/go-fastping +[license]: https://github.com/tatsushid/go-fastping/blob/master/LICENSE diff --git a/vendor/github.com/tatsushid/go-fastping/fastping.go b/vendor/github.com/tatsushid/go-fastping/fastping.go new file mode 100644 index 0000000000..96950ca012 --- /dev/null +++ b/vendor/github.com/tatsushid/go-fastping/fastping.go @@ -0,0 +1,685 @@ +// Package fastping is an ICMP ping library inspired by AnyEvent::FastPing Perl +// module to send ICMP ECHO REQUEST packets quickly. Original Perl module is +// available at +// http://search.cpan.org/~mlehmann/AnyEvent-FastPing-2.01/ +// +// It hasn't been fully implemented original functions yet. +// +// Here is an example: +// +// p := fastping.NewPinger() +// ra, err := net.ResolveIPAddr("ip4:icmp", os.Args[1]) +// if err != nil { +// fmt.Println(err) +// os.Exit(1) +// } +// p.AddIPAddr(ra) +// p.OnRecv = func(addr *net.IPAddr, rtt time.Duration) { +// fmt.Printf("IP Addr: %s receive, RTT: %v\n", addr.String(), rtt) +// } +// p.OnIdle = func() { +// fmt.Println("finish") +// } +// err = p.Run() +// if err != nil { +// fmt.Println(err) +// } +// +// It sends an ICMP packet and wait a response. If it receives a response, +// it calls "receive" callback. After that, MaxRTT time passed, it calls +// "idle" callback. If you need more example, please see "cmd/ping/ping.go". +// +// This library needs to run as a superuser for sending ICMP packets when +// privileged raw ICMP endpoints is used so in such a case, to run go test +// for the package, please run like a following +// +// sudo go test +// +package fastping + +import ( + "errors" + "fmt" + "log" + "math/rand" + "net" + "sync" + "syscall" + "time" + + "golang.org/x/net/icmp" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +const ( + TimeSliceLength = 8 + ProtocolICMP = 1 + ProtocolIPv6ICMP = 58 +) + +var ( + ipv4Proto = map[string]string{"ip": "ip4:icmp", "udp": "udp4"} + ipv6Proto = map[string]string{"ip": "ip6:ipv6-icmp", "udp": "udp6"} +) + +func byteSliceOfSize(n int) []byte { + b := make([]byte, n) + for i := 0; i < len(b); i++ { + b[i] = 1 + } + + return b +} + +func timeToBytes(t time.Time) []byte { + nsec := t.UnixNano() + b := make([]byte, 8) + for i := uint8(0); i < 8; i++ { + b[i] = byte((nsec >> ((7 - i) * 8)) & 0xff) + } + return b +} + +func bytesToTime(b []byte) time.Time { + var nsec int64 + for i := uint8(0); i < 8; i++ { + nsec += int64(b[i]) << ((7 - i) * 8) + } + return time.Unix(nsec/1000000000, nsec%1000000000) +} + +func isIPv4(ip net.IP) bool { + return len(ip.To4()) == net.IPv4len +} + +func isIPv6(ip net.IP) bool { + return len(ip) == net.IPv6len +} + +func ipv4Payload(b []byte) []byte { + if len(b) < ipv4.HeaderLen { + return b + } + hdrlen := int(b[0]&0x0f) << 2 + return b[hdrlen:] +} + +type packet struct { + bytes []byte + addr net.Addr +} + +type context struct { + stop chan bool + done chan bool + err error +} + +func newContext() *context { + return &context{ + stop: make(chan bool), + done: make(chan bool), + } +} + +// Pinger represents ICMP packet sender/receiver +type Pinger struct { + id int + seq int + // key string is IPAddr.String() + addrs map[string]*net.IPAddr + network string + source string + source6 string + hasIPv4 bool + hasIPv6 bool + ctx *context + mu sync.Mutex + + // Size in bytes of the payload to send + Size int + // Number of (nano,milli)seconds of an idle timeout. Once it passed, + // the library calls an idle callback function. It is also used for an + // interval time of RunLoop() method + MaxRTT time.Duration + // OnRecv is called with a response packet's source address and its + // elapsed time when Pinger receives a response packet. + OnRecv func(*net.IPAddr, time.Duration) + // OnIdle is called when MaxRTT time passed + OnIdle func() + // If Debug is true, it prints debug messages to stdout. + Debug bool +} + +// NewPinger returns a new Pinger struct pointer +func NewPinger() *Pinger { + rand.Seed(time.Now().UnixNano()) + return &Pinger{ + id: rand.Intn(0xffff), + seq: rand.Intn(0xffff), + addrs: make(map[string]*net.IPAddr), + network: "ip", + source: "", + source6: "", + hasIPv4: false, + hasIPv6: false, + Size: TimeSliceLength, + MaxRTT: time.Second, + OnRecv: nil, + OnIdle: nil, + Debug: false, + } +} + +// Network sets a network endpoints for ICMP ping and returns the previous +// setting. network arg should be "ip" or "udp" string or if others are +// specified, it returns an error. If this function isn't called, Pinger +// uses "ip" as default. +func (p *Pinger) Network(network string) (string, error) { + origNet := p.network + switch network { + case "ip": + fallthrough + case "udp": + p.network = network + default: + return origNet, errors.New(network + " can't be used as ICMP endpoint") + } + return origNet, nil +} + +// Source sets ipv4/ipv6 source IP for sending ICMP packets and returns the previous +// setting. Empty value indicates to use system default one (for both ipv4 and ipv6). +func (p *Pinger) Source(source string) (string, error) { + // using ipv4 previous value for new empty one + origSource := p.source + if "" == source { + p.mu.Lock() + p.source = "" + p.source6 = "" + p.mu.Unlock() + return origSource, nil + } + + addr := net.ParseIP(source) + if addr == nil { + return origSource, errors.New(source + " is not a valid textual representation of an IPv4/IPv6 address") + } + + if isIPv4(addr) { + p.mu.Lock() + p.source = source + p.mu.Unlock() + } else if isIPv6(addr) { + origSource = p.source6 + p.mu.Lock() + p.source6 = source + p.mu.Unlock() + } else { + return origSource, errors.New(source + " is not a valid textual representation of an IPv4/IPv6 address") + } + + return origSource, nil +} + +// AddIP adds an IP address to Pinger. ipaddr arg should be a string like +// "192.0.2.1". +func (p *Pinger) AddIP(ipaddr string) error { + addr := net.ParseIP(ipaddr) + if addr == nil { + return fmt.Errorf("%s is not a valid textual representation of an IP address", ipaddr) + } + p.mu.Lock() + p.addrs[addr.String()] = &net.IPAddr{IP: addr} + if isIPv4(addr) { + p.hasIPv4 = true + } else if isIPv6(addr) { + p.hasIPv6 = true + } + p.mu.Unlock() + return nil +} + +// AddIPAddr adds an IP address to Pinger. ip arg should be a net.IPAddr +// pointer. +func (p *Pinger) AddIPAddr(ip *net.IPAddr) { + p.mu.Lock() + p.addrs[ip.String()] = ip + if isIPv4(ip.IP) { + p.hasIPv4 = true + } else if isIPv6(ip.IP) { + p.hasIPv6 = true + } + p.mu.Unlock() +} + +// RemoveIP removes an IP address from Pinger. ipaddr arg should be a string +// like "192.0.2.1". +func (p *Pinger) RemoveIP(ipaddr string) error { + addr := net.ParseIP(ipaddr) + if addr == nil { + return fmt.Errorf("%s is not a valid textual representation of an IP address", ipaddr) + } + p.mu.Lock() + delete(p.addrs, addr.String()) + p.mu.Unlock() + return nil +} + +// RemoveIPAddr removes an IP address from Pinger. ip arg should be a net.IPAddr +// pointer. +func (p *Pinger) RemoveIPAddr(ip *net.IPAddr) { + p.mu.Lock() + delete(p.addrs, ip.String()) + p.mu.Unlock() +} + +// AddHandler adds event handler to Pinger. event arg should be "receive" or +// "idle" string. +// +// **CAUTION** This function is deprecated. Please use OnRecv and OnIdle field +// of Pinger struct to set following handlers. +// +// "receive" handler should be +// +// func(addr *net.IPAddr, rtt time.Duration) +// +// type function. The handler is called with a response packet's source address +// and its elapsed time when Pinger receives a response packet. +// +// "idle" handler should be +// +// func() +// +// type function. The handler is called when MaxRTT time passed. For more +// detail, please see Run() and RunLoop(). +func (p *Pinger) AddHandler(event string, handler interface{}) error { + switch event { + case "receive": + if hdl, ok := handler.(func(*net.IPAddr, time.Duration)); ok { + p.mu.Lock() + p.OnRecv = hdl + p.mu.Unlock() + return nil + } + return errors.New("receive event handler should be `func(*net.IPAddr, time.Duration)`") + case "idle": + if hdl, ok := handler.(func()); ok { + p.mu.Lock() + p.OnIdle = hdl + p.mu.Unlock() + return nil + } + return errors.New("idle event handler should be `func()`") + } + return errors.New("No such event: " + event) +} + +// Run invokes a single send/receive procedure. It sends packets to all hosts +// which have already been added by AddIP() etc. and wait those responses. When +// it receives a response, it calls "receive" handler registered by AddHander(). +// After MaxRTT seconds, it calls "idle" handler and returns to caller with +// an error value. It means it blocks until MaxRTT seconds passed. For the +// purpose of sending/receiving packets over and over, use RunLoop(). +func (p *Pinger) Run() error { + p.mu.Lock() + p.ctx = newContext() + p.mu.Unlock() + p.run(true) + p.mu.Lock() + defer p.mu.Unlock() + return p.ctx.err +} + +// RunLoop invokes send/receive procedure repeatedly. It sends packets to all +// hosts which have already been added by AddIP() etc. and wait those responses. +// When it receives a response, it calls "receive" handler registered by +// AddHander(). After MaxRTT seconds, it calls "idle" handler, resend packets +// and wait those response. MaxRTT works as an interval time. +// +// This is a non-blocking method so immediately returns. If you want to monitor +// and stop sending packets, use Done() and Stop() methods. For example, +// +// p.RunLoop() +// ticker := time.NewTicker(time.Millisecond * 250) +// select { +// case <-p.Done(): +// if err := p.Err(); err != nil { +// log.Fatalf("Ping failed: %v", err) +// } +// case <-ticker.C: +// break +// } +// ticker.Stop() +// p.Stop() +// +// For more details, please see "cmd/ping/ping.go". +func (p *Pinger) RunLoop() { + p.mu.Lock() + p.ctx = newContext() + p.mu.Unlock() + go p.run(false) +} + +// Done returns a channel that is closed when RunLoop() is stopped by an error +// or Stop(). It must be called after RunLoop() call. If not, it causes panic. +func (p *Pinger) Done() <-chan bool { + return p.ctx.done +} + +// Stop stops RunLoop(). It must be called after RunLoop(). If not, it causes +// panic. +func (p *Pinger) Stop() { + p.debugln("Stop(): close(p.ctx.stop)") + close(p.ctx.stop) + p.debugln("Stop(): <-p.ctx.done") + <-p.ctx.done +} + +// Err returns an error that is set by RunLoop(). It must be called after +// RunLoop(). If not, it causes panic. +func (p *Pinger) Err() error { + p.mu.Lock() + defer p.mu.Unlock() + return p.ctx.err +} + +func (p *Pinger) listen(netProto string, source string) *icmp.PacketConn { + conn, err := icmp.ListenPacket(netProto, source) + if err != nil { + p.mu.Lock() + p.ctx.err = err + p.mu.Unlock() + p.debugln("Run(): close(p.ctx.done)") + close(p.ctx.done) + return nil + } + return conn +} + +func (p *Pinger) run(once bool) { + p.debugln("Run(): Start") + var conn, conn6 *icmp.PacketConn + if p.hasIPv4 { + if conn = p.listen(ipv4Proto[p.network], p.source); conn == nil { + return + } + defer conn.Close() + } + + if p.hasIPv6 { + if conn6 = p.listen(ipv6Proto[p.network], p.source6); conn6 == nil { + return + } + defer conn6.Close() + } + + recv := make(chan *packet, 1) + recvCtx := newContext() + wg := new(sync.WaitGroup) + + p.debugln("Run(): call recvICMP()") + if conn != nil { + wg.Add(1) + go p.recvICMP(conn, recv, recvCtx, wg) + } + if conn6 != nil { + wg.Add(1) + go p.recvICMP(conn6, recv, recvCtx, wg) + } + + p.debugln("Run(): call sendICMP()") + queue, err := p.sendICMP(conn, conn6) + + ticker := time.NewTicker(p.MaxRTT) + +mainloop: + for { + select { + case <-p.ctx.stop: + p.debugln("Run(): <-p.ctx.stop") + break mainloop + case <-recvCtx.done: + p.debugln("Run(): <-recvCtx.done") + p.mu.Lock() + err = recvCtx.err + p.mu.Unlock() + break mainloop + case <-ticker.C: + p.mu.Lock() + handler := p.OnIdle + p.mu.Unlock() + if handler != nil { + handler() + } + if once || err != nil { + break mainloop + } + p.debugln("Run(): call sendICMP()") + queue, err = p.sendICMP(conn, conn6) + case r := <-recv: + p.debugln("Run(): <-recv") + p.procRecv(r, queue) + } + } + + ticker.Stop() + + p.debugln("Run(): close(recvCtx.stop)") + close(recvCtx.stop) + p.debugln("Run(): wait recvICMP()") + wg.Wait() + + p.mu.Lock() + p.ctx.err = err + p.mu.Unlock() + + p.debugln("Run(): close(p.ctx.done)") + close(p.ctx.done) + p.debugln("Run(): End") +} + +func (p *Pinger) sendICMP(conn, conn6 *icmp.PacketConn) (map[string]*net.IPAddr, error) { + p.debugln("sendICMP(): Start") + p.mu.Lock() + p.id = rand.Intn(0xffff) + p.seq = rand.Intn(0xffff) + p.mu.Unlock() + queue := make(map[string]*net.IPAddr) + wg := new(sync.WaitGroup) + for key, addr := range p.addrs { + var typ icmp.Type + var cn *icmp.PacketConn + if isIPv4(addr.IP) { + typ = ipv4.ICMPTypeEcho + cn = conn + } else if isIPv6(addr.IP) { + typ = ipv6.ICMPTypeEchoRequest + cn = conn6 + } else { + continue + } + if cn == nil { + continue + } + + t := timeToBytes(time.Now()) + + if p.Size-TimeSliceLength != 0 { + t = append(t, byteSliceOfSize(p.Size-TimeSliceLength)...) + } + + p.mu.Lock() + bytes, err := (&icmp.Message{ + Type: typ, Code: 0, + Body: &icmp.Echo{ + ID: p.id, Seq: p.seq, + Data: t, + }, + }).Marshal(nil) + p.mu.Unlock() + if err != nil { + wg.Wait() + return queue, err + } + + queue[key] = addr + var dst net.Addr = addr + if p.network == "udp" { + dst = &net.UDPAddr{IP: addr.IP, Zone: addr.Zone} + } + + p.debugln("sendICMP(): Invoke goroutine") + wg.Add(1) + go func(conn *icmp.PacketConn, ra net.Addr, b []byte) { + for { + if _, err := conn.WriteTo(bytes, ra); err != nil { + if neterr, ok := err.(*net.OpError); ok { + if neterr.Err == syscall.ENOBUFS { + continue + } + } + } + break + } + p.debugln("sendICMP(): WriteTo End") + wg.Done() + }(cn, dst, bytes) + } + wg.Wait() + p.debugln("sendICMP(): End") + return queue, nil +} + +func (p *Pinger) recvICMP(conn *icmp.PacketConn, recv chan<- *packet, ctx *context, wg *sync.WaitGroup) { + p.debugln("recvICMP(): Start") + for { + select { + case <-ctx.stop: + p.debugln("recvICMP(): <-ctx.stop") + wg.Done() + p.debugln("recvICMP(): wg.Done()") + return + default: + } + + bytes := make([]byte, 512) + conn.SetReadDeadline(time.Now().Add(time.Millisecond * 100)) + p.debugln("recvICMP(): ReadFrom Start") + _, ra, err := conn.ReadFrom(bytes) + p.debugln("recvICMP(): ReadFrom End") + if err != nil { + if neterr, ok := err.(*net.OpError); ok { + if neterr.Timeout() { + p.debugln("recvICMP(): Read Timeout") + continue + } else { + p.debugln("recvICMP(): OpError happen", err) + p.mu.Lock() + ctx.err = err + p.mu.Unlock() + p.debugln("recvICMP(): close(ctx.done)") + close(ctx.done) + p.debugln("recvICMP(): wg.Done()") + wg.Done() + return + } + } + } + p.debugln("recvICMP(): p.recv <- packet") + + select { + case recv <- &packet{bytes: bytes, addr: ra}: + case <-ctx.stop: + p.debugln("recvICMP(): <-ctx.stop") + wg.Done() + p.debugln("recvICMP(): wg.Done()") + return + } + } +} + +func (p *Pinger) procRecv(recv *packet, queue map[string]*net.IPAddr) { + var ipaddr *net.IPAddr + switch adr := recv.addr.(type) { + case *net.IPAddr: + ipaddr = adr + case *net.UDPAddr: + ipaddr = &net.IPAddr{IP: adr.IP, Zone: adr.Zone} + default: + return + } + + addr := ipaddr.String() + p.mu.Lock() + if _, ok := p.addrs[addr]; !ok { + p.mu.Unlock() + return + } + p.mu.Unlock() + + var bytes []byte + var proto int + if isIPv4(ipaddr.IP) { + if p.network == "ip" { + bytes = ipv4Payload(recv.bytes) + } else { + bytes = recv.bytes + } + proto = ProtocolICMP + } else if isIPv6(ipaddr.IP) { + bytes = recv.bytes + proto = ProtocolIPv6ICMP + } else { + return + } + + var m *icmp.Message + var err error + if m, err = icmp.ParseMessage(proto, bytes); err != nil { + return + } + + if m.Type != ipv4.ICMPTypeEchoReply && m.Type != ipv6.ICMPTypeEchoReply { + return + } + + var rtt time.Duration + switch pkt := m.Body.(type) { + case *icmp.Echo: + p.mu.Lock() + if pkt.ID == p.id && pkt.Seq == p.seq { + rtt = time.Since(bytesToTime(pkt.Data[:TimeSliceLength])) + } + p.mu.Unlock() + default: + return + } + + if _, ok := queue[addr]; ok { + delete(queue, addr) + p.mu.Lock() + handler := p.OnRecv + p.mu.Unlock() + if handler != nil { + handler(ipaddr, rtt) + } + } +} + +func (p *Pinger) debugln(args ...interface{}) { + p.mu.Lock() + defer p.mu.Unlock() + if p.Debug { + log.Println(args...) + } +} + +func (p *Pinger) debugf(format string, args ...interface{}) { + p.mu.Lock() + defer p.mu.Unlock() + if p.Debug { + log.Printf(format, args...) + } +} diff --git a/vendor/golang.org/x/net/icmp/dstunreach.go b/vendor/golang.org/x/net/icmp/dstunreach.go new file mode 100644 index 0000000000..8615cf54a4 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/dstunreach.go @@ -0,0 +1,59 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "golang.org/x/net/internal/iana" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// A DstUnreach represents an ICMP destination unreachable message +// body. +type DstUnreach struct { + Data []byte // data, known as original datagram field + Extensions []Extension // extensions +} + +// Len implements the Len method of MessageBody interface. +func (p *DstUnreach) Len(proto int) int { + if p == nil { + return 0 + } + l, _ := multipartMessageBodyDataLen(proto, true, p.Data, p.Extensions) + return l +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *DstUnreach) Marshal(proto int) ([]byte, error) { + var typ Type + switch proto { + case iana.ProtocolICMP: + typ = ipv4.ICMPTypeDestinationUnreachable + case iana.ProtocolIPv6ICMP: + typ = ipv6.ICMPTypeDestinationUnreachable + default: + return nil, errInvalidProtocol + } + if !validExtensions(typ, p.Extensions) { + return nil, errInvalidExtension + } + return marshalMultipartMessageBody(proto, true, p.Data, p.Extensions) +} + +// parseDstUnreach parses b as an ICMP destination unreachable message +// body. +func parseDstUnreach(proto int, typ Type, b []byte) (MessageBody, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + p := &DstUnreach{} + var err error + p.Data, p.Extensions, err = parseMultipartMessageBody(proto, typ, b) + if err != nil { + return nil, err + } + return p, nil +} diff --git a/vendor/golang.org/x/net/icmp/echo.go b/vendor/golang.org/x/net/icmp/echo.go new file mode 100644 index 0000000000..b591864278 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/echo.go @@ -0,0 +1,173 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "encoding/binary" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// An Echo represents an ICMP echo request or reply message body. +type Echo struct { + ID int // identifier + Seq int // sequence number + Data []byte // data +} + +// Len implements the Len method of MessageBody interface. +func (p *Echo) Len(proto int) int { + if p == nil { + return 0 + } + return 4 + len(p.Data) +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *Echo) Marshal(proto int) ([]byte, error) { + b := make([]byte, 4+len(p.Data)) + binary.BigEndian.PutUint16(b[:2], uint16(p.ID)) + binary.BigEndian.PutUint16(b[2:4], uint16(p.Seq)) + copy(b[4:], p.Data) + return b, nil +} + +// parseEcho parses b as an ICMP echo request or reply message body. +func parseEcho(proto int, _ Type, b []byte) (MessageBody, error) { + bodyLen := len(b) + if bodyLen < 4 { + return nil, errMessageTooShort + } + p := &Echo{ID: int(binary.BigEndian.Uint16(b[:2])), Seq: int(binary.BigEndian.Uint16(b[2:4]))} + if bodyLen > 4 { + p.Data = make([]byte, bodyLen-4) + copy(p.Data, b[4:]) + } + return p, nil +} + +// An ExtendedEchoRequest represents an ICMP extended echo request +// message body. +type ExtendedEchoRequest struct { + ID int // identifier + Seq int // sequence number + Local bool // must be true when identifying by name or index + Extensions []Extension // extensions +} + +// Len implements the Len method of MessageBody interface. +func (p *ExtendedEchoRequest) Len(proto int) int { + if p == nil { + return 0 + } + l, _ := multipartMessageBodyDataLen(proto, false, nil, p.Extensions) + return l +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *ExtendedEchoRequest) Marshal(proto int) ([]byte, error) { + var typ Type + switch proto { + case iana.ProtocolICMP: + typ = ipv4.ICMPTypeExtendedEchoRequest + case iana.ProtocolIPv6ICMP: + typ = ipv6.ICMPTypeExtendedEchoRequest + default: + return nil, errInvalidProtocol + } + if !validExtensions(typ, p.Extensions) { + return nil, errInvalidExtension + } + b, err := marshalMultipartMessageBody(proto, false, nil, p.Extensions) + if err != nil { + return nil, err + } + binary.BigEndian.PutUint16(b[:2], uint16(p.ID)) + b[2] = byte(p.Seq) + if p.Local { + b[3] |= 0x01 + } + return b, nil +} + +// parseExtendedEchoRequest parses b as an ICMP extended echo request +// message body. +func parseExtendedEchoRequest(proto int, typ Type, b []byte) (MessageBody, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + p := &ExtendedEchoRequest{ID: int(binary.BigEndian.Uint16(b[:2])), Seq: int(b[2])} + if b[3]&0x01 != 0 { + p.Local = true + } + var err error + _, p.Extensions, err = parseMultipartMessageBody(proto, typ, b) + if err != nil { + return nil, err + } + return p, nil +} + +// An ExtendedEchoReply represents an ICMP extended echo reply message +// body. +type ExtendedEchoReply struct { + ID int // identifier + Seq int // sequence number + State int // 3-bit state working together with Message.Code + Active bool // probed interface is active + IPv4 bool // probed interface runs IPv4 + IPv6 bool // probed interface runs IPv6 +} + +// Len implements the Len method of MessageBody interface. +func (p *ExtendedEchoReply) Len(proto int) int { + if p == nil { + return 0 + } + return 4 +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *ExtendedEchoReply) Marshal(proto int) ([]byte, error) { + b := make([]byte, 4) + binary.BigEndian.PutUint16(b[:2], uint16(p.ID)) + b[2] = byte(p.Seq) + b[3] = byte(p.State<<5) & 0xe0 + if p.Active { + b[3] |= 0x04 + } + if p.IPv4 { + b[3] |= 0x02 + } + if p.IPv6 { + b[3] |= 0x01 + } + return b, nil +} + +// parseExtendedEchoReply parses b as an ICMP extended echo reply +// message body. +func parseExtendedEchoReply(proto int, _ Type, b []byte) (MessageBody, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + p := &ExtendedEchoReply{ + ID: int(binary.BigEndian.Uint16(b[:2])), + Seq: int(b[2]), + State: int(b[3]) >> 5, + } + if b[3]&0x04 != 0 { + p.Active = true + } + if b[3]&0x02 != 0 { + p.IPv4 = true + } + if b[3]&0x01 != 0 { + p.IPv6 = true + } + return p, nil +} diff --git a/vendor/golang.org/x/net/icmp/endpoint.go b/vendor/golang.org/x/net/icmp/endpoint.go new file mode 100644 index 0000000000..47f5b698d5 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/endpoint.go @@ -0,0 +1,113 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "net" + "runtime" + "time" + + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +var _ net.PacketConn = &PacketConn{} + +// A PacketConn represents a packet network endpoint that uses either +// ICMPv4 or ICMPv6. +type PacketConn struct { + c net.PacketConn + p4 *ipv4.PacketConn + p6 *ipv6.PacketConn +} + +func (c *PacketConn) ok() bool { return c != nil && c.c != nil } + +// IPv4PacketConn returns the ipv4.PacketConn of c. +// It returns nil when c is not created as the endpoint for ICMPv4. +func (c *PacketConn) IPv4PacketConn() *ipv4.PacketConn { + if !c.ok() { + return nil + } + return c.p4 +} + +// IPv6PacketConn returns the ipv6.PacketConn of c. +// It returns nil when c is not created as the endpoint for ICMPv6. +func (c *PacketConn) IPv6PacketConn() *ipv6.PacketConn { + if !c.ok() { + return nil + } + return c.p6 +} + +// ReadFrom reads an ICMP message from the connection. +func (c *PacketConn) ReadFrom(b []byte) (int, net.Addr, error) { + if !c.ok() { + return 0, nil, errInvalidConn + } + // Please be informed that ipv4.NewPacketConn enables + // IP_STRIPHDR option by default on Darwin. + // See golang.org/issue/9395 for further information. + if (runtime.GOOS == "darwin" || runtime.GOOS == "ios") && c.p4 != nil { + n, _, peer, err := c.p4.ReadFrom(b) + return n, peer, err + } + return c.c.ReadFrom(b) +} + +// WriteTo writes the ICMP message b to dst. +// The provided dst must be net.UDPAddr when c is a non-privileged +// datagram-oriented ICMP endpoint. +// Otherwise it must be net.IPAddr. +func (c *PacketConn) WriteTo(b []byte, dst net.Addr) (int, error) { + if !c.ok() { + return 0, errInvalidConn + } + return c.c.WriteTo(b, dst) +} + +// Close closes the endpoint. +func (c *PacketConn) Close() error { + if !c.ok() { + return errInvalidConn + } + return c.c.Close() +} + +// LocalAddr returns the local network address. +func (c *PacketConn) LocalAddr() net.Addr { + if !c.ok() { + return nil + } + return c.c.LocalAddr() +} + +// SetDeadline sets the read and write deadlines associated with the +// endpoint. +func (c *PacketConn) SetDeadline(t time.Time) error { + if !c.ok() { + return errInvalidConn + } + return c.c.SetDeadline(t) +} + +// SetReadDeadline sets the read deadline associated with the +// endpoint. +func (c *PacketConn) SetReadDeadline(t time.Time) error { + if !c.ok() { + return errInvalidConn + } + return c.c.SetReadDeadline(t) +} + +// SetWriteDeadline sets the write deadline associated with the +// endpoint. +func (c *PacketConn) SetWriteDeadline(t time.Time) error { + if !c.ok() { + return errInvalidConn + } + return c.c.SetWriteDeadline(t) +} diff --git a/vendor/golang.org/x/net/icmp/extension.go b/vendor/golang.org/x/net/icmp/extension.go new file mode 100644 index 0000000000..eeb85c3fc0 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/extension.go @@ -0,0 +1,170 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "encoding/binary" + + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// An Extension represents an ICMP extension. +type Extension interface { + // Len returns the length of ICMP extension. + // The provided proto must be either the ICMPv4 or ICMPv6 + // protocol number. + Len(proto int) int + + // Marshal returns the binary encoding of ICMP extension. + // The provided proto must be either the ICMPv4 or ICMPv6 + // protocol number. + Marshal(proto int) ([]byte, error) +} + +const extensionVersion = 2 + +func validExtensionHeader(b []byte) bool { + v := int(b[0]&0xf0) >> 4 + s := binary.BigEndian.Uint16(b[2:4]) + if s != 0 { + s = checksum(b) + } + if v != extensionVersion || s != 0 { + return false + } + return true +} + +// parseExtensions parses b as a list of ICMP extensions. +// The length attribute l must be the length attribute field in +// received icmp messages. +// +// It will return a list of ICMP extensions and an adjusted length +// attribute that represents the length of the padded original +// datagram field. Otherwise, it returns an error. +func parseExtensions(typ Type, b []byte, l int) ([]Extension, int, error) { + // Still a lot of non-RFC 4884 compliant implementations are + // out there. Set the length attribute l to 128 when it looks + // inappropriate for backwards compatibility. + // + // A minimal extension at least requires 8 octets; 4 octets + // for an extension header, and 4 octets for a single object + // header. + // + // See RFC 4884 for further information. + switch typ { + case ipv4.ICMPTypeExtendedEchoRequest, ipv6.ICMPTypeExtendedEchoRequest: + if len(b) < 8 || !validExtensionHeader(b) { + return nil, -1, errNoExtension + } + l = 0 + default: + if 128 > l || l+8 > len(b) { + l = 128 + } + if l+8 > len(b) { + return nil, -1, errNoExtension + } + if !validExtensionHeader(b[l:]) { + if l == 128 { + return nil, -1, errNoExtension + } + l = 128 + if !validExtensionHeader(b[l:]) { + return nil, -1, errNoExtension + } + } + } + var exts []Extension + for b = b[l+4:]; len(b) >= 4; { + ol := int(binary.BigEndian.Uint16(b[:2])) + if 4 > ol || ol > len(b) { + break + } + switch b[2] { + case classMPLSLabelStack: + ext, err := parseMPLSLabelStack(b[:ol]) + if err != nil { + return nil, -1, err + } + exts = append(exts, ext) + case classInterfaceInfo: + ext, err := parseInterfaceInfo(b[:ol]) + if err != nil { + return nil, -1, err + } + exts = append(exts, ext) + case classInterfaceIdent: + ext, err := parseInterfaceIdent(b[:ol]) + if err != nil { + return nil, -1, err + } + exts = append(exts, ext) + default: + ext := &RawExtension{Data: make([]byte, ol)} + copy(ext.Data, b[:ol]) + exts = append(exts, ext) + } + b = b[ol:] + } + return exts, l, nil +} + +func validExtensions(typ Type, exts []Extension) bool { + switch typ { + case ipv4.ICMPTypeDestinationUnreachable, ipv4.ICMPTypeTimeExceeded, ipv4.ICMPTypeParameterProblem, + ipv6.ICMPTypeDestinationUnreachable, ipv6.ICMPTypeTimeExceeded: + for i := range exts { + switch exts[i].(type) { + case *MPLSLabelStack, *InterfaceInfo, *RawExtension: + default: + return false + } + } + return true + case ipv4.ICMPTypeExtendedEchoRequest, ipv6.ICMPTypeExtendedEchoRequest: + var n int + for i := range exts { + switch exts[i].(type) { + case *InterfaceIdent: + n++ + case *RawExtension: + default: + return false + } + } + // Not a single InterfaceIdent object or a combo of + // RawExtension and InterfaceIdent objects is not + // allowed. + if n == 1 && len(exts) > 1 { + return false + } + return true + default: + return false + } +} + +// A RawExtension represents a raw extension. +// +// A raw extension is excluded from message processing and can be used +// to construct applications such as protocol conformance testing. +type RawExtension struct { + Data []byte // data +} + +// Len implements the Len method of Extension interface. +func (p *RawExtension) Len(proto int) int { + if p == nil { + return 0 + } + return len(p.Data) +} + +// Marshal implements the Marshal method of Extension interface. +func (p *RawExtension) Marshal(proto int) ([]byte, error) { + return p.Data, nil +} diff --git a/vendor/golang.org/x/net/icmp/helper_posix.go b/vendor/golang.org/x/net/icmp/helper_posix.go new file mode 100644 index 0000000000..75e7557570 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/helper_posix.go @@ -0,0 +1,75 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package icmp + +import ( + "net" + "strconv" + "syscall" +) + +func sockaddr(family int, address string) (syscall.Sockaddr, error) { + switch family { + case syscall.AF_INET: + a, err := net.ResolveIPAddr("ip4", address) + if err != nil { + return nil, err + } + if len(a.IP) == 0 { + a.IP = net.IPv4zero + } + if a.IP = a.IP.To4(); a.IP == nil { + return nil, net.InvalidAddrError("non-ipv4 address") + } + sa := &syscall.SockaddrInet4{} + copy(sa.Addr[:], a.IP) + return sa, nil + case syscall.AF_INET6: + a, err := net.ResolveIPAddr("ip6", address) + if err != nil { + return nil, err + } + if len(a.IP) == 0 { + a.IP = net.IPv6unspecified + } + if a.IP.Equal(net.IPv4zero) { + a.IP = net.IPv6unspecified + } + if a.IP = a.IP.To16(); a.IP == nil || a.IP.To4() != nil { + return nil, net.InvalidAddrError("non-ipv6 address") + } + sa := &syscall.SockaddrInet6{ZoneId: zoneToUint32(a.Zone)} + copy(sa.Addr[:], a.IP) + return sa, nil + default: + return nil, net.InvalidAddrError("unexpected family") + } +} + +func zoneToUint32(zone string) uint32 { + if zone == "" { + return 0 + } + if ifi, err := net.InterfaceByName(zone); err == nil { + return uint32(ifi.Index) + } + n, err := strconv.Atoi(zone) + if err != nil { + return 0 + } + return uint32(n) +} + +func last(s string, b byte) int { + i := len(s) + for i--; i >= 0; i-- { + if s[i] == b { + break + } + } + return i +} diff --git a/vendor/golang.org/x/net/icmp/interface.go b/vendor/golang.org/x/net/icmp/interface.go new file mode 100644 index 0000000000..b3dd72fb0a --- /dev/null +++ b/vendor/golang.org/x/net/icmp/interface.go @@ -0,0 +1,322 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "encoding/binary" + "net" + "strings" + + "golang.org/x/net/internal/iana" +) + +const ( + classInterfaceInfo = 2 +) + +const ( + attrMTU = 1 << iota + attrName + attrIPAddr + attrIfIndex +) + +// An InterfaceInfo represents interface and next-hop identification. +type InterfaceInfo struct { + Class int // extension object class number + Type int // extension object sub-type + Interface *net.Interface + Addr *net.IPAddr +} + +func (ifi *InterfaceInfo) nameLen() int { + if len(ifi.Interface.Name) > 63 { + return 64 + } + l := 1 + len(ifi.Interface.Name) + return (l + 3) &^ 3 +} + +func (ifi *InterfaceInfo) attrsAndLen(proto int) (attrs, l int) { + l = 4 + if ifi.Interface != nil && ifi.Interface.Index > 0 { + attrs |= attrIfIndex + l += 4 + if len(ifi.Interface.Name) > 0 { + attrs |= attrName + l += ifi.nameLen() + } + if ifi.Interface.MTU > 0 { + attrs |= attrMTU + l += 4 + } + } + if ifi.Addr != nil { + switch proto { + case iana.ProtocolICMP: + if ifi.Addr.IP.To4() != nil { + attrs |= attrIPAddr + l += 4 + net.IPv4len + } + case iana.ProtocolIPv6ICMP: + if ifi.Addr.IP.To16() != nil && ifi.Addr.IP.To4() == nil { + attrs |= attrIPAddr + l += 4 + net.IPv6len + } + } + } + return +} + +// Len implements the Len method of Extension interface. +func (ifi *InterfaceInfo) Len(proto int) int { + _, l := ifi.attrsAndLen(proto) + return l +} + +// Marshal implements the Marshal method of Extension interface. +func (ifi *InterfaceInfo) Marshal(proto int) ([]byte, error) { + attrs, l := ifi.attrsAndLen(proto) + b := make([]byte, l) + if err := ifi.marshal(proto, b, attrs, l); err != nil { + return nil, err + } + return b, nil +} + +func (ifi *InterfaceInfo) marshal(proto int, b []byte, attrs, l int) error { + binary.BigEndian.PutUint16(b[:2], uint16(l)) + b[2], b[3] = classInterfaceInfo, byte(ifi.Type) + for b = b[4:]; len(b) > 0 && attrs != 0; { + switch { + case attrs&attrIfIndex != 0: + b = ifi.marshalIfIndex(proto, b) + attrs &^= attrIfIndex + case attrs&attrIPAddr != 0: + b = ifi.marshalIPAddr(proto, b) + attrs &^= attrIPAddr + case attrs&attrName != 0: + b = ifi.marshalName(proto, b) + attrs &^= attrName + case attrs&attrMTU != 0: + b = ifi.marshalMTU(proto, b) + attrs &^= attrMTU + } + } + return nil +} + +func (ifi *InterfaceInfo) marshalIfIndex(proto int, b []byte) []byte { + binary.BigEndian.PutUint32(b[:4], uint32(ifi.Interface.Index)) + return b[4:] +} + +func (ifi *InterfaceInfo) parseIfIndex(b []byte) ([]byte, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + ifi.Interface.Index = int(binary.BigEndian.Uint32(b[:4])) + return b[4:], nil +} + +func (ifi *InterfaceInfo) marshalIPAddr(proto int, b []byte) []byte { + switch proto { + case iana.ProtocolICMP: + binary.BigEndian.PutUint16(b[:2], uint16(iana.AddrFamilyIPv4)) + copy(b[4:4+net.IPv4len], ifi.Addr.IP.To4()) + b = b[4+net.IPv4len:] + case iana.ProtocolIPv6ICMP: + binary.BigEndian.PutUint16(b[:2], uint16(iana.AddrFamilyIPv6)) + copy(b[4:4+net.IPv6len], ifi.Addr.IP.To16()) + b = b[4+net.IPv6len:] + } + return b +} + +func (ifi *InterfaceInfo) parseIPAddr(b []byte) ([]byte, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + afi := int(binary.BigEndian.Uint16(b[:2])) + b = b[4:] + switch afi { + case iana.AddrFamilyIPv4: + if len(b) < net.IPv4len { + return nil, errMessageTooShort + } + ifi.Addr.IP = make(net.IP, net.IPv4len) + copy(ifi.Addr.IP, b[:net.IPv4len]) + b = b[net.IPv4len:] + case iana.AddrFamilyIPv6: + if len(b) < net.IPv6len { + return nil, errMessageTooShort + } + ifi.Addr.IP = make(net.IP, net.IPv6len) + copy(ifi.Addr.IP, b[:net.IPv6len]) + b = b[net.IPv6len:] + } + return b, nil +} + +func (ifi *InterfaceInfo) marshalName(proto int, b []byte) []byte { + l := byte(ifi.nameLen()) + b[0] = l + copy(b[1:], []byte(ifi.Interface.Name)) + return b[l:] +} + +func (ifi *InterfaceInfo) parseName(b []byte) ([]byte, error) { + if 4 > len(b) || len(b) < int(b[0]) { + return nil, errMessageTooShort + } + l := int(b[0]) + if l%4 != 0 || 4 > l || l > 64 { + return nil, errInvalidExtension + } + var name [63]byte + copy(name[:], b[1:l]) + ifi.Interface.Name = strings.Trim(string(name[:]), "\000") + return b[l:], nil +} + +func (ifi *InterfaceInfo) marshalMTU(proto int, b []byte) []byte { + binary.BigEndian.PutUint32(b[:4], uint32(ifi.Interface.MTU)) + return b[4:] +} + +func (ifi *InterfaceInfo) parseMTU(b []byte) ([]byte, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + ifi.Interface.MTU = int(binary.BigEndian.Uint32(b[:4])) + return b[4:], nil +} + +func parseInterfaceInfo(b []byte) (Extension, error) { + ifi := &InterfaceInfo{ + Class: int(b[2]), + Type: int(b[3]), + } + if ifi.Type&(attrIfIndex|attrName|attrMTU) != 0 { + ifi.Interface = &net.Interface{} + } + if ifi.Type&attrIPAddr != 0 { + ifi.Addr = &net.IPAddr{} + } + attrs := ifi.Type & (attrIfIndex | attrIPAddr | attrName | attrMTU) + for b = b[4:]; len(b) > 0 && attrs != 0; { + var err error + switch { + case attrs&attrIfIndex != 0: + b, err = ifi.parseIfIndex(b) + attrs &^= attrIfIndex + case attrs&attrIPAddr != 0: + b, err = ifi.parseIPAddr(b) + attrs &^= attrIPAddr + case attrs&attrName != 0: + b, err = ifi.parseName(b) + attrs &^= attrName + case attrs&attrMTU != 0: + b, err = ifi.parseMTU(b) + attrs &^= attrMTU + } + if err != nil { + return nil, err + } + } + if ifi.Interface != nil && ifi.Interface.Name != "" && ifi.Addr != nil && ifi.Addr.IP.To16() != nil && ifi.Addr.IP.To4() == nil { + ifi.Addr.Zone = ifi.Interface.Name + } + return ifi, nil +} + +const ( + classInterfaceIdent = 3 + typeInterfaceByName = 1 + typeInterfaceByIndex = 2 + typeInterfaceByAddress = 3 +) + +// An InterfaceIdent represents interface identification. +type InterfaceIdent struct { + Class int // extension object class number + Type int // extension object sub-type + Name string // interface name + Index int // interface index + AFI int // address family identifier; see address family numbers in IANA registry + Addr []byte // address +} + +// Len implements the Len method of Extension interface. +func (ifi *InterfaceIdent) Len(_ int) int { + switch ifi.Type { + case typeInterfaceByName: + l := len(ifi.Name) + if l > 255 { + l = 255 + } + return 4 + (l+3)&^3 + case typeInterfaceByIndex: + return 4 + 4 + case typeInterfaceByAddress: + return 4 + 4 + (len(ifi.Addr)+3)&^3 + default: + return 4 + } +} + +// Marshal implements the Marshal method of Extension interface. +func (ifi *InterfaceIdent) Marshal(proto int) ([]byte, error) { + b := make([]byte, ifi.Len(proto)) + if err := ifi.marshal(proto, b); err != nil { + return nil, err + } + return b, nil +} + +func (ifi *InterfaceIdent) marshal(proto int, b []byte) error { + l := ifi.Len(proto) + binary.BigEndian.PutUint16(b[:2], uint16(l)) + b[2], b[3] = classInterfaceIdent, byte(ifi.Type) + switch ifi.Type { + case typeInterfaceByName: + copy(b[4:], ifi.Name) + case typeInterfaceByIndex: + binary.BigEndian.PutUint32(b[4:4+4], uint32(ifi.Index)) + case typeInterfaceByAddress: + binary.BigEndian.PutUint16(b[4:4+2], uint16(ifi.AFI)) + b[4+2] = byte(len(ifi.Addr)) + copy(b[4+4:], ifi.Addr) + } + return nil +} + +func parseInterfaceIdent(b []byte) (Extension, error) { + ifi := &InterfaceIdent{ + Class: int(b[2]), + Type: int(b[3]), + } + switch ifi.Type { + case typeInterfaceByName: + ifi.Name = strings.Trim(string(b[4:]), "\x00") + case typeInterfaceByIndex: + if len(b[4:]) < 4 { + return nil, errInvalidExtension + } + ifi.Index = int(binary.BigEndian.Uint32(b[4 : 4+4])) + case typeInterfaceByAddress: + if len(b[4:]) < 4 { + return nil, errInvalidExtension + } + ifi.AFI = int(binary.BigEndian.Uint16(b[4 : 4+2])) + l := int(b[4+2]) + if len(b[4+4:]) < l { + return nil, errInvalidExtension + } + ifi.Addr = make([]byte, l) + copy(ifi.Addr, b[4+4:]) + } + return ifi, nil +} diff --git a/vendor/golang.org/x/net/icmp/ipv4.go b/vendor/golang.org/x/net/icmp/ipv4.go new file mode 100644 index 0000000000..0ad40fef2f --- /dev/null +++ b/vendor/golang.org/x/net/icmp/ipv4.go @@ -0,0 +1,69 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "encoding/binary" + "net" + "runtime" + + "golang.org/x/net/internal/socket" + "golang.org/x/net/ipv4" +) + +// freebsdVersion is set in sys_freebsd.go. +// See http://www.freebsd.org/doc/en/books/porters-handbook/freebsd-versions.html. +var freebsdVersion uint32 + +// ParseIPv4Header returns the IPv4 header of the IPv4 packet that +// triggered an ICMP error message. +// This is found in the Data field of the ICMP error message body. +// +// The provided b must be in the format used by a raw ICMP socket on +// the local system. +// This may differ from the wire format, and the format used by a raw +// IP socket, depending on the system. +// +// To parse an IPv6 header, use ipv6.ParseHeader. +func ParseIPv4Header(b []byte) (*ipv4.Header, error) { + if len(b) < ipv4.HeaderLen { + return nil, errHeaderTooShort + } + hdrlen := int(b[0]&0x0f) << 2 + if hdrlen > len(b) { + return nil, errBufferTooShort + } + h := &ipv4.Header{ + Version: int(b[0] >> 4), + Len: hdrlen, + TOS: int(b[1]), + ID: int(binary.BigEndian.Uint16(b[4:6])), + FragOff: int(binary.BigEndian.Uint16(b[6:8])), + TTL: int(b[8]), + Protocol: int(b[9]), + Checksum: int(binary.BigEndian.Uint16(b[10:12])), + Src: net.IPv4(b[12], b[13], b[14], b[15]), + Dst: net.IPv4(b[16], b[17], b[18], b[19]), + } + switch runtime.GOOS { + case "darwin", "ios": + h.TotalLen = int(socket.NativeEndian.Uint16(b[2:4])) + case "freebsd": + if freebsdVersion >= 1000000 { + h.TotalLen = int(binary.BigEndian.Uint16(b[2:4])) + } else { + h.TotalLen = int(socket.NativeEndian.Uint16(b[2:4])) + } + default: + h.TotalLen = int(binary.BigEndian.Uint16(b[2:4])) + } + h.Flags = ipv4.HeaderFlags(h.FragOff&0xe000) >> 13 + h.FragOff = h.FragOff & 0x1fff + if hdrlen-ipv4.HeaderLen > 0 { + h.Options = make([]byte, hdrlen-ipv4.HeaderLen) + copy(h.Options, b[ipv4.HeaderLen:]) + } + return h, nil +} diff --git a/vendor/golang.org/x/net/icmp/ipv6.go b/vendor/golang.org/x/net/icmp/ipv6.go new file mode 100644 index 0000000000..2e8cfeb131 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/ipv6.go @@ -0,0 +1,23 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "net" + + "golang.org/x/net/internal/iana" +) + +const ipv6PseudoHeaderLen = 2*net.IPv6len + 8 + +// IPv6PseudoHeader returns an IPv6 pseudo header for checksum +// calculation. +func IPv6PseudoHeader(src, dst net.IP) []byte { + b := make([]byte, ipv6PseudoHeaderLen) + copy(b, src.To16()) + copy(b[net.IPv6len:], dst.To16()) + b[len(b)-1] = byte(iana.ProtocolIPv6ICMP) + return b +} diff --git a/vendor/golang.org/x/net/icmp/listen_posix.go b/vendor/golang.org/x/net/icmp/listen_posix.go new file mode 100644 index 0000000000..327abf843f --- /dev/null +++ b/vendor/golang.org/x/net/icmp/listen_posix.go @@ -0,0 +1,103 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris windows + +package icmp + +import ( + "net" + "os" + "runtime" + "syscall" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +const sysIP_STRIPHDR = 0x17 // for now only darwin supports this option + +// ListenPacket listens for incoming ICMP packets addressed to +// address. See net.Dial for the syntax of address. +// +// For non-privileged datagram-oriented ICMP endpoints, network must +// be "udp4" or "udp6". The endpoint allows to read, write a few +// limited ICMP messages such as echo request and echo reply. +// Currently only Darwin and Linux support this. +// +// Examples: +// ListenPacket("udp4", "192.168.0.1") +// ListenPacket("udp4", "0.0.0.0") +// ListenPacket("udp6", "fe80::1%en0") +// ListenPacket("udp6", "::") +// +// For privileged raw ICMP endpoints, network must be "ip4" or "ip6" +// followed by a colon and an ICMP protocol number or name. +// +// Examples: +// ListenPacket("ip4:icmp", "192.168.0.1") +// ListenPacket("ip4:1", "0.0.0.0") +// ListenPacket("ip6:ipv6-icmp", "fe80::1%en0") +// ListenPacket("ip6:58", "::") +func ListenPacket(network, address string) (*PacketConn, error) { + var family, proto int + switch network { + case "udp4": + family, proto = syscall.AF_INET, iana.ProtocolICMP + case "udp6": + family, proto = syscall.AF_INET6, iana.ProtocolIPv6ICMP + default: + i := last(network, ':') + if i < 0 { + i = len(network) + } + switch network[:i] { + case "ip4": + proto = iana.ProtocolICMP + case "ip6": + proto = iana.ProtocolIPv6ICMP + } + } + var cerr error + var c net.PacketConn + switch family { + case syscall.AF_INET, syscall.AF_INET6: + s, err := syscall.Socket(family, syscall.SOCK_DGRAM, proto) + if err != nil { + return nil, os.NewSyscallError("socket", err) + } + if (runtime.GOOS == "darwin" || runtime.GOOS == "ios") && family == syscall.AF_INET { + if err := syscall.SetsockoptInt(s, iana.ProtocolIP, sysIP_STRIPHDR, 1); err != nil { + syscall.Close(s) + return nil, os.NewSyscallError("setsockopt", err) + } + } + sa, err := sockaddr(family, address) + if err != nil { + syscall.Close(s) + return nil, err + } + if err := syscall.Bind(s, sa); err != nil { + syscall.Close(s) + return nil, os.NewSyscallError("bind", err) + } + f := os.NewFile(uintptr(s), "datagram-oriented icmp") + c, cerr = net.FilePacketConn(f) + f.Close() + default: + c, cerr = net.ListenPacket(network, address) + } + if cerr != nil { + return nil, cerr + } + switch proto { + case iana.ProtocolICMP: + return &PacketConn{c: c, p4: ipv4.NewPacketConn(c)}, nil + case iana.ProtocolIPv6ICMP: + return &PacketConn{c: c, p6: ipv6.NewPacketConn(c)}, nil + default: + return &PacketConn{c: c}, nil + } +} diff --git a/vendor/golang.org/x/net/icmp/listen_stub.go b/vendor/golang.org/x/net/icmp/listen_stub.go new file mode 100644 index 0000000000..3acd91dc5c --- /dev/null +++ b/vendor/golang.org/x/net/icmp/listen_stub.go @@ -0,0 +1,33 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !aix,!darwin,!dragonfly,!freebsd,!linux,!netbsd,!openbsd,!solaris,!windows + +package icmp + +// ListenPacket listens for incoming ICMP packets addressed to +// address. See net.Dial for the syntax of address. +// +// For non-privileged datagram-oriented ICMP endpoints, network must +// be "udp4" or "udp6". The endpoint allows to read, write a few +// limited ICMP messages such as echo request and echo reply. +// Currently only Darwin and Linux support this. +// +// Examples: +// ListenPacket("udp4", "192.168.0.1") +// ListenPacket("udp4", "0.0.0.0") +// ListenPacket("udp6", "fe80::1%en0") +// ListenPacket("udp6", "::") +// +// For privileged raw ICMP endpoints, network must be "ip4" or "ip6" +// followed by a colon and an ICMP protocol number or name. +// +// Examples: +// ListenPacket("ip4:icmp", "192.168.0.1") +// ListenPacket("ip4:1", "0.0.0.0") +// ListenPacket("ip6:ipv6-icmp", "fe80::1%en0") +// ListenPacket("ip6:58", "::") +func ListenPacket(network, address string) (*PacketConn, error) { + return nil, errNotImplemented +} diff --git a/vendor/golang.org/x/net/icmp/message.go b/vendor/golang.org/x/net/icmp/message.go new file mode 100644 index 0000000000..40db65d0cd --- /dev/null +++ b/vendor/golang.org/x/net/icmp/message.go @@ -0,0 +1,162 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package icmp provides basic functions for the manipulation of +// messages used in the Internet Control Message Protocols, +// ICMPv4 and ICMPv6. +// +// ICMPv4 and ICMPv6 are defined in RFC 792 and RFC 4443. +// Multi-part message support for ICMP is defined in RFC 4884. +// ICMP extensions for MPLS are defined in RFC 4950. +// ICMP extensions for interface and next-hop identification are +// defined in RFC 5837. +// PROBE: A utility for probing interfaces is defined in RFC 8335. +package icmp // import "golang.org/x/net/icmp" + +import ( + "encoding/binary" + "errors" + "net" + "runtime" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// BUG(mikio): This package is not implemented on JS, NaCl and Plan 9. + +var ( + errInvalidConn = errors.New("invalid connection") + errInvalidProtocol = errors.New("invalid protocol") + errMessageTooShort = errors.New("message too short") + errHeaderTooShort = errors.New("header too short") + errBufferTooShort = errors.New("buffer too short") + errInvalidBody = errors.New("invalid body") + errNoExtension = errors.New("no extension") + errInvalidExtension = errors.New("invalid extension") + errNotImplemented = errors.New("not implemented on " + runtime.GOOS + "/" + runtime.GOARCH) +) + +func checksum(b []byte) uint16 { + csumcv := len(b) - 1 // checksum coverage + s := uint32(0) + for i := 0; i < csumcv; i += 2 { + s += uint32(b[i+1])<<8 | uint32(b[i]) + } + if csumcv&1 == 0 { + s += uint32(b[csumcv]) + } + s = s>>16 + s&0xffff + s = s + s>>16 + return ^uint16(s) +} + +// A Type represents an ICMP message type. +type Type interface { + Protocol() int +} + +// A Message represents an ICMP message. +type Message struct { + Type Type // type, either ipv4.ICMPType or ipv6.ICMPType + Code int // code + Checksum int // checksum + Body MessageBody // body +} + +// Marshal returns the binary encoding of the ICMP message m. +// +// For an ICMPv4 message, the returned message always contains the +// calculated checksum field. +// +// For an ICMPv6 message, the returned message contains the calculated +// checksum field when psh is not nil, otherwise the kernel will +// compute the checksum field during the message transmission. +// When psh is not nil, it must be the pseudo header for IPv6. +func (m *Message) Marshal(psh []byte) ([]byte, error) { + var mtype byte + switch typ := m.Type.(type) { + case ipv4.ICMPType: + mtype = byte(typ) + case ipv6.ICMPType: + mtype = byte(typ) + default: + return nil, errInvalidProtocol + } + b := []byte{mtype, byte(m.Code), 0, 0} + proto := m.Type.Protocol() + if proto == iana.ProtocolIPv6ICMP && psh != nil { + b = append(psh, b...) + } + if m.Body != nil && m.Body.Len(proto) != 0 { + mb, err := m.Body.Marshal(proto) + if err != nil { + return nil, err + } + b = append(b, mb...) + } + if proto == iana.ProtocolIPv6ICMP { + if psh == nil { // cannot calculate checksum here + return b, nil + } + off, l := 2*net.IPv6len, len(b)-len(psh) + binary.BigEndian.PutUint32(b[off:off+4], uint32(l)) + } + s := checksum(b) + // Place checksum back in header; using ^= avoids the + // assumption the checksum bytes are zero. + b[len(psh)+2] ^= byte(s) + b[len(psh)+3] ^= byte(s >> 8) + return b[len(psh):], nil +} + +var parseFns = map[Type]func(int, Type, []byte) (MessageBody, error){ + ipv4.ICMPTypeDestinationUnreachable: parseDstUnreach, + ipv4.ICMPTypeTimeExceeded: parseTimeExceeded, + ipv4.ICMPTypeParameterProblem: parseParamProb, + + ipv4.ICMPTypeEcho: parseEcho, + ipv4.ICMPTypeEchoReply: parseEcho, + ipv4.ICMPTypeExtendedEchoRequest: parseExtendedEchoRequest, + ipv4.ICMPTypeExtendedEchoReply: parseExtendedEchoReply, + + ipv6.ICMPTypeDestinationUnreachable: parseDstUnreach, + ipv6.ICMPTypePacketTooBig: parsePacketTooBig, + ipv6.ICMPTypeTimeExceeded: parseTimeExceeded, + ipv6.ICMPTypeParameterProblem: parseParamProb, + + ipv6.ICMPTypeEchoRequest: parseEcho, + ipv6.ICMPTypeEchoReply: parseEcho, + ipv6.ICMPTypeExtendedEchoRequest: parseExtendedEchoRequest, + ipv6.ICMPTypeExtendedEchoReply: parseExtendedEchoReply, +} + +// ParseMessage parses b as an ICMP message. +// The provided proto must be either the ICMPv4 or ICMPv6 protocol +// number. +func ParseMessage(proto int, b []byte) (*Message, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + var err error + m := &Message{Code: int(b[1]), Checksum: int(binary.BigEndian.Uint16(b[2:4]))} + switch proto { + case iana.ProtocolICMP: + m.Type = ipv4.ICMPType(b[0]) + case iana.ProtocolIPv6ICMP: + m.Type = ipv6.ICMPType(b[0]) + default: + return nil, errInvalidProtocol + } + if fn, ok := parseFns[m.Type]; !ok { + m.Body, err = parseRawBody(proto, b[4:]) + } else { + m.Body, err = fn(proto, m.Type, b[4:]) + } + if err != nil { + return nil, err + } + return m, nil +} diff --git a/vendor/golang.org/x/net/icmp/messagebody.go b/vendor/golang.org/x/net/icmp/messagebody.go new file mode 100644 index 0000000000..e2d9bfa01b --- /dev/null +++ b/vendor/golang.org/x/net/icmp/messagebody.go @@ -0,0 +1,52 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +// A MessageBody represents an ICMP message body. +type MessageBody interface { + // Len returns the length of ICMP message body. + // The provided proto must be either the ICMPv4 or ICMPv6 + // protocol number. + Len(proto int) int + + // Marshal returns the binary encoding of ICMP message body. + // The provided proto must be either the ICMPv4 or ICMPv6 + // protocol number. + Marshal(proto int) ([]byte, error) +} + +// A RawBody represents a raw message body. +// +// A raw message body is excluded from message processing and can be +// used to construct applications such as protocol conformance +// testing. +type RawBody struct { + Data []byte // data +} + +// Len implements the Len method of MessageBody interface. +func (p *RawBody) Len(proto int) int { + if p == nil { + return 0 + } + return len(p.Data) +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *RawBody) Marshal(proto int) ([]byte, error) { + return p.Data, nil +} + +// parseRawBody parses b as an ICMP message body. +func parseRawBody(proto int, b []byte) (MessageBody, error) { + p := &RawBody{Data: make([]byte, len(b))} + copy(p.Data, b) + return p, nil +} + +// A DefaultMessageBody represents the default message body. +// +// Deprecated: Use RawBody instead. +type DefaultMessageBody = RawBody diff --git a/vendor/golang.org/x/net/icmp/mpls.go b/vendor/golang.org/x/net/icmp/mpls.go new file mode 100644 index 0000000000..f9f4841bce --- /dev/null +++ b/vendor/golang.org/x/net/icmp/mpls.go @@ -0,0 +1,77 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import "encoding/binary" + +// MPLSLabel represents an MPLS label stack entry. +type MPLSLabel struct { + Label int // label value + TC int // traffic class; formerly experimental use + S bool // bottom of stack + TTL int // time to live +} + +const ( + classMPLSLabelStack = 1 + typeIncomingMPLSLabelStack = 1 +) + +// MPLSLabelStack represents an MPLS label stack. +type MPLSLabelStack struct { + Class int // extension object class number + Type int // extension object sub-type + Labels []MPLSLabel +} + +// Len implements the Len method of Extension interface. +func (ls *MPLSLabelStack) Len(proto int) int { + return 4 + (4 * len(ls.Labels)) +} + +// Marshal implements the Marshal method of Extension interface. +func (ls *MPLSLabelStack) Marshal(proto int) ([]byte, error) { + b := make([]byte, ls.Len(proto)) + if err := ls.marshal(proto, b); err != nil { + return nil, err + } + return b, nil +} + +func (ls *MPLSLabelStack) marshal(proto int, b []byte) error { + l := ls.Len(proto) + binary.BigEndian.PutUint16(b[:2], uint16(l)) + b[2], b[3] = classMPLSLabelStack, typeIncomingMPLSLabelStack + off := 4 + for _, ll := range ls.Labels { + b[off], b[off+1], b[off+2] = byte(ll.Label>>12), byte(ll.Label>>4&0xff), byte(ll.Label<<4&0xf0) + b[off+2] |= byte(ll.TC << 1 & 0x0e) + if ll.S { + b[off+2] |= 0x1 + } + b[off+3] = byte(ll.TTL) + off += 4 + } + return nil +} + +func parseMPLSLabelStack(b []byte) (Extension, error) { + ls := &MPLSLabelStack{ + Class: int(b[2]), + Type: int(b[3]), + } + for b = b[4:]; len(b) >= 4; b = b[4:] { + ll := MPLSLabel{ + Label: int(b[0])<<12 | int(b[1])<<4 | int(b[2])>>4, + TC: int(b[2]&0x0e) >> 1, + TTL: int(b[3]), + } + if b[2]&0x1 != 0 { + ll.S = true + } + ls.Labels = append(ls.Labels, ll) + } + return ls, nil +} diff --git a/vendor/golang.org/x/net/icmp/multipart.go b/vendor/golang.org/x/net/icmp/multipart.go new file mode 100644 index 0000000000..5f36675594 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/multipart.go @@ -0,0 +1,129 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import "golang.org/x/net/internal/iana" + +// multipartMessageBodyDataLen takes b as an original datagram and +// exts as extensions, and returns a required length for message body +// and a required length for a padded original datagram in wire +// format. +func multipartMessageBodyDataLen(proto int, withOrigDgram bool, b []byte, exts []Extension) (bodyLen, dataLen int) { + bodyLen = 4 // length of leading octets + var extLen int + var rawExt bool // raw extension may contain an empty object + for _, ext := range exts { + extLen += ext.Len(proto) + if _, ok := ext.(*RawExtension); ok { + rawExt = true + } + } + if extLen > 0 && withOrigDgram { + dataLen = multipartMessageOrigDatagramLen(proto, b) + } else { + dataLen = len(b) + } + if extLen > 0 || rawExt { + bodyLen += 4 // length of extension header + } + bodyLen += dataLen + extLen + return bodyLen, dataLen +} + +// multipartMessageOrigDatagramLen takes b as an original datagram, +// and returns a required length for a padded orignal datagram in wire +// format. +func multipartMessageOrigDatagramLen(proto int, b []byte) int { + roundup := func(b []byte, align int) int { + // According to RFC 4884, the padded original datagram + // field must contain at least 128 octets. + if len(b) < 128 { + return 128 + } + r := len(b) + return (r + align - 1) &^ (align - 1) + } + switch proto { + case iana.ProtocolICMP: + return roundup(b, 4) + case iana.ProtocolIPv6ICMP: + return roundup(b, 8) + default: + return len(b) + } +} + +// marshalMultipartMessageBody takes data as an original datagram and +// exts as extesnsions, and returns a binary encoding of message body. +// It can be used for non-multipart message bodies when exts is nil. +func marshalMultipartMessageBody(proto int, withOrigDgram bool, data []byte, exts []Extension) ([]byte, error) { + bodyLen, dataLen := multipartMessageBodyDataLen(proto, withOrigDgram, data, exts) + b := make([]byte, bodyLen) + copy(b[4:], data) + if len(exts) > 0 { + b[4+dataLen] = byte(extensionVersion << 4) + off := 4 + dataLen + 4 // leading octets, data, extension header + for _, ext := range exts { + switch ext := ext.(type) { + case *MPLSLabelStack: + if err := ext.marshal(proto, b[off:]); err != nil { + return nil, err + } + off += ext.Len(proto) + case *InterfaceInfo: + attrs, l := ext.attrsAndLen(proto) + if err := ext.marshal(proto, b[off:], attrs, l); err != nil { + return nil, err + } + off += ext.Len(proto) + case *InterfaceIdent: + if err := ext.marshal(proto, b[off:]); err != nil { + return nil, err + } + off += ext.Len(proto) + case *RawExtension: + copy(b[off:], ext.Data) + off += ext.Len(proto) + } + } + s := checksum(b[4+dataLen:]) + b[4+dataLen+2] ^= byte(s) + b[4+dataLen+3] ^= byte(s >> 8) + if withOrigDgram { + switch proto { + case iana.ProtocolICMP: + b[1] = byte(dataLen / 4) + case iana.ProtocolIPv6ICMP: + b[0] = byte(dataLen / 8) + } + } + } + return b, nil +} + +// parseMultipartMessageBody parses b as either a non-multipart +// message body or a multipart message body. +func parseMultipartMessageBody(proto int, typ Type, b []byte) ([]byte, []Extension, error) { + var l int + switch proto { + case iana.ProtocolICMP: + l = 4 * int(b[1]) + case iana.ProtocolIPv6ICMP: + l = 8 * int(b[0]) + } + if len(b) == 4 { + return nil, nil, nil + } + exts, l, err := parseExtensions(typ, b[4:], l) + if err != nil { + l = len(b) - 4 + } + var data []byte + if l > 0 { + data = make([]byte, l) + copy(data, b[4:]) + } + return data, exts, nil +} diff --git a/vendor/golang.org/x/net/icmp/packettoobig.go b/vendor/golang.org/x/net/icmp/packettoobig.go new file mode 100644 index 0000000000..afbf24f1ba --- /dev/null +++ b/vendor/golang.org/x/net/icmp/packettoobig.go @@ -0,0 +1,43 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import "encoding/binary" + +// A PacketTooBig represents an ICMP packet too big message body. +type PacketTooBig struct { + MTU int // maximum transmission unit of the nexthop link + Data []byte // data, known as original datagram field +} + +// Len implements the Len method of MessageBody interface. +func (p *PacketTooBig) Len(proto int) int { + if p == nil { + return 0 + } + return 4 + len(p.Data) +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *PacketTooBig) Marshal(proto int) ([]byte, error) { + b := make([]byte, 4+len(p.Data)) + binary.BigEndian.PutUint32(b[:4], uint32(p.MTU)) + copy(b[4:], p.Data) + return b, nil +} + +// parsePacketTooBig parses b as an ICMP packet too big message body. +func parsePacketTooBig(proto int, _ Type, b []byte) (MessageBody, error) { + bodyLen := len(b) + if bodyLen < 4 { + return nil, errMessageTooShort + } + p := &PacketTooBig{MTU: int(binary.BigEndian.Uint32(b[:4]))} + if bodyLen > 4 { + p.Data = make([]byte, bodyLen-4) + copy(p.Data, b[4:]) + } + return p, nil +} diff --git a/vendor/golang.org/x/net/icmp/paramprob.go b/vendor/golang.org/x/net/icmp/paramprob.go new file mode 100644 index 0000000000..f16fd33ec2 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/paramprob.go @@ -0,0 +1,72 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "encoding/binary" + + "golang.org/x/net/internal/iana" + "golang.org/x/net/ipv4" +) + +// A ParamProb represents an ICMP parameter problem message body. +type ParamProb struct { + Pointer uintptr // offset within the data where the error was detected + Data []byte // data, known as original datagram field + Extensions []Extension // extensions +} + +// Len implements the Len method of MessageBody interface. +func (p *ParamProb) Len(proto int) int { + if p == nil { + return 0 + } + l, _ := multipartMessageBodyDataLen(proto, true, p.Data, p.Extensions) + return l +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *ParamProb) Marshal(proto int) ([]byte, error) { + switch proto { + case iana.ProtocolICMP: + if !validExtensions(ipv4.ICMPTypeParameterProblem, p.Extensions) { + return nil, errInvalidExtension + } + b, err := marshalMultipartMessageBody(proto, true, p.Data, p.Extensions) + if err != nil { + return nil, err + } + b[0] = byte(p.Pointer) + return b, nil + case iana.ProtocolIPv6ICMP: + b := make([]byte, p.Len(proto)) + binary.BigEndian.PutUint32(b[:4], uint32(p.Pointer)) + copy(b[4:], p.Data) + return b, nil + default: + return nil, errInvalidProtocol + } +} + +// parseParamProb parses b as an ICMP parameter problem message body. +func parseParamProb(proto int, typ Type, b []byte) (MessageBody, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + p := &ParamProb{} + if proto == iana.ProtocolIPv6ICMP { + p.Pointer = uintptr(binary.BigEndian.Uint32(b[:4])) + p.Data = make([]byte, len(b)-4) + copy(p.Data, b[4:]) + return p, nil + } + p.Pointer = uintptr(b[0]) + var err error + p.Data, p.Extensions, err = parseMultipartMessageBody(proto, typ, b) + if err != nil { + return nil, err + } + return p, nil +} diff --git a/vendor/golang.org/x/net/icmp/sys_freebsd.go b/vendor/golang.org/x/net/icmp/sys_freebsd.go new file mode 100644 index 0000000000..c75f3ddaa7 --- /dev/null +++ b/vendor/golang.org/x/net/icmp/sys_freebsd.go @@ -0,0 +1,11 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import "syscall" + +func init() { + freebsdVersion, _ = syscall.SysctlUint32("kern.osreldate") +} diff --git a/vendor/golang.org/x/net/icmp/timeexceeded.go b/vendor/golang.org/x/net/icmp/timeexceeded.go new file mode 100644 index 0000000000..ffa986fdea --- /dev/null +++ b/vendor/golang.org/x/net/icmp/timeexceeded.go @@ -0,0 +1,57 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package icmp + +import ( + "golang.org/x/net/internal/iana" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// A TimeExceeded represents an ICMP time exceeded message body. +type TimeExceeded struct { + Data []byte // data, known as original datagram field + Extensions []Extension // extensions +} + +// Len implements the Len method of MessageBody interface. +func (p *TimeExceeded) Len(proto int) int { + if p == nil { + return 0 + } + l, _ := multipartMessageBodyDataLen(proto, true, p.Data, p.Extensions) + return l +} + +// Marshal implements the Marshal method of MessageBody interface. +func (p *TimeExceeded) Marshal(proto int) ([]byte, error) { + var typ Type + switch proto { + case iana.ProtocolICMP: + typ = ipv4.ICMPTypeTimeExceeded + case iana.ProtocolIPv6ICMP: + typ = ipv6.ICMPTypeTimeExceeded + default: + return nil, errInvalidProtocol + } + if !validExtensions(typ, p.Extensions) { + return nil, errInvalidExtension + } + return marshalMultipartMessageBody(proto, true, p.Data, p.Extensions) +} + +// parseTimeExceeded parses b as an ICMP time exceeded message body. +func parseTimeExceeded(proto int, typ Type, b []byte) (MessageBody, error) { + if len(b) < 4 { + return nil, errMessageTooShort + } + p := &TimeExceeded{} + var err error + p.Data, p.Extensions, err = parseMultipartMessageBody(proto, typ, b) + if err != nil { + return nil, err + } + return p, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 776e685bb1..198be92c94 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -604,6 +604,8 @@ github.com/stretchr/testify/assert github.com/stretchr/testify/mock github.com/stretchr/testify/require github.com/stretchr/testify/suite +# github.com/tatsushid/go-fastping v0.0.0-20160109021039-d7bb493dee3e +github.com/tatsushid/go-fastping # github.com/tencentcloud/tencentcloud-sdk-go v3.0.135+incompatible github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common/errors @@ -737,6 +739,7 @@ golang.org/x/net/http/httpguts golang.org/x/net/http/httpproxy golang.org/x/net/http2 golang.org/x/net/http2/hpack +golang.org/x/net/icmp golang.org/x/net/idna golang.org/x/net/internal/iana golang.org/x/net/internal/socket @@ -1134,7 +1137,7 @@ sigs.k8s.io/yaml yunion.io/x/executor/apis yunion.io/x/executor/client yunion.io/x/executor/server -# yunion.io/x/jsonutils v0.0.0-20201110084044-3e4e1cb49769 +# yunion.io/x/jsonutils v0.0.0-20210709075951-798a67800349 yunion.io/x/jsonutils # yunion.io/x/log v0.0.0-20201210064738-43181789dc74 yunion.io/x/log @@ -1177,7 +1180,7 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20210401131841-cdb9ab689824 +# yunion.io/x/sqlchemy v0.0.0-20210619142628-653684d2c4f8 yunion.io/x/sqlchemy # yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce yunion.io/x/structarg diff --git a/vendor/yunion.io/x/jsonutils/jsonutils.go b/vendor/yunion.io/x/jsonutils/jsonutils.go index b610037cc3..cc5f1977c8 100644 --- a/vendor/yunion.io/x/jsonutils/jsonutils.go +++ b/vendor/yunion.io/x/jsonutils/jsonutils.go @@ -239,7 +239,7 @@ func parseString(str []byte, offset int) (string, bool, int, error) { ret2: for i < len(str) { switch str[i] { - case ' ', ':', ',', '\t', '\n', '}', ']': + case ' ', ':', ',', '\t', '\r', '\n', '}', ']': break ret2 default: i++ diff --git a/vendor/yunion.io/x/sqlchemy/parser.go b/vendor/yunion.io/x/sqlchemy/parser.go index 0f37d7967a..3e069a9cac 100644 --- a/vendor/yunion.io/x/sqlchemy/parser.go +++ b/vendor/yunion.io/x/sqlchemy/parser.go @@ -35,7 +35,7 @@ func structField2ColumnSpec(field *reflectutils.SStructFieldValue) IColumnSpec { retCol = getFiledTypeCol(fieldType.Elem(), fieldname, tagmap, true) } if retCol == nil { - panic("unsupported colume data type %s" + fieldType.Name()) + panic(fmt.Sprintf("unsupported colume %s data type %s", fieldname, fieldType.Name())) } return retCol } diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 193199f7d7..9eb6bca34c 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -196,9 +196,9 @@ func (tbl *STable) Fields() []IQueryField { func (c *STableField) Expression() string { if len(c.alias) > 0 { - return fmt.Sprintf("%s.%s as `%s`", c.table.Alias(), c.spec.Name(), c.alias) + return fmt.Sprintf("`%s`.`%s` as `%s`", c.table.Alias(), c.spec.Name(), c.alias) } else { - return fmt.Sprintf("%s.%s", c.table.Alias(), c.spec.Name()) + return fmt.Sprintf("`%s`.`%s`", c.table.Alias(), c.spec.Name()) } } diff --git a/vendor/yunion.io/x/sqlchemy/union.go b/vendor/yunion.io/x/sqlchemy/union.go index b755ba8cd6..7af8c7174a 100644 --- a/vendor/yunion.io/x/sqlchemy/union.go +++ b/vendor/yunion.io/x/sqlchemy/union.go @@ -30,7 +30,11 @@ type SUnionQueryField struct { } func (sqf *SUnionQueryField) Expression() string { - return sqf.name + if len(sqf.alias) > 0 { + return fmt.Sprintf("`%s`.`%s` as `%s`", sqf.union.Alias(), sqf.name, sqf.alias) + } else { + return fmt.Sprintf("`%s`.`%s`", sqf.union.Alias(), sqf.name) + } } func (sqf *SUnionQueryField) Name() string { @@ -42,7 +46,7 @@ func (sqf *SUnionQueryField) Name() string { } func (sqf *SUnionQueryField) Reference() string { - return fmt.Sprintf("`%s`.`%s`", sqf.union.Alias(), sqf.name) + return fmt.Sprintf("`%s`.`%s`", sqf.union.Alias(), sqf.Name()) } func (sqf *SUnionQueryField) Label(label string) IQueryField {