From 2943344b48ae5bf067e9632f5be193a8d634bd30 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Mon, 19 May 2025 22:28:20 +0800 Subject: [PATCH] feature: monitor bucket formance and resource status (#22552) Co-authored-by: Qiu Jian --- cmd/climc/shell/compute/buckets.go | 529 +++--------------- go.mod | 6 +- go.sum | 12 +- pkg/apigateway/handler/init.go | 25 +- pkg/apis/compute/bucket.go | 67 ++- pkg/cloudmon/misc/bucketprobe.go | 177 ++++++ pkg/cloudmon/misc/pinger.go | 8 +- pkg/cloudmon/misc/send.go | 40 ++ pkg/cloudmon/misc/statusprobe.go | 137 +++++ pkg/cloudmon/options/options.go | 11 + pkg/cloudmon/service/service.go | 5 + pkg/compute/models/buckets.go | 2 + pkg/mcclient/modules/compute/mod_buckets.go | 68 +++ pkg/mcclient/modules/loader/doc.go | 15 + pkg/mcclient/modules/loader/loader.go | 44 ++ pkg/mcclient/options/compute/buckets.go | 444 +++++++++++++++ pkg/monitor/dbinit/measurements/all.go | 2 + pkg/monitor/dbinit/measurements/oss.go | 38 ++ pkg/monitor/dbinit/measurements/service.go | 16 + vendor/modules.txt | 6 +- .../cloudmux/pkg/cloudprovider/objectstore.go | 5 + vendor/yunion.io/x/executor/server/server.go | 15 + .../x/pkg/util/fileutils/filesize.go | 4 + .../x/pkg/util/streamutils/streamutils.go | 35 +- 24 files changed, 1179 insertions(+), 532 deletions(-) create mode 100644 pkg/cloudmon/misc/bucketprobe.go create mode 100644 pkg/cloudmon/misc/send.go create mode 100644 pkg/cloudmon/misc/statusprobe.go create mode 100644 pkg/mcclient/modules/loader/doc.go create mode 100644 pkg/mcclient/modules/loader/loader.go create mode 100644 pkg/mcclient/options/compute/buckets.go diff --git a/cmd/climc/shell/compute/buckets.go b/cmd/climc/shell/compute/buckets.go index 9056e19c95..8162019fc2 100644 --- a/cmd/climc/shell/compute/buckets.go +++ b/cmd/climc/shell/compute/buckets.go @@ -19,178 +19,71 @@ import ( "io" "os" - "yunion.io/x/cloudmux/pkg/multicloud/objectstore" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/fileutils" "yunion.io/x/pkg/util/printutils" - api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/cmd/climc/shell" + "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/mcclient" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" - "yunion.io/x/onecloud/pkg/mcclient/options" + computeoptions "yunion.io/x/onecloud/pkg/mcclient/options/compute" ) func init() { - type BucketListOptions struct { - options.BaseListOptions - DistinctField string `help:"query specified distinct field"` - } - R(&BucketListOptions{}, "bucket-list", "List all buckets", func(s *mcclient.ClientSession, args *BucketListOptions) error { - params, err := options.ListStructToParams(args) - if err != nil { - return err - } - if len(args.DistinctField) > 0 { - params.Add(jsonutils.NewString(args.DistinctField), "extra_field") - result, err := modules.Buckets.Get(s, "distinct-field", params) - if err != nil { - return err - } - fmt.Println(result) - return nil - } - result, err := modules.Buckets.List(s, params) - if err != nil { - return err - } - printList(result, modules.Buckets.GetColumns(s)) - return nil - }) - - type BucketIdOptions struct { - ID string `help:"ID or name of bucket"` - } - R(&BucketIdOptions{}, "bucket-show", "Id details of bucket", func(s *mcclient.ClientSession, args *BucketIdOptions) error { - result, err := modules.Buckets.Get(s, args.ID, nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - R(&BucketIdOptions{}, "bucket-syncstatus", "Sync bucket statust", func(s *mcclient.ClientSession, args *BucketIdOptions) error { - result, err := modules.Buckets.PerformAction(s, args.ID, "syncstatus", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketUpdateOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - Name string `help:"new name of bucket" json:"name"` - Desc string `help:"Description of bucket" json:"description" token:"desc"` - } - R(&BucketUpdateOptions{}, "bucket-update", "update bucket", func(s *mcclient.ClientSession, args *BucketUpdateOptions) error { - params := jsonutils.Marshal(args) - result, err := modules.Buckets.Update(s, args.ID, params) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketDeleteOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketDeleteOptions{}, "bucket-delete", "delete bucket", func(s *mcclient.ClientSession, args *BucketDeleteOptions) error { - result, err := modules.Buckets.Delete(s, args.ID, nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketCreateOptions struct { - NAME string `help:"name of bucket" json:"name"` - CLOUDREGION string `help:"location of bucket" json:"cloudregion"` - MANAGER string `help:"cloud provider" json:"manager"` - - StorageClass string `help:"bucket storage class"` - Acl string `help:"bucket ACL"` - } - R(&BucketCreateOptions{}, "bucket-create", "Create a bucket", func(s *mcclient.ClientSession, args *BucketCreateOptions) error { - params, err := options.StructToParams(args) - if err != nil { - return err - } - result, err := modules.Buckets.Create(s, params) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketListObjectsOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - Prefix string `help:"List objects with prefix"` - Recursive bool `help:"List objects recursively"` - Limit int `help:"maximal items per request"` - PagingMarker string `help:"paging marker"` - } - R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error { - params, err := options.StructToParams(args) - if err != nil { - return err - } - result, err := modules.Buckets.GetSpecific(s, args.ID, "objects", params) - if err != nil { - return err - } - + cmd := shell.NewResourceCmd(&modules.Buckets) + cmd.List(&computeoptions.BucketListOptions{}) + cmd.GetProperty(&computeoptions.BucketGetPropertyOptions{}) + cmd.Show(&computeoptions.BucketIdOptions{}) + cmd.Perform("syncstatus", &computeoptions.BucketIdOptions{}) + cmd.Update(&computeoptions.BucketUpdateOptions{}) + cmd.Delete(&computeoptions.BucketIdOptions{}) + cmd.Create(&computeoptions.BucketCreateOptions{}) + cmd.GetWithCustomOptionShow("objects", func(data jsonutils.JSONObject, args shell.IGetOpt) { listResult := printutils.ListResult{} - err = result.Unmarshal(&listResult) + err := data.Unmarshal(&listResult) if err != nil { - return err + return } printList(&listResult, []string{}) - return nil - }) + }, &computeoptions.BucketListObjectsOptions{}) + cmd.Perform("delete", &computeoptions.BucketDeleteObjectsOptions{}) + cmd.Perform("makedir", &computeoptions.BucketMakeDirOptions{}) + cmd.Perform("temp-url", &computeoptions.BucketPresignObjectsOptions{}) + cmd.Perform("acl", &computeoptions.BucketSetAclOptions{}) + cmd.GetWithCustomOptionShow("acl", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketAclOptions{}) + cmd.Perform("sync", &computeoptions.BucketSyncOptions{}) + cmd.Perform("limit", &computeoptions.BucketLimitOptions{}) + cmd.GetWithCustomOptionShow("access-info", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketAccessInfoOptions{}) + cmd.Perform("metadata", &computeoptions.BucketSetMetadataOptions{}) + cmd.Perform("set-website", &computeoptions.BucketSetWebsiteOption{}) + cmd.GetWithCustomOptionShow("website", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketGetWebsiteConfOption{}) + cmd.Perform("delete-website", &computeoptions.BucketDeleteWebsiteConfOption{}) + cmd.Perform("set-cors", &computeoptions.BucketSetCorsOption{}) + cmd.GetWithCustomOptionShow("cors", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketGetCorsOption{}) + cmd.Perform("delete-cors", &computeoptions.BucketDeleteCorsOption{}) + cmd.Perform("set-referer", &computeoptions.BucketSetRefererOption{}) + cmd.GetWithCustomOptionShow("referer", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketGetRefererOption{}) + cmd.GetWithCustomOptionShow("cdn-domain", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketGetCdnDomainOption{}) + cmd.GetWithCustomOptionShow("policy", func(data jsonutils.JSONObject, args shell.IGetOpt) { + printObject(data) + }, &computeoptions.BucketGetPolicyOption{}) + cmd.Perform("set-policy", &computeoptions.BucketSetPolicyOption{}) + cmd.Perform("delete-policy", &computeoptions.BucketDeletePolicyOption{}) - type BucketDeleteObjectsOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - KEYS []string `help:"List of objects to delete"` - } - R(&BucketDeleteObjectsOptions{}, "bucket-object-delete", "Delete objects in a bucket", func(s *mcclient.ClientSession, args *BucketDeleteObjectsOptions) error { - params := jsonutils.Marshal(args) - result, err := modules.Buckets.PerformAction(s, args.ID, "delete", params) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketMakeDirOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - KEY string `help:"DIR key to create"` - } - R(&BucketMakeDirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(s *mcclient.ClientSession, args *BucketMakeDirOptions) error { - params := jsonutils.Marshal(args) - result, err := modules.Buckets.PerformAction(s, args.ID, "makedir", params) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketUploadObjectsOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - KEY string `help:"Key of object to upload"` - Path string `help:"Path to file to upload" required:"true"` - - ContentLength int64 `help:"Content lenght (bytes)" default:"-1"` - StorageClass string `help:"storage CLass"` - Acl string `help:"object acl." choices:"private|public-read|public-read-write"` - - objectstore.ObjectHeaderOptions - } - R(&BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *BucketUploadObjectsOptions) error { + R(&computeoptions.BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *computeoptions.BucketUploadObjectsOptions) error { var body io.Reader if len(args.Path) > 0 { file, err := os.Open(args.Path) @@ -223,332 +116,36 @@ func init() { return nil }) - type BucketPresignObjectsOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - KEY string `help:"Key of object to upload"` - Method string `help:"Request method" choices:"GET|PUT|DELETE"` - ExpireSeconds int `help:"expire in seconds" default:"60"` - } - R(&BucketPresignObjectsOptions{}, "bucket-object-tempurl", "Get temporal URL for an object in a bucket", func(s *mcclient.ClientSession, args *BucketPresignObjectsOptions) error { - params, err := options.StructToParams(args) + R(&computeoptions.BucketPerfMonOptions{}, "bucket-perf-mon", "Bucket performance monitor", func(s *mcclient.ClientSession, args *computeoptions.BucketPerfMonOptions) error { + result, err := modules.Buckets.Get(s, args.ID, nil) if err != nil { return err } - result, err := modules.Buckets.PerformAction(s, args.ID, "temp-url", params) + bucketDetails := compute.BucketDetails{} + err = result.Unmarshal(&bucketDetails) if err != nil { return err } - printObject(result) - return nil - }) - type BucketSetAclOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - ACL string `help:"ACL to set" choices:"default|private|public-read|public-read-write" json:"acl"` - Key []string `help:"Optional object key" json:"key"` - } - R(&BucketSetAclOptions{}, "bucket-set-acl", "Set ACL of bucket or object", func(s *mcclient.ClientSession, args *BucketSetAclOptions) error { - params := jsonutils.Marshal(args) - result, err := modules.Buckets.PerformAction(s, args.ID, "acl", params) + bucket, err := modules.GetIBucket(s.GetContext(), s, &bucketDetails) if err != nil { return err } - printObject(result) - return nil - }) - type BucketAclOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - Key string `help:"Optional object key"` - } - R(&BucketAclOptions{}, "bucket-acl", "Get ACL of bucket or object", func(s *mcclient.ClientSession, args *BucketAclOptions) error { - params, err := options.StructToParams(args) + payload, err := fileutils.GetSizeBytes(args.Payload, 1024) if err != nil { return err } - result, err := modules.Buckets.GetSpecific(s, args.ID, "acl", params) + + stats, err := modules.ProbeBucketStats(s.GetContext(), bucket, "test", int64(payload)) if err != nil { return err } - printObject(result) - return nil - }) - type BucketSyncOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - StatsOnly bool `help:"sync statistics only"` - } - R(&BucketSyncOptions{}, "bucket-sync", "Sync bucket", func(s *mcclient.ClientSession, args *BucketSyncOptions) error { - params, err := options.StructToParams(args) - if err != nil { - return err - } - result, err := modules.Buckets.PerformAction(s, args.ID, "sync", params) - if err != nil { - return err - } - printObject(result) - return nil - }) + fmt.Printf("Upload delay %f ms throughput %f MB/s\n", stats.UploadDelayMs(), stats.UploadThroughputMbps(payload/1024/1024)) + fmt.Printf("Download delay %f ms throughput %f MB/s\n", stats.DownloadDelayMs(), stats.DownloadThroughputMbps(payload/1024/1024)) + fmt.Printf("Delete delay %f ms\n", stats.DeleteDelayMs()) - type BucketLimitOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - SizeBytes int64 `help:"size limit in bytes"` - ObjectCount int64 `help:"object count limit"` - } - R(&BucketLimitOptions{}, "bucket-limit", "Set limit of bucket", func(s *mcclient.ClientSession, args *BucketLimitOptions) error { - limit := jsonutils.Marshal(args) - params := jsonutils.NewDict() - params.Set("limit", limit) - result, err := modules.Buckets.PerformAction(s, args.ID, "limit", params) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketAccessInfoOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketAccessInfoOptions{}, "bucket-access-info", "Show backend access info of a bucket", func(s *mcclient.ClientSession, args *BucketAccessInfoOptions) error { - result, err := modules.Buckets.GetSpecific(s, args.ID, "access-info", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketSetMetadataOptions struct { - ID string `help:"ID or name of bucket" json:"-"` - - Key []string `help:"Optional object key" json:"key"` - - objectstore.ObjectHeaderOptions - } - R(&BucketSetMetadataOptions{}, "bucket-set-metadata", "Set metadata of object", func(s *mcclient.ClientSession, args *BucketSetMetadataOptions) error { - input := api.BucketMetadataInput{} - input.Key = args.Key - input.Metadata = args.ObjectHeaderOptions.Options2Header() - err := input.Validate() - if err != nil { - return err - } - result, err := modules.Buckets.PerformAction(s, args.ID, "metadata", jsonutils.Marshal(input)) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketSetWebsiteOption struct { - ID string `help:"ID or name of bucket" json:"-"` - // 主页 - Index string `help:"main page"` - // 错误时返回的文档 - ErrorDocument string `help:"error return"` - // http或https - Protocol string `help:"force https" choices:"http|https"` - } - R(&BucketSetWebsiteOption{}, "bucket-set-website", "Set bucket website", func(s *mcclient.ClientSession, args *BucketSetWebsiteOption) error { - conf := api.BucketWebsiteConf{ - Index: args.Index, - ErrorDocument: args.ErrorDocument, - Protocol: args.Protocol, - } - result, err := modules.Buckets.PerformAction(s, args.ID, "set-website", jsonutils.Marshal(conf)) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketGetWebsiteConfOption struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketGetWebsiteConfOption{}, "bucket-get-website", "Get bucket website", func(s *mcclient.ClientSession, args *BucketGetWebsiteConfOption) error { - result, err := modules.Buckets.GetSpecific(s, args.ID, "website", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketDeleteWebsiteConfOption struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketDeleteWebsiteConfOption{}, "bucket-delete-website", "Delete bucket website", func(s *mcclient.ClientSession, args *BucketDeleteWebsiteConfOption) error { - result, err := modules.Buckets.PerformAction(s, args.ID, "delete-website", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketSetCorsOption struct { - ID string `help:"ID or name of bucket" json:"-"` - AllowedMethods []string `help:"allowed http method" choices:"PUT|GET|POST|DELETE|HEAD"` - // 允许的源站，可以设为* - AllowedOrigins []string - AllowedHeaders []string - MaxAgeSeconds int - ExposeHeaders []string - RuleId string - } - R(&BucketSetCorsOption{}, "bucket-set-cors", "Set bucket cors", func(s *mcclient.ClientSession, args *BucketSetCorsOption) error { - - rule := api.BucketCORSRule{ - AllowedOrigins: args.AllowedOrigins, - AllowedMethods: args.AllowedMethods, - AllowedHeaders: args.AllowedHeaders, - MaxAgeSeconds: args.MaxAgeSeconds, - ExposeHeaders: args.ExposeHeaders, - Id: args.RuleId, - } - rules := api.BucketCORSRules{Data: []api.BucketCORSRule{rule}} - result, err := modules.Buckets.PerformAction(s, args.ID, "set-cors", jsonutils.Marshal(rules)) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketGetCorsOption struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketGetCorsOption{}, "bucket-get-cors", "Get bucket cors", func(s *mcclient.ClientSession, args *BucketGetCorsOption) error { - result, err := modules.Buckets.GetSpecific(s, args.ID, "cors", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketDeleteCorsOption struct { - ID string `help:"ID or name of bucket" json:"-"` - Id []string `"help:Id of rules to delete"` - } - R(&BucketDeleteCorsOption{}, "bucket-delete-cors", "Delete bucket cors", func(s *mcclient.ClientSession, args *BucketDeleteCorsOption) error { - input := api.BucketCORSRuleDeleteInput{} - input.Id = args.Id - result, err := modules.Buckets.PerformAction(s, args.ID, "delete-cors", jsonutils.Marshal(input)) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketSetRefererOption struct { - ID string `help:"ID or name of bucket" json:"-"` - // 域名列表 - DomainList []string - // 是否允许空referer 访问 - AllowEmptyRefer bool `help:"all empty refer access"` - Enabled bool - RerererType string `help:"Referer type" choices:"Black-List|White-List"` - } - R(&BucketSetRefererOption{}, "bucket-set-referer", "Set bucket referer", func(s *mcclient.ClientSession, args *BucketSetRefererOption) error { - conf := api.BucketRefererConf{ - Enabled: args.Enabled, - AllowEmptyRefer: args.AllowEmptyRefer, - RefererType: args.RerererType, - DomainList: args.DomainList, - } - result, err := modules.Buckets.PerformAction(s, args.ID, "set-referer", jsonutils.Marshal(conf)) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketGetRefererOption struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketGetRefererOption{}, "bucket-get-referer", "get bucket referer", func(s *mcclient.ClientSession, args *BucketGetRefererOption) error { - result, err := modules.Buckets.GetSpecific(s, args.ID, "referer", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketGetCdnDomainOption struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketGetRefererOption{}, "bucket-get-cdn-domain", "get bucket cdn domain", func(s *mcclient.ClientSession, args *BucketGetRefererOption) error { - result, err := modules.Buckets.GetSpecific(s, args.ID, "cdn-domain", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketGetPolicyOption struct { - ID string `help:"ID or name of bucket" json:"-"` - } - R(&BucketGetPolicyOption{}, "bucket-get-policy", "get bucket policy", func(s *mcclient.ClientSession, args *BucketGetPolicyOption) error { - result, err := modules.Buckets.GetSpecific(s, args.ID, "policy", nil) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketSetPolicyOption struct { - ID string `help:"ID or name of bucket" json:"-"` - // 格式主账号id:子账号id - PrincipalId []string `help:"ext account id, accountId:subaccountId"` - // Read|ReadWrite|FullControl - CannedAction string `help:"authority action" choice:"Read|FullControl"` - // Allow|Deny - Effect string `help:"allow or deny" choice:"Allow|Deny"` - // 被授权的资源地址 - ResourcePath []string - // ip 条件 - IpEquals []string - IpNotEquals []string - } - R(&BucketSetPolicyOption{}, "bucket-set-policy", "set bucket policy", func(s *mcclient.ClientSession, args *BucketSetPolicyOption) error { - opts := api.BucketPolicyStatementInput{} - opts.CannedAction = args.CannedAction - opts.Effect = args.Effect - opts.IpEquals = args.IpEquals - opts.IpNotEquals = args.IpNotEquals - opts.ResourcePath = args.ResourcePath - opts.PrincipalId = args.PrincipalId - - result, err := modules.Buckets.PerformAction(s, args.ID, "set-policy", jsonutils.Marshal(opts)) - if err != nil { - return err - } - printObject(result) - return nil - }) - - type BucketDeletePolicyOption struct { - ID string `help:"ID or name of bucket" json:"-"` - Id []string - } - R(&BucketDeletePolicyOption{}, "bucket-delete-policy", "delete bucket policy", func(s *mcclient.ClientSession, args *BucketDeletePolicyOption) error { - input := api.BucketPolicyDeleteInput{} - input.Id = args.Id - result, err := modules.Buckets.PerformAction(s, args.ID, "delete-policy", jsonutils.Marshal(input)) - if err != nil { - return err - } - printObject(result) return nil }) } diff --git a/go.mod b/go.mod index 3dae486b26..5ba90f0a25 100644 --- a/go.mod +++ b/go.mod @@ -93,12 +93,12 @@ require ( k8s.io/cri-api v0.22.17 k8s.io/klog/v2 v2.20.0 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2 - yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090 + yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 - yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e + yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 yunion.io/x/sqlchemy v1.1.3-0.20250513031856-ce9f71063b3a yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c diff --git a/go.sum b/go.sum index 7b5ff4cd07..afe88cf802 100644 --- a/go.sum +++ b/go.sum @@ -1376,10 +1376,10 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2 h1:Zrqat1z8qiSqZXitZlUOs1uD4xN+pHXkrWir1vYBXr8= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ= -yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8= -yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090 h1:E/I9CkZumtHaEBweSYc9bq5rFZ25i+s17bAZskXLGDc= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ= +yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 h1:msG4SiDSVU7CrXH06WuHlNEZXIooTcmNbfrIGHuIHBU= +yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 h1:/+THlvf/MvgCW+7KeCDCr33e81KSRa5JmdZ1IIyLOXQ= yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo= @@ -1391,8 +1391,8 @@ yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVB yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc= yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e h1:OiBxJR18bq5fFK/IZWYRX+5HTMs2KgQJe0a3BZfdIVU= -yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= +yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 h1:iKWkBMKazSijYNhOaSh4qBuIu+PmXYhEAMNwrxaXL4Q= +yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v1.1.3-0.20250513031856-ce9f71063b3a h1:mDW1VyYJxZ4ORITZGDWacUiBNyqX6rMObnQk77NvUOg= diff --git a/pkg/apigateway/handler/init.go b/pkg/apigateway/handler/init.go index 08190ce92a..a34bc6d7a5 100644 --- a/pkg/apigateway/handler/init.go +++ b/pkg/apigateway/handler/init.go @@ -15,28 +15,5 @@ package handler import ( - _ "yunion.io/x/onecloud/pkg/mcclient/modules/ansible" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudevent" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudid" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudproxy" - modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/devtool" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/etcd" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/identity" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/image" - "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/logger" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/notify" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/quota" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduledtask" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduler" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/webconsole" - _ "yunion.io/x/onecloud/pkg/mcclient/modules/yunionconf" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/loader" ) - -func init() { - modules.InitUsages() - modules.Usages.RegisterManager(modules.UsageManagerK8s, k8s.Usages) -} diff --git a/pkg/apis/compute/bucket.go b/pkg/apis/compute/bucket.go index 04b9a742a1..a79d4dc92c 100644 --- a/pkg/apis/compute/bucket.go +++ b/pkg/apis/compute/bucket.go @@ -17,6 +17,7 @@ package compute import ( "net/http" "reflect" + "time" "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/cloudmux/pkg/cloudprovider" @@ -53,6 +54,8 @@ type BucketCreateInput struct { CloudproviderResourceInput StorageClass string `json:"storage_class"` + + EnablePerfMon *bool `json:"enable_perf_mon"` } type BucketDetails struct { @@ -66,28 +69,28 @@ type BucketDetails struct { AccessUrls []cloudprovider.SBucketAccessUrl `json:"access_urls"` } -func (self BucketDetails) GetMetricTags() map[string]string { +func (bucket BucketDetails) GetMetricTags() map[string]string { ret := map[string]string{ - "id": self.Id, - "brand": self.Brand, - "cloudregion": self.Cloudregion, - "cloudregion_id": self.CloudregionId, - "domain_id": self.DomainId, - "oss_id": self.Id, - "oss_name": self.Name, - "project_domain": self.ProjectDomain, - "region_ext_id": self.RegionExtId, - "status": self.Status, - "tenant": self.Project, - "tenant_id": self.ProjectId, - "account": self.Account, - "account_id": self.AccountId, - "external_id": self.ExternalId, + "id": bucket.Id, + "brand": bucket.Brand, + "cloudregion": bucket.Cloudregion, + "cloudregion_id": bucket.CloudregionId, + "domain_id": bucket.DomainId, + "oss_id": bucket.Id, + "oss_name": bucket.Name, + "project_domain": bucket.ProjectDomain, + "region_ext_id": bucket.RegionExtId, + "status": bucket.Status, + "tenant": bucket.Project, + "tenant_id": bucket.ProjectId, + "account": bucket.Account, + "account_id": bucket.AccountId, + "external_id": bucket.ExternalId, } - return AppendMetricTags(ret, self.MetadataResourceInfo, self.ProjectizedResourceInfo) + return AppendMetricTags(ret, bucket.MetadataResourceInfo, bucket.ProjectizedResourceInfo) } -func (self BucketDetails) GetMetricPairs() map[string]string { +func (bucket BucketDetails) GetMetricPairs() map[string]string { ret := map[string]string{} return ret } @@ -150,6 +153,8 @@ type BucketSyncstatusInput struct { type BucketUpdateInput struct { apis.SharableVirtualResourceBaseUpdateInput + + EnablePerfMon *bool `json:"enable_perf_mon"` } type BucketPerformTempUrlInput struct { @@ -370,3 +375,29 @@ func init() { return &SBackupStorageAccessInfo{} }) } + +type BucketProbeResult struct { + UploadTime time.Duration + DownloadTime time.Duration + DeleteTime time.Duration +} + +func (result BucketProbeResult) UploadDelayMs() float64 { + return float64(result.UploadTime) / float64(time.Millisecond) +} + +func (result BucketProbeResult) DownloadDelayMs() float64 { + return float64(result.DownloadTime) / float64(time.Millisecond) +} + +func (result BucketProbeResult) DeleteDelayMs() float64 { + return float64(result.DeleteTime) / float64(time.Millisecond) +} + +func (result BucketProbeResult) UploadThroughputMbps(sizeMBytes int) float64 { + return float64(sizeMBytes) * 8 / float64(result.UploadTime.Seconds()) +} + +func (result BucketProbeResult) DownloadThroughputMbps(sizeMBytes int) float64 { + return float64(sizeMBytes) * 8 / float64(result.DownloadTime.Seconds()) +} diff --git a/pkg/cloudmon/misc/bucketprobe.go b/pkg/cloudmon/misc/bucketprobe.go new file mode 100644 index 0000000000..4970017438 --- /dev/null +++ b/pkg/cloudmon/misc/bucketprobe.go @@ -0,0 +1,177 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package misc + +import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + computeapi "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + baseoptions "yunion.io/x/onecloud/pkg/mcclient/options" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func BucketProbe(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + if options.Options.EnableBucketProbeDebug { + log.Debugf("BucketProbe start") + } + if !options.Options.EnableBucketProbe { + if options.Options.EnableBucketProbeDebug { + log.Debugf("BucketProbe is disabled") + } + return + } + + sess := auth.GetSession(ctx, userCred, options.Options.Region) + + metrics, err := gatherBucketMetrics(ctx, sess) + if err != nil { + log.Errorf("BucketProbe gatherBucketMetrics failed: %s", err) + return + } + + err = sendMetrics(sess, metrics, "telegraf") + if err != nil { + log.Errorf("StatusProbe SendMetrics error: %s", err) + } +} + +func gatherBucketMetrics(ctx context.Context, sess *mcclient.ClientSession) ([]influxdb.SMetricData, error) { + allMetrics := []influxdb.SMetricData{} + + params := baseoptions.BaseListOptions{} + params.Scope = "max" + limit := 1000 + params.Limit = &limit + params.Filter = []string{ + "enable_perf_mon.equals(1)", + } + boolTrue := true + params.Details = &boolTrue + + total := -1 + offset := 0 + for total < 0 || offset < total { + params.Offset = &offset + results, err := computemodules.Buckets.List(sess, jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "computemodules.Buckets.List") + } + total = results.Total + offset = results.Offset + len(results.Data) + + for _, bucket := range results.Data { + bucketDetails := computeapi.BucketDetails{} + err = bucket.Unmarshal(&bucketDetails) + if err != nil { + log.Errorf("BucketProbe failed: %s", err) + continue + } + + metrics, err := probeBucketStats(ctx, sess, &bucketDetails) + if err != nil { + log.Errorf("BucketProbe failed: %s", err) + continue + } + allMetrics = append(allMetrics, metrics...) + } + } + + return allMetrics, nil +} + +func probeBucketStats(ctx context.Context, sess *mcclient.ClientSession, bucketDetails *computeapi.BucketDetails) ([]influxdb.SMetricData, error) { + bucket, err := computemodules.GetIBucket(ctx, sess, bucketDetails) + if err != nil { + return nil, errors.Wrap(err, "getIBucket") + } + + resultDelay, err := computemodules.ProbeBucketStats(ctx, bucket, options.Options.BucketProbeTestKey, 0) + if err != nil { + return nil, errors.Wrap(err, "doProbeBucketStats zero") + } + + resultRate, err := computemodules.ProbeBucketStats(ctx, bucket, options.Options.BucketProbeTestKey, int64(options.Options.BucketProbeTestSizeMb)*1024*1024) + if err != nil { + return nil, errors.Wrap(err, "doProbeBucketStats with payload") + } + + metricTags := []influxdb.SKeyValue{} + for k, v := range bucketDetails.GetMetricTags() { + if len(v) == 0 { + continue + } + metricTags = append(metricTags, influxdb.SKeyValue{ + Key: k, + Value: v, + }) + } + + metrics := []influxdb.SKeyValue{} + for k, v := range bucketDetails.GetMetricTags() { + if len(v) == 0 { + continue + } + metrics = append(metrics, influxdb.SKeyValue{ + Key: k, + Value: v, + }) + } + + metrics = append(metrics, + influxdb.SKeyValue{ + Key: "upload_delay_ms", + Value: fmt.Sprintf("%f", resultDelay.UploadDelayMs()), + }, + influxdb.SKeyValue{ + Key: "download_delay_ms", + Value: fmt.Sprintf("%f", resultDelay.DownloadDelayMs()), + }, + influxdb.SKeyValue{ + Key: "delete_delay_ms", + Value: fmt.Sprintf("%f", resultDelay.DeleteDelayMs()), + }, + influxdb.SKeyValue{ + Key: "upload_rate_mbps", + Value: fmt.Sprintf("%f", resultRate.UploadThroughputMbps(options.Options.BucketProbeTestSizeMb)), + }, + influxdb.SKeyValue{ + Key: "download_rate_mbps", + Value: fmt.Sprintf("%f", resultRate.DownloadThroughputMbps(options.Options.BucketProbeTestSizeMb)), + }, + ) + + if options.Options.EnableBucketProbeDebug { + log.Debugf("BucketProbe for bucket %s metrics: %s", bucketDetails.Name, jsonutils.Marshal(metrics)) + } + + return []influxdb.SMetricData{ + { + Name: "bucket_perf", + Tags: metricTags, + Metrics: metrics, + Timestamp: time.Now(), + }, + }, nil +} diff --git a/pkg/cloudmon/misc/pinger.go b/pkg/cloudmon/misc/pinger.go index aaa4418a21..0df6c9b9b7 100644 --- a/pkg/cloudmon/misc/pinger.go +++ b/pkg/cloudmon/misc/pinger.go @@ -110,9 +110,11 @@ func getNetworkAddrMap(s *mcclient.ClientSession, netId string) (map[string]api. return nil, errors.Wrap(err, "GetSpecific addresses") } addrList := make([]api.SNetworkUsedAddress, 0) - err = addrListJson.Unmarshal(&addrList, "addresses") - if err != nil { - return nil, errors.Wrap(err, "Unmarshal addreses") + if addrListJson.Contains("addresses") { + err = addrListJson.Unmarshal(&addrList, "addresses") + if err != nil { + return nil, errors.Wrap(err, "Unmarshal addreses") + } } addrMap := make(map[string]api.SNetworkUsedAddress) for i := range addrList { diff --git a/pkg/cloudmon/misc/send.go b/pkg/cloudmon/misc/send.go new file mode 100644 index 0000000000..c5ace6ba90 --- /dev/null +++ b/pkg/cloudmon/misc/send.go @@ -0,0 +1,40 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package misc + +import ( + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/cloudcommon/tsdb" + "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func sendMetrics(sess *mcclient.ClientSession, metrics []influxdb.SMetricData, database string) error { + if len(metrics) == 0 { + return nil + } + + urls, err := tsdb.GetDefaultServiceSourceURLs(sess, options.Options.SessionEndpointType) + if err != nil { + return errors.Wrap(err, "GetServiceURLs") + } + err = influxdb.SendMetrics(urls, database, metrics, false) + if err != nil { + return errors.Wrap(err, "SendMetrics") + } + return nil +} diff --git a/pkg/cloudmon/misc/statusprobe.go b/pkg/cloudmon/misc/statusprobe.go new file mode 100644 index 0000000000..53a0236531 --- /dev/null +++ b/pkg/cloudmon/misc/statusprobe.go @@ -0,0 +1,137 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package misc + +import ( + "context" + "fmt" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + + "yunion.io/x/onecloud/pkg/apis" + "yunion.io/x/onecloud/pkg/cloudmon/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/mcclient/modulebase" + baseOptions "yunion.io/x/onecloud/pkg/mcclient/options" + "yunion.io/x/onecloud/pkg/util/influxdb" +) + +func StatusProbe(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { + if options.Options.EnableStatusProbeDebug { + log.Debugf("Start resource status probe") + } + + if !options.Options.EnableStatusProbe { + if options.Options.EnableStatusProbeDebug { + log.Debugf("Resource status probe is disabled") + } + return + } + + sess := auth.GetSession(ctx, userCred, options.Options.Region) + metrics := make([]influxdb.SMetricData, 0) + for _, model := range options.Options.StatusProbeModels { + mts, err := doModelStatusProbe(sess, model) + if err != nil { + log.Errorf("doModelStatusProbe failed: %s", err) + } + metrics = append(metrics, mts...) + } + + err := sendMetrics(sess, metrics, "system") + if err != nil { + log.Errorf("StatusProbe SendMetrics error: %s", err) + } +} + +func doModelStatusProbe(sess *mcclient.ClientSession, modelName string) ([]influxdb.SMetricData, error) { + model, err := modulebase.GetModule(sess, modelName) + if err != nil { + return nil, errors.Wrap(err, "GetModule") + } + + listOpts := baseOptions.BaseListOptions{} + listOpts.Scope = "max" + listOpts.SummaryStats = true + limit := 0 + listOpts.Limit = &limit + + results, err := model.List(sess, jsonutils.Marshal(listOpts)) + if err != nil { + return nil, errors.Wrap(err, "List") + } + + statusInfoTotal := struct { + apis.TotalCountBase + StatusInfo []apis.StatusStatisticStatusInfo + }{} + + err = results.Totals.Unmarshal(&statusInfoTotal) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal statusInfoTotal") + } + + log.Infof("statusInfoTotal: %s", jsonutils.Marshal(statusInfoTotal)) + + metrics := make([]influxdb.SMetricData, 0) + + totalCount := int64(0) + pendingDeletedCount := int64(0) + for _, statusInfo := range statusInfoTotal.StatusInfo { + metrics = append(metrics, genStatusMetricData(model, statusInfo.Status, statusInfo.TotalCount, statusInfo.PendingDeletedCount)) + totalCount += statusInfo.TotalCount + pendingDeletedCount += statusInfo.PendingDeletedCount + } + metrics = append(metrics, genStatusMetricData(model, "total", totalCount, pendingDeletedCount)) + + if options.Options.EnableStatusProbeDebug { + log.Debugf("StatusProbe for model %s metrics: %s", modelName, jsonutils.Marshal(metrics)) + } + + return metrics, nil +} + +func genStatusMetricData(model modulebase.Manager, status string, count int64, pendingDeletedCount int64) influxdb.SMetricData { + return influxdb.SMetricData{ + Name: "status_probe", + Tags: influxdb.TKeyValuePairs{ + influxdb.SKeyValue{ + Key: "service", + Value: model.ServiceType(), + }, + influxdb.SKeyValue{ + Key: "model", + Value: model.GetKeyword(), + }, + influxdb.SKeyValue{ + Key: "status", + Value: status, + }, + }, + Metrics: influxdb.TKeyValuePairs{ + influxdb.SKeyValue{ + Key: "count", + Value: fmt.Sprintf("%d", count), + }, + influxdb.SKeyValue{ + Key: "pending_deleted", + Value: fmt.Sprintf("%d", pendingDeletedCount), + }, + }, + } +} diff --git a/pkg/cloudmon/options/options.go b/pkg/cloudmon/options/options.go index 9ba4d973d8..444b42b6cd 100644 --- a/pkg/cloudmon/options/options.go +++ b/pkg/cloudmon/options/options.go @@ -37,6 +37,17 @@ type CloudMonOptions struct { CloudAccountCollectMetricsBatchCount int `help:"Cloud Account Collect Metrics Batch Count" default:"10"` CloudResourceCollectMetricsBatchCount int `help:"Cloud Resource Collect Metrics BatchC ount" default:"40"` OracleCloudResourceCollectMetricsBatchCount int `help:"OracleCloud Resource Collect Metrics BatchC ount" default:"1"` + + StatusProbeIntervalMinutes int `help:"Status Probe Interval unit:minute" default:"15"` + StatusProbeModels []string `help:"Status Probe Models" default:"compute-servers,compute-hosts"` + EnableStatusProbe bool `help:"Enable Status Probe" default:"false"` + EnableStatusProbeDebug bool `help:"Enable Status Probe Debug" default:"false"` + + BucketProbeIntervalMinutes int `help:"Bucket Probe Interval unit:minute" default:"15"` + EnableBucketProbe bool `help:"Enable Bucket Probe" default:"false"` + EnableBucketProbeDebug bool `help:"Enable Bucket Probe Debug" default:"false"` + BucketProbeTestKey string `help:"Bucket Probe Test Key" default:"bucket_performance_test_object"` + BucketProbeTestSizeMb int `help:"Bucket Probe Test Size" default:"4"` } type PingProbeOptions struct { diff --git a/pkg/cloudmon/service/service.go b/pkg/cloudmon/service/service.go index 8830cb2e75..50fd0cc29b 100644 --- a/pkg/cloudmon/service/service.go +++ b/pkg/cloudmon/service/service.go @@ -33,6 +33,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/cloudmon/resources" "yunion.io/x/onecloud/pkg/mcclient/auth" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/loader" ) func StartService() { @@ -57,6 +58,10 @@ func StartService() { cron.AddJobAtIntervalsWithStartRun("PingProb", time.Duration(opts.PingProbIntervalHours)*time.Hour, misc.PingProbe, true) + cron.AddJobAtIntervalsWithStartRun("StatusProbe", time.Duration(opts.StatusProbeIntervalMinutes)*time.Minute, misc.StatusProbe, true) + + cron.AddJobAtIntervalsWithStartRun("BucketProbe", time.Duration(opts.BucketProbeIntervalMinutes)*time.Minute, misc.BucketProbe, true) + cron.AddJobEveryFewDays("UsageMetricCollect", 1, 23, 10, 10, misc.UsegReport, false) cron.AddJobEveryFewDays("AlertHistoryMetricCollect", 1, 23, 59, 59, misc.AlertHistoryReport, false) diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 0193f42454..e27fb3d95d 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -92,6 +92,8 @@ type SBucket struct { ObjectCntLimit int `nullable:"false" default:"0" list:"user"` AccessUrls jsonutils.JSONObject `nullable:"true" list:"user"` + + EnablePerfMon bool `default:"false" list:"user" update:"user" create:"optional"` } func (manager *SBucketManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration { diff --git a/pkg/mcclient/modules/compute/mod_buckets.go b/pkg/mcclient/modules/compute/mod_buckets.go index 3060d1ee97..578a79422f 100644 --- a/pkg/mcclient/modules/compute/mod_buckets.go +++ b/pkg/mcclient/modules/compute/mod_buckets.go @@ -15,12 +15,16 @@ package compute import ( + "context" "fmt" "io" + "math/rand" "net/http" "strconv" + "time" "yunion.io/x/cloudmux/pkg/cloudprovider" + _ "yunion.io/x/cloudmux/pkg/multicloud/objectstore/provider" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/httputils" @@ -81,3 +85,67 @@ func init() { modules.RegisterCompute(&Buckets) } + +func GetIBucket(ctx context.Context, sess *mcclient.ClientSession, bucketDetails *api.BucketDetails) (cloudprovider.ICloudBucket, error) { + provider, err := Cloudproviders.GetProvider(ctx, sess, bucketDetails.ManagerId) + if err != nil { + return nil, errors.Wrap(err, "computemodules.Cloudproviders.GetProvider") + } + + iregion, err := func() (cloudprovider.ICloudRegion, error) { + if provider.GetFactory().IsOnPremise() { + return provider.GetOnPremiseIRegion() + } else { + return provider.GetIRegionById(bucketDetails.RegionExternalId) + } + }() + if err != nil { + return nil, errors.Wrap(err, "GetIRegion") + } + + bucket, err := iregion.GetIBucketById(bucketDetails.ExternalId) + if err != nil { + return nil, errors.Wrap(err, "iregion.GetIBucketById") + } + + return bucket, nil +} + +type nullWriter struct{} + +func (w *nullWriter) WriteAt(p []byte, off int64) (n int, err error) { + return len(p), nil +} + +func getRandReader() io.Reader { + return rand.New(rand.NewSource(time.Now().UnixNano())) +} + +func ProbeBucketStats(ctx context.Context, bucket cloudprovider.ICloudBucket, testKey string, sizeBytes int64) (*api.BucketProbeResult, error) { + result := &api.BucketProbeResult{} + start := time.Now() + + // force upload object in one shot + err := cloudprovider.UploadObject(ctx, bucket, testKey, sizeBytes*2, getRandReader(), sizeBytes, cloudprovider.ACLPrivate, "", nil, false) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.UploadObject") + } + + result.UploadTime = time.Since(start) + + _, err = cloudprovider.DownloadObjectParallel(ctx, bucket, testKey, nil, &nullWriter{}, 0, 0, false, 1) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.DownloadObjectParallel") + } + + result.DownloadTime = time.Since(start) - result.UploadTime + + err = bucket.DeleteObject(ctx, testKey) + if err != nil { + return nil, errors.Wrap(err, "bucket.DeleteObject") + } + + result.DeleteTime = time.Since(start) - result.DownloadTime + + return result, nil +} diff --git a/pkg/mcclient/modules/loader/doc.go b/pkg/mcclient/modules/loader/doc.go new file mode 100644 index 0000000000..046e8f18d2 --- /dev/null +++ b/pkg/mcclient/modules/loader/doc.go @@ -0,0 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loader // import "yunion.io/x/onecloud/pkg/mcclient/modules/loader" diff --git a/pkg/mcclient/modules/loader/loader.go b/pkg/mcclient/modules/loader/loader.go new file mode 100644 index 0000000000..9cc57ca09a --- /dev/null +++ b/pkg/mcclient/modules/loader/loader.go @@ -0,0 +1,44 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loader + +import ( + _ "yunion.io/x/onecloud/pkg/mcclient/modules/ansible" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudevent" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudid" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudproxy" + modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/devtool" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/etcd" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/identity" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/image" + "yunion.io/x/onecloud/pkg/mcclient/modules/k8s" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/logger" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/monitor" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/notify" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/quota" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduledtask" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduler" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/vpcagent" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/webconsole" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/websocket" + _ "yunion.io/x/onecloud/pkg/mcclient/modules/yunionconf" +) + +func init() { + modules.InitUsages() + modules.Usages.RegisterManager(modules.UsageManagerK8s, k8s.Usages) +} diff --git a/pkg/mcclient/options/compute/buckets.go b/pkg/mcclient/options/compute/buckets.go new file mode 100644 index 0000000000..2192e4379f --- /dev/null +++ b/pkg/mcclient/options/compute/buckets.go @@ -0,0 +1,444 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package compute + +import ( + "yunion.io/x/cloudmux/pkg/multicloud/objectstore" + "yunion.io/x/jsonutils" + + "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/mcclient/options" +) + +type BucketListOptions struct { + options.BaseListOptions + DistinctField string `help:"query specified distinct field"` +} + +func (opts *BucketListOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.ListStructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketGetPropertyOptions struct { + DistinctField string `help:"query specified distinct field"` +} + +func (opts *BucketGetPropertyOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.ListStructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +func (opts *BucketGetPropertyOptions) Property() string { + return "distinct-field" +} + +type BucketIdOptions struct { + options.BaseIdOptions +} + +type BucketUpdateOptions struct { + BucketIdOptions + + Name string `help:"new name of bucket" json:"name"` + Desc string `help:"Description of bucket" json:"description" token:"desc"` + EnablePerfMon bool `help:"enable performance monitor" json:"-"` +} + +func (opts *BucketUpdateOptions) Params() (jsonutils.JSONObject, error) { + params := jsonutils.Marshal(opts) + if opts.EnablePerfMon { + params.(*jsonutils.JSONDict).Add(jsonutils.JSONTrue, "enable_perf_mon") + } + return params, nil +} + +type BucketCreateOptions struct { + NAME string `help:"name of bucket" json:"name"` + CLOUDREGION string `help:"location of bucket" json:"cloudregion"` + MANAGER string `help:"cloud provider" json:"manager"` + + StorageClass string `help:"bucket storage class"` + Acl string `help:"bucket ACL"` + + EnablePerfMon bool `help:"enable performance monitor"` +} + +func (opts *BucketCreateOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketListObjectsOptions struct { + BucketIdOptions + + Prefix string `help:"List objects with prefix"` + Recursive bool `help:"List objects recursively"` + Limit int `help:"maximal items per request"` + PagingMarker string `help:"paging marker"` +} + +func (opts *BucketListObjectsOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.ListStructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketDeleteObjectsOptions struct { + BucketIdOptions + + KEYS []string `help:"List of objects to delete"` +} + +func (opts *BucketDeleteObjectsOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketMakeDirOptions struct { + BucketIdOptions + + KEY string `help:"DIR key to create"` +} + +func (opts *BucketMakeDirOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketPresignObjectsOptions struct { + BucketIdOptions + + KEY string `help:"Key of object to upload"` + Method string `help:"Request method" choices:"GET|PUT|DELETE"` + ExpireSeconds int `help:"expire in seconds" default:"60"` +} + +func (opts *BucketPresignObjectsOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketSetAclOptions struct { + BucketIdOptions + + ACL string `help:"ACL to set" choices:"default|private|public-read|public-read-write" json:"acl"` + Key []string `help:"Optional object key" json:"key"` +} + +func (opts *BucketSetAclOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketAclOptions struct { + BucketIdOptions + + Key string `help:"Optional object key"` +} + +func (opts *BucketAclOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketSyncOptions struct { + BucketIdOptions + + StatsOnly bool `help:"sync statistics only"` +} + +func (opts *BucketSyncOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketLimitOptions struct { + BucketIdOptions + + SizeBytes int64 `help:"size limit in bytes"` + ObjectCount int64 `help:"object count limit"` +} + +func (opts *BucketLimitOptions) Params() (jsonutils.JSONObject, error) { + limit := jsonutils.Marshal(opts) + params := jsonutils.NewDict() + params.Set("limit", limit) + return params, nil +} + +type BucketAccessInfoOptions struct { + BucketIdOptions +} + +func (opts *BucketAccessInfoOptions) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketSetMetadataOptions struct { + BucketIdOptions + + Key []string `help:"Optional object key" json:"key"` + + objectstore.ObjectHeaderOptions +} + +func (opts *BucketSetMetadataOptions) Params() (jsonutils.JSONObject, error) { + input := compute.BucketMetadataInput{} + input.Key = opts.Key + input.Metadata = opts.ObjectHeaderOptions.Options2Header() + err := input.Validate() + if err != nil { + return nil, err + } + return jsonutils.Marshal(input), nil +} + +type BucketSetWebsiteOption struct { + BucketIdOptions + // 主页 + Index string `help:"main page"` + // 错误时返回的文档 + ErrorDocument string `help:"error return"` + // http或https + Protocol string `help:"force https" choices:"http|https"` +} + +func (opts *BucketSetWebsiteOption) Params() (jsonutils.JSONObject, error) { + conf := compute.BucketWebsiteConf{ + Index: opts.Index, + ErrorDocument: opts.ErrorDocument, + Protocol: opts.Protocol, + } + return jsonutils.Marshal(conf), nil +} + +type BucketGetWebsiteConfOption struct { + BucketIdOptions +} + +func (opts *BucketGetWebsiteConfOption) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketDeleteWebsiteConfOption struct { + BucketIdOptions +} + +func (opts *BucketDeleteWebsiteConfOption) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketSetCorsOption struct { + BucketIdOptions + + AllowedMethods []string `help:"allowed http method" choices:"PUT|GET|POST|DELETE|HEAD"` + // 允许的源站，可以设为* + AllowedOrigins []string + AllowedHeaders []string + MaxAgeSeconds int + ExposeHeaders []string + RuleId string +} + +func (args *BucketSetCorsOption) Params() (jsonutils.JSONObject, error) { + rule := compute.BucketCORSRule{ + AllowedOrigins: args.AllowedOrigins, + AllowedMethods: args.AllowedMethods, + AllowedHeaders: args.AllowedHeaders, + MaxAgeSeconds: args.MaxAgeSeconds, + ExposeHeaders: args.ExposeHeaders, + Id: args.RuleId, + } + rules := compute.BucketCORSRules{Data: []compute.BucketCORSRule{rule}} + return jsonutils.Marshal(rules), nil +} + +type BucketGetCorsOption struct { + BucketIdOptions +} + +func (opts *BucketGetCorsOption) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketDeleteCorsOption struct { + BucketIdOptions + + RuleId []string `help:"Id of rules to delete"` +} + +func (opts *BucketDeleteCorsOption) Params() (jsonutils.JSONObject, error) { + input := compute.BucketCORSRuleDeleteInput{} + input.Id = opts.RuleId + return jsonutils.Marshal(input), nil +} + +type BucketSetRefererOption struct { + BucketIdOptions + // 域名列表 + DomainList []string + // 是否允许空referer 访问 + AllowEmptyRefer bool `help:"all empty refer access"` + Enabled bool + RerererType string `help:"Referer type" choices:"Black-List|White-List"` +} + +func (args *BucketSetRefererOption) Params() (jsonutils.JSONObject, error) { + conf := compute.BucketRefererConf{ + Enabled: args.Enabled, + AllowEmptyRefer: args.AllowEmptyRefer, + RefererType: args.RerererType, + DomainList: args.DomainList, + } + return jsonutils.Marshal(conf), nil +} + +type BucketGetRefererOption struct { + BucketIdOptions +} + +func (opts *BucketGetRefererOption) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketGetCdnDomainOption struct { + BucketIdOptions +} + +func (opts *BucketGetCdnDomainOption) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketGetPolicyOption struct { + BucketIdOptions +} + +func (opts *BucketGetPolicyOption) Params() (jsonutils.JSONObject, error) { + params, err := options.StructToParams(opts) + if err != nil { + return nil, err + } + return params, nil +} + +type BucketSetPolicyOption struct { + BucketIdOptions + // 格式主账号id:子账号id + PrincipalId []string `help:"ext account id, accountId:subaccountId"` + // Read|ReadWrite|FullControl + CannedAction string `help:"authority action" choice:"Read|FullControl"` + // Allow|Deny + Effect string `help:"allow or deny" choice:"Allow|Deny"` + // 被授权的资源地址 + ResourcePath []string + // ip 条件 + IpEquals []string + IpNotEquals []string +} + +func (args *BucketSetPolicyOption) Params() (jsonutils.JSONObject, error) { + opts := compute.BucketPolicyStatementInput{} + opts.CannedAction = args.CannedAction + opts.Effect = args.Effect + opts.IpEquals = args.IpEquals + opts.IpNotEquals = args.IpNotEquals + opts.ResourcePath = args.ResourcePath + opts.PrincipalId = args.PrincipalId + return jsonutils.Marshal(opts), nil +} + +type BucketDeletePolicyOption struct { + BucketIdOptions + PolicyId []string `help:"policy id to delete"` +} + +func (args *BucketDeletePolicyOption) Params() (jsonutils.JSONObject, error) { + input := compute.BucketPolicyDeleteInput{} + input.Id = args.PolicyId + return jsonutils.Marshal(input), nil +} + +type BucketUploadObjectsOptions struct { + BucketIdOptions + + KEY string `help:"Key of object to upload"` + Path string `help:"Path to file to upload" required:"true"` + + ContentLength int64 `help:"Content lenght (bytes)" default:"-1"` + StorageClass string `help:"storage CLass"` + Acl string `help:"object acl." choices:"private|public-read|public-read-write"` + + objectstore.ObjectHeaderOptions +} + +type BucketPerfMonOptions struct { + BucketIdOptions + + Payload string `help:"test payload in bytes, e.g. 1, 32, 1024M" default:"4M"` +} diff --git a/pkg/monitor/dbinit/measurements/all.go b/pkg/monitor/dbinit/measurements/all.go index 66d73a5c67..4df96b04a0 100644 --- a/pkg/monitor/dbinit/measurements/all.go +++ b/pkg/monitor/dbinit/measurements/all.go @@ -35,6 +35,7 @@ var All = []SMeasurement{ serviceHttpCode, serviceProcessStats, dbStats, + statusProbe, vmCpu, vmMem, @@ -83,6 +84,7 @@ var All = []SMeasurement{ ossLatency, ossNetio, ossReq, + ossPerfMon, ping, diff --git a/pkg/monitor/dbinit/measurements/oss.go b/pkg/monitor/dbinit/measurements/oss.go index cb248424ad..0a3ea1c607 100644 --- a/pkg/monitor/dbinit/measurements/oss.go +++ b/pkg/monitor/dbinit/measurements/oss.go @@ -60,3 +60,41 @@ var ossReq = SMeasurement{ }, }, } + +var ossPerfMon = SMeasurement{ + Context: []SMonitorContext{ + { + Name: "bucket_perf", + DisplayName: "Object storage bucket performance monitor", + ResourceType: monitor.METRIC_RES_TYPE_OSS, + Database: monitor.METRIC_DATABASE_TELE, + }, + }, + Metrics: []SMetric{ + { + Name: "upload_delay_ms", + DisplayName: "Bucket upload delay in milliseconds", + Unit: monitor.METRIC_UNIT_MS, + }, + { + Name: "download_delay_ms", + DisplayName: "Bucket download delay in milliseconds", + Unit: monitor.METRIC_UNIT_MS, + }, + { + Name: "delete_delay_ms", + DisplayName: "Bucket delete delay in milliseconds", + Unit: monitor.METRIC_UNIT_MS, + }, + { + Name: "upload_rate_mbps", + DisplayName: "Bucket upload rate in megabits per second", + Unit: monitor.METRIC_UNIT_MBPS, + }, + { + Name: "download_rate_mbps", + DisplayName: "Bucket download rate in megabits per second", + Unit: monitor.METRIC_UNIT_MBPS, + }, + }, +} diff --git a/pkg/monitor/dbinit/measurements/service.go b/pkg/monitor/dbinit/measurements/service.go index ca293b16ef..cc44e11aad 100644 --- a/pkg/monitor/dbinit/measurements/service.go +++ b/pkg/monitor/dbinit/measurements/service.go @@ -38,3 +38,19 @@ var worker = SMeasurement{ }, }, } + +var statusProbe = SMeasurement{ + Context: []SMonitorContext{ + { + "status_probe", "Resource status probe results", monitor.METRIC_RES_TYPE_SYSTEM, monitor.METRIC_DATABASE_SYSTEM, + }, + }, + Metrics: []SMetric{ + { + "count", "Resouce count for each status", monitor.METRIC_UNIT_NULL, + }, + { + "pending_deleted", "Pending deleted resource count for each status", monitor.METRIC_UNIT_NULL, + }, + }, +} diff --git a/vendor/modules.txt b/vendor/modules.txt index c156ff5edc..299b94def9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1785,7 +1785,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090 ## explicit; go 1.21 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1865,7 +1865,7 @@ yunion.io/x/cloudmux/pkg/multicloud/volcengine yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider yunion.io/x/cloudmux/pkg/multicloud/zstack yunion.io/x/cloudmux/pkg/multicloud/zstack/provider -# yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 +# yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 ## explicit; go 1.12 yunion.io/x/executor/apis yunion.io/x/executor/client @@ -1882,7 +1882,7 @@ yunion.io/x/log/hooks yunion.io/x/ovsdb/cli_util yunion.io/x/ovsdb/schema/ovn_nb yunion.io/x/ovsdb/types -# yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e +# yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 ## explicit; go 1.18 yunion.io/x/pkg/appctx yunion.io/x/pkg/errors diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/objectstore.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/objectstore.go index 0a795a457d..bd6627be37 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/objectstore.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/objectstore.go @@ -1154,6 +1154,11 @@ func DownloadObjectParallelWithProgress(ctx context.Context, bucket ICloudBucket blocksz = MAX_PUT_OBJECT_SIZEBYTES } sizeBytes := obj.GetSizeBytes() + if sizeBytes < 0 { + return 0, errors.Wrapf(errors.ErrServer, "object size is negative (%d)", sizeBytes) + } else if sizeBytes == 0 { + return 0, nil + } if rangeOpt == nil { rangeOpt = &SGetObjectRange{ Start: 0, diff --git a/vendor/yunion.io/x/executor/server/server.go b/vendor/yunion.io/x/executor/server/server.go index 655a5a3aaa..336fcebccc 100644 --- a/vendor/yunion.io/x/executor/server/server.go +++ b/vendor/yunion.io/x/executor/server/server.go @@ -198,6 +198,21 @@ func (e *Executor) SendInput(s apis.Executor_SendInputServer) error { for { input, err := s.Recv() if err == io.EOF { + if input != nil && m == nil { + icm, ok := cmds.Load(input.Sn) + if !ok { + return errors.Errorf("unknown sn %d", input.Sn) + } + m = icm.(*Commander) + if m.stdin == nil { + return errors.New("Process stdin not init") + } + } + if m != nil { + if e := m.stdin.Close(); e != nil { + return errors.Wrap(e, "close stdin") + } + } return s.SendAndClose(&apis.Error{}) } else if err != nil { return s.SendAndClose(&apis.Error{ diff --git a/vendor/yunion.io/x/pkg/util/fileutils/filesize.go b/vendor/yunion.io/x/pkg/util/fileutils/filesize.go index 01bddf3db4..4d368f9c44 100644 --- a/vendor/yunion.io/x/pkg/util/fileutils/filesize.go +++ b/vendor/yunion.io/x/pkg/util/fileutils/filesize.go @@ -63,3 +63,7 @@ func GetSizeKb(sizeStr string, defaultSize byte, base int) (int, error) { size, err := parseSizeStr(sizeStr, defaultSize, base) return size / base, err } + +func GetSizeBytes(sizeStr string, base int) (int, error) { + return parseSizeStr(sizeStr, 'B', base) +} diff --git a/vendor/yunion.io/x/pkg/util/streamutils/streamutils.go b/vendor/yunion.io/x/pkg/util/streamutils/streamutils.go index 7bee4d9dad..068fe41a0c 100644 --- a/vendor/yunion.io/x/pkg/util/streamutils/streamutils.go +++ b/vendor/yunion.io/x/pkg/util/streamutils/streamutils.go @@ -33,21 +33,30 @@ type SStreamProperty struct { type sXZReadAheadReader struct { offset int64 header []byte + hdrEof bool upstream io.Reader } func newXZReadAheadReader(stream io.Reader) (*sXZReadAheadReader, error) { xzHdr := make([]byte, xz.HeaderLen) n, err := stream.Read(xzHdr) + hdrEof := false if err != nil { - return nil, errors.Wrap(err, "Read XZ hader") - } - if n != len(xzHdr) { - return nil, errors.Wrap(errors.ErrEOF, "too few header bytes") + if errors.Cause(err) == io.EOF { + // delay the EOF + hdrEof = true + xzHdr = xzHdr[:n] + } else { + return nil, errors.Wrap(err, "Read XZ header") + } + } else if n != len(xzHdr) { + hdrEof = true + xzHdr = xzHdr[:n] } return &sXZReadAheadReader{ offset: 0, header: xzHdr, + hdrEof: hdrEof, upstream: stream, }, nil } @@ -57,6 +66,7 @@ func (s *sXZReadAheadReader) IsXz() bool { } func (s *sXZReadAheadReader) Read(buf []byte) (int, error) { + bufOffset := 0 if s.offset < int64(len(s.header)) { // read from header rdSize := len(s.header) - int(s.offset) @@ -65,12 +75,19 @@ func (s *sXZReadAheadReader) Read(buf []byte) (int, error) { } n := copy(buf, s.header[s.offset:s.offset+int64(rdSize)]) s.offset += int64(n) - return n, nil - } else { - n, err := s.upstream.Read(buf) - s.offset += int64(n) - return n, err + bufOffset = n } + // read buffer is full + if bufOffset >= len(buf) { + return bufOffset, nil + } + if s.offset >= int64(len(s.header)) && s.hdrEof { + return bufOffset, io.EOF + } + + n, err := s.upstream.Read(buf[bufOffset:]) + s.offset += int64(n) + return n + bufOffset, err } func StreamPipe(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64)) (*SStreamProperty, error) {