feature: monitor bucket formance and resource status (#22552)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

cmd/climc/shell/compute/buckets.go

@@ -19,178 +19,71 @@ import (
 	"io"
 	"os"
 
-	"yunion.io/x/cloudmux/pkg/multicloud/objectstore"
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/util/fileutils"
 	"yunion.io/x/pkg/util/printutils"
 
-	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/cmd/climc/shell"
+	"yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
-	"yunion.io/x/onecloud/pkg/mcclient/options"
+	computeoptions "yunion.io/x/onecloud/pkg/mcclient/options/compute"
 )
 
 func init() {
-	type BucketListOptions struct {
-		options.BaseListOptions
-		DistinctField string `help:"query specified distinct field"`
-	}
-	R(&BucketListOptions{}, "bucket-list", "List all buckets", func(s *mcclient.ClientSession, args *BucketListOptions) error {
-		params, err := options.ListStructToParams(args)
-		if err != nil {
-			return err
-		}
-		if len(args.DistinctField) > 0 {
-			params.Add(jsonutils.NewString(args.DistinctField), "extra_field")
-			result, err := modules.Buckets.Get(s, "distinct-field", params)
-			if err != nil {
-				return err
-			}
-			fmt.Println(result)
-			return nil
-		}
-		result, err := modules.Buckets.List(s, params)
-		if err != nil {
-			return err
-		}
-		printList(result, modules.Buckets.GetColumns(s))
-		return nil
-	})
-
-	type BucketIdOptions struct {
-		ID string `help:"ID or name of bucket"`
-	}
-	R(&BucketIdOptions{}, "bucket-show", "Id details of bucket", func(s *mcclient.ClientSession, args *BucketIdOptions) error {
-		result, err := modules.Buckets.Get(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	R(&BucketIdOptions{}, "bucket-syncstatus", "Sync bucket statust", func(s *mcclient.ClientSession, args *BucketIdOptions) error {
-		result, err := modules.Buckets.PerformAction(s, args.ID, "syncstatus", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketUpdateOptions struct {
-		ID   string `help:"ID or name of bucket" json:"-"`
-		Name string `help:"new name of bucket" json:"name"`
-		Desc string `help:"Description of bucket" json:"description" token:"desc"`
-	}
-	R(&BucketUpdateOptions{}, "bucket-update", "update bucket", func(s *mcclient.ClientSession, args *BucketUpdateOptions) error {
-		params := jsonutils.Marshal(args)
-		result, err := modules.Buckets.Update(s, args.ID, params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketDeleteOptions struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketDeleteOptions{}, "bucket-delete", "delete bucket", func(s *mcclient.ClientSession, args *BucketDeleteOptions) error {
-		result, err := modules.Buckets.Delete(s, args.ID, nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketCreateOptions struct {
-		NAME        string `help:"name of bucket" json:"name"`
-		CLOUDREGION string `help:"location of bucket" json:"cloudregion"`
-		MANAGER     string `help:"cloud provider" json:"manager"`
-
-		StorageClass string `help:"bucket storage class"`
-		Acl          string `help:"bucket ACL"`
-	}
-	R(&BucketCreateOptions{}, "bucket-create", "Create a bucket", func(s *mcclient.ClientSession, args *BucketCreateOptions) error {
-		params, err := options.StructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.Buckets.Create(s, params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketListObjectsOptions struct {
-		ID           string `help:"ID or name of bucket" json:"-"`
-		Prefix       string `help:"List objects with prefix"`
-		Recursive    bool   `help:"List objects recursively"`
-		Limit        int    `help:"maximal items per request"`
-		PagingMarker string `help:"paging marker"`
-	}
-	R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error {
-		params, err := options.StructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "objects", params)
-		if err != nil {
-			return err
-		}
-
+	cmd := shell.NewResourceCmd(&modules.Buckets)
+	cmd.List(&computeoptions.BucketListOptions{})
+	cmd.GetProperty(&computeoptions.BucketGetPropertyOptions{})
+	cmd.Show(&computeoptions.BucketIdOptions{})
+	cmd.Perform("syncstatus", &computeoptions.BucketIdOptions{})
+	cmd.Update(&computeoptions.BucketUpdateOptions{})
+	cmd.Delete(&computeoptions.BucketIdOptions{})
+	cmd.Create(&computeoptions.BucketCreateOptions{})
+	cmd.GetWithCustomOptionShow("objects", func(data jsonutils.JSONObject, args shell.IGetOpt) {
 		listResult := printutils.ListResult{}
-		err = result.Unmarshal(&listResult)
+		err := data.Unmarshal(&listResult)
 		if err != nil {
-			return err
+			return
 		}
 		printList(&listResult, []string{})
-		return nil
-	})
+	}, &computeoptions.BucketListObjectsOptions{})
+	cmd.Perform("delete", &computeoptions.BucketDeleteObjectsOptions{})
+	cmd.Perform("makedir", &computeoptions.BucketMakeDirOptions{})
+	cmd.Perform("temp-url", &computeoptions.BucketPresignObjectsOptions{})
+	cmd.Perform("acl", &computeoptions.BucketSetAclOptions{})
+	cmd.GetWithCustomOptionShow("acl", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketAclOptions{})
+	cmd.Perform("sync", &computeoptions.BucketSyncOptions{})
+	cmd.Perform("limit", &computeoptions.BucketLimitOptions{})
+	cmd.GetWithCustomOptionShow("access-info", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketAccessInfoOptions{})
+	cmd.Perform("metadata", &computeoptions.BucketSetMetadataOptions{})
+	cmd.Perform("set-website", &computeoptions.BucketSetWebsiteOption{})
+	cmd.GetWithCustomOptionShow("website", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketGetWebsiteConfOption{})
+	cmd.Perform("delete-website", &computeoptions.BucketDeleteWebsiteConfOption{})
+	cmd.Perform("set-cors", &computeoptions.BucketSetCorsOption{})
+	cmd.GetWithCustomOptionShow("cors", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketGetCorsOption{})
+	cmd.Perform("delete-cors", &computeoptions.BucketDeleteCorsOption{})
+	cmd.Perform("set-referer", &computeoptions.BucketSetRefererOption{})
+	cmd.GetWithCustomOptionShow("referer", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketGetRefererOption{})
+	cmd.GetWithCustomOptionShow("cdn-domain", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketGetCdnDomainOption{})
+	cmd.GetWithCustomOptionShow("policy", func(data jsonutils.JSONObject, args shell.IGetOpt) {
+		printObject(data)
+	}, &computeoptions.BucketGetPolicyOption{})
+	cmd.Perform("set-policy", &computeoptions.BucketSetPolicyOption{})
+	cmd.Perform("delete-policy", &computeoptions.BucketDeletePolicyOption{})
 
-	type BucketDeleteObjectsOptions struct {
-		ID   string   `help:"ID or name of bucket" json:"-"`
-		KEYS []string `help:"List of objects to delete"`
-	}
-	R(&BucketDeleteObjectsOptions{}, "bucket-object-delete", "Delete objects in a bucket", func(s *mcclient.ClientSession, args *BucketDeleteObjectsOptions) error {
-		params := jsonutils.Marshal(args)
-		result, err := modules.Buckets.PerformAction(s, args.ID, "delete", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketMakeDirOptions struct {
-		ID  string `help:"ID or name of bucket" json:"-"`
-		KEY string `help:"DIR key to create"`
-	}
-	R(&BucketMakeDirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(s *mcclient.ClientSession, args *BucketMakeDirOptions) error {
-		params := jsonutils.Marshal(args)
-		result, err := modules.Buckets.PerformAction(s, args.ID, "makedir", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketUploadObjectsOptions struct {
-		ID   string `help:"ID or name of bucket" json:"-"`
-		KEY  string `help:"Key of object to upload"`
-		Path string `help:"Path to file to upload" required:"true"`
-
-		ContentLength int64  `help:"Content lenght (bytes)" default:"-1"`
-		StorageClass  string `help:"storage CLass"`
-		Acl           string `help:"object acl." choices:"private|public-read|public-read-write"`
-
-		objectstore.ObjectHeaderOptions
-	}
-	R(&BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *BucketUploadObjectsOptions) error {
+	R(&computeoptions.BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *computeoptions.BucketUploadObjectsOptions) error {
 		var body io.Reader
 		if len(args.Path) > 0 {
 			file, err := os.Open(args.Path)
@@ -223,332 +116,36 @@ func init() {
 		return nil
 	})
 
-	type BucketPresignObjectsOptions struct {
-		ID            string `help:"ID or name of bucket" json:"-"`
-		KEY           string `help:"Key of object to upload"`
-		Method        string `help:"Request method" choices:"GET|PUT|DELETE"`
-		ExpireSeconds int    `help:"expire in seconds" default:"60"`
-	}
-	R(&BucketPresignObjectsOptions{}, "bucket-object-tempurl", "Get temporal URL for an object in a bucket", func(s *mcclient.ClientSession, args *BucketPresignObjectsOptions) error {
-		params, err := options.StructToParams(args)
+	R(&computeoptions.BucketPerfMonOptions{}, "bucket-perf-mon", "Bucket performance monitor", func(s *mcclient.ClientSession, args *computeoptions.BucketPerfMonOptions) error {
+		result, err := modules.Buckets.Get(s, args.ID, nil)
 		if err != nil {
 			return err
 		}
-		result, err := modules.Buckets.PerformAction(s, args.ID, "temp-url", params)
+		bucketDetails := compute.BucketDetails{}
+		err = result.Unmarshal(&bucketDetails)
 		if err != nil {
 			return err
 		}
-		printObject(result)
-		return nil
-	})
 
-	type BucketSetAclOptions struct {
-		ID  string   `help:"ID or name of bucket" json:"-"`
-		ACL string   `help:"ACL to set" choices:"default|private|public-read|public-read-write" json:"acl"`
-		Key []string `help:"Optional object key" json:"key"`
-	}
-	R(&BucketSetAclOptions{}, "bucket-set-acl", "Set ACL of bucket or object", func(s *mcclient.ClientSession, args *BucketSetAclOptions) error {
-		params := jsonutils.Marshal(args)
-		result, err := modules.Buckets.PerformAction(s, args.ID, "acl", params)
+		bucket, err := modules.GetIBucket(s.GetContext(), s, &bucketDetails)
 		if err != nil {
 			return err
 		}
-		printObject(result)
-		return nil
-	})
 
-	type BucketAclOptions struct {
-		ID  string `help:"ID or name of bucket" json:"-"`
-		Key string `help:"Optional object key"`
-	}
-	R(&BucketAclOptions{}, "bucket-acl", "Get ACL of bucket or object", func(s *mcclient.ClientSession, args *BucketAclOptions) error {
-		params, err := options.StructToParams(args)
+		payload, err := fileutils.GetSizeBytes(args.Payload, 1024)
 		if err != nil {
 			return err
 		}
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "acl", params)
+
+		stats, err := modules.ProbeBucketStats(s.GetContext(), bucket, "test", int64(payload))
 		if err != nil {
 			return err
 		}
-		printObject(result)
-		return nil
-	})
 
-	type BucketSyncOptions struct {
-		ID        string `help:"ID or name of bucket" json:"-"`
-		StatsOnly bool   `help:"sync statistics only"`
-	}
-	R(&BucketSyncOptions{}, "bucket-sync", "Sync bucket", func(s *mcclient.ClientSession, args *BucketSyncOptions) error {
-		params, err := options.StructToParams(args)
-		if err != nil {
-			return err
-		}
-		result, err := modules.Buckets.PerformAction(s, args.ID, "sync", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
+		fmt.Printf("Upload delay %f ms throughput %f MB/s\n", stats.UploadDelayMs(), stats.UploadThroughputMbps(payload/1024/1024))
+		fmt.Printf("Download delay %f ms throughput %f MB/s\n", stats.DownloadDelayMs(), stats.DownloadThroughputMbps(payload/1024/1024))
+		fmt.Printf("Delete delay %f ms\n", stats.DeleteDelayMs())
 
-	type BucketLimitOptions struct {
-		ID          string `help:"ID or name of bucket" json:"-"`
-		SizeBytes   int64  `help:"size limit in bytes"`
-		ObjectCount int64  `help:"object count limit"`
-	}
-	R(&BucketLimitOptions{}, "bucket-limit", "Set limit of bucket", func(s *mcclient.ClientSession, args *BucketLimitOptions) error {
-		limit := jsonutils.Marshal(args)
-		params := jsonutils.NewDict()
-		params.Set("limit", limit)
-		result, err := modules.Buckets.PerformAction(s, args.ID, "limit", params)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketAccessInfoOptions struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketAccessInfoOptions{}, "bucket-access-info", "Show backend access info of a bucket", func(s *mcclient.ClientSession, args *BucketAccessInfoOptions) error {
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "access-info", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketSetMetadataOptions struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-
-		Key []string `help:"Optional object key" json:"key"`
-
-		objectstore.ObjectHeaderOptions
-	}
-	R(&BucketSetMetadataOptions{}, "bucket-set-metadata", "Set metadata of object", func(s *mcclient.ClientSession, args *BucketSetMetadataOptions) error {
-		input := api.BucketMetadataInput{}
-		input.Key = args.Key
-		input.Metadata = args.ObjectHeaderOptions.Options2Header()
-		err := input.Validate()
-		if err != nil {
-			return err
-		}
-		result, err := modules.Buckets.PerformAction(s, args.ID, "metadata", jsonutils.Marshal(input))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketSetWebsiteOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-		// 主页
-		Index string `help:"main page"`
-		// 错误时返回的文档
-		ErrorDocument string `help:"error return"`
-		// http或https
-		Protocol string `help:"force https" choices:"http|https"`
-	}
-	R(&BucketSetWebsiteOption{}, "bucket-set-website", "Set bucket website", func(s *mcclient.ClientSession, args *BucketSetWebsiteOption) error {
-		conf := api.BucketWebsiteConf{
-			Index:         args.Index,
-			ErrorDocument: args.ErrorDocument,
-			Protocol:      args.Protocol,
-		}
-		result, err := modules.Buckets.PerformAction(s, args.ID, "set-website", jsonutils.Marshal(conf))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketGetWebsiteConfOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketGetWebsiteConfOption{}, "bucket-get-website", "Get bucket website", func(s *mcclient.ClientSession, args *BucketGetWebsiteConfOption) error {
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "website", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketDeleteWebsiteConfOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketDeleteWebsiteConfOption{}, "bucket-delete-website", "Delete bucket website", func(s *mcclient.ClientSession, args *BucketDeleteWebsiteConfOption) error {
-		result, err := modules.Buckets.PerformAction(s, args.ID, "delete-website", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketSetCorsOption struct {
-		ID             string   `help:"ID or name of bucket" json:"-"`
-		AllowedMethods []string `help:"allowed http method" choices:"PUT|GET|POST|DELETE|HEAD"`
-		// 允许的源站，可以设为*
-		AllowedOrigins []string
-		AllowedHeaders []string
-		MaxAgeSeconds  int
-		ExposeHeaders  []string
-		RuleId         string
-	}
-	R(&BucketSetCorsOption{}, "bucket-set-cors", "Set bucket cors", func(s *mcclient.ClientSession, args *BucketSetCorsOption) error {
-
-		rule := api.BucketCORSRule{
-			AllowedOrigins: args.AllowedOrigins,
-			AllowedMethods: args.AllowedMethods,
-			AllowedHeaders: args.AllowedHeaders,
-			MaxAgeSeconds:  args.MaxAgeSeconds,
-			ExposeHeaders:  args.ExposeHeaders,
-			Id:             args.RuleId,
-		}
-		rules := api.BucketCORSRules{Data: []api.BucketCORSRule{rule}}
-		result, err := modules.Buckets.PerformAction(s, args.ID, "set-cors", jsonutils.Marshal(rules))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketGetCorsOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketGetCorsOption{}, "bucket-get-cors", "Get bucket cors", func(s *mcclient.ClientSession, args *BucketGetCorsOption) error {
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "cors", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketDeleteCorsOption struct {
-		ID string   `help:"ID or name of bucket" json:"-"`
-		Id []string `"help:Id of rules to delete"`
-	}
-	R(&BucketDeleteCorsOption{}, "bucket-delete-cors", "Delete bucket cors", func(s *mcclient.ClientSession, args *BucketDeleteCorsOption) error {
-		input := api.BucketCORSRuleDeleteInput{}
-		input.Id = args.Id
-		result, err := modules.Buckets.PerformAction(s, args.ID, "delete-cors", jsonutils.Marshal(input))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketSetRefererOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-		// 域名列表
-		DomainList []string
-		// 是否允许空referer 访问
-		AllowEmptyRefer bool `help:"all empty refer access"`
-		Enabled         bool
-		RerererType     string `help:"Referer type" choices:"Black-List|White-List"`
-	}
-	R(&BucketSetRefererOption{}, "bucket-set-referer", "Set bucket referer", func(s *mcclient.ClientSession, args *BucketSetRefererOption) error {
-		conf := api.BucketRefererConf{
-			Enabled:         args.Enabled,
-			AllowEmptyRefer: args.AllowEmptyRefer,
-			RefererType:     args.RerererType,
-			DomainList:      args.DomainList,
-		}
-		result, err := modules.Buckets.PerformAction(s, args.ID, "set-referer", jsonutils.Marshal(conf))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketGetRefererOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketGetRefererOption{}, "bucket-get-referer", "get bucket referer", func(s *mcclient.ClientSession, args *BucketGetRefererOption) error {
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "referer", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketGetCdnDomainOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketGetRefererOption{}, "bucket-get-cdn-domain", "get bucket cdn domain", func(s *mcclient.ClientSession, args *BucketGetRefererOption) error {
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "cdn-domain", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketGetPolicyOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-	}
-	R(&BucketGetPolicyOption{}, "bucket-get-policy", "get bucket policy", func(s *mcclient.ClientSession, args *BucketGetPolicyOption) error {
-		result, err := modules.Buckets.GetSpecific(s, args.ID, "policy", nil)
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketSetPolicyOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-		// 格式主账号id:子账号id
-		PrincipalId []string `help:"ext account id, accountId:subaccountId"`
-		// Read|ReadWrite|FullControl
-		CannedAction string `help:"authority action" choice:"Read|FullControl"`
-		// Allow|Deny
-		Effect string `help:"allow or deny" choice:"Allow|Deny"`
-		// 被授权的资源地址
-		ResourcePath []string
-		// ip 条件
-		IpEquals    []string
-		IpNotEquals []string
-	}
-	R(&BucketSetPolicyOption{}, "bucket-set-policy", "set bucket policy", func(s *mcclient.ClientSession, args *BucketSetPolicyOption) error {
-		opts := api.BucketPolicyStatementInput{}
-		opts.CannedAction = args.CannedAction
-		opts.Effect = args.Effect
-		opts.IpEquals = args.IpEquals
-		opts.IpNotEquals = args.IpNotEquals
-		opts.ResourcePath = args.ResourcePath
-		opts.PrincipalId = args.PrincipalId
-
-		result, err := modules.Buckets.PerformAction(s, args.ID, "set-policy", jsonutils.Marshal(opts))
-		if err != nil {
-			return err
-		}
-		printObject(result)
-		return nil
-	})
-
-	type BucketDeletePolicyOption struct {
-		ID string `help:"ID or name of bucket" json:"-"`
-		Id []string
-	}
-	R(&BucketDeletePolicyOption{}, "bucket-delete-policy", "delete bucket policy", func(s *mcclient.ClientSession, args *BucketDeletePolicyOption) error {
-		input := api.BucketPolicyDeleteInput{}
-		input.Id = args.Id
-		result, err := modules.Buckets.PerformAction(s, args.ID, "delete-policy", jsonutils.Marshal(input))
-		if err != nil {
-			return err
-		}
-		printObject(result)
 		return nil
 	})
 }

go.mod

@@ -93,12 +93,12 @@ require (
 	k8s.io/cri-api v0.22.17
 	k8s.io/klog/v2 v2.20.0
 	moul.io/http2curl/v2 v2.3.0
-	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2
-	yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256
+	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090
+	yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0
 	yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1
 	yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
-	yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e
+	yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0
 	yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
 	yunion.io/x/sqlchemy v1.1.3-0.20250513031856-ce9f71063b3a
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c

go.sum

@@ -1376,10 +1376,10 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2 h1:Zrqat1z8qiSqZXitZlUOs1uD4xN+pHXkrWir1vYBXr8=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ=
-yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8=
-yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090 h1:E/I9CkZumtHaEBweSYc9bq5rFZ25i+s17bAZskXLGDc=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090/go.mod h1:FXxAEbdNfWXX9gjME3K2nJhkydHY5EKEUZb+RLEzVwQ=
+yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 h1:msG4SiDSVU7CrXH06WuHlNEZXIooTcmNbfrIGHuIHBU=
+yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1 h1:/+THlvf/MvgCW+7KeCDCr33e81KSRa5JmdZ1IIyLOXQ=
 yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
@@ -1391,8 +1391,8 @@ yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVB
 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e h1:OiBxJR18bq5fFK/IZWYRX+5HTMs2KgQJe0a3BZfdIVU=
-yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
+yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0 h1:iKWkBMKazSijYNhOaSh4qBuIu+PmXYhEAMNwrxaXL4Q=
+yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v1.1.3-0.20250513031856-ce9f71063b3a h1:mDW1VyYJxZ4ORITZGDWacUiBNyqX6rMObnQk77NvUOg=

pkg/apigateway/handler/init.go

@@ -15,28 +15,5 @@
 package handler
 
 import (
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/ansible"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudevent"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudid"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudproxy"
-	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/devtool"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/etcd"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/image"
-	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/logger"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/quota"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduledtask"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduler"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/webconsole"
-	_ "yunion.io/x/onecloud/pkg/mcclient/modules/yunionconf"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/loader"
 )
-
-func init() {
-	modules.InitUsages()
-	modules.Usages.RegisterManager(modules.UsageManagerK8s, k8s.Usages)
-}

pkg/apis/compute/bucket.go

@@ -17,6 +17,7 @@ package compute
 import (
 	"net/http"
 	"reflect"
+	"time"
 
 	"yunion.io/x/cloudmux/pkg/apis/compute"
 	"yunion.io/x/cloudmux/pkg/cloudprovider"
@@ -53,6 +54,8 @@ type BucketCreateInput struct {
 	CloudproviderResourceInput
 
 	StorageClass string `json:"storage_class"`
+
+	EnablePerfMon *bool `json:"enable_perf_mon"`
 }
 
 type BucketDetails struct {
@@ -66,28 +69,28 @@ type BucketDetails struct {
 	AccessUrls []cloudprovider.SBucketAccessUrl `json:"access_urls"`
 }
 
-func (self BucketDetails) GetMetricTags() map[string]string {
+func (bucket BucketDetails) GetMetricTags() map[string]string {
 	ret := map[string]string{
-		"id":             self.Id,
-		"brand":          self.Brand,
-		"cloudregion":    self.Cloudregion,
-		"cloudregion_id": self.CloudregionId,
-		"domain_id":      self.DomainId,
-		"oss_id":         self.Id,
-		"oss_name":       self.Name,
-		"project_domain": self.ProjectDomain,
-		"region_ext_id":  self.RegionExtId,
-		"status":         self.Status,
-		"tenant":         self.Project,
-		"tenant_id":      self.ProjectId,
-		"account":        self.Account,
-		"account_id":     self.AccountId,
-		"external_id":    self.ExternalId,
+		"id":             bucket.Id,
+		"brand":          bucket.Brand,
+		"cloudregion":    bucket.Cloudregion,
+		"cloudregion_id": bucket.CloudregionId,
+		"domain_id":      bucket.DomainId,
+		"oss_id":         bucket.Id,
+		"oss_name":       bucket.Name,
+		"project_domain": bucket.ProjectDomain,
+		"region_ext_id":  bucket.RegionExtId,
+		"status":         bucket.Status,
+		"tenant":         bucket.Project,
+		"tenant_id":      bucket.ProjectId,
+		"account":        bucket.Account,
+		"account_id":     bucket.AccountId,
+		"external_id":    bucket.ExternalId,
 	}
-	return AppendMetricTags(ret, self.MetadataResourceInfo, self.ProjectizedResourceInfo)
+	return AppendMetricTags(ret, bucket.MetadataResourceInfo, bucket.ProjectizedResourceInfo)
 }
 
-func (self BucketDetails) GetMetricPairs() map[string]string {
+func (bucket BucketDetails) GetMetricPairs() map[string]string {
 	ret := map[string]string{}
 	return ret
 }
@@ -150,6 +153,8 @@ type BucketSyncstatusInput struct {
 
 type BucketUpdateInput struct {
 	apis.SharableVirtualResourceBaseUpdateInput
+
+	EnablePerfMon *bool `json:"enable_perf_mon"`
 }
 
 type BucketPerformTempUrlInput struct {
@@ -370,3 +375,29 @@ func init() {
 		return &SBackupStorageAccessInfo{}
 	})
 }
+
+type BucketProbeResult struct {
+	UploadTime   time.Duration
+	DownloadTime time.Duration
+	DeleteTime   time.Duration
+}
+
+func (result BucketProbeResult) UploadDelayMs() float64 {
+	return float64(result.UploadTime) / float64(time.Millisecond)
+}
+
+func (result BucketProbeResult) DownloadDelayMs() float64 {
+	return float64(result.DownloadTime) / float64(time.Millisecond)
+}
+
+func (result BucketProbeResult) DeleteDelayMs() float64 {
+	return float64(result.DeleteTime) / float64(time.Millisecond)
+}
+
+func (result BucketProbeResult) UploadThroughputMbps(sizeMBytes int) float64 {
+	return float64(sizeMBytes) * 8 / float64(result.UploadTime.Seconds())
+}
+
+func (result BucketProbeResult) DownloadThroughputMbps(sizeMBytes int) float64 {
+	return float64(sizeMBytes) * 8 / float64(result.DownloadTime.Seconds())
+}

pkg/cloudmon/misc/bucketprobe.go

@@ -0,0 +1,177 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package misc
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	baseoptions "yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func BucketProbe(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if options.Options.EnableBucketProbeDebug {
+		log.Debugf("BucketProbe start")
+	}
+	if !options.Options.EnableBucketProbe {
+		if options.Options.EnableBucketProbeDebug {
+			log.Debugf("BucketProbe is disabled")
+		}
+		return
+	}
+
+	sess := auth.GetSession(ctx, userCred, options.Options.Region)
+
+	metrics, err := gatherBucketMetrics(ctx, sess)
+	if err != nil {
+		log.Errorf("BucketProbe gatherBucketMetrics failed: %s", err)
+		return
+	}
+
+	err = sendMetrics(sess, metrics, "telegraf")
+	if err != nil {
+		log.Errorf("StatusProbe SendMetrics error: %s", err)
+	}
+}
+
+func gatherBucketMetrics(ctx context.Context, sess *mcclient.ClientSession) ([]influxdb.SMetricData, error) {
+	allMetrics := []influxdb.SMetricData{}
+
+	params := baseoptions.BaseListOptions{}
+	params.Scope = "max"
+	limit := 1000
+	params.Limit = &limit
+	params.Filter = []string{
+		"enable_perf_mon.equals(1)",
+	}
+	boolTrue := true
+	params.Details = &boolTrue
+
+	total := -1
+	offset := 0
+	for total < 0 || offset < total {
+		params.Offset = &offset
+		results, err := computemodules.Buckets.List(sess, jsonutils.Marshal(params))
+		if err != nil {
+			return nil, errors.Wrap(err, "computemodules.Buckets.List")
+		}
+		total = results.Total
+		offset = results.Offset + len(results.Data)
+
+		for _, bucket := range results.Data {
+			bucketDetails := computeapi.BucketDetails{}
+			err = bucket.Unmarshal(&bucketDetails)
+			if err != nil {
+				log.Errorf("BucketProbe failed: %s", err)
+				continue
+			}
+
+			metrics, err := probeBucketStats(ctx, sess, &bucketDetails)
+			if err != nil {
+				log.Errorf("BucketProbe failed: %s", err)
+				continue
+			}
+			allMetrics = append(allMetrics, metrics...)
+		}
+	}
+
+	return allMetrics, nil
+}
+
+func probeBucketStats(ctx context.Context, sess *mcclient.ClientSession, bucketDetails *computeapi.BucketDetails) ([]influxdb.SMetricData, error) {
+	bucket, err := computemodules.GetIBucket(ctx, sess, bucketDetails)
+	if err != nil {
+		return nil, errors.Wrap(err, "getIBucket")
+	}
+
+	resultDelay, err := computemodules.ProbeBucketStats(ctx, bucket, options.Options.BucketProbeTestKey, 0)
+	if err != nil {
+		return nil, errors.Wrap(err, "doProbeBucketStats zero")
+	}
+
+	resultRate, err := computemodules.ProbeBucketStats(ctx, bucket, options.Options.BucketProbeTestKey, int64(options.Options.BucketProbeTestSizeMb)*1024*1024)
+	if err != nil {
+		return nil, errors.Wrap(err, "doProbeBucketStats with payload")
+	}
+
+	metricTags := []influxdb.SKeyValue{}
+	for k, v := range bucketDetails.GetMetricTags() {
+		if len(v) == 0 {
+			continue
+		}
+		metricTags = append(metricTags, influxdb.SKeyValue{
+			Key:   k,
+			Value: v,
+		})
+	}
+
+	metrics := []influxdb.SKeyValue{}
+	for k, v := range bucketDetails.GetMetricTags() {
+		if len(v) == 0 {
+			continue
+		}
+		metrics = append(metrics, influxdb.SKeyValue{
+			Key:   k,
+			Value: v,
+		})
+	}
+
+	metrics = append(metrics,
+		influxdb.SKeyValue{
+			Key:   "upload_delay_ms",
+			Value: fmt.Sprintf("%f", resultDelay.UploadDelayMs()),
+		},
+		influxdb.SKeyValue{
+			Key:   "download_delay_ms",
+			Value: fmt.Sprintf("%f", resultDelay.DownloadDelayMs()),
+		},
+		influxdb.SKeyValue{
+			Key:   "delete_delay_ms",
+			Value: fmt.Sprintf("%f", resultDelay.DeleteDelayMs()),
+		},
+		influxdb.SKeyValue{
+			Key:   "upload_rate_mbps",
+			Value: fmt.Sprintf("%f", resultRate.UploadThroughputMbps(options.Options.BucketProbeTestSizeMb)),
+		},
+		influxdb.SKeyValue{
+			Key:   "download_rate_mbps",
+			Value: fmt.Sprintf("%f", resultRate.DownloadThroughputMbps(options.Options.BucketProbeTestSizeMb)),
+		},
+	)
+
+	if options.Options.EnableBucketProbeDebug {
+		log.Debugf("BucketProbe for bucket %s metrics: %s", bucketDetails.Name, jsonutils.Marshal(metrics))
+	}
+
+	return []influxdb.SMetricData{
+		{
+			Name:      "bucket_perf",
+			Tags:      metricTags,
+			Metrics:   metrics,
+			Timestamp: time.Now(),
+		},
+	}, nil
+}

pkg/cloudmon/misc/pinger.go

@@ -110,9 +110,11 @@ func getNetworkAddrMap(s *mcclient.ClientSession, netId string) (map[string]api.
 		return nil, errors.Wrap(err, "GetSpecific addresses")
 	}
 	addrList := make([]api.SNetworkUsedAddress, 0)
-	err = addrListJson.Unmarshal(&addrList, "addresses")
-	if err != nil {
-		return nil, errors.Wrap(err, "Unmarshal addreses")
+	if addrListJson.Contains("addresses") {
+		err = addrListJson.Unmarshal(&addrList, "addresses")
+		if err != nil {
+			return nil, errors.Wrap(err, "Unmarshal addreses")
+		}
 	}
 	addrMap := make(map[string]api.SNetworkUsedAddress)
 	for i := range addrList {

pkg/cloudmon/misc/send.go

@@ -0,0 +1,40 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package misc
+
+import (
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/tsdb"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func sendMetrics(sess *mcclient.ClientSession, metrics []influxdb.SMetricData, database string) error {
+	if len(metrics) == 0 {
+		return nil
+	}
+
+	urls, err := tsdb.GetDefaultServiceSourceURLs(sess, options.Options.SessionEndpointType)
+	if err != nil {
+		return errors.Wrap(err, "GetServiceURLs")
+	}
+	err = influxdb.SendMetrics(urls, database, metrics, false)
+	if err != nil {
+		return errors.Wrap(err, "SendMetrics")
+	}
+	return nil
+}

pkg/cloudmon/misc/statusprobe.go

@@ -0,0 +1,137 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package misc
+
+import (
+	"context"
+	"fmt"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudmon/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
+	baseOptions "yunion.io/x/onecloud/pkg/mcclient/options"
+	"yunion.io/x/onecloud/pkg/util/influxdb"
+)
+
+func StatusProbe(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) {
+	if options.Options.EnableStatusProbeDebug {
+		log.Debugf("Start resource status probe")
+	}
+
+	if !options.Options.EnableStatusProbe {
+		if options.Options.EnableStatusProbeDebug {
+			log.Debugf("Resource status probe is disabled")
+		}
+		return
+	}
+
+	sess := auth.GetSession(ctx, userCred, options.Options.Region)
+	metrics := make([]influxdb.SMetricData, 0)
+	for _, model := range options.Options.StatusProbeModels {
+		mts, err := doModelStatusProbe(sess, model)
+		if err != nil {
+			log.Errorf("doModelStatusProbe failed: %s", err)
+		}
+		metrics = append(metrics, mts...)
+	}
+
+	err := sendMetrics(sess, metrics, "system")
+	if err != nil {
+		log.Errorf("StatusProbe SendMetrics error: %s", err)
+	}
+}
+
+func doModelStatusProbe(sess *mcclient.ClientSession, modelName string) ([]influxdb.SMetricData, error) {
+	model, err := modulebase.GetModule(sess, modelName)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetModule")
+	}
+
+	listOpts := baseOptions.BaseListOptions{}
+	listOpts.Scope = "max"
+	listOpts.SummaryStats = true
+	limit := 0
+	listOpts.Limit = &limit
+
+	results, err := model.List(sess, jsonutils.Marshal(listOpts))
+	if err != nil {
+		return nil, errors.Wrap(err, "List")
+	}
+
+	statusInfoTotal := struct {
+		apis.TotalCountBase
+		StatusInfo []apis.StatusStatisticStatusInfo
+	}{}
+
+	err = results.Totals.Unmarshal(&statusInfoTotal)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unmarshal statusInfoTotal")
+	}
+
+	log.Infof("statusInfoTotal: %s", jsonutils.Marshal(statusInfoTotal))
+
+	metrics := make([]influxdb.SMetricData, 0)
+
+	totalCount := int64(0)
+	pendingDeletedCount := int64(0)
+	for _, statusInfo := range statusInfoTotal.StatusInfo {
+		metrics = append(metrics, genStatusMetricData(model, statusInfo.Status, statusInfo.TotalCount, statusInfo.PendingDeletedCount))
+		totalCount += statusInfo.TotalCount
+		pendingDeletedCount += statusInfo.PendingDeletedCount
+	}
+	metrics = append(metrics, genStatusMetricData(model, "total", totalCount, pendingDeletedCount))
+
+	if options.Options.EnableStatusProbeDebug {
+		log.Debugf("StatusProbe for model %s metrics: %s", modelName, jsonutils.Marshal(metrics))
+	}
+
+	return metrics, nil
+}
+
+func genStatusMetricData(model modulebase.Manager, status string, count int64, pendingDeletedCount int64) influxdb.SMetricData {
+	return influxdb.SMetricData{
+		Name: "status_probe",
+		Tags: influxdb.TKeyValuePairs{
+			influxdb.SKeyValue{
+				Key:   "service",
+				Value: model.ServiceType(),
+			},
+			influxdb.SKeyValue{
+				Key:   "model",
+				Value: model.GetKeyword(),
+			},
+			influxdb.SKeyValue{
+				Key:   "status",
+				Value: status,
+			},
+		},
+		Metrics: influxdb.TKeyValuePairs{
+			influxdb.SKeyValue{
+				Key:   "count",
+				Value: fmt.Sprintf("%d", count),
+			},
+			influxdb.SKeyValue{
+				Key:   "pending_deleted",
+				Value: fmt.Sprintf("%d", pendingDeletedCount),
+			},
+		},
+	}
+}

pkg/cloudmon/options/options.go

@@ -37,6 +37,17 @@ type CloudMonOptions struct {
 	CloudAccountCollectMetricsBatchCount        int `help:"Cloud Account Collect Metrics Batch Count" default:"10"`
 	CloudResourceCollectMetricsBatchCount       int `help:"Cloud Resource Collect Metrics BatchC ount" default:"40"`
 	OracleCloudResourceCollectMetricsBatchCount int `help:"OracleCloud Resource Collect Metrics BatchC ount" default:"1"`
+
+	StatusProbeIntervalMinutes int      `help:"Status Probe Interval unit:minute" default:"15"`
+	StatusProbeModels          []string `help:"Status Probe Models" default:"compute-servers,compute-hosts"`
+	EnableStatusProbe          bool     `help:"Enable Status Probe" default:"false"`
+	EnableStatusProbeDebug     bool     `help:"Enable Status Probe Debug" default:"false"`
+
+	BucketProbeIntervalMinutes int    `help:"Bucket Probe Interval unit:minute" default:"15"`
+	EnableBucketProbe          bool   `help:"Enable Bucket Probe" default:"false"`
+	EnableBucketProbeDebug     bool   `help:"Enable Bucket Probe Debug" default:"false"`
+	BucketProbeTestKey         string `help:"Bucket Probe Test Key" default:"bucket_performance_test_object"`
+	BucketProbeTestSizeMb      int    `help:"Bucket Probe Test Size" default:"4"`
 }
 
 type PingProbeOptions struct {

pkg/cloudmon/service/service.go

@@ -33,6 +33,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudmon/options"
 	"yunion.io/x/onecloud/pkg/cloudmon/resources"
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/loader"
 )
 
 func StartService() {
@@ -57,6 +58,10 @@ func StartService() {
 
 		cron.AddJobAtIntervalsWithStartRun("PingProb", time.Duration(opts.PingProbIntervalHours)*time.Hour, misc.PingProbe, true)
 
+		cron.AddJobAtIntervalsWithStartRun("StatusProbe", time.Duration(opts.StatusProbeIntervalMinutes)*time.Minute, misc.StatusProbe, true)
+
+		cron.AddJobAtIntervalsWithStartRun("BucketProbe", time.Duration(opts.BucketProbeIntervalMinutes)*time.Minute, misc.BucketProbe, true)
+
 		cron.AddJobEveryFewDays("UsageMetricCollect", 1, 23, 10, 10, misc.UsegReport, false)
 		cron.AddJobEveryFewDays("AlertHistoryMetricCollect", 1, 23, 59, 59, misc.AlertHistoryReport, false)
 

pkg/compute/models/buckets.go

@@ -92,6 +92,8 @@ type SBucket struct {
 	ObjectCntLimit int   `nullable:"false" default:"0" list:"user"`
 
 	AccessUrls jsonutils.JSONObject `nullable:"true" list:"user"`
+
+	EnablePerfMon bool `default:"false" list:"user" update:"user" create:"optional"`
 }
 
 func (manager *SBucketManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration {

pkg/mcclient/modules/compute/mod_buckets.go

@@ -15,12 +15,16 @@
 package compute
 
 import (
+	"context"
 	"fmt"
 	"io"
+	"math/rand"
 	"net/http"
 	"strconv"
+	"time"
 
 	"yunion.io/x/cloudmux/pkg/cloudprovider"
+	_ "yunion.io/x/cloudmux/pkg/multicloud/objectstore/provider"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/httputils"
 
@@ -81,3 +85,67 @@ func init() {
 
 	modules.RegisterCompute(&Buckets)
 }
+
+func GetIBucket(ctx context.Context, sess *mcclient.ClientSession, bucketDetails *api.BucketDetails) (cloudprovider.ICloudBucket, error) {
+	provider, err := Cloudproviders.GetProvider(ctx, sess, bucketDetails.ManagerId)
+	if err != nil {
+		return nil, errors.Wrap(err, "computemodules.Cloudproviders.GetProvider")
+	}
+
+	iregion, err := func() (cloudprovider.ICloudRegion, error) {
+		if provider.GetFactory().IsOnPremise() {
+			return provider.GetOnPremiseIRegion()
+		} else {
+			return provider.GetIRegionById(bucketDetails.RegionExternalId)
+		}
+	}()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetIRegion")
+	}
+
+	bucket, err := iregion.GetIBucketById(bucketDetails.ExternalId)
+	if err != nil {
+		return nil, errors.Wrap(err, "iregion.GetIBucketById")
+	}
+
+	return bucket, nil
+}
+
+type nullWriter struct{}
+
+func (w *nullWriter) WriteAt(p []byte, off int64) (n int, err error) {
+	return len(p), nil
+}
+
+func getRandReader() io.Reader {
+	return rand.New(rand.NewSource(time.Now().UnixNano()))
+}
+
+func ProbeBucketStats(ctx context.Context, bucket cloudprovider.ICloudBucket, testKey string, sizeBytes int64) (*api.BucketProbeResult, error) {
+	result := &api.BucketProbeResult{}
+	start := time.Now()
+
+	// force upload object in one shot
+	err := cloudprovider.UploadObject(ctx, bucket, testKey, sizeBytes*2, getRandReader(), sizeBytes, cloudprovider.ACLPrivate, "", nil, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudprovider.UploadObject")
+	}
+
+	result.UploadTime = time.Since(start)
+
+	_, err = cloudprovider.DownloadObjectParallel(ctx, bucket, testKey, nil, &nullWriter{}, 0, 0, false, 1)
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudprovider.DownloadObjectParallel")
+	}
+
+	result.DownloadTime = time.Since(start) - result.UploadTime
+
+	err = bucket.DeleteObject(ctx, testKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.DeleteObject")
+	}
+
+	result.DeleteTime = time.Since(start) - result.DownloadTime
+
+	return result, nil
+}

pkg/mcclient/modules/loader/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loader // import "yunion.io/x/onecloud/pkg/mcclient/modules/loader"

pkg/mcclient/modules/loader/loader.go

@@ -0,0 +1,44 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package loader
+
+import (
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/ansible"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudevent"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudid"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudnet"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/cloudproxy"
+	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/devtool"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/etcd"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/identity"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/image"
+	"yunion.io/x/onecloud/pkg/mcclient/modules/k8s"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/logger"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/monitor"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/notify"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/quota"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduledtask"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/scheduler"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/vpcagent"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/webconsole"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/websocket"
+	_ "yunion.io/x/onecloud/pkg/mcclient/modules/yunionconf"
+)
+
+func init() {
+	modules.InitUsages()
+	modules.Usages.RegisterManager(modules.UsageManagerK8s, k8s.Usages)
+}

pkg/mcclient/options/compute/buckets.go

@@ -0,0 +1,444 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+import (
+	"yunion.io/x/cloudmux/pkg/multicloud/objectstore"
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+type BucketListOptions struct {
+	options.BaseListOptions
+	DistinctField string `help:"query specified distinct field"`
+}
+
+func (opts *BucketListOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.ListStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketGetPropertyOptions struct {
+	DistinctField string `help:"query specified distinct field"`
+}
+
+func (opts *BucketGetPropertyOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.ListStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+func (opts *BucketGetPropertyOptions) Property() string {
+	return "distinct-field"
+}
+
+type BucketIdOptions struct {
+	options.BaseIdOptions
+}
+
+type BucketUpdateOptions struct {
+	BucketIdOptions
+
+	Name          string `help:"new name of bucket" json:"name"`
+	Desc          string `help:"Description of bucket" json:"description" token:"desc"`
+	EnablePerfMon bool   `help:"enable performance monitor" json:"-"`
+}
+
+func (opts *BucketUpdateOptions) Params() (jsonutils.JSONObject, error) {
+	params := jsonutils.Marshal(opts)
+	if opts.EnablePerfMon {
+		params.(*jsonutils.JSONDict).Add(jsonutils.JSONTrue, "enable_perf_mon")
+	}
+	return params, nil
+}
+
+type BucketCreateOptions struct {
+	NAME        string `help:"name of bucket" json:"name"`
+	CLOUDREGION string `help:"location of bucket" json:"cloudregion"`
+	MANAGER     string `help:"cloud provider" json:"manager"`
+
+	StorageClass string `help:"bucket storage class"`
+	Acl          string `help:"bucket ACL"`
+
+	EnablePerfMon bool `help:"enable performance monitor"`
+}
+
+func (opts *BucketCreateOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketListObjectsOptions struct {
+	BucketIdOptions
+
+	Prefix       string `help:"List objects with prefix"`
+	Recursive    bool   `help:"List objects recursively"`
+	Limit        int    `help:"maximal items per request"`
+	PagingMarker string `help:"paging marker"`
+}
+
+func (opts *BucketListObjectsOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.ListStructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketDeleteObjectsOptions struct {
+	BucketIdOptions
+
+	KEYS []string `help:"List of objects to delete"`
+}
+
+func (opts *BucketDeleteObjectsOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketMakeDirOptions struct {
+	BucketIdOptions
+
+	KEY string `help:"DIR key to create"`
+}
+
+func (opts *BucketMakeDirOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketPresignObjectsOptions struct {
+	BucketIdOptions
+
+	KEY           string `help:"Key of object to upload"`
+	Method        string `help:"Request method" choices:"GET|PUT|DELETE"`
+	ExpireSeconds int    `help:"expire in seconds" default:"60"`
+}
+
+func (opts *BucketPresignObjectsOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketSetAclOptions struct {
+	BucketIdOptions
+
+	ACL string   `help:"ACL to set" choices:"default|private|public-read|public-read-write" json:"acl"`
+	Key []string `help:"Optional object key" json:"key"`
+}
+
+func (opts *BucketSetAclOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketAclOptions struct {
+	BucketIdOptions
+
+	Key string `help:"Optional object key"`
+}
+
+func (opts *BucketAclOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketSyncOptions struct {
+	BucketIdOptions
+
+	StatsOnly bool `help:"sync statistics only"`
+}
+
+func (opts *BucketSyncOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketLimitOptions struct {
+	BucketIdOptions
+
+	SizeBytes   int64 `help:"size limit in bytes"`
+	ObjectCount int64 `help:"object count limit"`
+}
+
+func (opts *BucketLimitOptions) Params() (jsonutils.JSONObject, error) {
+	limit := jsonutils.Marshal(opts)
+	params := jsonutils.NewDict()
+	params.Set("limit", limit)
+	return params, nil
+}
+
+type BucketAccessInfoOptions struct {
+	BucketIdOptions
+}
+
+func (opts *BucketAccessInfoOptions) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketSetMetadataOptions struct {
+	BucketIdOptions
+
+	Key []string `help:"Optional object key" json:"key"`
+
+	objectstore.ObjectHeaderOptions
+}
+
+func (opts *BucketSetMetadataOptions) Params() (jsonutils.JSONObject, error) {
+	input := compute.BucketMetadataInput{}
+	input.Key = opts.Key
+	input.Metadata = opts.ObjectHeaderOptions.Options2Header()
+	err := input.Validate()
+	if err != nil {
+		return nil, err
+	}
+	return jsonutils.Marshal(input), nil
+}
+
+type BucketSetWebsiteOption struct {
+	BucketIdOptions
+	// 主页
+	Index string `help:"main page"`
+	// 错误时返回的文档
+	ErrorDocument string `help:"error return"`
+	// http或https
+	Protocol string `help:"force https" choices:"http|https"`
+}
+
+func (opts *BucketSetWebsiteOption) Params() (jsonutils.JSONObject, error) {
+	conf := compute.BucketWebsiteConf{
+		Index:         opts.Index,
+		ErrorDocument: opts.ErrorDocument,
+		Protocol:      opts.Protocol,
+	}
+	return jsonutils.Marshal(conf), nil
+}
+
+type BucketGetWebsiteConfOption struct {
+	BucketIdOptions
+}
+
+func (opts *BucketGetWebsiteConfOption) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketDeleteWebsiteConfOption struct {
+	BucketIdOptions
+}
+
+func (opts *BucketDeleteWebsiteConfOption) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketSetCorsOption struct {
+	BucketIdOptions
+
+	AllowedMethods []string `help:"allowed http method" choices:"PUT|GET|POST|DELETE|HEAD"`
+	// 允许的源站，可以设为*
+	AllowedOrigins []string
+	AllowedHeaders []string
+	MaxAgeSeconds  int
+	ExposeHeaders  []string
+	RuleId         string
+}
+
+func (args *BucketSetCorsOption) Params() (jsonutils.JSONObject, error) {
+	rule := compute.BucketCORSRule{
+		AllowedOrigins: args.AllowedOrigins,
+		AllowedMethods: args.AllowedMethods,
+		AllowedHeaders: args.AllowedHeaders,
+		MaxAgeSeconds:  args.MaxAgeSeconds,
+		ExposeHeaders:  args.ExposeHeaders,
+		Id:             args.RuleId,
+	}
+	rules := compute.BucketCORSRules{Data: []compute.BucketCORSRule{rule}}
+	return jsonutils.Marshal(rules), nil
+}
+
+type BucketGetCorsOption struct {
+	BucketIdOptions
+}
+
+func (opts *BucketGetCorsOption) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketDeleteCorsOption struct {
+	BucketIdOptions
+
+	RuleId []string `help:"Id of rules to delete"`
+}
+
+func (opts *BucketDeleteCorsOption) Params() (jsonutils.JSONObject, error) {
+	input := compute.BucketCORSRuleDeleteInput{}
+	input.Id = opts.RuleId
+	return jsonutils.Marshal(input), nil
+}
+
+type BucketSetRefererOption struct {
+	BucketIdOptions
+	// 域名列表
+	DomainList []string
+	// 是否允许空referer 访问
+	AllowEmptyRefer bool `help:"all empty refer access"`
+	Enabled         bool
+	RerererType     string `help:"Referer type" choices:"Black-List|White-List"`
+}
+
+func (args *BucketSetRefererOption) Params() (jsonutils.JSONObject, error) {
+	conf := compute.BucketRefererConf{
+		Enabled:         args.Enabled,
+		AllowEmptyRefer: args.AllowEmptyRefer,
+		RefererType:     args.RerererType,
+		DomainList:      args.DomainList,
+	}
+	return jsonutils.Marshal(conf), nil
+}
+
+type BucketGetRefererOption struct {
+	BucketIdOptions
+}
+
+func (opts *BucketGetRefererOption) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketGetCdnDomainOption struct {
+	BucketIdOptions
+}
+
+func (opts *BucketGetCdnDomainOption) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketGetPolicyOption struct {
+	BucketIdOptions
+}
+
+func (opts *BucketGetPolicyOption) Params() (jsonutils.JSONObject, error) {
+	params, err := options.StructToParams(opts)
+	if err != nil {
+		return nil, err
+	}
+	return params, nil
+}
+
+type BucketSetPolicyOption struct {
+	BucketIdOptions
+	// 格式主账号id:子账号id
+	PrincipalId []string `help:"ext account id, accountId:subaccountId"`
+	// Read|ReadWrite|FullControl
+	CannedAction string `help:"authority action" choice:"Read|FullControl"`
+	// Allow|Deny
+	Effect string `help:"allow or deny" choice:"Allow|Deny"`
+	// 被授权的资源地址
+	ResourcePath []string
+	// ip 条件
+	IpEquals    []string
+	IpNotEquals []string
+}
+
+func (args *BucketSetPolicyOption) Params() (jsonutils.JSONObject, error) {
+	opts := compute.BucketPolicyStatementInput{}
+	opts.CannedAction = args.CannedAction
+	opts.Effect = args.Effect
+	opts.IpEquals = args.IpEquals
+	opts.IpNotEquals = args.IpNotEquals
+	opts.ResourcePath = args.ResourcePath
+	opts.PrincipalId = args.PrincipalId
+	return jsonutils.Marshal(opts), nil
+}
+
+type BucketDeletePolicyOption struct {
+	BucketIdOptions
+	PolicyId []string `help:"policy id to delete"`
+}
+
+func (args *BucketDeletePolicyOption) Params() (jsonutils.JSONObject, error) {
+	input := compute.BucketPolicyDeleteInput{}
+	input.Id = args.PolicyId
+	return jsonutils.Marshal(input), nil
+}
+
+type BucketUploadObjectsOptions struct {
+	BucketIdOptions
+
+	KEY  string `help:"Key of object to upload"`
+	Path string `help:"Path to file to upload" required:"true"`
+
+	ContentLength int64  `help:"Content lenght (bytes)" default:"-1"`
+	StorageClass  string `help:"storage CLass"`
+	Acl           string `help:"object acl." choices:"private|public-read|public-read-write"`
+
+	objectstore.ObjectHeaderOptions
+}
+
+type BucketPerfMonOptions struct {
+	BucketIdOptions
+
+	Payload string `help:"test payload in bytes, e.g. 1, 32, 1024M" default:"4M"`
+}

pkg/monitor/dbinit/measurements/all.go

@@ -35,6 +35,7 @@ var All = []SMeasurement{
 	serviceHttpCode,
 	serviceProcessStats,
 	dbStats,
+	statusProbe,
 
 	vmCpu,
 	vmMem,
@@ -83,6 +84,7 @@ var All = []SMeasurement{
 	ossLatency,
 	ossNetio,
 	ossReq,
+	ossPerfMon,
 
 	ping,
 

pkg/monitor/dbinit/measurements/oss.go

@@ -60,3 +60,41 @@ var ossReq = SMeasurement{
 		},
 	},
 }
+
+var ossPerfMon = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			Name:         "bucket_perf",
+			DisplayName:  "Object storage bucket performance monitor",
+			ResourceType: monitor.METRIC_RES_TYPE_OSS,
+			Database:     monitor.METRIC_DATABASE_TELE,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			Name:        "upload_delay_ms",
+			DisplayName: "Bucket upload delay in milliseconds",
+			Unit:        monitor.METRIC_UNIT_MS,
+		},
+		{
+			Name:        "download_delay_ms",
+			DisplayName: "Bucket download delay in milliseconds",
+			Unit:        monitor.METRIC_UNIT_MS,
+		},
+		{
+			Name:        "delete_delay_ms",
+			DisplayName: "Bucket delete delay in milliseconds",
+			Unit:        monitor.METRIC_UNIT_MS,
+		},
+		{
+			Name:        "upload_rate_mbps",
+			DisplayName: "Bucket upload rate in megabits per second",
+			Unit:        monitor.METRIC_UNIT_MBPS,
+		},
+		{
+			Name:        "download_rate_mbps",
+			DisplayName: "Bucket download rate in megabits per second",
+			Unit:        monitor.METRIC_UNIT_MBPS,
+		},
+	},
+}

pkg/monitor/dbinit/measurements/service.go

@@ -38,3 +38,19 @@ var worker = SMeasurement{
 		},
 	},
 }
+
+var statusProbe = SMeasurement{
+	Context: []SMonitorContext{
+		{
+			"status_probe", "Resource status probe results", monitor.METRIC_RES_TYPE_SYSTEM, monitor.METRIC_DATABASE_SYSTEM,
+		},
+	},
+	Metrics: []SMetric{
+		{
+			"count", "Resouce count for each status", monitor.METRIC_UNIT_NULL,
+		},
+		{
+			"pending_deleted", "Pending deleted resource count for each status", monitor.METRIC_UNIT_NULL,
+		},
+	},
+}

vendor/modules.txt

@@ -1785,7 +1785,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.2.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
-# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250515074509-c7dba5d8b5e2
+# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250519035927-2b74cec67090
 ## explicit; go 1.21
 yunion.io/x/cloudmux/pkg/apis
 yunion.io/x/cloudmux/pkg/apis/billing
@@ -1865,7 +1865,7 @@ yunion.io/x/cloudmux/pkg/multicloud/volcengine
 yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider
 yunion.io/x/cloudmux/pkg/multicloud/zstack
 yunion.io/x/cloudmux/pkg/multicloud/zstack/provider
-# yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256
+# yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0
 ## explicit; go 1.12
 yunion.io/x/executor/apis
 yunion.io/x/executor/client
@@ -1882,7 +1882,7 @@ yunion.io/x/log/hooks
 yunion.io/x/ovsdb/cli_util
 yunion.io/x/ovsdb/schema/ovn_nb
 yunion.io/x/ovsdb/types
-# yunion.io/x/pkg v1.10.4-0.20250513085810-efeaaca81a0e
+# yunion.io/x/pkg v1.10.4-0.20250519013345-54017bf6c1f0
 ## explicit; go 1.18
 yunion.io/x/pkg/appctx
 yunion.io/x/pkg/errors

vendor/yunion.io/x/cloudmux/pkg/cloudprovider/objectstore.go

@@ -1154,6 +1154,11 @@ func DownloadObjectParallelWithProgress(ctx context.Context, bucket ICloudBucket
 		blocksz = MAX_PUT_OBJECT_SIZEBYTES
 	}
 	sizeBytes := obj.GetSizeBytes()
+	if sizeBytes < 0 {
+		return 0, errors.Wrapf(errors.ErrServer, "object size is negative (%d)", sizeBytes)
+	} else if sizeBytes == 0 {
+		return 0, nil
+	}
 	if rangeOpt == nil {
 		rangeOpt = &SGetObjectRange{
 			Start: 0,

vendor/yunion.io/x/executor/server/server.go

@@ -198,6 +198,21 @@ func (e *Executor) SendInput(s apis.Executor_SendInputServer) error {
 	for {
 		input, err := s.Recv()
 		if err == io.EOF {
+			if input != nil && m == nil {
+				icm, ok := cmds.Load(input.Sn)
+				if !ok {
+					return errors.Errorf("unknown sn %d", input.Sn)
+				}
+				m = icm.(*Commander)
+				if m.stdin == nil {
+					return errors.New("Process stdin not init")
+				}
+			}
+			if m != nil {
+				if e := m.stdin.Close(); e != nil {
+					return errors.Wrap(e, "close stdin")
+				}
+			}
 			return s.SendAndClose(&apis.Error{})
 		} else if err != nil {
 			return s.SendAndClose(&apis.Error{

vendor/yunion.io/x/pkg/util/fileutils/filesize.go

@@ -63,3 +63,7 @@ func GetSizeKb(sizeStr string, defaultSize byte, base int) (int, error) {
 	size, err := parseSizeStr(sizeStr, defaultSize, base)
 	return size / base, err
 }
+
+func GetSizeBytes(sizeStr string, base int) (int, error) {
+	return parseSizeStr(sizeStr, 'B', base)
+}

vendor/yunion.io/x/pkg/util/streamutils/streamutils.go

@@ -33,21 +33,30 @@ type SStreamProperty struct {
 type sXZReadAheadReader struct {
 	offset   int64
 	header   []byte
+	hdrEof   bool
 	upstream io.Reader
 }
 
 func newXZReadAheadReader(stream io.Reader) (*sXZReadAheadReader, error) {
 	xzHdr := make([]byte, xz.HeaderLen)
 	n, err := stream.Read(xzHdr)
+	hdrEof := false
 	if err != nil {
-		return nil, errors.Wrap(err, "Read XZ hader")
-	}
-	if n != len(xzHdr) {
-		return nil, errors.Wrap(errors.ErrEOF, "too few header bytes")
+		if errors.Cause(err) == io.EOF {
+			// delay the EOF
+			hdrEof = true
+			xzHdr = xzHdr[:n]
+		} else {
+			return nil, errors.Wrap(err, "Read XZ header")
+		}
+	} else if n != len(xzHdr) {
+		hdrEof = true
+		xzHdr = xzHdr[:n]
 	}
 	return &sXZReadAheadReader{
 		offset:   0,
 		header:   xzHdr,
+		hdrEof:   hdrEof,
 		upstream: stream,
 	}, nil
 }
@@ -57,6 +66,7 @@ func (s *sXZReadAheadReader) IsXz() bool {
 }
 
 func (s *sXZReadAheadReader) Read(buf []byte) (int, error) {
+	bufOffset := 0
 	if s.offset < int64(len(s.header)) {
 		// read from header
 		rdSize := len(s.header) - int(s.offset)
@@ -65,12 +75,19 @@ func (s *sXZReadAheadReader) Read(buf []byte) (int, error) {
 		}
 		n := copy(buf, s.header[s.offset:s.offset+int64(rdSize)])
 		s.offset += int64(n)
-		return n, nil
-	} else {
-		n, err := s.upstream.Read(buf)
-		s.offset += int64(n)
-		return n, err
+		bufOffset = n
 	}
+	// read buffer is full
+	if bufOffset >= len(buf) {
+		return bufOffset, nil
+	}
+	if s.offset >= int64(len(s.header)) && s.hdrEof {
+		return bufOffset, io.EOF
+	}
+
+	n, err := s.upstream.Read(buf[bufOffset:])
+	s.offset += int64(n)
+	return n + bufOffset, err
 }
 
 func StreamPipe(upstream io.Reader, writer io.Writer, CalChecksum bool, callback func(savedTotal int64)) (*SStreamProperty, error) {