Merge pull request #1769 from swordqiu/feature/qj-s3gateway

feature: s3gateway

Makefile

@@ -55,6 +55,12 @@ export GO111MODULE:=on
 export CGO_CFLAGS = ${X_CGO_CFLAGS}
 export CGO_LDFLAGS = ${X_CGO_LDFLAGS}
 
+UNAME := $(shell uname)
+
+ifeq ($(UNAME), Linux)
+XARGS_FLAGS = --no-run-if-empty
+endif
+
 all: build
 
 
@@ -116,7 +122,7 @@ clean:
 
 fmt:
 	@$(if $(ONECLOUD_CI_BUILD),:,find) . -type f -name "*.go" -not -path "./_output/*" \
-		-not -path "./vendor/*" | xargs --no-run-if-empty gofmt -s -w
+		-not -path "./vendor/*" | xargs $(XARGS_FLAGS) gofmt -s -w
 
 define depDeprecated
 OneCloud now requires using go-mod for dependency management.  dep target,

cmd/aliyuncli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/aliyun"
@@ -71,7 +70,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/awscli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/aws"
@@ -71,7 +70,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/azurecli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/azure"
@@ -75,7 +74,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/climc/climc.go

@@ -114,7 +114,8 @@ func getSubcommandsParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	fmt.Fprintf(os.Stderr, "Error: %s\n", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/climc/shell/buckets.go

@@ -0,0 +1,199 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"io"
+	"os"
+
+	"yunion.io/x/jsonutils"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/mcclient/options"
+)
+
+func init() {
+	type BucketListOptions struct {
+		options.BaseListOptions
+	}
+	R(&BucketListOptions{}, "bucket-list", "List all buckets", func(s *mcclient.ClientSession, args *BucketListOptions) error {
+		params, err := options.ListStructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.List(s, params)
+		if err != nil {
+			return err
+		}
+		printList(result, modules.Buckets.GetColumns(s))
+		return nil
+	})
+
+	type BucketShowOptions struct {
+		ID string `help:"ID or name of bucket"`
+	}
+	R(&BucketShowOptions{}, "bucket-show", "Show details of bucket", func(s *mcclient.ClientSession, args *BucketShowOptions) error {
+		result, err := modules.Buckets.Get(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketUpdateOptions struct {
+		ID   string `help:"ID or name of bucket" json:"-"`
+		Name string `help:"new name of bucket" json:"name"`
+		Desc string `help:"Description of bucket" json:"description" token:"desc"`
+	}
+	R(&BucketUpdateOptions{}, "bucket-update", "update bucket", func(s *mcclient.ClientSession, args *BucketUpdateOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Buckets.Update(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketDeleteOptions struct {
+		ID string `help:"ID or name of bucket" json:"-"`
+	}
+	R(&BucketDeleteOptions{}, "bucket-delete", "delete bucket", func(s *mcclient.ClientSession, args *BucketDeleteOptions) error {
+		result, err := modules.Buckets.Delete(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketCreateOptions struct {
+		NAME        string `help:"name of bucket" json:"name"`
+		CLOUDREGION string `help:"location of bucket" json:"cloudregion"`
+		MANAGER     string `help:"cloud provider" json:"manager"`
+
+		StorageClass string `help:"bucket storage class"`
+		Acl          string `help:"bucket ACL"`
+	}
+	R(&BucketCreateOptions{}, "bucket-create", "Create a bucket", func(s *mcclient.ClientSession, args *BucketCreateOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketListObjectsOptions struct {
+		ID        string `help:"ID or name of bucket" json:"-"`
+		Prefix    string `help:"List objects with prefix"`
+		Recursive bool   `help:"List objects recursively"`
+	}
+	R(&BucketListObjectsOptions{}, "bucket-object-list", "List objects in a bucket", func(s *mcclient.ClientSession, args *BucketListObjectsOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.GetSpecific(s, args.ID, "objects", params)
+		if err != nil {
+			return err
+		}
+
+		arrays, _ := result.GetArray("objects")
+		listResult := modules.ListResult{Data: arrays}
+		printList(&listResult, []string{})
+		return nil
+	})
+
+	type BucketDeleteObjectsOptions struct {
+		ID   string   `help:"ID or name of bucket" json:"-"`
+		KEYS []string `help:"List of objects to delete"`
+	}
+	R(&BucketDeleteObjectsOptions{}, "bucket-object-delete", "Delete objects in a bucket", func(s *mcclient.ClientSession, args *BucketDeleteObjectsOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Buckets.PerformAction(s, args.ID, "delete", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketMakeDirOptions struct {
+		ID  string `help:"ID or name of bucket" json:"-"`
+		KEY string `help:"DIR key to create"`
+	}
+	R(&BucketMakeDirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(s *mcclient.ClientSession, args *BucketMakeDirOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Buckets.PerformAction(s, args.ID, "makedir", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type BucketUploadObjectsOptions struct {
+		ID   string `help:"ID or name of bucket" json:"-"`
+		KEY  string `help:"Key of object to upload"`
+		Path string `help:"Path to file to upload"`
+
+		ContentType  string `help:"Content type"`
+		StorageClass string `help:"storage CLass"`
+	}
+	R(&BucketUploadObjectsOptions{}, "bucket-object-upload", "Upload an object into a bucket", func(s *mcclient.ClientSession, args *BucketUploadObjectsOptions) error {
+		var body io.Reader
+		if len(args.Path) > 0 {
+			file, err := os.Open(args.Path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+			body = file
+		} else {
+			body = os.Stdin
+		}
+		err := modules.Buckets.Upload(s, args.ID, args.KEY, body, args.ContentType, args.StorageClass)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketPresignObjectsOptions struct {
+		ID            string `help:"ID or name of bucket" json:"-"`
+		KEY           string `help:"Key of object to upload"`
+		Method        string `help:"Request method" choices:"GET|PUT|DELETE"`
+		ExpireSeconds int    `help:"expire in seconds" default:"60"`
+	}
+	R(&BucketPresignObjectsOptions{}, "bucket-object-tempurl", "Get temporal URL for an object in a bucket", func(s *mcclient.ClientSession, args *BucketPresignObjectsOptions) error {
+		params, err := options.StructToParams(args)
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.PerformAction(s, args.ID, "temp-url", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+}

cmd/climc/shell/cloudaccounts.go

@@ -178,6 +178,17 @@ func init() {
 		return nil
 	})
 
+	R(&options.SS3CloudAccountCreateOptions{}, "cloud-account-create-s3", "Create a generaic S3 object storage account", func(s *mcclient.ClientSession, args *options.SS3CloudAccountCreateOptions) error {
+		params := jsonutils.Marshal(args)
+		params.(*jsonutils.JSONDict).Add(jsonutils.NewString("S3"), "provider")
+		result, err := modules.Cloudaccounts.Create(s, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudaccountUpdateOptions struct {
 		ID        string `help:"ID or Name of cloud account"`
 		Name      string `help:"New name to update"`
@@ -316,6 +327,19 @@ func init() {
 		return nil
 	})
 
+	R(&options.SUcloudCloudAccountUpdateOptions{}, "cloud-account-update-ucloud", "update a Ucloud cloud account", func(s *mcclient.ClientSession, args *options.SUcloudCloudAccountUpdateOptions) error {
+		params := jsonutils.Marshal(args).(*jsonutils.JSONDict)
+		if params.Size() == 0 {
+			return InvalidUpdateError()
+		}
+		result, err := modules.Cloudaccounts.Update(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	R(&options.SZStackCloudAccountUpdateOptions{}, "cloud-account-update-zstack", "update a ZStack cloud account", func(s *mcclient.ClientSession, args *options.SZStackCloudAccountUpdateOptions) error {
 		params := jsonutils.Marshal(args).(*jsonutils.JSONDict)
 		if params.Size() == 0 {
@@ -329,6 +353,19 @@ func init() {
 		return nil
 	})
 
+	R(&options.SS3CloudAccountUpdateOptions{}, "cloud-account-update-s3", "update a generic S3 cloud account", func(s *mcclient.ClientSession, args *options.SS3CloudAccountUpdateOptions) error {
+		params := jsonutils.Marshal(args).(*jsonutils.JSONDict)
+		if params.Size() == 0 {
+			return InvalidUpdateError()
+		}
+		result, err := modules.Cloudaccounts.Update(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudaccountShowOptions struct {
 		ID string `help:"ID or Name of cloud account"`
 	}
@@ -483,6 +520,36 @@ func init() {
 		return nil
 	})
 
+	R(&options.SUcloudCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-ucloud", "Update credential of a Ucloud cloud account", func(s *mcclient.ClientSession, args *options.SUcloudCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SZStackCloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-zstack", "Update credential of a ZStack cloud account", func(s *mcclient.ClientSession, args *options.SZStackCloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	R(&options.SS3CloudAccountUpdateCredentialOptions{}, "cloud-account-update-credential-s3", "Update credential of a generic S3 cloud account", func(s *mcclient.ClientSession, args *options.SS3CloudAccountUpdateCredentialOptions) error {
+		params := jsonutils.Marshal(args)
+		result, err := modules.Cloudaccounts.PerformAction(s, args.ID, "update-credential", params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
 	type CloudaccountSyncOptions struct {
 		ID       string   `help:"ID or Name of cloud account"`
 		Force    bool     `help:"Force sync no matter what"`

cmd/cryptool/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -68,7 +67,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/esxicli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/esxi"
@@ -71,7 +70,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/huaweicli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/huawei"
@@ -73,7 +72,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/ipmicli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/baremetal/utils/ipmitool"
@@ -37,8 +36,9 @@ type BaseOptions struct {
 	SUBCOMMAND string `help:"ipmicli subcommand" subcommand:"true"`
 }
 
-func showErrorAndExit(err error) {
-	log.Errorf("%s", err)
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/ldapcli/main.go

@@ -73,7 +73,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/openstackcli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/openstack"
@@ -76,7 +75,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/qcloudcli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/qcloud"
@@ -72,7 +71,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/s3cli/main.go

@@ -0,0 +1,131 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"yunion.io/x/structarg"
+
+	"yunion.io/x/onecloud/pkg/multicloud/objectstore"
+	_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/shell"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+type BaseOptions struct {
+	Debug      bool   `help:"debug mode"`
+	Help       bool   `help:"Show help"`
+	AccessUrl  string `help:"Access url" default:"$S3_ACCESS_URL"`
+	AccessKey  string `help:"Access key" default:"$S3_ACCESS_KEY"`
+	Secret     string `help:"Secret" default:"$S3_SECRET"`
+	SUBCOMMAND string `help:"s3cli subcommand" subcommand:"true"`
+}
+
+func getSubcommandParser() (*structarg.ArgumentParser, error) {
+	parse, e := structarg.NewArgumentParser(&BaseOptions{},
+		"s3cli",
+		"Command-line interface to standard S3 API.",
+		`See "s3cli help COMMAND" for help on a specific command.`)
+
+	if e != nil {
+		return nil, e
+	}
+
+	subcmd := parse.GetSubcommand()
+	if subcmd == nil {
+		return nil, fmt.Errorf("No subcommand argument.")
+	}
+	type HelpOptions struct {
+		SUBCOMMAND string `help:"sub-command name"`
+	}
+	shellutils.R(&HelpOptions{}, "help", "Show help of a subcommand", func(args *HelpOptions) error {
+		helpstr, e := subcmd.SubHelpString(args.SUBCOMMAND)
+		if e != nil {
+			return e
+		} else {
+			fmt.Print(helpstr)
+			return nil
+		}
+	})
+	for _, v := range shellutils.CommandTable {
+		_, e := subcmd.AddSubParser(v.Options, v.Command, v.Desc, v.Callback)
+		if e != nil {
+			return nil, e
+		}
+	}
+	return parse, nil
+}
+
+func showErrorAndExit(e error) {
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
+	os.Exit(1)
+}
+
+func newClient(options *BaseOptions) (*objectstore.SObjectStoreClient, error) {
+	if len(options.AccessUrl) == 0 {
+		return nil, fmt.Errorf("Missing accessUrl")
+	}
+
+	if len(options.AccessKey) == 0 {
+		return nil, fmt.Errorf("Missing accessKey")
+	}
+
+	if len(options.Secret) == 0 {
+		return nil, fmt.Errorf("Missing secret")
+	}
+
+	return objectstore.NewObjectStoreClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
+}
+
+func main() {
+	parser, e := getSubcommandParser()
+	if e != nil {
+		showErrorAndExit(e)
+	}
+	e = parser.ParseArgs(os.Args[1:], false)
+	options := parser.Options().(*BaseOptions)
+
+	if options.Help {
+		fmt.Print(parser.HelpString())
+	} else {
+		subcmd := parser.GetSubcommand()
+		subparser := subcmd.GetSubParser()
+		if e != nil {
+			if subparser != nil {
+				fmt.Print(subparser.Usage())
+			} else {
+				fmt.Print(parser.Usage())
+			}
+			showErrorAndExit(e)
+		} else {
+			suboptions := subparser.Options()
+			if options.SUBCOMMAND == "help" {
+				e = subcmd.Invoke(suboptions)
+			} else {
+				var client *objectstore.SObjectStoreClient
+				client, e = newClient(options)
+				if e != nil {
+					showErrorAndExit(e)
+				}
+				e = subcmd.Invoke(client, suboptions)
+			}
+			if e != nil {
+				showErrorAndExit(e)
+			}
+		}
+	}
+}

cmd/s3gateway/main.go

@@ -0,0 +1,26 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"yunion.io/x/onecloud/pkg/s3gateway/service"
+	"yunion.io/x/onecloud/pkg/util/atexit"
+)
+
+func main() {
+	defer atexit.Handle()
+
+	service.StartService()
+}

cmd/ucloudcli/main.go

@@ -19,7 +19,6 @@ import (
 	"os"
 	"yunion.io/x/onecloud/pkg/util/ucloud"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -73,7 +72,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

cmd/zstackcli/main.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"os"
 
-	"yunion.io/x/log"
 	"yunion.io/x/structarg"
 
 	"yunion.io/x/onecloud/pkg/util/shellutils"
@@ -72,7 +71,8 @@ func getSubcommandParser() (*structarg.ArgumentParser, error) {
 }
 
 func showErrorAndExit(e error) {
-	log.Errorf("%s", e)
+	fmt.Fprintf(os.Stderr, "%s", e)
+	fmt.Fprintln(os.Stderr)
 	os.Exit(1)
 }
 

docs/bucket/bucket.yaml

@@ -0,0 +1,40 @@
+get:
+  summary: 获得指定对象存储桶的详情
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+  responses:
+    200:
+      description: 对象存储桶信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketGetResponse"
+  tags:
+    - buckets
+
+put:
+  summary: 更新指定对象存储桶的信息
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+    - name: bucket
+      in: body
+      required: true
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketPutRequestInput"
+  responses:
+    200:
+      description: 对象存储桶信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketGetResponse"
+  tags:
+    - buckets
+
+delete:
+  summary: 删除指定的对象存储桶
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+  responses:
+    200:
+      description: 被删除的对象存储桶信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketGetResponse"
+  tags:
+    - buckets

docs/bucket/buckets.yaml

@@ -0,0 +1,32 @@
+get:
+  summary: 按指定条件列出对象存储桶
+  parameters:
+    - $ref: '../parameters/common.yaml#/offset'
+    - $ref: '../parameters/common.yaml#/limit'
+    - $ref: '../parameters/common.yaml#/provider'
+    - $ref: '../parameters/common.yaml#/brand'
+    - $ref: '../parameters/common.yaml#/region'
+    - $ref: '../parameters/common.yaml#/account'
+  responses:
+    200:
+      description: 对象存储桶列表信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketListResponse"
+  tags:
+    - buckets
+
+post:
+  summary: 新建对象存储桶
+  parameters:
+    - name: bucket
+      in: body
+      required: true
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketCreateInput"
+  response:
+    200:
+      description: 对象存储桶信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketGetResponse"
+  tags:
+    - buckets

docs/bucket/deleteobjects.yaml

@@ -0,0 +1,16 @@
+post:
+  summary: 删除指定桶下的指定对象
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+    - name: bucket
+      in: body
+      required: true
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketObjectDeleteInput"
+  responses:
+    200:
+      description: 对象删除结果
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketObjectDeleteResponse"
+  tags:
+    - buckets

docs/bucket/listobjects.yaml

@@ -0,0 +1,13 @@
+get:
+  summary: 获得指定对象存储桶的对象列表
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+    - $ref: '../parameters/bucket.yaml#/prefix'
+    - $ref: '../parameters/bucket.yaml#/recursive'
+  responses:
+    200:
+      description: 对象存储桶的对象列表信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketObjectListResponse"
+  tags:
+    - buckets

docs/bucket/makedir.yaml

@@ -0,0 +1,14 @@
+post:
+  summary: 创建一个大小为0的对象，用来做目录的占位
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+    - name: bucket
+      in: body
+      required: true
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketMakedirInput"
+  responses:
+    200:
+      description: 创建成功
+  tags:
+    - buckets

docs/bucket/tempurl.yaml

@@ -0,0 +1,16 @@
+post:
+  summary: 生成访问指定对象的临时URL
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+    - name: bucket
+      in: body
+      required: true
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketObjectTempUrlInput"
+  responses:
+    200:
+      description: 指定对象的临时URL响应
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketObjectTempUrlResponse"
+  tags:
+    - buckets

docs/bucket/uploadobject.yaml

@@ -0,0 +1,12 @@
+post:
+  summary: 上传对象
+  parameters:
+    - $ref: "../parameters/bucket.yaml#/bucket_name"
+    - $ref: "../parameters/bucket.yaml#/x-bucket-object-key"
+    - $ref: "../parameters/bucket.yaml#/x-bucket-content-type"
+    - $ref: "../parameters/bucket.yaml#/x-bucket-storage-class"
+  responses:
+    200:
+      description: 上传成功
+  tags:
+    - buckets

docs/image/image.yaml

@@ -107,6 +107,7 @@ put:
         $ref: "../schemas/image.yaml#/ImageResponse"
   tags:
     - images 
+
 get:
   parameters:
     - in: query

docs/index.yaml

@@ -353,3 +353,18 @@ paths:
     $ref: "./networkinterface/networkinterfaces.yaml"
   /networkinterfaces/{networkinterfaceId}:
     $ref: "./networkinterface/networkinterface.yaml"
+
+  /buckets:
+    $ref: "./bucket/buckets.yaml"
+  /buckets/{bucketName}:
+    $ref: "./bucket/bucket.yaml"
+  /buckets/{bucketName}/objects:
+    $ref: "./bucket/listobjects.yaml"
+  /buckets/{bucketName}/delete:
+    $ref: "./bucket/deleteobjects.yaml"
+  /buckets/{bucketName}/makedir:
+    $ref: "./bucket/makedir.yaml"
+  /buckets/{bucketName}/upload:
+    $ref: "./bucket/uploadobject.yaml"
+  /buckets/{bucketName}/tempurl:
+    $ref: "./bucket/tempurl.yaml"

docs/parameters/bucket.yaml

@@ -0,0 +1,36 @@
+bucket_name:
+  name: bucket_name
+  type: string
+  required: true
+  in: path
+  description: 存储桶的名称
+
+prefix:
+  name: prefix
+  type: string
+  in: query
+  description: 对象的前缀过滤
+
+recursive:
+  name: recursive
+  type: bool
+  in: query
+  description: 是否展开对象列表，false则只显示当前目录层级下的对象，true则显示匹配前缀的所有对象
+
+x-bucket-object-key:
+  name: x-bucket-object-key
+  type: string
+  in: header
+  description: 对象的key
+
+x-bucket-content-type:
+  name: x-bucket-content-type
+  type: string
+  in: header
+  description: 对象的content-type
+
+x-bucket-storage-class:
+  name: x-bucket-storage-class
+  type: string
+  in: header
+  description: 对象的storage_class

docs/schemas/bucket.yaml

@@ -0,0 +1,175 @@
+BucketListResponse:
+  type: object
+  properties:
+    limit:
+      type: integer
+      example: 20
+    offset:
+      type: integer
+      example: 0
+    total:
+      type: integer
+      description: 总量
+    buckets:
+      type: array
+      items:
+        $ref: "#/Bucket"
+
+BucketObjectListResponse:
+  type: object
+  properties:
+    bucket:
+      type: object
+      properties:
+        objects:
+          type: array
+          items:
+            $ref: '#/BucketObject'
+
+BucketObject:
+  type: object
+  properties:
+    key:
+      type: string
+      description: 对象的key, 如果key以"/"结尾，则该key一般是目录占位对象
+    size_bytes:
+      type: integer
+      description: 对象大小
+    last_modified:
+      type: string
+      description: 该对象的最近修改时间
+    etag:
+      type: string
+      description: 该对象的md5 checksum
+    storage_class:
+      type: string
+      description: 该对象的存储等级字符串
+
+BucketGetResponse:
+  type: object
+  properties:
+    bucket:
+      type: object
+      $ref: "#/Bucket"
+
+BucketPutRequestInput:
+  type: object
+  properties:
+    name:
+      type: string
+      description: 存储桶的名称
+    description:
+      type: string
+      description: 存储桶的描述
+
+BucketCreateInput:
+  type: object
+  properties:
+    name:
+      type: string
+      required: true
+      description:  存储桶的名称
+    cloudregion:
+      type: string
+      required: true
+      description: 存储桶所在cloudregion的名称或ID，对于on premise的通用s3存储，cloudregion为default
+    manager:
+      type: string
+      required: true
+      description: 存储桶归属的账号订阅(cloudprovider)的名称或ID
+    storage_class:
+      type: string
+      description: 存储桶的存储类型，可能值为：standard, ia和archive
+    description:
+      type: string
+      description: 存储桶的描述
+
+Bucket:
+  type: object
+  description: 存储桶的描述信息
+  properties:
+    id:
+      type: string
+      description: 存储桶的ID
+      readOnly: true
+    name:
+      type: string
+      description: 对象存储桶的名称，全局唯一。并且名字唯一性受到所在云供应商的桶名字空间唯一性的约束。
+    can_delete:
+      type: boolean
+      description: 是否可以删除
+    storage_class:
+      type: string
+      description: 存储桶的默认存储类型
+    tenant_id:
+      type: string
+      description: 存储桶归属的项目ID
+    domain_id:
+      type: string
+      description: 存储桶归属的项目的域ID
+    location:
+      type: string
+      description: 存储桶的所在区域信息
+    account_id:
+      type: string
+      description: 存储桶的云账号的ID
+
+BucketObjectDeleteInput:
+  type: object
+  properties:
+    keys:
+      type: array
+      items:
+        type: string
+        description: 待删除的对象Key
+
+BucketObjectDeleteResponse:
+  type: object
+  properties:
+    bucket:
+      type: array
+      items:
+        $ref: ""
+
+BucketObjectDeleteInfo:
+  type: object
+  properties:
+    id:
+      type: string
+      description: 对象key
+    status:
+      type: integer
+      description: 删除结果，200为成功，400为失败
+    data:
+      type: string
+      description: 如果status为400，则data为错误原因
+
+BucketObjectTempUrlInput:
+  type: object
+  properties:
+    key:
+      type: string
+      description: 对象key
+    method:
+      type: string
+      description: 请求对象的HTTP方法，例如：GET|PUT|DELETE
+    expire_seconds:
+      type: integer
+      description: 该临时URL的超时时间，单位为秒
+
+BucketObjectTempUrlResponse:
+  type: object
+  properties:
+    bucket:
+      type: object
+      properties:
+        url:
+          type: string
+          description: 生成的临时URL
+
+BucketMakedirInput:
+  type: object
+  properties:
+    key:
+      type: string
+      description: 目录的名称，必需以"/"结尾

go.mod

@@ -52,6 +52,7 @@ require (
 	github.com/gin-contrib/sse v0.0.0-20170109093832-22d885f9ecc7 // indirect
 	github.com/gin-gonic/gin v1.3.0
 	github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 // indirect
+	github.com/go-ini/ini v1.44.0 // indirect
 	github.com/go-logfmt/logfmt v0.4.0 // indirect
 	github.com/go-ole/go-ole v1.2.2 // indirect
 	github.com/go-sql-driver/mysql v1.4.1
@@ -63,6 +64,7 @@ require (
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/golang/protobuf v1.3.1
 	github.com/google/btree v1.0.0 // indirect
+	github.com/google/go-querystring v1.0.0 // indirect
 	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
 	github.com/google/gopacket v1.1.17
 	github.com/google/uuid v1.1.0 // indirect
@@ -95,12 +97,14 @@ require (
 	github.com/mdlayher/raw v0.0.0-20190606144222-a54781e5f38f
 	github.com/mholt/caddy v0.10.11
 	github.com/miekg/dns v1.1.1
+	github.com/minio/minio-go v6.0.14+incompatible
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 // indirect
 	github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect
 	github.com/moul/http2curl v1.0.0
+	github.com/mozillazg/go-httpheader v0.2.1 // indirect
 	github.com/mozillazg/go-pinyin v0.15.0
-	github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80
 	github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 // indirect
 	github.com/opentracing/opentracing-go v1.0.2 // indirect
 	github.com/openzipkin/zipkin-go-opentracing v0.3.4 // indirect
@@ -120,6 +124,7 @@ require (
 	github.com/spf13/pflag v1.0.3 // indirect
 	github.com/stretchr/testify v1.3.0
 	github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e
+	github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f
 	github.com/tinylib/msgp v1.1.0 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
 	github.com/tredoe/osutil v0.0.0-20161130133508-7d3ee1afa71c
@@ -144,6 +149,7 @@ require (
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v8 v8.18.2 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
+	gopkg.in/ini.v1 v1.44.0 // indirect
 	gopkg.in/ldap.v3 v3.0.3
 	gopkg.in/yaml.v2 v2.2.2
 	k8s.io/api v0.0.0-20181004124137-fd83cbc87e76
@@ -156,5 +162,5 @@ require (
 	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
 	yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30
 	yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda
-	yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe
+	yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d
 )

go.sum

@@ -143,6 +143,8 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2 h1:Ujru
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493 h1:OTanQnFt0bi5iLFSdbEVA/idR6Q2WhCm+deb7ir2CcM=
 github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
+github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w=
+github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
 github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80nA=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-ole/go-ole v1.2.2 h1:QNWhweRd9D5Py2rRVboZ2L4SEoW/dyraWJCc8bgS8kE=
@@ -177,6 +179,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
+github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeqr2z77+8R2RKyh8PG66dcu1V0ck=
 github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
 github.com/google/gopacket v1.1.17 h1:rMrlX2ZY2UbvT+sdz3+6J+pp2z+msCq9MxTU6ymxbBY=
@@ -274,6 +278,10 @@ github.com/mholt/caddy v0.10.11 h1:s8X+R8DuBbrrMuUTcWSxlDe567B0s5EDmiDBKSYsioY=
 github.com/mholt/caddy v0.10.11/go.mod h1:Wb1PlT4DAYSqOEd03MsqkdkXnTxA8v9pKjdpxbqM1kY=
 github.com/miekg/dns v1.1.1 h1:DVkblRdiScEnEr0LR9nTnEQqHYycjkXW9bOjd+2EL2o=
 github.com/miekg/dns v1.1.1/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/minio/minio-go v6.0.14+incompatible h1:fnV+GD28LeqdN6vT2XdGKW8Qe/IfjJDswNVuni6km9o=
+github.com/minio/minio-go v6.0.14+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLDQ0W1YjYsBW+p8U2u7vzgW2SQVmlNazg=
@@ -282,12 +290,12 @@ github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9
 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
 github.com/moul/http2curl v1.0.0 h1:dRMWoAtb+ePxMlLkrCbAqh4TlPHXvoGUSQ323/9Zahs=
 github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ=
+github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ=
+github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
 github.com/mozillazg/go-pinyin v0.15.0 h1:sSwlnsogK/WMzcf0HnjgxyAI4GU6LFqwXnhr77q1Z80=
 github.com/mozillazg/go-pinyin v0.15.0/go.mod h1:bO+dztNW6O2lSJdYLha7LO3bujXzjjU3UvKb2IGANfg=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY=
 github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg=
-github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80 h1:1tGm26e9ktdEIa7LHD7xf7oMUCE0V2wP4URkUAvEttA=
-github.com/nelsonken/cos-go-sdk-v5 v0.0.0-20180622024522-5247afdb7a80/go.mod h1:UZaoQ2hntRH4P8MwrKOxcVXvNgg1N4atxfa7NP+1wWg=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492 h1:lM6RxxfUMrYL/f8bWEUqdXrANWtrL7Nndbm9iFN0DlU=
@@ -356,6 +364,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
 github.com/syncthing/syncthing v0.14.48-rc.4/go.mod h1:nw3siZwHPA6M8iSfjDCWQ402eqvEIasMQOE8nFOxy7M=
 github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e h1:CtKVGXKh2bfmepZ/YogAjvL/CrSy9NGZET500K7arf4=
 github.com/tencentcloud/tencentcloud-sdk-go v0.0.0-20181108132626-805d01dd0e2e/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4=
+github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f h1:TzE7Cs9HhTyfot4WIoMnbD1rWfD4Jkwy2M3Zs66CaRE=
+github.com/tencentyun/cos-go-sdk-v5 v0.0.0-20190717101923-c5c1f9751e7f/go.mod h1:/4BhymH1yO6ljUGQgcKsd7L3W+pdKRxoRiOuoZPLnGg=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e h1:T5PdfK/M1xyrHwynxMIVMWLS7f/qHwfslZphxtGnw7s=
 github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e/go.mod h1:XDKHRm5ThF8YJjx001LtgelzsoaEcvnA7lVWz9EeX3g=
 github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
@@ -453,6 +463,8 @@ gopkg.in/go-playground/validator.v8 v8.18.2 h1:lFB4DoMU6B626w8ny76MV7VX6W2VHct2G
 gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y=
 gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc=
 gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.44.0 h1:YRJzTUp0kSYWUVFF5XAbDFfyiqwsl0Vb9R8TVP5eRi0=
+gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ldap.v3 v3.0.3 h1:YKRHW/2sIl05JsCtx/5ZuUueFuJyoj/6+DGXe3wp6ro=
 gopkg.in/ldap.v3 v3.0.3/go.mod h1:oxD7NyBuxchC+SgJDE1Q5Od05eGt29SDQVBmV+HYbzw=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
@@ -486,5 +498,5 @@ yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30 h1:6CkrwtX4xeYFqqpdWtQPAVqEnD
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda h1:i+/3Hh+kVmPZM1P+j3wfViEzb0XlttWyNvDSYA5DDDU=
 yunion.io/x/sqlchemy v0.0.0-20190704155352-6aff6c803fda/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
-yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe h1:Wr2EXv72ynwtW+x0VCqWwpsfUBWecaZWnLMgaP3UiTo=
-yunion.io/x/structarg v0.0.0-20190625074850-3c0636a9fffe/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
+yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d h1:00kGV39weRaYPldUUh5mllj4aHcGMOZZx4m3CotRESw=
+yunion.io/x/structarg v0.0.0-20190717142057-5caf182cbb4d/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/compute/bucket.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package compute
+
+const (
+	BUCKET_STATUS_START_CREATE = "start_create"
+	BUCKET_STATUS_CREATING     = "creating"
+	BUCKET_STATUS_READY        = "ready"
+	BUCKET_STATUS_CREATE_FAIL  = "create_fail"
+	BUCKET_STATUS_START_DELETE = "start_delete"
+	BUCKET_STATUS_DELETING     = "deleting"
+	BUCKET_STATUS_DELETED      = "deleted"
+	BUCKET_STATUS_DELETE_FAIL  = "delete_fail"
+
+	BUCKET_UPLOAD_OBJECT_KEY_HEADER          = "X-Yunion-Bucket-Upload-Key"
+	BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER = "X-Yunion-Bucket-Upload-Storageclass"
+)

pkg/apis/compute/cloudaccount_const.go

@@ -40,6 +40,8 @@ const (
 	CLOUD_PROVIDER_UCLOUD    = "Ucloud"
 	CLOUD_PROVIDER_ZSTACK    = "ZStack"
 
+	CLOUD_PROVIDER_GENERICS3 = "S3"
+
 	CLOUD_PROVIDER_HEALTH_NORMAL        = "normal"        // 远端处于健康状态
 	CLOUD_PROVIDER_HEALTH_INSUFFICIENT  = "insufficient"  // 不足按需资源余额
 	CLOUD_PROVIDER_HEALTH_SUSPENDED     = "suspended"     // 远端处于冻结状态

pkg/apis/image/consts.go

@@ -17,6 +17,8 @@ package image
 type TImageType string
 
 const (
+	SERVICE_TYPE = "image"
+
 	// https://docs.openstack.org/glance/pike/user/statuses.html
 	//
 	IMAGE_STATUS_QUEUED     = "queued"

pkg/apis/s3gateway/consts.go

@@ -0,0 +1,19 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3gateway
+
+const (
+	SERVICE_TYPE = "s3gateway"
+)

pkg/apis/s3gateway/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3gateway // import "yunion.io/x/onecloud/pkg/apis/s3gateway"

pkg/appsrv/appsrv.go

@@ -250,7 +250,8 @@ func (app *Application) defaultHandle(w http.ResponseWriter, r *http.Request, ri
 				to = app.processTimeout
 			}
 			var (
-				ctx    context.Context    = app.context
+				ctx = app.context
+
 				cancel context.CancelFunc = nil
 			)
 			if to > 0 {

pkg/appsrv/handlerinfo.go

@@ -26,27 +26,38 @@ type handlerRequestCounter struct {
 	duration float64
 }
 
+type TProcessTimeoutCallback func(*SHandlerInfo, *http.Request) time.Duration
+
 type SHandlerInfo struct {
-	method         string
-	path           []string
-	name           string
-	handler        func(context.Context, http.ResponseWriter, *http.Request)
-	metadata       map[string]interface{}
-	tags           map[string]string
-	counter2XX     handlerRequestCounter
-	counter4XX     handlerRequestCounter
-	counter5XX     handlerRequestCounter
-	processTimeout time.Duration
-	workerMan      *SWorkerManager
-	skipLog        bool
+	method     string
+	path       []string
+	name       string
+	handler    func(context.Context, http.ResponseWriter, *http.Request)
+	metadata   map[string]interface{}
+	tags       map[string]string
+	counter2XX handlerRequestCounter
+	counter4XX handlerRequestCounter
+	counter5XX handlerRequestCounter
+	workerMan  *SWorkerManager
+	skipLog    bool
+
+	processTimeout         time.Duration
+	processTimeoutCallback TProcessTimeoutCallback
 }
 
 func (this *SHandlerInfo) FetchProcessTimeout(r *http.Request) time.Duration {
-	if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 {
-		return time.Hour * 2
-	} else {
-		return this.processTimeout
+	var tm time.Duration
+	if this.processTimeoutCallback != nil {
+		tm = this.processTimeoutCallback(this, r)
 	}
+	if tm < this.processTimeout {
+		tm = this.processTimeout
+	}
+	return tm
+}
+
+func (this *SHandlerInfo) SetProcessTimeoutCallback(callback TProcessTimeoutCallback) {
+	this.processTimeoutCallback = callback
 }
 
 func (this *SHandlerInfo) GetName(params map[string]string) string {

pkg/cloudcommon/db/interface.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/sqlchemy"
 
+	"time"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
@@ -90,6 +91,8 @@ type IModelManager interface {
 	InitializeData() error
 
 	CustomizeHandlerInfo(info *appsrv.SHandlerInfo)
+	SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration
+
 	FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error)
 	FetchUpdateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error)
 	IsCustomizedGetDetailsBody() bool

pkg/cloudcommon/db/modelbase.go

@@ -22,6 +22,8 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/sqlchemy"
 
+	"log"
+	"time"
 	"yunion.io/x/onecloud/pkg/appsrv"
 	"yunion.io/x/onecloud/pkg/cloudcommon/object"
 	"yunion.io/x/onecloud/pkg/httperrors"
@@ -53,7 +55,11 @@ func NewModelBaseManager(model interface{}, tableName string, keyword string, ke
 }
 
 func (manager *SModelBaseManager) GetIModelManager() IModelManager {
-	return manager.GetVirtualObject().(IModelManager)
+	virt := manager.GetVirtualObject()
+	if virt == nil {
+		log.Fatalf("%s.GetIModelManager got nil!", manager.keywordPlural)
+	}
+	return virt.(IModelManager)
 }
 
 func (manager *SModelBaseManager) SetAlias(alias string, aliasPlural string) {
@@ -203,7 +209,14 @@ func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query
 }
 
 func (manager *SModelBaseManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) {
-	// do nothing
+	info.SetProcessTimeoutCallback(manager.GetIModelManager().SetHandlerProcessTimeout)
+}
+
+func (manager *SModelBaseManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration {
+	if r.Method == http.MethodGet && len(r.URL.Query().Get("export_keys")) > 0 {
+		return time.Hour * 2
+	}
+	return -time.Second
 }
 
 func (manager *SModelBaseManager) FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) {

pkg/cloudcommon/validators/errors.go

@@ -128,6 +128,8 @@ func newError(typ ErrType, errFmt string, params ...interface{}) error {
 			return nil
 		case ERR_GENERAL, ERR_MODEL_MANAGER:
 			return httperrors.NewInternalServerError(errFmt, params...)
+		case ERR_MODEL_NOT_FOUND:
+			return httperrors.NewResourceNotFoundError(errFmt, params...)
 		default:
 			return httperrors.NewInputParameterError(errFmt, params...)
 		}

pkg/cloudcommon/validators/validators.go

@@ -21,6 +21,7 @@ package validators
 // uri
 
 import (
+	"database/sql"
 	"math"
 	"net"
 	"reflect"
@@ -35,6 +36,7 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/util/choices"
 )
@@ -472,7 +474,11 @@ func (v *ValidatorModelIdOrName) validate(data *jsonutils.JSONDict) error {
 	v.ModelManager = modelManager
 	model, err := modelManager.FetchByIdOrName(v, modelIdOrName)
 	if err != nil {
-		return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err)
+		if err == sql.ErrNoRows {
+			return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err)
+		} else {
+			return httperrors.NewGeneralError(err)
+		}
 	}
 	if v.noPendingDeleted {
 		if pd, ok := model.(db.IPendingDeletable); ok && pd.GetPendingDeleted() {

pkg/cloudprovider/consts.go

@@ -15,7 +15,7 @@
 package cloudprovider
 
 import (
-	"errors"
+	"yunion.io/x/pkg/errors"
 )
 
 const (
@@ -25,13 +25,13 @@ const (
 	CloudVMStatusChangeFlavor = "change_flavor"
 	CloudVMStatusDeploying    = "deploying"
 	CloudVMStatusOther        = "other"
-)
 
-var ErrNotFound = errors.New("id not found")
-var ErrDuplicateId = errors.New("duplicate id")
-var ErrInvalidStatus = errors.New("invalid status")
-var ErrTimeout = errors.New("timeout")
-var ErrNotImplemented = errors.New("Not implemented")
-var ErrNotSupported = errors.New("Not supported")
-var ErrInvalidProvider = errors.New("Invalid provider")
-var ErrNoBalancePermission = errors.New("No balance permission")
+	ErrNotFound            = errors.Error("id not found")
+	ErrDuplicateId         = errors.Error("duplicate id")
+	ErrInvalidStatus       = errors.Error("invalid status")
+	ErrTimeout             = errors.Error("timeout")
+	ErrNotImplemented      = errors.Error("Not implemented")
+	ErrNotSupported        = errors.Error("Not supported")
+	ErrInvalidProvider     = errors.Error("Invalid provider")
+	ErrNoBalancePermission = errors.Error("No balance permission")
+)

pkg/cloudprovider/objectstore.go

@@ -0,0 +1,179 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudprovider
+
+import (
+	"context"
+	"io"
+	"time"
+
+	"strings"
+	"yunion.io/x/pkg/errors"
+)
+
+type SBucketAccessUrl struct {
+	Url         string
+	Description string
+}
+
+type SBaseCloudObject struct {
+	Key          string
+	SizeBytes    int64
+	StorageClass string
+	ETag         string
+	LastModified time.Time
+	ContentType  string
+}
+
+type SListObjectResult struct {
+	Objects        []ICloudObject
+	NextMarker     string
+	CommonPrefixes []ICloudObject
+	IsTruncated    bool
+}
+
+type ICloudBucket interface {
+	IVirtualResource
+
+	GetGlobalId() string
+	GetName() string
+	GetAcl() string
+	GetLocation() string
+	GetIRegion() ICloudRegion
+	GetCreateAt() time.Time
+	GetStorageClass() string
+	GetAccessUrls() []SBucketAccessUrl
+
+	ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error)
+	GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error)
+	PutObject(ctx context.Context, key string, input io.Reader, contType string, storageClass string) error
+	DeleteObject(ctx context.Context, keys string) error
+	GetTempUrl(method string, key string, expire time.Duration) (string, error)
+	// ObjectExist(key string) (bool, error)
+}
+
+type ICloudObject interface {
+	GetIBucket() ICloudBucket
+
+	GetKey() string
+	GetSizeBytes() int64
+	GetLastModified() time.Time
+	GetStorageClass() string
+	GetETag() string
+	GetContentType() string
+}
+
+func ICloudObject2BaseCloudObject(obj ICloudObject) SBaseCloudObject {
+	return SBaseCloudObject{
+		Key:          obj.GetKey(),
+		SizeBytes:    obj.GetSizeBytes(),
+		StorageClass: obj.GetStorageClass(),
+		ETag:         obj.GetETag(),
+		LastModified: obj.GetLastModified(),
+		ContentType:  obj.GetContentType(),
+	}
+}
+
+func (o *SBaseCloudObject) GetKey() string {
+	return o.Key
+}
+
+func (o *SBaseCloudObject) GetSizeBytes() int64 {
+	return o.SizeBytes
+}
+
+func (o *SBaseCloudObject) GetLastModified() time.Time {
+	return o.LastModified
+}
+
+func (o *SBaseCloudObject) GetStorageClass() string {
+	return o.StorageClass
+}
+
+func (o *SBaseCloudObject) GetETag() string {
+	return o.ETag
+}
+
+func (o *SBaseCloudObject) GetContentType() string {
+	return o.ContentType
+}
+
+func GetIBucketById(region ICloudRegion, name string) (ICloudBucket, error) {
+	buckets, err := region.GetIBuckets()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetIBuckets")
+	}
+	for i := range buckets {
+		if buckets[i].GetGlobalId() == name {
+			return buckets[i], nil
+		}
+	}
+	return nil, ErrNotFound
+}
+
+func GetIObjects(bucket ICloudBucket, objectPrefix string, isRecursive bool) ([]ICloudObject, error) {
+	delimiter := "/"
+	if isRecursive {
+		delimiter = ""
+	}
+	ret := make([]ICloudObject, 0)
+	// Save marker for next request.
+	var marker string
+	for {
+		// Get list of objects a maximum of 1000 per request.
+		result, err := bucket.ListObjects(objectPrefix, marker, delimiter, 1000)
+		if err != nil {
+			return nil, errors.Wrap(err, "bucket.ListObjects")
+		}
+
+		// Send all objects
+		if len(result.Objects) > 0 {
+			ret = append(ret, result.Objects...)
+			marker = result.Objects[len(result.Objects)-1].GetKey()
+		}
+
+		// Send all common prefixes if any.
+		// NOTE: prefixes are only present if the request is delimited.
+		if len(result.CommonPrefixes) > 0 {
+			ret = append(ret, result.CommonPrefixes...)
+		}
+
+		// If next marker present, save it for next request.
+		if result.NextMarker != "" {
+			marker = result.NextMarker
+		}
+
+		// Listing ends result is not truncated, break the loop
+		if !result.IsTruncated {
+			break
+		}
+	}
+	return ret, nil
+}
+
+func Makedir(ctx context.Context, bucket ICloudBucket, key string) error {
+	segs := make([]string, 0)
+	for _, seg := range strings.Split(key, "/") {
+		if len(seg) > 0 {
+			segs = append(segs, seg)
+		}
+	}
+	path := strings.Join(segs, "/") + "/"
+	err := bucket.PutObject(ctx, path, strings.NewReader(""), "", "")
+	if err != nil {
+		return errors.Wrap(err, "PutObject")
+	}
+	return nil
+}

pkg/cloudprovider/resources.go

@@ -110,6 +110,12 @@ type ICloudRegion interface {
 
 	GetINetworkInterfaces() ([]ICloudNetworkInterface, error)
 
+	GetIBuckets() ([]ICloudBucket, error)
+	CreateIBucket(name string, storageClassStr string, acl string) error
+	DeleteIBucket(name string) error
+	IBucketExist(name string) (bool, error)
+	GetIBucketById(name string) (ICloudBucket, error)
+
 	GetProvider() string
 }
 

pkg/compute/models/buckets.go

@@ -0,0 +1,628 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"database/sql"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/pkg/s3utils"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/compare"
+	"yunion.io/x/sqlchemy"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+)
+
+type SBucketManager struct {
+	db.SVirtualResourceBaseManager
+}
+
+var BucketManager *SBucketManager
+
+func init() {
+	BucketManager = &SBucketManager{
+		SVirtualResourceBaseManager: db.NewVirtualResourceBaseManager(
+			SBucket{},
+			"buckets_tbl",
+			"bucket",
+			"buckets",
+		),
+	}
+	BucketManager.SetVirtualObject(BucketManager)
+}
+
+type SBucket struct {
+	db.SVirtualResourceBase
+	db.SExternalizedResourceBase
+
+	SManagedResourceBase
+
+	CloudregionId string `width:"36" charset:"ascii" nullable:"false" list:"user" create:"admin_required"`
+
+	StorageClass string `width:"36" charset:"ascii" nullable:"false" list:"user"`
+	Location     string `width:"36" charset:"ascii" nullable:"false" list:"user"`
+	Acl          string `width:"36" charset:"ascii" nullable:"false" list:"user"`
+}
+
+func (manager *SBucketManager) SetHandlerProcessTimeout(info *appsrv.SHandlerInfo, r *http.Request) time.Duration {
+	if r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/upload") && r.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER) != "" {
+		log.Debugf("upload object, set process timeout to 2 hour!!!")
+		return 2 * time.Hour
+	}
+	return manager.SVirtualResourceBaseManager.SetHandlerProcessTimeout(info, r)
+}
+
+func (manager *SBucketManager) fetchBuckets(provider *SCloudprovider, region *SCloudregion) ([]SBucket, error) {
+	q := manager.Query()
+	if provider != nil {
+		q = q.Equals("manager_id", provider.GetId())
+	}
+	if region != nil {
+		q = q.Equals("cloudregion_id", region.GetId())
+	}
+	buckets := make([]SBucket, 0)
+	err := db.FetchModelObjects(manager, q, &buckets)
+	if err != nil && err != sql.ErrNoRows {
+		return nil, errors.Wrap(err, "db.FetchModelObjects")
+	}
+	return buckets, nil
+}
+
+func (manager *SBucketManager) syncBuckets(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, buckets []cloudprovider.ICloudBucket) compare.SyncResult {
+	lockman.LockClass(ctx, manager, "")
+	defer lockman.ReleaseClass(ctx, manager, "")
+
+	syncResult := compare.SyncResult{}
+
+	dbBuckets, err := manager.fetchBuckets(provider, region)
+	if err != nil {
+		syncResult.Error(err)
+		return syncResult
+	}
+
+	removed := make([]SBucket, 0)
+	commondb := make([]SBucket, 0)
+	commonext := make([]cloudprovider.ICloudBucket, 0)
+	added := make([]cloudprovider.ICloudBucket, 0)
+
+	err = compare.CompareSets(dbBuckets, buckets, &removed, &commondb, &commonext, &added)
+	if err != nil {
+		syncResult.Error(err)
+		return syncResult
+	}
+
+	for i := 0; i < len(removed); i += 1 {
+		err = removed[i].syncRemoveCloudBucket(ctx, userCred)
+		if err != nil {
+			syncResult.DeleteError(err)
+		} else {
+			syncResult.Delete()
+		}
+	}
+	for i := 0; i < len(commondb); i += 1 {
+		err = commondb[i].syncWithCloudBucket(ctx, userCred, commonext[i], provider)
+		if err != nil {
+			syncResult.UpdateError(err)
+		} else {
+			syncResult.Update()
+		}
+	}
+	for i := 0; i < len(added); i += 1 {
+		_, err := manager.newFromCloudBucket(ctx, userCred, added[i], provider, region)
+		if err != nil {
+			syncResult.AddError(err)
+		} else {
+			syncResult.Add()
+		}
+	}
+
+	return syncResult
+}
+
+func (manager *SBucketManager) newFromCloudBucket(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	extBucket cloudprovider.ICloudBucket,
+	provider *SCloudprovider,
+	region *SCloudregion,
+) (*SBucket, error) {
+	bucket := SBucket{}
+	bucket.SetModelManager(manager, &bucket)
+
+	bucket.ExternalId = extBucket.GetGlobalId()
+	bucket.ManagerId = provider.Id
+	bucket.CloudregionId = region.Id
+	bucket.Status = api.BUCKET_STATUS_READY
+
+	newName, err := db.GenerateName(manager, nil, extBucket.GetName())
+	if err != nil {
+		return nil, errors.Wrap(err, "db.GenerateName")
+	}
+
+	bucket.Name = newName
+
+	created := extBucket.GetCreateAt()
+	if !created.IsZero() {
+		bucket.CreatedAt = created
+	}
+
+	bucket.Location = extBucket.GetLocation()
+	bucket.StorageClass = extBucket.GetStorageClass()
+	// bucket.Acl = extBucket.GetAcl()
+
+	bucket.IsEmulated = false
+
+	err = manager.TableSpec().Insert(&bucket)
+	if err != nil {
+		return nil, errors.Wrap(err, "Insert")
+	}
+
+	SyncCloudProject(userCred, &bucket, provider.GetOwnerId(), extBucket, provider.Id)
+
+	db.OpsLog.LogEvent(&bucket, db.ACT_CREATE, bucket.GetShortDesc(ctx), userCred)
+
+	return &bucket, nil
+}
+
+func (bucket *SBucket) syncWithCloudBucket(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	extBucket cloudprovider.ICloudBucket,
+	provider *SCloudprovider,
+) error {
+	diff, err := db.UpdateWithLock(ctx, bucket, func() error {
+		// bucket.Acl = extBucket.GetAcl()
+		bucket.Location = extBucket.GetLocation()
+		bucket.StorageClass = extBucket.GetStorageClass()
+
+		bucket.Status = api.BUCKET_STATUS_READY
+		return nil
+	})
+	if err != nil {
+		return errors.Wrap(err, "db.UpdateWithLock")
+	}
+
+	db.OpsLog.LogSyncUpdate(bucket, diff, userCred)
+
+	if provider != nil {
+		SyncCloudProject(userCred, bucket, provider.GetOwnerId(), extBucket, provider.Id)
+	}
+
+	return nil
+}
+
+func (bucket *SBucket) syncRemoveCloudBucket(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+) error {
+	lockman.LockObject(ctx, bucket)
+	defer lockman.ReleaseObject(ctx, bucket)
+
+	err := bucket.RealDelete(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "RealDelete")
+	}
+	return nil
+}
+
+func (bucket *SBucket) Delete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	// override
+	log.Infof("bucket delete do nothing")
+	return nil
+}
+
+func (bucket *SBucket) RealDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	return bucket.SVirtualResourceBase.Delete(ctx, userCred)
+}
+
+func (bucket *SBucket) RemoteDelete(ctx context.Context, userCred mcclient.TokenCredential) error {
+	iregion, err := bucket.GetIRegion()
+	if err != nil {
+		return errors.Wrap(err, "bucket.GetIRegion")
+	}
+	err = iregion.DeleteIBucket(bucket.ExternalId)
+	if err != nil {
+		return errors.Wrap(err, "iregion.DeleteIBucket")
+	}
+	err = bucket.RealDelete(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "bucket.RealDelete")
+	}
+	return nil
+}
+
+func (bucket *SBucket) CustomizeDelete(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) error {
+	return bucket.StartBucketDeleteTask(ctx, userCred, "")
+}
+
+func (bucket *SBucket) StartBucketDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string) error {
+	params := jsonutils.NewDict()
+	task, err := taskman.TaskManager.NewTask(ctx, "BucketDeleteTask", bucket, userCred, params, parentTaskId, "", nil)
+	if err != nil {
+		log.Errorf("%s", err)
+		return err
+	}
+	bucket.SetStatus(userCred, api.CLOUD_PROVIDER_START_DELETE, "StartBucketDeleteTask")
+	task.ScheduleRun(nil)
+	return nil
+}
+
+func (bucket *SBucket) GetRegion() (*SCloudregion, error) {
+	region, err := CloudregionManager.FetchById(bucket.CloudregionId)
+	if err != nil {
+		return nil, errors.Wrap(err, "CloudregionManager.FetchById")
+	}
+	return region.(*SCloudregion), nil
+}
+
+func (bucket *SBucket) GetIRegion() (cloudprovider.ICloudRegion, error) {
+	provider, err := bucket.GetDriver()
+	if err != nil {
+		return nil, err
+	}
+	if provider.GetFactory().IsOnPremise() {
+		return provider.GetOnPremiseIRegion()
+	} else {
+		region, err := bucket.GetRegion()
+		if err != nil {
+			return nil, errors.Wrap(err, "bucket.GetRegion")
+		}
+		return provider.GetIRegionById(region.GetExternalId())
+	}
+}
+
+func (bucket *SBucket) GetIBucket() (cloudprovider.ICloudBucket, error) {
+	iregion, err := bucket.GetIRegion()
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetIRegion")
+	}
+	return iregion.GetIBucketById(bucket.ExternalId)
+}
+
+func isValidBucketName(name string) error {
+	return s3utils.CheckValidBucketNameStrict(name)
+}
+
+func (manager *SBucketManager) ValidateCreateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data *jsonutils.JSONDict,
+) (*jsonutils.JSONDict, error) {
+	for _, v := range []validators.IValidator{
+		validators.NewModelIdOrNameValidator("cloudregion", CloudregionManager.Keyword(), ownerId),
+		validators.NewModelIdOrNameValidator("manager", CloudproviderManager.Keyword(), ownerId),
+	} {
+		err := v.Validate(data)
+		if err != nil {
+			return nil, err
+		}
+	}
+	nameStr, _ := data.GetString("name")
+	if len(nameStr) == 0 {
+		return nil, httperrors.NewInputParameterError("missing name")
+	}
+	err := isValidBucketName(nameStr)
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err)
+	}
+	return manager.SVirtualResourceBaseManager.ValidateCreateData(ctx, userCred, ownerId, query, data)
+}
+
+func (bucket *SBucket) PostCreate(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	ownerId mcclient.IIdentityProvider,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) {
+	bucket.SetStatus(userCred, api.BUCKET_STATUS_START_CREATE, "PostCreate")
+	task, err := taskman.TaskManager.NewTask(ctx, "BucketCreateTask", bucket, userCred, nil, "", "", nil)
+	if err != nil {
+		log.Errorf("BucketCreateTask newTask error %s", err)
+	} else {
+		task.ScheduleRun(nil)
+	}
+}
+
+func (bucket *SBucket) ValidateUpdateData(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data *jsonutils.JSONDict,
+) (*jsonutils.JSONDict, error) {
+	nameStr, _ := data.GetString("name")
+	if len(nameStr) > 0 {
+		err := isValidBucketName(nameStr)
+		if err != nil {
+			return nil, httperrors.NewInputParameterError("invalid bucket name: %s", err)
+		}
+	}
+	return bucket.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
+}
+
+func (bucket *SBucket) RemoteCreate(ctx context.Context, userCred mcclient.TokenCredential) error {
+	iregion, err := bucket.GetIRegion()
+	if err != nil {
+		return errors.Wrap(err, "bucket.GetIRegion")
+	}
+	err = iregion.CreateIBucket(bucket.Name, bucket.StorageClass, bucket.Acl)
+	if err != nil {
+		return errors.Wrap(err, "iregion.CreateIBucket")
+	}
+	err = db.SetExternalId(bucket, userCred, bucket.Name)
+	if err != nil {
+		return errors.Wrap(err, "db.SetExternalId")
+	}
+	extBucket, err := iregion.GetIBucketById(bucket.Name)
+	if err != nil {
+		return errors.Wrap(err, "iregion.GetIBucketByName")
+	}
+	err = bucket.syncWithCloudBucket(ctx, userCred, extBucket, nil)
+	if err != nil {
+		return errors.Wrap(err, "bucket.syncWithCloudBucket")
+	}
+	return nil
+}
+
+func (bucket *SBucket) GetCustomizeColumns(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+	extra := bucket.SVirtualResourceBase.GetCustomizeColumns(ctx, userCred, query)
+	return bucket.getMoreDetails(extra)
+}
+
+func (bucket *SBucket) GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*jsonutils.JSONDict, error) {
+	extra, err := bucket.SVirtualResourceBase.GetExtraDetails(ctx, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+	return bucket.getMoreDetails(extra), nil
+}
+
+func (bucket *SBucket) getMoreDetails(extra *jsonutils.JSONDict) *jsonutils.JSONDict {
+	info := bucket.getCloudProviderInfo()
+	extra.Update(jsonutils.Marshal(&info))
+
+	return extra
+}
+
+func (bucket *SBucket) getCloudProviderInfo() SCloudProviderInfo {
+	region, _ := bucket.GetRegion()
+	provider := bucket.GetCloudprovider()
+	return MakeCloudProviderInfo(region, nil, provider)
+}
+
+func (manager *SBucketManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	var err error
+
+	q, err = managedResourceFilterByAccount(q, query, "", nil)
+	if err != nil {
+		return nil, err
+	}
+	q = managedResourceFilterByCloudType(q, query, "", nil)
+
+	q, err = managedResourceFilterByDomain(q, query, "", nil)
+	if err != nil {
+		return nil, err
+	}
+
+	q, err = manager.SVirtualResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
+	if err != nil {
+		return nil, err
+	}
+
+	return q, nil
+}
+
+func (bucket *SBucket) AllowGetDetailsObjects(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) GetDetailsObjects(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+	}
+	prefix, _ := query.GetString("prefix")
+	isRecursive := jsonutils.QueryBoolean(query, "recursive", false)
+	objects, err := iBucket.GetIObjects(prefix, isRecursive)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to get objects: %s", err)
+	}
+	retArray := jsonutils.NewArray()
+	for i := range objects {
+		retArray.Add(jsonutils.Marshal(cloudprovider.ICloudObject2BaseCloudObject(objects[i])))
+	}
+	ret := jsonutils.NewDict()
+	ret.Add(retArray, "objects")
+	return ret, nil
+}
+
+func (bucket *SBucket) AllowPerformTempUrl(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) PerformTempUrl(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	method, _ := data.GetString("method")
+	key, _ := data.GetString("key")
+	expire, _ := data.Int("expire_seconds")
+
+	if len(method) == 0 {
+		method = "GET"
+	}
+	if len(key) == 0 {
+		return nil, httperrors.NewInputParameterError("missing key")
+	}
+	if expire == 0 {
+		expire = 60 // default 60 seconds
+	}
+
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+	}
+	tmpUrl, err := iBucket.GetTempUrl(method, key, time.Duration(expire)*time.Second)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to generate temp url: %s", err)
+	}
+	ret := jsonutils.NewDict()
+	ret.Add(jsonutils.NewString(tmpUrl), "url")
+	return ret, nil
+}
+
+func (bucket *SBucket) AllowPerformMakedir(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) PerformMakedir(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	key, _ := data.GetString("key")
+	if key[len(key)-1] != '/' {
+		return nil, httperrors.NewInputParameterError("directory must ends with /")
+	}
+	err := s3utils.CheckValidObjectName(key)
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("invalid key: %s", err)
+	}
+
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+	}
+
+	err = cloudprovider.Makedir(ctx, iBucket, key)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to mkdir: %s", err)
+	}
+
+	return nil, nil
+}
+
+func (bucket *SBucket) AllowPerformDelete(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) PerformDelete(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	keys, _ := data.Get("keys")
+	if keys == nil {
+		return nil, httperrors.NewInputParameterError("missing keys")
+	}
+	keyStrs := keys.(*jsonutils.JSONArray).GetStringArray()
+	if len(keyStrs) == 0 {
+		return nil, httperrors.NewInputParameterError("empty keys")
+	}
+
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+	}
+	ok := jsonutils.NewDict()
+	results := modules.BatchDo(keyStrs, func(key string) (jsonutils.JSONObject, error) {
+		err := iBucket.DeleteObject(ctx, key)
+		if err != nil {
+			return nil, err
+		} else {
+			return ok, nil
+		}
+	})
+	return modules.SubmitResults2JSON(results), nil
+}
+
+func (bucket *SBucket) AllowPerformUpload(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) PerformUpload(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	appParams := appsrv.AppContextGetParams(ctx)
+
+	key := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER)
+	err := s3utils.CheckValidObjectName(key)
+	if err != nil {
+		return nil, httperrors.NewInputParameterError("invalid object key: %s", err)
+	}
+
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+	}
+
+	contType := appParams.Request.Header.Get("Content-Type")
+	storageClass := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER)
+	err = iBucket.PutObject(ctx, key, appParams.Request.Body, contType, storageClass)
+	if err != nil {
+		return nil, httperrors.NewInternalServerError("put object error %s", err)
+	}
+
+	return nil, nil
+}

pkg/compute/models/cloudproviders.go

@@ -1087,6 +1087,7 @@ func (self *SCloudprovider) RealDelete(ctx context.Context, userCred mcclient.To
 	var err error
 
 	for _, manager := range []IPurgeableManager{
+		BucketManager,
 		HostManager,
 		SnapshotManager,
 		SnapshotPolicyManager,

pkg/compute/models/cloudregions.go

@@ -654,3 +654,7 @@ func (self *SCloudregion) getMinDataDiskCount() int {
 func (self *SCloudregion) getMaxDataDiskCount() int {
 	return options.Options.MaxDataDiskCount
 }
+
+func (manager *SCloudregionManager) FetchDefaultRegion() *SCloudregion {
+	return manager.FetchRegionById(api.DEFAULT_REGION_ID)
+}

pkg/compute/models/cloudsync.go

@@ -169,6 +169,28 @@ func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, sync
 	// db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred)
 }
 
+func syncRegionBuckets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) {
+	buckets, err := remoteRegion.GetIBuckets()
+	if err != nil {
+		msg := fmt.Sprintf("GetIBuckets for region %s failed %s", remoteRegion.GetName(), err)
+		log.Errorf(msg)
+		return
+	}
+
+	result := BucketManager.syncBuckets(ctx, userCred, provider, localRegion, buckets)
+
+	syncResults.Add(BucketManager, result)
+
+	msg := result.Result()
+	notes := fmt.Sprintf("GetIBuckets for region %s result: %s", localRegion.Name, msg)
+	log.Infof(notes)
+	if result.IsError() {
+		return
+	}
+	db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred)
+	// logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true)
+}
+
 func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) {
 	vpcs, err := remoteRegion.GetIVpcs()
 	if err != nil {
@@ -956,6 +978,8 @@ func syncPublicCloudProviderInfo(
 
 	// no need to lock public cloud region as cloud region for public cloud is readonly
 
+	syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, remoteRegion)
+
 	// 需要先同步vpc，避免私有云eip找不到network
 	syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange)
 
@@ -1018,6 +1042,9 @@ func syncOnPremiseCloudProviderInfo(
 	syncRange *SSyncRange,
 ) error {
 	log.Debugf("Start sync on-premise provider %s(%s)", provider.Name, provider.Provider)
+
+	syncProjects(ctx, userCred, syncResults, driver, provider)
+
 	iregion, err := driver.GetOnPremiseIRegion()
 	if err != nil {
 		msg := fmt.Sprintf("GetOnPremiseIRegion for provider %s failed %s", provider.GetName(), err)
@@ -1025,6 +1052,9 @@ func syncOnPremiseCloudProviderInfo(
 		return err
 	}
 
+	localRegion := CloudregionManager.FetchDefaultRegion()
+	syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, iregion)
+
 	ihosts, err := iregion.GetIHosts()
 	if err != nil {
 		msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err)

pkg/compute/models/guest_actions.go

@@ -82,12 +82,17 @@ func (self *SGuest) GetDetailsVnc(ctx context.Context, userCred mcclient.TokenCr
 func (self *SGuest) AllowPerformMonitor(ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
-	data jsonutils.JSONObject) bool {
+	data jsonutils.JSONObject,
+) bool {
 	return self.IsOwner(userCred) || db.IsAdminAllowPerform(userCred, self, "monitor")
 }
 
-func (self *SGuest) PerformMonitor(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject,
-	data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+func (self *SGuest) PerformMonitor(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	data jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
 	if utils.IsInStringArray(self.Status, []string{api.VM_RUNNING, api.VM_BLOCK_STREAM}) {
 		cmd, err := data.GetString("command")
 		if err != nil {

pkg/compute/models/natdtable.go

@@ -46,6 +46,7 @@ func init() {
 			"natdtables",
 		),
 	}
+	NatDTableManager.SetVirtualObject(NatDTableManager)
 }
 
 type SNatDEntry struct {

pkg/compute/models/natgateways.go

@@ -46,6 +46,7 @@ func init() {
 			"natgateways",
 		),
 	}
+	NatGatewayManager.SetVirtualObject(NatGatewayManager)
 }
 
 type SNatGateway struct {

pkg/compute/models/natstable.go

@@ -46,6 +46,7 @@ func init() {
 			"natstables",
 		),
 	}
+	NatSTableManager.SetVirtualObject(NatSTableManager)
 }
 
 type SNatSEntry struct {

pkg/compute/models/purge.go

@@ -1039,3 +1039,30 @@ func (manager *SNetworkInterfaceManager) purgeAll(ctx context.Context, userCred
 	}
 	return nil
 }
+
+func (bucket *SBucket) purge(ctx context.Context, userCred mcclient.TokenCredential) error {
+	lockman.LockObject(ctx, bucket)
+	defer lockman.ReleaseObject(ctx, bucket)
+
+	err := bucket.ValidateDeleteCondition(ctx)
+	if err != nil {
+		return err
+	}
+
+	return bucket.RealDelete(ctx, userCred)
+}
+
+func (bucketManager *SBucketManager) purgeAll(ctx context.Context, userCred mcclient.TokenCredential, providerId string) error {
+	buckets := make([]SBucket, 0)
+	err := fetchByManagerId(bucketManager, providerId, &buckets)
+	if err != nil {
+		return err
+	}
+	for i := range buckets {
+		err := buckets[i].purge(ctx, userCred)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

pkg/compute/models/snapshotpolicy.go

@@ -62,6 +62,7 @@ func init() {
 			"snapshotpolicies",
 		),
 	}
+	SnapshotPolicyManager.SetVirtualObject(SnapshotPolicyManager)
 }
 
 func (manager *SSnapshotPolicyManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {

pkg/compute/models/vpcs.go

@@ -476,22 +476,32 @@ func (manager *SVpcManager) InitializeData() error {
 }
 
 func (manager *SVpcManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	regionId, err := data.GetString("cloudregion_id")
-	if err != nil {
+	regionId := jsonutils.GetAnyString(data, []string{"region", "cloudregion", "cloudregion_id"})
+	if len(regionId) == 0 {
 		return nil, httperrors.NewMissingParameterError("cloudregion_id")
 	}
-	region := CloudregionManager.FetchRegionById(regionId)
-	if region == nil {
-		return nil, httperrors.NewInputParameterError("Invalid cloudregion_id")
+	regionObj, err := CloudregionManager.FetchByIdOrName(userCred, regionId)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, httperrors.NewResourceNotFoundError2(CloudregionManager.Keyword(), regionId)
+		} else {
+			return nil, httperrors.NewGeneralError(err)
+		}
 	}
+	region := regionObj.(*SCloudregion)
+	data.Add(jsonutils.NewString(region.GetId()), "cloudregion_id")
 	if region.isManaged() {
 		managerStr := jsonutils.GetAnyString(data, []string{"manager_id", "manager"})
 		if len(managerStr) == 0 {
 			return nil, httperrors.NewMissingParameterError("manager_id")
 		}
-		managerObj := CloudproviderManager.FetchCloudproviderByIdOrName(managerStr)
+		managerObj, err := CloudproviderManager.FetchByIdOrName(userCred, managerStr)
 		if err != nil {
-			return nil, httperrors.NewResourceNotFoundError("Cloud provider/manager %s not found", managerStr)
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(CloudproviderManager.Keyword(), managerStr)
+			} else {
+				return nil, httperrors.NewGeneralError(err)
+			}
 		}
 		data.Add(jsonutils.NewString(managerObj.GetId()), "manager_id")
 	} else {

pkg/compute/service/handlers.go

@@ -64,6 +64,7 @@ func InitHandlers(app *appsrv.Application) {
 	for _, manager := range []db.IModelManager{
 		db.OpsLog,
 		db.Metadata,
+		models.BucketManager,
 		models.CloudaccountManager,
 		models.CloudproviderManager,
 		models.CloudregionManager,

pkg/compute/service/service.go

@@ -34,6 +34,7 @@ import (
 	_ "yunion.io/x/onecloud/pkg/compute/regiondrivers"
 	_ "yunion.io/x/onecloud/pkg/compute/storagedrivers"
 	_ "yunion.io/x/onecloud/pkg/compute/tasks"
+	_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider"
 	_ "yunion.io/x/onecloud/pkg/util/aliyun/provider"
 	_ "yunion.io/x/onecloud/pkg/util/aws/provider"
 	_ "yunion.io/x/onecloud/pkg/util/azure/provider"

pkg/compute/tasks/bucket_create_task.go

@@ -0,0 +1,58 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type BucketCreateTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(BucketCreateTask{})
+}
+
+func (task *BucketCreateTask) taskFailed(ctx context.Context, bucket *models.SBucket, err error) {
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_CREATE_FAIL, err.Error())
+	db.OpsLog.LogEvent(bucket, db.ACT_ALLOCATE_FAIL, err.Error(), task.UserCred)
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_ALLOCATE, err.Error(), task.UserCred, false)
+	task.SetStageFailed(ctx, err.Error())
+}
+
+func (task *BucketCreateTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	bucket := obj.(*models.SBucket)
+
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_CREATING, "StartBucketCreateTask")
+
+	err := bucket.RemoteCreate(ctx, task.UserCred)
+	if err != nil {
+		task.taskFailed(ctx, bucket, err)
+		return
+	}
+
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_READY, "BucketCreateTask")
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_ALLOCATE, nil, task.UserCred, true)
+	task.SetStageComplete(ctx, nil)
+}

pkg/compute/tasks/bucket_delete_task.go

@@ -0,0 +1,57 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tasks
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type BucketDeleteTask struct {
+	taskman.STask
+}
+
+func init() {
+	taskman.RegisterTask(BucketDeleteTask{})
+}
+
+func (task *BucketDeleteTask) taskFailed(ctx context.Context, bucket *models.SBucket, err error) {
+	bucket.SetStatus(task.UserCred, api.VPC_STATUS_DELETE_FAILED, err.Error())
+	db.OpsLog.LogEvent(bucket, db.ACT_DELOCATE_FAIL, err.Error(), task.UserCred)
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_DELETE, err.Error(), task.UserCred, false)
+	task.SetStageFailed(ctx, err.Error())
+}
+
+func (task *BucketDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) {
+	bucket := obj.(*models.SBucket)
+
+	bucket.SetStatus(task.UserCred, api.BUCKET_STATUS_DELETING, "StartBucketDeleteTask")
+
+	err := bucket.RemoteDelete(ctx, task.UserCred)
+	if err != nil {
+		task.taskFailed(ctx, bucket, err)
+		return
+	}
+
+	logclient.AddActionLogWithStartable(task, bucket, logclient.ACT_DELETE, nil, task.UserCred, true)
+	task.SetStageComplete(ctx, nil)
+}

pkg/hostman/hostdeployer/apis/deploy.pb.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // source: deploy.proto
 

pkg/image/models/images.go

@@ -122,6 +122,8 @@ type SImage struct {
 }
 
 func (manager *SImageManager) CustomizeHandlerInfo(info *appsrv.SHandlerInfo) {
+	manager.SVirtualResourceBaseManager.CustomizeHandlerInfo(info)
+
 	switch info.GetName(nil) {
 	case "get_details", "create", "update":
 		info.SetProcessTimeout(time.Minute * 120).SetWorkerManager(imgStreamingWorkerMan)

pkg/image/service/service.go

@@ -23,6 +23,7 @@ import (
 
 	"yunion.io/x/log"
 
+	api "yunion.io/x/onecloud/pkg/apis/image"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
 	"yunion.io/x/onecloud/pkg/cloudcommon/cronman"
@@ -37,16 +38,12 @@ import (
 	"yunion.io/x/onecloud/pkg/util/sysutils"
 )
 
-const (
-	SERVICE_TYPE = "image"
-)
-
 func StartService() {
 	opts := &options.Options
 	commonOpts := &opts.CommonOptions
 	baseOpts := &opts.BaseOptions
 	dbOpts := &opts.DBOptions
-	common_options.ParseOptions(opts, os.Args, "glance-api.conf", SERVICE_TYPE)
+	common_options.ParseOptions(opts, os.Args, "glance-api.conf", api.SERVICE_TYPE)
 
 	isRoot := sysutils.IsRootPermission()
 	if !isRoot {

pkg/mcclient/modules/mod_buckets.go

@@ -0,0 +1,66 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package modules
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/pkg/errors"
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+)
+
+type SBucketManager struct {
+	ResourceManager
+}
+
+func (manager *SBucketManager) Upload(s *mcclient.ClientSession, bucketId string, key string, body io.Reader, contType string, storageClass string) error {
+	method := httputils.POST
+	path := fmt.Sprintf("/%s/%s/upload", manager.URLPath(), bucketId)
+	headers := http.Header{}
+	headers.Set(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER, key)
+	if len(contType) > 0 {
+		headers.Set("Content-Type", contType)
+	}
+	if len(storageClass) > 0 {
+		headers.Set(api.BUCKET_UPLOAD_OBJECT_STORAGECLASS_HEADER, storageClass)
+	}
+
+	_, err := manager.rawRequest(s, method, path, headers, body)
+	if err != nil {
+		return errors.Wrap(err, "rawRequest")
+	}
+	return nil
+}
+
+var (
+	Buckets SBucketManager
+)
+
+func init() {
+	Buckets = SBucketManager{
+		NewComputeManager("bucket", "buckets",
+			[]string{"ID", "Name", "Storage_Class",
+				"Status", "location", "acl",
+				"region", "manager_id",
+			},
+			[]string{}),
+	}
+
+	registerCompute(&Buckets)
+}

pkg/mcclient/options/cloudaccounts.go

@@ -131,6 +131,12 @@ type SZStackCloudAccountCreateOptions struct {
 	AuthURL string `help:"ZStack auth_url" positional:"true" json:"auth_url"`
 }
 
+type SS3CloudAccountCreateOptions struct {
+	SCloudAccountCreateBaseOptions
+	SAccessKeyCredential
+	Endpoint string `help:"S3 endpoint" poisitional:"true" json:"endpoint"`
+}
+
 // update credential options
 
 type SCloudAccountUpdateCredentialBaseOptions struct {
@@ -172,6 +178,21 @@ type SHuaweiCloudAccountUpdateCredentialOptions struct {
 	SAccessKeyCredential
 }
 
+type SUcloudCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SAccessKeyCredential
+}
+
+type SZStackCloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SUserPasswordCredential
+}
+
+type SS3CloudAccountUpdateCredentialOptions struct {
+	SCloudAccountUpdateCredentialBaseOptions
+	SAccessKeyCredential
+}
+
 // update
 
 type SCloudAccountUpdateBaseOptions struct {
@@ -218,7 +239,14 @@ type SHuaweiCloudAccountUpdateOptions struct {
 	SCloudAccountUpdateBaseOptions
 }
 
+type SUcloudCloudAccountUpdateOptions struct {
+	SCloudAccountUpdateBaseOptions
+}
+
 type SZStackCloudAccountUpdateOptions struct {
 	SCloudAccountUpdateBaseOptions
-	SUserPasswordCredential
+}
+
+type SS3CloudAccountUpdateOptions struct {
+	SCloudAccountUpdateBaseOptions
 }

pkg/multicloud/no_storage_region.go

@@ -0,0 +1,43 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package multicloud
+
+import "yunion.io/x/onecloud/pkg/cloudprovider"
+
+type SNoObjectStorageRegion struct{}
+
+///////////////// S3 ///////////////////
+
+func (cli *SNoObjectStorageRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) DeleteIBucket(name string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) IBucketExist(name string) (bool, error) {
+	return false, cloudprovider.ErrNotSupported
+}
+
+func (cli *SNoObjectStorageRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+////////////////// END S3 fake API //////////

pkg/multicloud/objectstore/buckets.go

@@ -0,0 +1,148 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package objectstore
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"path"
+	"time"
+
+	"github.com/minio/minio-go"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	client *SObjectStoreClient
+
+	Name         string
+	Location     string
+	CreatedAt    time.Time
+	StorageClass string
+	Acl          string
+}
+
+func (bucket *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (bucket *SBucket) GetGlobalId() string {
+	return bucket.Name
+}
+
+func (bucket *SBucket) GetName() string {
+	return bucket.Name
+}
+
+func (bucket *SBucket) GetAcl() string {
+	return bucket.Acl
+}
+
+func (bucket *SBucket) GetLocation() string {
+	return bucket.Location
+}
+
+func (bucket *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return bucket.client
+}
+
+func (bucket *SBucket) GetCreateAt() time.Time {
+	return bucket.CreatedAt
+}
+
+func (bucket *SBucket) GetStorageClass() string {
+	return bucket.StorageClass
+}
+
+func (bucket *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         path.Join(bucket.client.endpoint, bucket.Name),
+			Description: fmt.Sprintf("%s", bucket.Location),
+		},
+	}
+}
+
+func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
+	isRecursive := true
+	if delimiter == "/" {
+		isRecursive = false
+	}
+	result := cloudprovider.SListObjectResult{}
+	var err error
+	result.Objects, err = bucket.GetIObjects(prefix, isRecursive)
+	return result, err
+}
+
+func (bucket *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
+	doneCh := make(chan struct{})
+	defer close(doneCh)
+
+	ret := make([]cloudprovider.ICloudObject, 0)
+	objectCh := bucket.client.client.ListObjects(bucket.Name, prefix, isRecursive, doneCh)
+	for object := range objectCh {
+		if object.Err != nil {
+			return nil, errors.Wrap(object.Err, "ListObjects")
+		}
+		obj := &SObject{
+			bucket: bucket,
+			SBaseCloudObject: cloudprovider.SBaseCloudObject{
+				StorageClass: object.StorageClass,
+				Key:          object.Key,
+				SizeBytes:    object.Size,
+				ETag:         object.ETag,
+				LastModified: object.LastModified,
+				ContentType:  object.ContentType,
+			},
+		}
+		ret = append(ret, obj)
+	}
+	return ret, nil
+}
+
+func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.Reader, contType string, storageClass string) error {
+	opts := minio.PutObjectOptions{}
+	if len(contType) > 0 {
+		opts.ContentType = contType
+	}
+	if len(storageClass) > 0 {
+		opts.StorageClass = storageClass
+	}
+	_, err := bucket.client.client.PutObjectWithContext(ctx, bucket.Name, key, input, -1, opts)
+	return err
+}
+
+func (bucket *SBucket) DeleteObject(ctx context.Context, key string) error {
+	err := bucket.client.client.RemoveObject(bucket.Name, key)
+	if err != nil {
+		return errors.Wrap(err, "RemoveObject")
+	}
+	return nil
+}
+
+func (bucket *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
+	if method != "GET" && method != "PUT" && method != "DELETE" {
+		return "", errors.Error("unsupported method")
+	}
+	url, err := bucket.client.client.Presign(method, bucket.Name, key, expire, nil)
+	if err != nil {
+		return "", errors.Wrap(err, "Presign")
+	}
+	return url.String(), nil
+}

pkg/multicloud/objectstore/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package objectstore // import "yunion.io/x/onecloud/pkg/multicloud/objectstore"

pkg/multicloud/objectstore/object.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package objectstore
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SObject struct {
+	bucket *SBucket
+
+	cloudprovider.SBaseCloudObject
+}
+
+func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
+	return o.bucket
+}

pkg/multicloud/objectstore/objectstore.go

@@ -0,0 +1,343 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package objectstore
+
+import (
+	"net/url"
+	"time"
+
+	"github.com/minio/minio-go"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/secrules"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/object"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/multicloud"
+	"yunion.io/x/onecloud/pkg/util/httputils"
+)
+
+type SObjectStoreClient struct {
+	object.SObject
+
+	cloudprovider.SFakeOnPremiseRegion
+	multicloud.SRegion
+
+	providerId   string
+	providerName string
+	endpoint     string
+	accessKey    string
+	secret       string
+
+	client *minio.Client
+
+	Debug bool
+}
+
+func NewObjectStoreClient(providerId string, providerName string, endpoint string, accessKey string, secret string, isDebug bool) (*SObjectStoreClient, error) {
+	client := SObjectStoreClient{
+		providerId:   providerId,
+		providerName: providerName,
+		endpoint:     endpoint,
+		accessKey:    accessKey,
+		secret:       secret,
+		Debug:        isDebug,
+	}
+	parts, err := url.Parse(endpoint)
+	if err != nil {
+		return nil, errors.Wrap(err, "url.Parse endpoint")
+	}
+	useSsl := false
+	if parts.Scheme == "https" {
+		useSsl = true
+	}
+	cli, err := minio.New(parts.Host, accessKey, secret, useSsl)
+	if err != nil {
+		return nil, errors.Wrap(err, "minio.New")
+	}
+
+	tr := httputils.GetTransport(true, time.Second*5)
+	cli.SetCustomTransport(tr)
+
+	client.client = cli
+
+	return &client, nil
+}
+
+func (cli *SObjectStoreClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	subAccount := cloudprovider.SSubAccount{
+		Account:      cli.accessKey,
+		Name:         cli.providerName,
+		HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL,
+	}
+	return []cloudprovider.SSubAccount{subAccount}, nil
+}
+
+func (cli *SObjectStoreClient) GetIRegion() cloudprovider.ICloudRegion {
+	return cli.GetVirtualObject().(cloudprovider.ICloudRegion)
+}
+
+func (cli *SObjectStoreClient) GetVersion() string {
+	return ""
+}
+
+func (cli *SObjectStoreClient) About() jsonutils.JSONObject {
+	about := jsonutils.NewDict()
+	return about
+}
+
+func (cli *SObjectStoreClient) GetProvider() string {
+	return api.CLOUD_PROVIDER_GENERICS3
+}
+
+///////////////////////////////// fake impletementations //////////////////////
+
+func (cli *SObjectStoreClient) GetIZones() ([]cloudprovider.ICloudZone, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIVpcs() ([]cloudprovider.ICloudVpc, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIEips() ([]cloudprovider.ICloudEIP, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIVpcById(id string) (cloudprovider.ICloudVpc, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIZoneById(id string) (cloudprovider.ICloudZone, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIEipById(id string) (cloudprovider.ICloudEIP, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIVMById(id string) (cloudprovider.ICloudVM, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIDiskById(id string) (cloudprovider.ICloudDisk, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) DeleteSecurityGroup(vpcId, secgroupId string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) SyncSecurityGroup(secgroupId string, vpcId string, name string, desc string, rules []secrules.SecurityRule) (string, error) {
+	return "", cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateIVpc(name string, desc string, cidr string) (cloudprovider.ICloudVpc, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateEIP(eip *cloudprovider.SEip) (cloudprovider.ICloudEIP, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateSnapshotPolicy(*cloudprovider.SnapshotPolicyInput) (string, error) {
+	return "", cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) DeleteSnapshotPolicy(string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) ApplySnapshotPolicyToDisks(snapshotPolicyId string, diskIds []string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CancelSnapshotPolicyToDisks(diskIds []string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshotPolicies() ([]cloudprovider.ICloudSnapshotPolicy, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISnapshotPolicyById(snapshotPolicyId string) (cloudprovider.ICloudSnapshotPolicy, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIHosts() ([]cloudprovider.ICloudHost, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIHostById(id string) (cloudprovider.ICloudHost, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStorages() ([]cloudprovider.ICloudStorage, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStoragecaches() ([]cloudprovider.ICloudStoragecache, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetIStoragecacheById(id string) (cloudprovider.ICloudStoragecache, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancers() ([]cloudprovider.ICloudLoadbalancer, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerAcls() ([]cloudprovider.ICloudLoadbalancerAcl, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerCertificates() ([]cloudprovider.ICloudLoadbalancerCertificate, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerById(loadbalancerId string) (cloudprovider.ICloudLoadbalancer, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerAclById(aclId string) (cloudprovider.ICloudLoadbalancerAcl, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetILoadBalancerCertificateById(certId string) (cloudprovider.ICloudLoadbalancerCertificate, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbalancer) (cloudprovider.ICloudLoadbalancer, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateILoadBalancerCertificate(cert *cloudprovider.SLoadbalancerCertificate) (cloudprovider.ICloudLoadbalancerCertificate, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISkuById(skuId string) (cloudprovider.ICloudSku, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) GetISkus(zoneId string) ([]cloudprovider.ICloudSku, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (cli *SObjectStoreClient) CreateISku(sku *cloudprovider.SServerSku) (cloudprovider.ICloudSku, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+////////////////////////////////// S3 API ///////////////////////////////////
+
+func (cli *SObjectStoreClient) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	buckets, err := cli.client.ListBuckets()
+	if err != nil {
+		return nil, errors.Wrap(err, "client.ListBuckets")
+	}
+	ret := make([]cloudprovider.ICloudBucket, len(buckets))
+	for i := range buckets {
+		b := SBucket{
+			client:    cli,
+			Name:      buckets[i].Name,
+			CreatedAt: buckets[i].CreationDate,
+		}
+		ret[i] = &b
+	}
+	return ret, nil
+}
+
+func (cli *SObjectStoreClient) CreateIBucket(name string, storageClass string, acl string) error {
+	err := cli.client.MakeBucket(name, "")
+	if err != nil {
+		return errors.Wrap(err, "MakeBucket")
+	}
+	return nil
+}
+
+func minioErrCode(err error) int {
+	if srvErr, ok := err.(minio.ErrorResponse); ok {
+		return srvErr.StatusCode
+	}
+	if srvErr, ok := err.(*minio.ErrorResponse); ok {
+		return srvErr.StatusCode
+	}
+	return -1
+}
+
+func (cli *SObjectStoreClient) DeleteIBucket(name string) error {
+	err := cli.client.RemoveBucket(name)
+	if err != nil {
+		if minioErrCode(err) == 404 {
+			return nil
+		}
+		return errors.Wrap(err, "RemoveBucket")
+	}
+	return nil
+}
+
+func (cli *SObjectStoreClient) GetIBucketPolicy(name string) (string, error) {
+	policy, err := cli.client.GetBucketPolicy(name)
+	if err != nil {
+		return "", errors.Wrap(err, "GetBucketPolicy")
+	}
+	return policy, nil
+}
+
+func (cli *SObjectStoreClient) SetIBucketPolicy(name string, policy string) error {
+	err := cli.client.SetBucketPolicy(name, policy)
+	if err != nil {
+		return errors.Wrap(err, "SetBucketPolicy")
+	}
+	return nil
+}
+
+func (cli *SObjectStoreClient) GetIBucketLiftcycle(name string) (string, error) {
+	liftcycle, err := cli.client.GetBucketLifecycle(name)
+	if err != nil {
+		return "", errors.Wrap(err, "GetBucketLifecycle")
+	}
+	return liftcycle, nil
+}
+
+func (cli *SObjectStoreClient) IBucketExist(name string) (bool, error) {
+	exist, err := cli.client.BucketExists(name)
+	if err != nil {
+		return false, errors.Wrap(err, "BucketExists")
+	}
+	return exist, nil
+}
+
+func (cli *SObjectStoreClient) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketById(cli, name)
+}

pkg/multicloud/objectstore/provider/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package provider // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider"

pkg/multicloud/objectstore/provider/provider.go

@@ -0,0 +1,136 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package provider
+
+import (
+	"context"
+
+	"yunion.io/x/jsonutils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/multicloud/objectstore"
+)
+
+type SObjectStoreProviderFactory struct {
+	cloudprovider.SPremiseBaseProviderFactory
+}
+
+func (self *SObjectStoreProviderFactory) GetId() string {
+	return api.CLOUD_PROVIDER_GENERICS3
+}
+
+func (self *SObjectStoreProviderFactory) GetName() string {
+	return api.CLOUD_PROVIDER_GENERICS3
+}
+
+func (self *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) error {
+	accessKeyID, _ := data.GetString("access_key_id")
+	if len(accessKeyID) == 0 {
+		return httperrors.NewMissingParameterError("access_key_id")
+	}
+	accessKeySecret, _ := data.GetString("access_key_secret")
+	if len(accessKeySecret) == 0 {
+		return httperrors.NewMissingParameterError("access_key_secret")
+	}
+	endpointURL, _ := data.GetString("endpoint")
+	if len(endpointURL) == 0 {
+		return httperrors.NewMissingParameterError("endpoint")
+	}
+	data.Set("account", jsonutils.NewString(accessKeyID))
+	data.Set("secret", jsonutils.NewString(accessKeySecret))
+	data.Set("access_url", jsonutils.NewString(endpointURL))
+	return nil
+}
+
+func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+	accessKeyID, _ := data.GetString("access_key_id")
+	if len(accessKeyID) == 0 {
+		return nil, httperrors.NewMissingParameterError("access_key_id")
+	}
+	accessKeySecret, _ := data.GetString("access_key_secret")
+	if len(accessKeySecret) == 0 {
+		return nil, httperrors.NewMissingParameterError("access_key_secret")
+	}
+	account := &cloudprovider.SCloudaccount{
+		Account: accessKeyID,
+		Secret:  accessKeySecret,
+	}
+	return account, nil
+}
+
+func (self *SObjectStoreProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {
+	client, err := objectstore.NewObjectStoreClient(providerId, providerName, url, account, secret, false)
+	if err != nil {
+		return nil, err
+	}
+	client.SetVirtualObject(client)
+	return &SObjectStoreProvider{
+		SBaseProvider: cloudprovider.NewBaseProvider(self),
+		client:        client,
+	}, nil
+}
+
+func (self *SObjectStoreProviderFactory) GetClientRC(url, account, secret string) (map[string]string, error) {
+	return map[string]string{
+		"OBJECTSTORE_ACCESSKEY": account,
+		"OBJECTSTORE_SECRET":    secret,
+		"OBJECTSTORE_ENDPOINT":  url,
+	}, nil
+}
+
+func init() {
+	factory := SObjectStoreProviderFactory{}
+	cloudprovider.RegisterFactory(&factory)
+}
+
+type SObjectStoreProvider struct {
+	cloudprovider.SBaseProvider
+	client *objectstore.SObjectStoreClient
+}
+
+func (self *SObjectStoreProvider) GetIRegions() []cloudprovider.ICloudRegion {
+	return nil
+}
+
+func (self *SObjectStoreProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (self *SObjectStoreProvider) GetBalance() (float64, string, error) {
+	return 0.0, api.CLOUD_PROVIDER_HEALTH_NORMAL, cloudprovider.ErrNotSupported
+}
+
+func (self *SObjectStoreProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) {
+	return self.client, nil
+}
+
+func (self *SObjectStoreProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (self *SObjectStoreProvider) GetSysInfo() (jsonutils.JSONObject, error) {
+	return self.client.About(), nil
+}
+
+func (self *SObjectStoreProvider) GetVersion() string {
+	return self.client.GetVersion()
+}
+
+func (self *SObjectStoreProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
+	return self.client.GetSubAccounts()
+}

pkg/multicloud/objectstore/shell.go

@@ -0,0 +1,204 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package objectstore
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"os"
+	"time"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/util/printutils"
+	"yunion.io/x/onecloud/pkg/util/shellutils"
+)
+
+func S3Shell() {
+	type BucketListOptions struct {
+	}
+	shellutils.R(&BucketListOptions{}, "bucket-list", "List all bucket", func(cli cloudprovider.ICloudRegion, args *BucketListOptions) error {
+		buckets, err := cli.GetIBuckets()
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type BucketCreateOptions struct {
+		NAME string `help:"name of bucket to create"`
+	}
+	shellutils.R(&BucketCreateOptions{}, "bucket-create", "Create bucket", func(cli cloudprovider.ICloudRegion, args *BucketCreateOptions) error {
+		err := cli.CreateIBucket(args.NAME, "", "")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketDeleteOptions struct {
+		NAME string `help:"name of bucket to delete"`
+	}
+	shellutils.R(&BucketDeleteOptions{}, "bucket-delete", "Delete bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteOptions) error {
+		err := cli.DeleteIBucket(args.NAME)
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type BucketObjectsOptions struct {
+		BUCKET    string `help:"name of bucket to list objects"`
+		Prefix    string `help:"prefix"`
+		Marker    string `help:"marker"`
+		Demiliter string `help:"delimiter"`
+		Max       int    `help:"Max count"`
+	}
+	shellutils.R(&BucketObjectsOptions{}, "bucket-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketObjectsOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		result, err := bucket.ListObjects(args.Prefix, args.Marker, args.Demiliter, args.Max)
+		if err != nil {
+			return err
+		}
+		if result.IsTruncated {
+			fmt.Printf("NextMarker: %s IsTruncated: %v\n", result.NextMarker, result.IsTruncated)
+		}
+		fmt.Println("Common prefixes:")
+		printutils.PrintGetterList(result.CommonPrefixes, []string{"key", "size_bytes"})
+		fmt.Println("Objects:")
+		printutils.PrintGetterList(result.Objects, []string{"key", "size_bytes"})
+		return nil
+	})
+
+	type BucketListObjectsOptions struct {
+		BUCKET string `help:"name of bucket to list objects"`
+		Prefix string `help:"prefix"`
+	}
+	shellutils.R(&BucketListObjectsOptions{}, "bucket-list-object", "List objects in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		objects, err := bucket.GetIObjects(args.Prefix, true)
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(objects, []string{"key", "size_bytes"})
+		return nil
+	})
+
+	shellutils.R(&BucketListObjectsOptions{}, "bucket-dir-object", "List objects in a bucket like directory", func(cli cloudprovider.ICloudRegion, args *BucketListObjectsOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		objects, err := bucket.GetIObjects(args.Prefix, false)
+		if err != nil {
+			return err
+		}
+		printutils.PrintGetterList(objects, []string{"key", "size_bytes"})
+		return nil
+	})
+
+	type BucketMakrdirOptions struct {
+		BUCKET string `help:"name of bucket to put object"`
+		DIR    string `help:"dir to make"`
+	}
+	shellutils.R(&BucketMakrdirOptions{}, "bucket-mkdir", "Mkdir in a bucket", func(cli cloudprovider.ICloudRegion, args *BucketMakrdirOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		err = cloudprovider.Makedir(context.Background(), bucket, args.DIR)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Mkdir success\n")
+		return nil
+	})
+
+	type BucketPutObjectOptions struct {
+		BUCKET string `help:"name of bucket to put object"`
+		KEY    string `help:"key of object"`
+		Path   string `help:"Path of file to upload"`
+
+		ContentType  string `help:"content-type"`
+		StorageClass string `help:"storage class"`
+	}
+	shellutils.R(&BucketPutObjectOptions{}, "put-object", "Put object into a bucket", func(cli cloudprovider.ICloudRegion, args *BucketPutObjectOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		var input io.ReadSeeker
+		if len(args.Path) > 0 {
+			file, err := os.Open(args.Path)
+			if err != nil {
+				return err
+			}
+			defer file.Close()
+
+			input = file
+		} else {
+			input = os.Stdout
+		}
+		err = bucket.PutObject(context.Background(), args.KEY, input, args.ContentType, args.StorageClass)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Upload success\n")
+		return nil
+	})
+
+	type BucketDeleteObjectOptions struct {
+		BUCKET string `help:"name of bucket to put object"`
+		KEY    string `help:"key of object"`
+	}
+	shellutils.R(&BucketDeleteObjectOptions{}, "delete-object", "Delete object from a bucket", func(cli cloudprovider.ICloudRegion, args *BucketDeleteObjectOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		err = bucket.DeleteObject(context.Background(), args.KEY)
+		if err != nil {
+			return err
+		}
+		fmt.Printf("Delete success\n")
+		return nil
+	})
+
+	type BucketTempUrlOption struct {
+		BUCKET   string `help:"name of bucket to put object"`
+		METHOD   string `help:"http method" choices:"GET|PUT|DELETE"`
+		KEY      string `help:"key of object"`
+		Duration int    `help:"duration in seconds" default:"60"`
+	}
+	shellutils.R(&BucketTempUrlOption{}, "temp-url", "generate temp url", func(cli cloudprovider.ICloudRegion, args *BucketTempUrlOption) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		urlStr, err := bucket.GetTempUrl(args.METHOD, args.KEY, time.Duration(args.Duration)*time.Second)
+		if err != nil {
+			return err
+		}
+		fmt.Println(urlStr)
+		return nil
+	})
+}

pkg/multicloud/objectstore/shell/bucket.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import "yunion.io/x/onecloud/pkg/multicloud/objectstore"
+
+func init() {
+	objectstore.S3Shell()
+}

pkg/multicloud/objectstore/shell/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/shell"

pkg/multicloud/objectstore/shell/printutils.go

@@ -0,0 +1,25 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import "yunion.io/x/onecloud/pkg/util/printutils"
+
+func printList(data interface{}, columns []string) {
+	printutils.PrintGetterList(data, columns)
+}
+
+func printObject(obj interface{}) {
+	printutils.PrintInterfaceObject(obj)
+}

pkg/s3gateway/models/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models // import "yunion.io/x/onecloud/pkg/s3gateway/models"

pkg/s3gateway/models/initdb.go

@@ -0,0 +1,37 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+)
+
+func InitDB() error {
+	for _, manager := range []db.IModelManager{
+		/*
+		 * Important!!!
+		 * initialization order matters, do not change the order
+		 */
+	} {
+		err := manager.InitializeData()
+		if err != nil {
+			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
+			// return err skip error table
+		}
+	}
+	return nil
+}

pkg/s3gateway/options/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options // import "yunion.io/x/onecloud/pkg/s3gateway/options"

pkg/s3gateway/options/options.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package options
+
+import (
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+)
+
+type SS3GatewayOptions struct {
+	common_options.CommonOptions
+
+	common_options.DBOptions
+}
+
+var (
+	Options SS3GatewayOptions
+)

pkg/s3gateway/service/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service // import "yunion.io/x/onecloud/pkg/s3gateway/service"

pkg/s3gateway/service/handlers.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	// "yunion.io/x/onecloud/pkg/s3gateway/models"
+)
+
+func initHandlers(app *appsrv.Application) {
+	db.InitAllManagers()
+
+	// quotas.AddQuotaHandler(models.QuotaManager, API_VERSION, app)
+	// usages.AddUsageHandler(API_VERSION, app)
+	taskman.AddTaskHandler("", app)
+
+	for _, manager := range []db.IModelManager{
+		taskman.TaskManager,
+		taskman.SubTaskManager,
+		taskman.TaskObjectManager,
+		db.Metadata,
+	} {
+		db.RegisterModelManager(manager)
+	}
+
+	for _, manager := range []db.IModelManager{
+		db.OpsLog,
+	} {
+		db.RegisterModelManager(manager)
+		handler := db.NewModelHandler(manager)
+		dispatcher.AddModelDispatcher("", app, handler)
+	}
+}

pkg/s3gateway/service/service.go

@@ -0,0 +1,73 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package service
+
+import (
+	"os"
+
+	"yunion.io/x/log"
+
+	api "yunion.io/x/onecloud/pkg/apis/s3gateway"
+	"yunion.io/x/onecloud/pkg/cloudcommon"
+	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+	"yunion.io/x/onecloud/pkg/s3gateway/options"
+)
+
+func StartService() {
+	opts := &options.Options
+	commonOpts := &opts.CommonOptions
+	baseOpts := &opts.BaseOptions
+	dbOpts := &opts.DBOptions
+	common_options.ParseOptions(opts, os.Args, "s3gateway.conf", api.SERVICE_TYPE)
+
+	app_common.InitAuth(commonOpts, func() {
+		log.Infof("Auth complete!!")
+	})
+
+	cloudcommon.InitDB(dbOpts)
+
+	app := app_common.InitApp(&opts.BaseOptions, true)
+	initHandlers(app)
+
+	cloudcommon.InitDB(&opts.DBOptions)
+
+	if !db.CheckSync(opts.AutoSyncTable) {
+		log.Fatalf("database schema not in sync!")
+	}
+
+	models.InitDB()
+
+	if opts.ExitAfterDBInit {
+		log.Infof("Exiting after db initialization ...")
+		os.Exit(0)
+	}
+
+	/*if !opts.IsSlaveNode {
+		cron := cronman.GetCronJobManager(true)
+		cron.AddJob1("CleanPendingDeleteImages", time.Duration(options.Options.PendingDeleteCheckSeconds)*time.Second, models.ImageManager.CleanPendingDeleteImages)
+		cron.AddJob1("CalculateQuotaUsages", time.Duration(opts.CalculateQuotaUsageIntervalSeconds)*time.Second, models.QuotaManager.CalculateQuotaUsages)
+
+		cron.Start()
+	}*/
+
+	cloudcommon.AppDBInit(app)
+	app_common.ServeForeverWithCleanup(app, baseOpts, func() {
+		cloudcommon.CloseDB()
+
+	})
+}

pkg/scheduler/algorithm/predicates/predicates.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at

pkg/util/aliyun/bucket.go

@@ -0,0 +1,201 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aliyun
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/aliyun/aliyun-oss-go-sdk/oss"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	region *SRegion
+
+	Name         string
+	Location     string
+	CreationDate time.Time
+	StorageClass string
+	Acl          string
+}
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SBucket) GetName() string {
+	return b.Name
+}
+
+func (b *SBucket) GetAcl() string {
+	return b.Acl
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Location
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return b.CreationDate
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return b.StorageClass
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         fmt.Sprintf("https://%s.aliyuncs.com", b.Location),
+			Description: "ExtranetEndpoint",
+		},
+		{
+			Url:         fmt.Sprintf("https://%s-internal.aliyuncs.com", b.Location),
+			Description: "IntranetEndpoint",
+		},
+	}
+}
+
+func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
+	result := cloudprovider.SListObjectResult{}
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return result, errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return result, errors.Wrap(err, "Bucket")
+	}
+	opts := make([]oss.Option, 0)
+	if len(prefix) > 0 {
+		opts = append(opts, oss.Prefix(prefix))
+	}
+	if len(delimiter) > 0 {
+		opts = append(opts, oss.Delimiter(delimiter))
+	}
+	if len(marker) > 0 {
+		opts = append(opts, oss.Marker(marker))
+	}
+	if maxCount > 0 {
+		opts = append(opts, oss.MaxKeys(maxCount))
+	}
+	oResult, err := bucket.ListObjects(opts...)
+	if err != nil {
+		return result, errors.Wrap(err, "ListObjects")
+	}
+	result.Objects = make([]cloudprovider.ICloudObject, 0)
+	for _, object := range oResult.Objects {
+		obj := &SObject{
+			bucket: b,
+			SBaseCloudObject: cloudprovider.SBaseCloudObject{
+				StorageClass: object.StorageClass,
+				Key:          object.Key,
+				SizeBytes:    object.Size,
+				ETag:         object.ETag,
+				LastModified: object.LastModified,
+				ContentType:  object.Type,
+			},
+		}
+		result.Objects = append(result.Objects, obj)
+	}
+	if oResult.CommonPrefixes != nil {
+		result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(oResult.CommonPrefixes))
+		for i, commPrefix := range oResult.CommonPrefixes {
+			result.CommonPrefixes[i] = &SObject{
+				bucket:           b,
+				SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: commPrefix},
+			}
+		}
+	}
+	result.IsTruncated = oResult.IsTruncated
+	result.NextMarker = oResult.NextMarker
+	return result, nil
+}
+
+func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
+	return cloudprovider.GetIObjects(b, prefix, isRecursive)
+}
+
+func (b *SBucket) PutObject(ctx context.Context, key string, input io.Reader, contType string, storageClassStr string) error {
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return errors.Wrap(err, "Bucket")
+	}
+	opts := make([]oss.Option, 0)
+	if len(contType) > 0 {
+		opts = append(opts, oss.ContentType(contType))
+	}
+	if len(storageClassStr) > 0 {
+		storageClass, err := str2StorageClass(storageClassStr)
+		if err != nil {
+			return errors.Wrap(err, "str2StorageClass")
+		}
+		opts = append(opts, oss.ObjectStorageClass(storageClass))
+	}
+	return bucket.PutObject(key, input, opts...)
+}
+
+func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return errors.Wrap(err, "Bucket")
+	}
+	err = bucket.DeleteObject(key)
+	if err != nil {
+		return errors.Wrap(err, "DeleteObject")
+	}
+	return nil
+}
+
+func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
+	if method != "GET" && method != "PUT" && method != "DELETE" {
+		return "", errors.Error("unsupported method")
+	}
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return "", errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return "", errors.Wrap(err, "Bucket")
+	}
+	urlStr, err := bucket.SignURL(key, oss.HTTPMethod(method), int64(expire/time.Second))
+	if err != nil {
+		return "", errors.Wrap(err, "SignURL")
+	}
+	return urlStr, nil
+}

pkg/util/aliyun/objects.go

@@ -0,0 +1,29 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aliyun
+
+import (
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SObject struct {
+	bucket *SBucket
+
+	cloudprovider.SBaseCloudObject
+}
+
+func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
+	return o.bucket
+}

pkg/util/aliyun/region.go

@@ -24,6 +24,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/secrules"
 	"yunion.io/x/pkg/utils"
@@ -917,3 +918,150 @@ func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAc
 	}
 	return iAcl, region.AddAccessControlListEntry(aclId, acl.Entrys)
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetOssClient")
+	}
+	result, err := osscli.ListBuckets()
+	if err != nil {
+		return nil, errors.Wrap(err, "oss.ListBuckets")
+	}
+
+	ret := make([]cloudprovider.ICloudBucket, 0)
+	for _, bInfo := range result.Buckets {
+		if bInfo.Location[4:] != region.GetId() {
+			continue
+		}
+		acl := string(oss.ACLPrivate)
+		aclResp, err := osscli.GetBucketACL(bInfo.Name)
+		if err == nil {
+			acl = aclResp.ACL
+		}
+		b := SBucket{
+			region: region,
+
+			Name:         bInfo.Name,
+			Location:     bInfo.Location,
+			CreationDate: bInfo.CreationDate,
+			StorageClass: bInfo.StorageClass,
+			Acl:          acl,
+		}
+		ret = append(ret, &b)
+	}
+	return ret, nil
+}
+
+func str2StorageClass(storageClassStr string) (oss.StorageClassType, error) {
+	storageClass := oss.StorageStandard
+	if strings.EqualFold(storageClassStr, string(oss.StorageStandard)) {
+		//
+	} else if strings.EqualFold(storageClassStr, string(oss.StorageIA)) {
+		storageClass = oss.StorageIA
+	} else if strings.EqualFold(storageClassStr, string(oss.StorageArchive)) {
+		storageClass = oss.StorageArchive
+	} else {
+		return storageClass, errors.Error("not supported storageClass")
+	}
+	return storageClass, nil
+}
+
+func str2Acl(aclStr string) (oss.ACLType, error) {
+	acl := oss.ACLPrivate
+	if strings.EqualFold(aclStr, string(oss.ACLPrivate)) {
+		// private, default
+	} else if strings.EqualFold(aclStr, string(oss.ACLPublicRead)) {
+		acl = oss.ACLPublicRead
+	} else if strings.EqualFold(aclStr, string(oss.ACLPublicReadWrite)) {
+		acl = oss.ACLPublicReadWrite
+	} else {
+		return acl, errors.Error("not supported acl")
+	}
+	return acl, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, aclStr string) error {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "region.GetOssClient")
+	}
+	opts := make([]oss.Option, 0)
+	if len(storageClassStr) > 0 {
+		storageClass, err := str2StorageClass(storageClassStr)
+		if err != nil {
+			return err
+		}
+		opts = append(opts, oss.StorageClass(storageClass))
+	}
+	if len(aclStr) > 0 {
+		acl, err := str2Acl(aclStr)
+		if err != nil {
+			return err
+		}
+		opts = append(opts, oss.ACL(acl))
+	}
+	err = osscli.CreateBucket(name, opts...)
+	if err != nil {
+		return errors.Wrap(err, "oss.CreateBucket")
+	}
+	return nil
+}
+
+func ossErrorCode(err error) int {
+	if srvErr, ok := err.(oss.ServiceError); ok {
+		return srvErr.StatusCode
+	}
+	if srvErr, ok := err.(*oss.ServiceError); ok {
+		return srvErr.StatusCode
+	}
+	return -1
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "region.GetOssClient")
+	}
+	err = osscli.DeleteBucket(name)
+	if err != nil {
+		if ossErrorCode(err) == 404 {
+			return nil
+		}
+		return errors.Wrap(err, "DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return false, errors.Wrap(err, "region.GetOssClient")
+	}
+	exist, err := osscli.IsBucketExist(name)
+	if err != nil {
+		return false, errors.Wrap(err, "IsBucketExist")
+	}
+	return exist, nil
+}
+
+func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
+	osscli, err := region.GetOssClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetOssClient")
+	}
+	bi, err := osscli.GetBucketInfo(name)
+	if err != nil {
+		return nil, errors.Wrap(err, "Bucket")
+	}
+	bInfo := bi.BucketInfo
+	b := SBucket{
+		region:       region,
+		Name:         bInfo.Name,
+		Location:     bInfo.Location,
+		CreationDate: bInfo.CreationDate,
+		StorageClass: bInfo.StorageClass,
+		Acl:          bInfo.ACL,
+	}
+	return &b, nil
+}

pkg/util/aliyun/shell/bucket.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import "yunion.io/x/onecloud/pkg/multicloud/objectstore"
+
+func init() {
+	objectstore.S3Shell()
+}

pkg/util/aliyun/shell/oss.go

@@ -59,15 +59,11 @@ func init() {
 	type OssListOptions struct {
 	}
 	shellutils.R(&OssListOptions{}, "oss-list", "List OSS buckets", func(cli *aliyun.SRegion, args *OssListOptions) error {
-		oss, err := cli.GetOssClient()
+		buckets, err := cli.GetIBuckets()
 		if err != nil {
 			return err
 		}
-		result, err := oss.ListBuckets()
-		if err != nil {
-			return err
-		}
-		printList(result.Buckets, len(result.Buckets), 0, 50, nil)
+		printList(buckets, len(buckets), 0, 50, nil)
 		return nil
 	})
 
@@ -92,12 +88,25 @@ func init() {
 		return nil
 	})
 
-	shellutils.R(&OssListBucketOptions{}, "oss-create-bucket", "Create a OSS bucket", func(cli *aliyun.SRegion, args *OssListBucketOptions) error {
-		oss, err := cli.GetOssClient()
+	type OssCreateBucketOptions struct {
+		BUCKET       string `help:"bucket name"`
+		StorageClass string `help:"storage class" choices:"Standard|IA|Archive"`
+
+		Acl string `help:"ACL" choices:"private|public-read|public-read-write"`
+	}
+	shellutils.R(&OssCreateBucketOptions{}, "oss-create-bucket", "Create a OSS bucket", func(cli *aliyun.SRegion, args *OssCreateBucketOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, args.StorageClass, args.Acl)
 		if err != nil {
 			return err
 		}
-		err = oss.CreateBucket(args.BUCKET)
+		return nil
+	})
+
+	type OssDeleteBucketOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&OssDeleteBucketOptions{}, "oss-delete-bucket", "Delete a OSS bucket", func(cli *aliyun.SRegion, args *OssDeleteBucketOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
 		if err != nil {
 			return err
 		}

pkg/util/aws/aws.go

@@ -154,9 +154,9 @@ func (self *SAwsClient) GetRegion(regionId string) *SRegion {
 	if len(regionId) == 0 {
 		regionId = AWS_INTERNATIONAL_DEFAULT_REGION
 		switch self.accessUrl {
-		case "InternationalCloud":
+		case AWS_INTERNATIONAL_CLOUDENV:
 			regionId = AWS_INTERNATIONAL_DEFAULT_REGION
-		case "ChinaCloud":
+		case AWS_CHINA_CLOUDENV:
 			regionId = AWS_CHINA_DEFAULT_REGION
 		}
 	}

pkg/util/aws/bucket.go

@@ -0,0 +1,216 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/request"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+)
+
+type SBucket struct {
+	region *SRegion
+
+	Name         string
+	Location     string
+	CreationDate time.Time
+}
+
+func (b *SBucket) GetProjectId() string {
+	return ""
+}
+
+func (b *SBucket) GetGlobalId() string {
+	return b.Name
+}
+
+func (b *SBucket) GetName() string {
+	return b.Name
+}
+
+func (b *SBucket) GetLocation() string {
+	return b.Location
+}
+
+func (b *SBucket) GetIRegion() cloudprovider.ICloudRegion {
+	return b.region
+}
+
+func (b *SBucket) GetCreateAt() time.Time {
+	return b.CreationDate
+}
+
+func (b *SBucket) GetStorageClass() string {
+	return ""
+}
+
+func (b *SBucket) GetAcl() string {
+	return ""
+}
+
+func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
+	return []cloudprovider.SBucketAccessUrl{
+		{
+			Url:         fmt.Sprintf("https://%s.%s", b.Name, b.region.getS3Endpoint()),
+			Description: "bucket domain",
+		},
+		{
+			Url:         fmt.Sprintf("https://%s/%s", b.region.getS3Endpoint(), b.Name),
+			Description: "s3 domain",
+		},
+	}
+}
+
+func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
+	result := cloudprovider.SListObjectResult{}
+	s3cli, err := b.region.GetS3Client()
+	if err != nil {
+		return result, errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.ListObjectsInput{}
+	input.SetBucket(b.Name)
+	if len(prefix) > 0 {
+		input.SetPrefix(prefix)
+	}
+	if len(marker) > 0 {
+		input.SetMarker(marker)
+	}
+	if len(delimiter) > 0 {
+		input.SetDelimiter(delimiter)
+	}
+	if maxCount > 0 {
+		input.SetMaxKeys(int64(maxCount))
+	}
+	oResult, err := s3cli.ListObjects(input)
+	if err != nil {
+		return result, errors.Wrap(err, "ListObjects")
+	}
+	result.Objects = make([]cloudprovider.ICloudObject, 0)
+	for _, object := range oResult.Contents {
+		obj := &SObject{
+			bucket: b,
+			SBaseCloudObject: cloudprovider.SBaseCloudObject{
+				StorageClass: *object.StorageClass,
+				Key:          *object.Key,
+				SizeBytes:    *object.Size,
+				ETag:         *object.ETag,
+				LastModified: *object.LastModified,
+				ContentType:  "",
+			},
+		}
+		result.Objects = append(result.Objects, obj)
+	}
+	if oResult.CommonPrefixes != nil {
+		result.CommonPrefixes = make([]cloudprovider.ICloudObject, len(oResult.CommonPrefixes))
+		for i, commPrefix := range oResult.CommonPrefixes {
+			result.CommonPrefixes[i] = &SObject{
+				bucket:           b,
+				SBaseCloudObject: cloudprovider.SBaseCloudObject{Key: *commPrefix.Prefix},
+			}
+		}
+	}
+	if oResult.IsTruncated != nil {
+		result.IsTruncated = *oResult.IsTruncated
+	}
+	if oResult.NextMarker != nil {
+		result.NextMarker = *oResult.NextMarker
+	}
+	return result, nil
+}
+
+func (b *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
+	return cloudprovider.GetIObjects(b, prefix, isRecursive)
+}
+
+func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, contType string, storageClassStr string) error {
+	sess, err := session.NewSession(&aws.Config{Region: aws.String(b.region.GetId())})
+	if err != nil {
+		return errors.Wrap(err, "session.NewSession")
+	}
+
+	svc := s3manager.NewUploader(sess)
+	input := &s3manager.UploadInput{
+		Bucket: aws.String(b.Name),
+		Key:    aws.String(key),
+		Body:   reader,
+	}
+	if len(contType) > 0 {
+		input.ContentType = aws.String(contType)
+	}
+	if len(storageClassStr) > 0 {
+		input.StorageClass = aws.String(storageClassStr)
+	}
+
+	_, err = svc.Upload(input)
+	if err != nil {
+		return errors.Wrap(err, "svc.Upload")
+	}
+
+	return nil
+}
+
+func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
+	s3cli, err := b.region.GetS3Client()
+	if err != nil {
+		return errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.DeleteObjectInput{}
+	input.SetBucket(b.Name)
+	input.SetKey(key)
+	_, err = s3cli.DeleteObjectWithContext(ctx, input)
+	if err != nil {
+		return errors.Wrap(err, "DeleteObject")
+	}
+	return nil
+}
+
+func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
+	s3cli, err := b.region.GetS3Client()
+	if err != nil {
+		return "", errors.Wrap(err, "GetS3Client")
+	}
+	var request *request.Request
+	switch method {
+	case "GET":
+		input := &s3.GetObjectInput{}
+		input.SetBucket(b.Name)
+		input.SetKey(key)
+		request, _ = s3cli.GetObjectRequest(input)
+	case "PUT":
+		input := &s3.PutObjectInput{}
+		input.SetBucket(b.Name)
+		input.SetKey(key)
+		request, _ = s3cli.PutObjectRequest(input)
+	case "DELETE":
+		input := &s3.DeleteObjectInput{}
+		input.SetBucket(b.Name)
+		input.SetKey(key)
+		request, _ = s3cli.DeleteObjectRequest(input)
+	default:
+		return "", errors.Error("unsupported method")
+	}
+	return request.Presign(expire)
+}

pkg/util/aws/region.go

@@ -16,6 +16,7 @@ package aws
 
 import (
 	"fmt"
+	"strings"
 
 	sdk "github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -26,6 +27,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -524,3 +526,109 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala
 func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
 	return nil, cloudprovider.ErrNotImplemented
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetS3Client")
+	}
+	output, err := s3cli.ListBuckets(&s3.ListBucketsInput{})
+	if err != nil {
+		return nil, errors.Wrap(err, "ListBuckets")
+	}
+	ret := make([]cloudprovider.ICloudBucket, 0)
+	for _, bInfo := range output.Buckets {
+		input := &s3.GetBucketLocationInput{}
+		input.Bucket = bInfo.Name
+		output, err := s3cli.GetBucketLocation(input)
+		if err != nil {
+			log.Errorf("s3cli.GetBucketLocation error %s", err)
+			continue
+		}
+		if *output.LocationConstraint != region.GetId() {
+			continue
+		}
+		b := SBucket{
+			region:       region,
+			Name:         *bInfo.Name,
+			Location:     region.GetId(),
+			CreationDate: *bInfo.CreationDate,
+		}
+		ret = append(ret, &b)
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, acl string) error {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.CreateBucketInput{}
+	input.Bucket = &name
+	input.CreateBucketConfiguration = &s3.CreateBucketConfiguration{}
+	location := region.GetId()
+	input.CreateBucketConfiguration.LocationConstraint = &location
+	_, err = s3cli.CreateBucket(input)
+	if err != nil {
+		return errors.Wrap(err, "CreateBucket")
+	}
+	// if *output.Location != region.GetId() {
+	// 	log.Warningf("Request location %s != got locaiton %s", region.GetId(), *output.Location)
+	// }
+	return nil
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.DeleteBucketInput{}
+	input.Bucket = &name
+	_, err = s3cli.DeleteBucket(input)
+	if err != nil {
+		if strings.Index(err.Error(), "NoSuchBucket") >= 0 {
+			return nil
+		}
+		return errors.Wrap(err, "DeleteBucket")
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	s3cli, err := region.GetS3Client()
+	if err != nil {
+		return false, errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.HeadBucketInput{}
+	input.Bucket = &name
+	_, err = s3cli.HeadBucket(input)
+	if err != nil {
+		return false, errors.Wrap(err, "IsBucketExist")
+	}
+	return true, nil
+}
+
+func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketById(region, name)
+}
+
+func (region *SRegion) getBaseEndpoint() string {
+	if len(region.RegionEndpoint) > 4 {
+		return region.RegionEndpoint[4:]
+	}
+	return ""
+}
+
+func (region *SRegion) getS3Endpoint() string {
+	base := region.getBaseEndpoint()
+	if len(base) > 0 {
+		return "s3." + base
+	}
+	return ""
+}
+
+func (region *SRegion) getEc2Endpoint() string {
+	return region.RegionEndpoint
+}

pkg/util/aws/s3object.go

@@ -0,0 +1,27 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package aws
+
+import "yunion.io/x/onecloud/pkg/cloudprovider"
+
+type SObject struct {
+	bucket *SBucket
+
+	cloudprovider.SBaseCloudObject
+}
+
+func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
+	return o.bucket
+}

pkg/util/aws/shell/bucket.go

@@ -0,0 +1,21 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import "yunion.io/x/onecloud/pkg/multicloud/objectstore"
+
+func init() {
+	objectstore.S3Shell()
+}

pkg/util/aws/shell/s3.go

@@ -16,9 +16,12 @@ package shell
 
 import (
 	"fmt"
-	"github.com/aws/aws-sdk-go/service/s3"
 	"os"
+
+	"github.com/aws/aws-sdk-go/service/s3"
+
 	"yunion.io/x/onecloud/pkg/util/aws"
+	"yunion.io/x/onecloud/pkg/util/printutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
 	"yunion.io/x/onecloud/pkg/util/streamutils"
 )
@@ -27,15 +30,34 @@ func init() {
 	type S3BucketListOptions struct {
 	}
 	shellutils.R(&S3BucketListOptions{}, "s3-list", "List all buckets", func(cli *aws.SRegion, args *S3BucketListOptions) error {
-		s3cli, err := cli.GetS3Client()
+		buckets, err := cli.GetIBuckets()
 		if err != nil {
 			return err
 		}
-		output, err := s3cli.ListBuckets(&s3.ListBucketsInput{})
+		printList(buckets, 0, 0, 0, nil)
+		printutils.PrintGetterList(buckets, nil)
+		return nil
+	})
+
+	type S3CreateBucketOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&S3CreateBucketOptions{}, "s3-create-bucket", "Create a bucket", func(cli *aws.SRegion, args *S3CreateBucketOptions) error {
+		err := cli.CreateIBucket(args.BUCKET, "", "")
+		if err != nil {
+			return err
+		}
+		return nil
+	})
+
+	type S3DeleteBucketOptions struct {
+		BUCKET string `help:"bucket name"`
+	}
+	shellutils.R(&S3DeleteBucketOptions{}, "s3-delete-bucket", "Delete a bucket", func(cli *aws.SRegion, args *S3DeleteBucketOptions) error {
+		err := cli.DeleteIBucket(args.BUCKET)
 		if err != nil {
 			return err
 		}
-		printList(output.Buckets, 0, 0, 0, nil)
 		return nil
 	})
 

pkg/util/azure/blobobject.go

@@ -0,0 +1,27 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package azure
+
+import "yunion.io/x/onecloud/pkg/cloudprovider"
+
+type SObject struct {
+	container *SContainer
+
+	cloudprovider.SBaseCloudObject
+}
+
+func (o *SObject) GetIBucket() cloudprovider.ICloudBucket {
+	return o.container.storageaccount
+}

pkg/util/azure/classic_disk.go

@@ -22,6 +22,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
+	"github.com/Azure/azure-sdk-for-go/storage"
 	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -67,7 +68,7 @@ func (self *SRegion) GetStorageAccountDisksWithSnapshots(storageaccount SStorage
 	}
 	for _, container := range containers {
 		if container.Name == "vhds" {
-			files, err := container.ListFiles()
+			files, err := container.ListAllFiles(&storage.IncludeBlobDataset{Snapshots: true, Metadata: true})
 			if err != nil {
 				log.Errorf("List storage %s container %s files error: %v", storageaccount.Name, container.Name, err)
 				return nil, nil, err

pkg/util/azure/region.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/util/secrules"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -608,3 +609,48 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala
 func (region *SRegion) CreateILoadBalancerAcl(acl *cloudprovider.SLoadbalancerAccessControlList) (cloudprovider.ICloudLoadbalancerAcl, error) {
 	return nil, cloudprovider.ErrNotImplemented
 }
+
+func (region *SRegion) GetIBuckets() ([]cloudprovider.ICloudBucket, error) {
+	accounts, err := region.GetStorageAccounts()
+	if err != nil {
+		return nil, errors.Wrap(err, "region.GetStorageAccounts")
+	}
+	ret := make([]cloudprovider.ICloudBucket, len(accounts))
+	for i := range accounts {
+		ret[i] = &accounts[i]
+	}
+	return ret, nil
+}
+
+func (region *SRegion) CreateIBucket(name string, storageClassStr string, acl string) error {
+	_, err := region.createStorageAccount(name, storageClassStr)
+	if err != nil {
+		return errors.Wrap(err, "region.createStorageAccount")
+	}
+	return nil
+}
+
+func (region *SRegion) DeleteIBucket(name string) error {
+	accounts, err := region.GetStorageAccounts()
+	if err != nil {
+		return errors.Wrap(err, "GetStorageAccounts")
+	}
+	for i := range accounts {
+		if accounts[i].Name == name {
+			err = region.client.Delete(accounts[i].ID)
+			if err != nil {
+				return errors.Wrap(err, "region.client.Delete")
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+func (region *SRegion) IBucketExist(name string) (bool, error) {
+	return region.checkStorageAccountNameExist(name)
+}
+
+func (region *SRegion) GetIBucketById(name string) (cloudprovider.ICloudBucket, error) {
+	return cloudprovider.GetIBucketById(region, name)
+}

pkg/util/azure/shell/bucket.go

@@ -0,0 +1,23 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package shell
+
+import (
+	"yunion.io/x/onecloud/pkg/multicloud/objectstore"
+)
+
+func init() {
+	objectstore.S3Shell()
+}

pkg/util/azure/shell/storageaccount.go

@@ -23,14 +23,12 @@ import (
 
 func init() {
 	type StorageAccountListOptions struct {
-		Limit  int `help:"page size"`
-		Offset int `help:"page offset"`
 	}
 	shellutils.R(&StorageAccountListOptions{}, "storage-account-list", "List storage account", func(cli *azure.SRegion, args *StorageAccountListOptions) error {
 		if accounts, err := cli.GetStorageAccounts(); err != nil {
 			return err
 		} else {
-			printList(accounts, len(accounts), args.Offset, args.Limit, []string{})
+			printList(accounts, len(accounts), 0, 0, []string{})
 			return nil
 		}
 	})
@@ -100,7 +98,7 @@ func init() {
 		if err != nil {
 			return err
 		}
-		blobs, err := container.ListFiles()
+		blobs, err := container.ListAllFiles(nil)
 		if err != nil {
 			return err
 		}
@@ -152,4 +150,14 @@ func init() {
 		return nil
 	})
 
+	type SStorageAccountSkuOptions struct {
+	}
+	shellutils.R(&SStorageAccountSkuOptions{}, "storage-account-skus", "List skus of storage account", func(cli *azure.SRegion, args *SStorageAccountSkuOptions) error {
+		skus, err := cli.GetStorageAccountSkus()
+		if err != nil {
+			return err
+		}
+		printList(skus, 0, 0, 0, nil)
+		return nil
+	})
 }