misc imporvements

cmd/climc/shell/buckets.go

@@ -21,10 +21,12 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modulebase"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/mcclient/options"
+	objectshell "yunion.io/x/onecloud/pkg/multicloud/objectstore"
 )
 
 func init() {
@@ -290,4 +292,27 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type BucketSetMetadataOptions struct {
+		ID string `help:"ID or name of bucket" json:"-"`
+
+		Key []string `help:"Optional object key" json:"key"`
+
+		objectshell.ObjectHeaderOptions
+	}
+	R(&BucketSetMetadataOptions{}, "bucket-set-metadata", "Set metadata of object", func(s *mcclient.ClientSession, args *BucketSetMetadataOptions) error {
+		input := api.BucketMetadataInput{}
+		input.Key = args.Key
+		input.Metadata = args.ObjectHeaderOptions.Options2Header()
+		err := input.Validate()
+		if err != nil {
+			return err
+		}
+		result, err := modules.Buckets.PerformAction(s, args.ID, "metadata", jsonutils.Marshal(input))
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

pkg/apis/compute/bucket.go

@@ -15,7 +15,13 @@
 package compute
 
 import (
+	"net/http"
+
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
 const (
@@ -47,3 +53,39 @@ type BucketDetail struct {
 	apis.Meta
 	SBucket
 }
+
+type BucketObjectsActionInput struct {
+	Key []string
+}
+
+type BucketAclInput struct {
+	BucketObjectsActionInput
+
+	Acl cloudprovider.TBucketACLType
+}
+
+func (input *BucketAclInput) Validate() error {
+	switch input.Acl {
+	case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
+		// do nothing
+	default:
+		return errors.Wrap(httperrors.ErrInputParameter, "acl")
+	}
+	return nil
+}
+
+type BucketMetadataInput struct {
+	BucketObjectsActionInput
+
+	Metadata http.Header
+}
+
+func (input *BucketMetadataInput) Validate() error {
+	if len(input.Key) == 0 {
+		return errors.Wrap(httperrors.ErrEmptyRequest, "key")
+	}
+	if len(input.Metadata) == 0 {
+		return errors.Wrap(httperrors.ErrEmptyRequest, "metadata")
+	}
+	return nil
+}

pkg/apis/compute/cloudaccount.go

@@ -19,39 +19,10 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/apis"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 )
 
-type CloudaccountCredentialInput struct {
-	ProjectName string //OpenStack
-	DomainName  string //OpenStack
-	Username    string //OpenStack Esxi ZStack
-	Password    string //OpenStack Esxi ZStack
-	AuthUrl     string //OpenStack ZStack
-
-	AccessKeyId     string //Huawei Aliyun Ucloud Aws
-	AccessKeySecret string //Huawei Aliyun Ucloud Aws
-	Environment     string //Huawei Azure Aws
-
-	DirectoryId  string //Azure
-	ClientId     string //Azure
-	ClientSecret string //Azure
-
-	Host string //Esxi
-	Port int    //Esxi
-
-	Endpoint string
-
-	AppId     string //Qcloud
-	SecretId  string //Qcloud
-	SecretKey string //Qcloud
-
-	ClientEmail  string //Google
-	ProjectId    string //Google
-	PrivateKeyId string //Google
-	PrivateKey   string //Google
-}
-
 type CloudaccountCreateInput struct {
 	apis.EnabledStatusStandaloneResourceCreateInput
 
@@ -59,18 +30,14 @@ type CloudaccountCreateInput struct {
 	Brand               string
 	IsPublicCloud       bool
 	IsOnPremise         bool
-	Account             string
-	Secret              string
-	AccessUrl           string
 	TenantId            string
-	Description         string
-	Enabled             bool
 	EnableAutoSync      bool
 	SyncIntervalSeconds int
 	AutoCreateProject   bool
 	Options             *jsonutils.JSONObject
 
-	CloudaccountCredentialInput
+	cloudprovider.SCloudaccount
+	cloudprovider.SCloudaccountCredential
 }
 
 type CloudaccountShareModeInput struct {

pkg/cloudprovider/cloudprovider.go

@@ -22,7 +22,6 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
-	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -31,9 +30,40 @@ const (
 	ErrNoSuchProvder = errors.Error("no such provider")
 )
 
+type SCloudaccountCredential struct {
+	ProjectName string //OpenStack
+	DomainName  string //OpenStack
+	Username    string //OpenStack Esxi ZStack
+	Password    string //OpenStack Esxi ZStack
+	AuthUrl     string //OpenStack ZStack
+
+	AccessKeyId     string //Huawei Aliyun Ucloud Aws
+	AccessKeySecret string //Huawei Aliyun Ucloud Aws
+	Environment     string //Huawei Azure Aws
+
+	DirectoryId  string //Azure
+	ClientId     string //Azure
+	ClientSecret string //Azure
+
+	Host string //Esxi
+	Port int    //Esxi
+
+	Endpoint string
+
+	AppId     string //Qcloud
+	SecretId  string //Qcloud
+	SecretKey string //Qcloud
+
+	ClientEmail  string //Google
+	ProjectId    string //Google
+	PrivateKeyId string //Google
+	PrivateKey   string //Google
+}
+
 type SCloudaccount struct {
-	Account string
-	Secret  string
+	Account   string
+	Secret    string
+	AccessUrl string
 }
 
 type ICloudProviderFactory interface {
@@ -45,8 +75,8 @@ type ICloudProviderFactory interface {
 	GetName() string
 
 	ValidateChangeBandwidth(instanceId string, bandwidth int64) error
-	ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error
-	ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*SCloudaccount, error)
+	ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input SCloudaccountCredential) (SCloudaccount, error)
+	ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input SCloudaccountCredential, cloudaccount string) (SCloudaccount, error)
 	GetSupportedBrands() []string
 
 	IsPublicCloud() bool
@@ -209,14 +239,6 @@ func (factory *baseProviderFactory) GetSupportedBrands() []string {
 	return []string{}
 }
 
-func (factory *baseProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
-	return httperrors.NewNotImplementedError("Not Implemented ValidateCreateCloudaccountData")
-}
-
-func (factory *baseProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject, cloudaccount string) (*SCloudaccount, error) {
-	return nil, httperrors.NewNotImplementedError("Not Implemented ValidateUpdateCloudaccountCredential")
-}
-
 func (factory *baseProviderFactory) GetProvider(providerId, providerName, url, username, password string) (ICloudProvider, error) {
 	return nil, httperrors.NewNotImplementedError("Not Implemented GetProvider")
 }

pkg/cloudprovider/objectstore.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"regexp"
 	"strconv"
 	"strings"
@@ -27,7 +28,6 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/s3cli"
-	"net/http"
 )
 
 type TBucketACLType string
@@ -44,12 +44,14 @@ const (
 	ACLPublicReadWrite = TBucketACLType(s3cli.CANNED_ACL_PUBLIC_READ_WRITE)
 	ACLUnknown         = TBucketACLType("")
 
-	META_HEADER_CONTENT_TYPE        = "Content-Type"
 	META_HEADER_CACHE_CONTROL       = "Cache-Control"
+	META_HEADER_CONTENT_TYPE        = "Content-Type"
 	META_HEADER_CONTENT_DISPOSITION = "Content-Disposition"
 	META_HEADER_CONTENT_ENCODING    = "Content-Encoding"
 	META_HEADER_CONTENT_LANGUAGE    = "Content-Language"
 	META_HEADER_CONTENT_MD5         = "Content-MD5"
+
+	META_HEADER_PREFIX = "X-Yunion-Meta-"
 )
 
 type SBucketStats struct {
@@ -171,7 +173,9 @@ type ICloudObject interface {
 	GetLastModified() time.Time
 	GetStorageClass() string
 	GetETag() string
+
 	GetMeta() http.Header
+	SetMeta(ctx context.Context, meta http.Header) error
 
 	GetAcl() TBucketACLType
 	SetAcl(acl TBucketACLType) error
@@ -222,6 +226,10 @@ func (o *SBaseCloudObject) GetMeta() http.Header {
 	return o.Meta
 }
 
+//func (o *SBaseCloudObject) SetMeta(meta http.Header) error {
+//    return nil
+//}
+
 func GetIBucketById(region ICloudRegion, name string) (ICloudBucket, error) {
 	buckets, err := region.GetIBuckets()
 	if err != nil {
@@ -553,3 +561,51 @@ func ObjectSetMeta(ctx context.Context,
 ) error {
 	return bucket.CopyObject(ctx, obj.GetKey(), bucket.GetName(), obj.GetKey(), obj.GetAcl(), obj.GetStorageClass(), meta)
 }
+
+func MetaToHttpHeader(metaPrefix string, meta http.Header) http.Header {
+	hdr := http.Header{}
+	for k, v := range meta {
+		if len(v) == 0 || len(v[0]) == 0 {
+			continue
+		}
+		k = http.CanonicalHeaderKey(k)
+		switch k {
+		case META_HEADER_CACHE_CONTROL,
+			META_HEADER_CONTENT_TYPE,
+			META_HEADER_CONTENT_DISPOSITION,
+			META_HEADER_CONTENT_ENCODING,
+			META_HEADER_CONTENT_LANGUAGE,
+			META_HEADER_CONTENT_MD5:
+			hdr.Set(k, v[0])
+		default:
+			hdr.Set(fmt.Sprintf("%s%s", metaPrefix, k), v[0])
+		}
+	}
+	return hdr
+}
+
+func FetchMetaFromHttpHeader(metaPrefix string, headers http.Header) http.Header {
+	metaPrefix = http.CanonicalHeaderKey(metaPrefix)
+	meta := http.Header{}
+	for hdr, vals := range headers {
+		hdr = http.CanonicalHeaderKey(hdr)
+		if strings.HasPrefix(hdr, metaPrefix) {
+			for _, val := range vals {
+				meta.Add(hdr[len(metaPrefix):], val)
+			}
+		}
+	}
+	for _, hdr := range []string{
+		META_HEADER_CONTENT_TYPE,
+		META_HEADER_CONTENT_ENCODING,
+		META_HEADER_CONTENT_DISPOSITION,
+		META_HEADER_CONTENT_LANGUAGE,
+		META_HEADER_CACHE_CONTROL,
+	} {
+		val := headers.Get(hdr)
+		if len(val) > 0 {
+			meta.Set(hdr, val)
+		}
+	}
+	return meta
+}

pkg/compute/models/buckets.go

@@ -877,7 +877,8 @@ func (bucket *SBucket) PerformUpload(
 		}
 	}
 
-	contType := appParams.Request.Header.Get("Content-Type")
+	meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, appParams.Request.Header)
+
 	sizeStr := appParams.Request.Header.Get("Content-Length")
 	if len(sizeStr) == 0 {
 		return nil, httperrors.NewInputParameterError("missing Content-Length")
@@ -935,10 +936,6 @@ func (bucket *SBucket) PerformUpload(
 		}
 	}
 
-	meta := http.Header{}
-	if len(contType) > 0 {
-		meta.Add(cloudprovider.META_HEADER_CONTENT_TYPE, contType)
-	}
 	err = cloudprovider.UploadObject(ctx, iBucket, key, 0, appParams.Request.Body, sizeBytes, cloudprovider.TBucketACLType(aclStr), storageClass, meta, false)
 	if err != nil {
 		return nil, httperrors.NewInternalServerError("put object error %s", err)
@@ -959,7 +956,7 @@ func (bucket *SBucket) PerformUpload(
 func (bucket *SBucket) AllowPerformAcl(ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
-	data jsonutils.JSONObject,
+	input api.BucketAclInput,
 ) bool {
 	return bucket.IsOwner(userCred)
 }
@@ -968,32 +965,20 @@ func (bucket *SBucket) PerformAcl(
 	ctx context.Context,
 	userCred mcclient.TokenCredential,
 	query jsonutils.JSONObject,
-	data jsonutils.JSONObject,
+	input api.BucketAclInput,
 ) (jsonutils.JSONObject, error) {
-	if len(bucket.ExternalId) == 0 {
-		return nil, httperrors.NewInvalidStatusError("no external bucket")
-	}
-
-	aclStr, _ := data.GetString("acl")
-	switch cloudprovider.TBucketACLType(aclStr) {
-	case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
-		// do nothing
-	default:
-		return nil, httperrors.NewInputParameterError("invalid acl: %s", aclStr)
-	}
-
-	iBucket, err := bucket.GetIBucket()
+	err := input.Validate()
 	if err != nil {
-		if errors.Cause(err) == httperrors.ErrInvalidStatus {
-			return nil, httperrors.NewInvalidStatusError("%s", err)
-		} else {
-			return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
-		}
+		return nil, err
 	}
 
-	objKey, _ := data.Get("key")
-	if objKey == nil {
-		err = iBucket.SetAcl(cloudprovider.TBucketACLType(aclStr))
+	iBucket, objects, err := bucket.processObjectsActionInput(input.BucketObjectsActionInput)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(objects) == 0 {
+		err = iBucket.SetAcl(input.Acl)
 		if err != nil {
 			return nil, httperrors.NewInternalServerError("setAcl error %s", err)
 		}
@@ -1004,42 +989,20 @@ func (bucket *SBucket) PerformAcl(
 		}
 		return nil, nil
 	}
-	var keys []string
-	switch jsonObj := objKey.(type) {
-	case *jsonutils.JSONString:
-		key, _ := jsonObj.GetString()
-		keys = []string{key}
-	case *jsonutils.JSONArray:
-		keys = jsonObj.GetStringArray()
-	}
-	var objects []cloudprovider.ICloudObject
-	for _, key := range keys {
-		if strings.HasSuffix(key, "/") {
-			objs, err := cloudprovider.GetIObjects(iBucket, key, true)
-			if err != nil {
-				return nil, httperrors.NewInternalServerError("iBucket.GetIObjects error %s", err)
-			}
-			objects = append(objects, objs...)
-		} else {
-			object, err := cloudprovider.GetIObject(iBucket, key)
-			if err != nil {
-				if err == cloudprovider.ErrNotFound {
-					return nil, httperrors.NewResourceNotFoundError("object %s not found", objKey)
-				} else {
-					return nil, httperrors.NewInternalServerError("iBucket.GetIObject error %s", err)
-				}
-			}
-			objects = append(objects, object)
-		}
-	}
+
+	errs := make([]error, 0)
 	for _, object := range objects {
-		err := object.SetAcl(cloudprovider.TBucketACLType(aclStr))
+		err := object.SetAcl(input.Acl)
 		if err != nil {
-			return nil, httperrors.NewInternalServerError("setAcl error %s", err)
+			errs = append(errs, errors.Wrap(err, object.GetKey()))
 		}
 	}
 
-	return nil, nil
+	if len(errs) > 0 {
+		return nil, errors.NewAggregate(errs)
+	} else {
+		return nil, nil
+	}
 }
 
 func (bucket *SBucket) AllowPerformSync(ctx context.Context,
@@ -1271,3 +1234,74 @@ func (bucket *SBucket) GetUsages() []db.IUsage {
 		&usage,
 	}
 }
+
+func (bucket *SBucket) AllowPerformMetadata(ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.BucketMetadataInput,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) PerformMetadata(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+	input api.BucketMetadataInput,
+) (jsonutils.JSONObject, error) {
+	err := input.Validate()
+	if err != nil {
+		return nil, err
+	}
+	_, objects, err := bucket.processObjectsActionInput(input.BucketObjectsActionInput)
+	if err != nil {
+		return nil, err
+	}
+	errs := make([]error, 0)
+	for _, obj := range objects {
+		err := obj.SetMeta(ctx, input.Metadata)
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, obj.GetKey()))
+		}
+	}
+	if len(errs) > 0 {
+		return nil, errors.NewAggregate(errs)
+	} else {
+		return nil, nil
+	}
+}
+
+func (bucket *SBucket) processObjectsActionInput(input api.BucketObjectsActionInput) (cloudprovider.ICloudBucket, []cloudprovider.ICloudObject, error) {
+	if len(bucket.ExternalId) == 0 {
+		return nil, nil, httperrors.NewInvalidStatusError("no external bucket")
+	}
+	iBucket, err := bucket.GetIBucket()
+	if err != nil {
+		if errors.Cause(err) == httperrors.ErrInvalidStatus {
+			return nil, nil, httperrors.NewInvalidStatusError("%s", err)
+		} else {
+			return nil, nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
+		}
+	}
+	objects := make([]cloudprovider.ICloudObject, 0)
+	for _, key := range input.Key {
+		if strings.HasSuffix(key, "/") {
+			objs, err := cloudprovider.GetIObjects(iBucket, key, true)
+			if err != nil {
+				return nil, nil, httperrors.NewInternalServerError("iBucket.GetIObjects error %s", err)
+			}
+			objects = append(objects, objs...)
+		} else {
+			object, err := cloudprovider.GetIObject(iBucket, key)
+			if err != nil {
+				if err == cloudprovider.ErrNotFound {
+					return nil, nil, httperrors.NewResourceNotFoundError("object %s not found", key)
+				} else {
+					return nil, nil, httperrors.NewInternalServerError("iBucket.GetIObject error %s", err)
+				}
+			}
+			objects = append(objects, object)
+		}
+	}
+	return iBucket, objects, nil
+}

pkg/compute/models/cloudaccounts.go

@@ -283,7 +283,7 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 		return input, httperrors.NewInputParameterError("Unsupported provider %s", input.Provider)
 	}
 	providerDriver, _ := cloudprovider.GetProviderFactory(input.Provider)
-	err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, &input)
+	input.SCloudaccount, err = providerDriver.ValidateCreateCloudaccountData(ctx, userCred, input.SCloudaccountCredential)
 	if err != nil {
 		return input, err
 	}
@@ -440,7 +440,7 @@ func (self *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
 		return nil, httperrors.NewBadRequestError("failed to found provider factory error: %v", err)
 	}
 
-	input := &api.CloudaccountCredentialInput{}
+	input := cloudprovider.SCloudaccountCredential{}
 	err = data.Unmarshal(input)
 	if err != nil {
 		return nil, httperrors.NewInputParameterError("failed to unmarshal input params: %v", err)

pkg/httperrors/consts.go

@@ -50,8 +50,9 @@ const (
 	ErrUnsupportedOperation = errors.Error("UnsupportOperationError")
 	ErrNotSupported         = errors.ErrNotSupported
 
-	ErrNotEmpty   = errors.Error("NotEmptyError")
-	ErrBadRequest = errors.Error("BadRequestError")
+	ErrNotEmpty     = errors.Error("NotEmptyError")
+	ErrBadRequest   = errors.Error("BadRequestError")
+	ErrEmptyRequest = errors.Error("EmptyRequestError")
 
 	ErrUnauthorized      = errors.Error("UnauthorizedError")
 	ErrInvalidCredential = errors.Error("InvalidCredentialError")
@@ -120,8 +121,9 @@ var (
 		ErrUnsupportedOperation: 406,
 		ErrNotSupported:         406,
 
-		ErrNotEmpty:   406,
-		ErrBadRequest: 400,
+		ErrNotEmpty:     406,
+		ErrBadRequest:   400,
+		ErrEmptyRequest: 400,
 
 		ErrUnauthorized:      401,
 		ErrInvalidCredential: 401,

pkg/multicloud/aliyun/bucket.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"time"
 
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
@@ -27,7 +28,6 @@ import (
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
-	"net/http"
 )
 
 type SBucket struct {

pkg/multicloud/aliyun/objects.go

@@ -15,13 +15,19 @@
 package aliyun
 
 import (
+	"context"
+	"net/http"
+
 	"github.com/aliyun/aliyun-oss-go-sdk/oss"
-	"github.com/pkg/errors"
 
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"net/http"
+)
+
+const (
+	OSS_META_HEADER = "x-oss-meta-"
 )
 
 type SObject struct {
@@ -97,6 +103,10 @@ func (o *SObject) GetMeta() http.Header {
 		log.Errorf("bucket.GetObjectACL error %s", err)
 		return nil
 	}
-	o.Meta = result
-	return result
-}
+	o.Meta = cloudprovider.FetchMetaFromHttpHeader(OSS_META_HEADER, result)
+	return o.Meta
+}
+
+func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error {
+	return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta)
+}

pkg/multicloud/aliyun/provider/provider.go

@@ -18,6 +18,7 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -42,30 +43,32 @@ func (self *SAliyunProviderFactory) IsCloudeventRegional() bool {
 	return true
 }
 
-func (self *SAliyunProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SAliyunProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	input.Account = input.AccessKeyId
-	input.Secret = input.AccessKeySecret
-	return nil
+	output.Account = input.AccessKeyId
+	output.Secret = input.AccessKeySecret
+	return output, nil
 }
 
-func (self *SAliyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SAliyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.AccessKeyId,
 		Secret:  input.AccessKeySecret,
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SAliyunProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/aws/bucket.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"time"
 
@@ -31,7 +32,6 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
-	"net/http"
 )
 
 type SBucket struct {
@@ -443,6 +443,7 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src
 		cannedAcl = b.GetAcl()
 	}
 	input.SetACL(string(cannedAcl))
+	var metaDir string
 	if meta != nil {
 		metaHdr := make(map[string]*string)
 		for k, v := range meta {
@@ -467,7 +468,11 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src
 		if len(metaHdr) > 0 {
 			input.SetMetadata(metaHdr)
 		}
+		metaDir = "REPLACE"
+	} else {
+		metaDir = "COPY"
 	}
+	input.SetMetadataDirective(metaDir)
 	_, err = s3cli.CopyObject(input)
 	if err != nil {
 		return errors.Wrap(err, "CopyObject")

pkg/multicloud/aws/provider/provider.go

@@ -18,6 +18,7 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -42,34 +43,36 @@ func (self *SAwsProviderFactory) IsSupportPrepaidResources() bool {
 	return false
 }
 
-func (self *SAwsProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SAwsProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
 	if len(input.Environment) == 0 {
-		return httperrors.NewMissingParameterError("environment")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "environment")
 	}
-	input.Account = input.AccessKeyId
-	input.Secret = input.AccessKeySecret
-	input.AccessUrl = input.Environment
-	return nil
+	output.Account = input.AccessKeyId
+	output.Secret = input.AccessKeySecret
+	output.AccessUrl = input.Environment
+	return output, nil
 }
 
-func (self *SAwsProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SAwsProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.AccessKeyId,
 		Secret:  input.AccessKeySecret,
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SAwsProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/aws/s3object.go

@@ -15,13 +15,15 @@
 package aws
 
 import (
+	"context"
+	"net/http"
+
 	"github.com/aws/aws-sdk-go/service/s3"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"net/http"
 )
 
 type SObject struct {
@@ -69,7 +71,6 @@ func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error {
 }
 
 func (o *SObject) GetMeta() http.Header {
-	log.Infof("GetMeta for %s/%s", o.bucket.Name, o.Key)
 	if o.Meta != nil {
 		return o.Meta
 	}
@@ -107,5 +108,9 @@ func (o *SObject) GetMeta() http.Header {
 	if output.ContentLanguage != nil && len(*output.ContentLanguage) > 0 {
 		ret.Set(cloudprovider.META_HEADER_CONTENT_LANGUAGE, *output.ContentLanguage)
 	}
-	return nil
-}
+	return ret
+}
+
+func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error {
+	return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta)
+}

pkg/multicloud/azure/blobobject.go

@@ -15,6 +15,14 @@
 package azure
 
 import (
+	"context"
+	"net/http"
+
+	"github.com/Azure/azure-sdk-for-go/storage"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
@@ -35,3 +43,75 @@ func (o *SObject) GetAcl() cloudprovider.TBucketACLType {
 func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error {
 	return nil
 }
+
+func (o *SObject) getBlobName() string {
+	if len(o.Key) <= len(o.container.Name)+1 {
+		return ""
+	} else {
+		return o.Key[len(o.container.Name)+1:]
+	}
+}
+
+func (o *SObject) getBlobRef() (*storage.Blob, error) {
+	blobName := o.getBlobName()
+	if len(blobName) == 0 {
+		return nil, nil
+	}
+	contRef, err := o.container.getContainerRef()
+	if err != nil {
+		return nil, errors.Wrap(err, "src getContainerRef")
+	}
+	blobRef := contRef.GetBlobReference(blobName)
+	return blobRef, nil
+}
+
+func (o *SObject) GetMeta() http.Header {
+	if o.Meta != nil {
+		return o.Meta
+	}
+	blobRef, err := o.getBlobRef()
+	if err != nil {
+		log.Errorf("o.getBlobRef fail %s", err)
+		return nil
+	}
+	if blobRef == nil {
+		return nil
+	}
+	err = blobRef.GetMetadata(nil)
+	if err != nil {
+		log.Errorf("blobRef.GetMetadata fail %s", err)
+	}
+	err = blobRef.GetProperties(nil)
+	if err != nil {
+		log.Errorf("blobRef.GetProperties fail %s", err)
+	}
+	meta := getBlobRefMeta(blobRef)
+	o.Meta = meta
+	return o.Meta
+}
+
+func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error {
+	blobRef, err := o.getBlobRef()
+	if err != nil {
+		return errors.Wrap(err, "o.getBlobRef")
+	}
+	if blobRef == nil {
+		return cloudprovider.ErrNotSupported
+	}
+	propChanged, metaChanged := setBlobRefMeta(blobRef, meta)
+	if propChanged {
+		propOpts := storage.SetBlobPropertiesOptions{}
+		err := blobRef.SetProperties(&propOpts)
+		if err != nil {
+			return errors.Wrap(err, "blob.SetProperties")
+		}
+	}
+	if metaChanged {
+		metaOpts := storage.SetBlobMetadataOptions{}
+		err := blobRef.SetMetadata(&metaOpts)
+		if err != nil {
+			return errors.Wrap(err, "blob.SetMetadata")
+		}
+	}
+	return nil
+}

pkg/multicloud/azure/provider/provider.go

@@ -21,6 +21,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -61,37 +62,39 @@ func (self *SAzureProviderFactory) IsSupportPrepaidResources() bool {
 	return false
 }
 
-func (self *SAzureProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SAzureProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.DirectoryId) == 0 {
-		return httperrors.NewMissingParameterError("directory_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "directory_id")
 	}
 	if len(input.ClientId) == 0 {
-		return httperrors.NewMissingParameterError("client_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "client_id")
 	}
 	if len(input.ClientSecret) == 0 {
-		return httperrors.NewMissingParameterError("client_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "client_secret")
 	}
 	if len(input.Environment) == 0 {
-		return httperrors.NewMissingParameterError("environment")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "environment")
 	}
-	input.Account = input.DirectoryId
-	input.Secret = fmt.Sprintf("%s/%s", input.ClientId, input.ClientSecret)
-	input.AccessUrl = input.Environment
-	return nil
+	output.Account = input.DirectoryId
+	output.Secret = fmt.Sprintf("%s/%s", input.ClientId, input.ClientSecret)
+	output.AccessUrl = input.Environment
+	return output, nil
 }
 
-func (self *SAzureProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SAzureProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.ClientId) == 0 {
-		return nil, httperrors.NewMissingParameterError("client_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "client_id")
 	}
 	if len(input.ClientSecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("client_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "client_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: cloudaccount,
 		Secret:  fmt.Sprintf("%s/%s", input.ClientId, input.ClientSecret),
 	}
-	return account, nil
+	return output, nil
 }
 
 func parseAccount(account, secret string) (tenantId string, appId string, appKey string, subId string) {

pkg/multicloud/azure/storageaccount.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"io"
 	"math/rand"
+	"net/http"
 	"path"
 	"strconv"
 	"strings"
@@ -37,7 +38,6 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/multicloud"
-	"net/http"
 )
 
 type SContainer struct {
@@ -525,6 +525,43 @@ func (self *SContainer) CopySnapshot(snapshotId, fileName string) (*storage.Blob
 	return blobRef, blobRef.GetProperties(&storage.GetBlobPropertiesOptions{})
 }
 
+func setBlobRefMeta(blobRef *storage.Blob, meta http.Header) (bool, bool) {
+	propChanged := false
+	metaChanged := false
+	for k, v := range meta {
+		if len(v) == 0 || len(v[0]) == 0 {
+			continue
+		}
+		switch http.CanonicalHeaderKey(k) {
+		case cloudprovider.META_HEADER_CACHE_CONTROL:
+			blobRef.Properties.CacheControl = v[0]
+			propChanged = true
+		case cloudprovider.META_HEADER_CONTENT_TYPE:
+			blobRef.Properties.ContentType = v[0]
+			propChanged = true
+		case cloudprovider.META_HEADER_CONTENT_MD5:
+			blobRef.Properties.ContentMD5 = v[0]
+			propChanged = true
+		case cloudprovider.META_HEADER_CONTENT_ENCODING:
+			blobRef.Properties.ContentEncoding = v[0]
+			propChanged = true
+		case cloudprovider.META_HEADER_CONTENT_LANGUAGE:
+			blobRef.Properties.ContentLanguage = v[0]
+			propChanged = true
+		case cloudprovider.META_HEADER_CONTENT_DISPOSITION:
+			blobRef.Properties.ContentDisposition = v[0]
+			propChanged = true
+		default:
+			if blobRef.Metadata == nil {
+				blobRef.Metadata = storage.BlobMetadata{}
+			}
+			blobRef.Metadata[k] = v[0]
+			metaChanged = true
+		}
+	}
+	return propChanged, metaChanged
+}
+
 func (self *SContainer) UploadStream(key string, reader io.Reader, meta http.Header) error {
 	blobService, err := self.storageaccount.getBlobServiceClient()
 	if err != nil {
@@ -534,30 +571,7 @@ func (self *SContainer) UploadStream(key string, reader io.Reader, meta http.Hea
 	blobRef := containerRef.GetBlobReference(key)
 	blobRef.Properties.BlobType = storage.BlobTypeBlock
 	if meta != nil {
-		for k, v := range meta {
-			if len(v) == 0 || len(v[0]) == 0{
-				continue
-			}
-			switch http.CanonicalHeaderKey(k) {
-			case cloudprovider.META_HEADER_CACHE_CONTROL:
-				blobRef.Properties.CacheControl = v[0]
-			case cloudprovider.META_HEADER_CONTENT_TYPE:
-				blobRef.Properties.ContentType = v[0]
-			case cloudprovider.META_HEADER_CONTENT_MD5:
-				blobRef.Properties.ContentMD5 = v[0]
-			case cloudprovider.META_HEADER_CONTENT_ENCODING:
-				blobRef.Properties.ContentEncoding = v[0]
-			case cloudprovider.META_HEADER_CONTENT_LANGUAGE:
-				blobRef.Properties.ContentLanguage = v[0]
-			case cloudprovider.META_HEADER_CONTENT_DISPOSITION:
-				blobRef.Properties.ContentDisposition = v[0]
-			default:
-				if blobRef.Metadata == nil {
-					blobRef.Metadata = storage.BlobMetadata{}
-				}
-				blobRef.Metadata[k] = v[0]
-			}
-		}
+		setBlobRefMeta(blobRef, meta)
 	}
 	return blobRef.CreateBlockBlobFromReader(reader, &storage.PutBlobOptions{})
 }
@@ -853,6 +867,32 @@ func (b *SStorageAccount) GetIObjects(prefix string, isRecursive bool) ([]cloudp
 	return cloudprovider.GetIObjects(b, prefix, isRecursive)
 }
 
+func getBlobRefMeta(blob *storage.Blob) http.Header {
+	meta := http.Header{}
+	for k, v := range blob.Metadata {
+		meta.Add(k, v)
+	}
+	if len(blob.Properties.CacheControl) > 0 {
+		meta.Set(cloudprovider.META_HEADER_CACHE_CONTROL, blob.Properties.CacheControl)
+	}
+	if len(blob.Properties.ContentType) > 0 {
+		meta.Set(cloudprovider.META_HEADER_CONTENT_TYPE, blob.Properties.ContentType)
+	}
+	if len(blob.Properties.ContentDisposition) > 0 {
+		meta.Set(cloudprovider.META_HEADER_CONTENT_DISPOSITION, blob.Properties.ContentDisposition)
+	}
+	if len(blob.Properties.ContentLanguage) > 0 {
+		meta.Set(cloudprovider.META_HEADER_CONTENT_LANGUAGE, blob.Properties.ContentLanguage)
+	}
+	if len(blob.Properties.ContentEncoding) > 0 {
+		meta.Set(cloudprovider.META_HEADER_CONTENT_ENCODING, blob.Properties.ContentEncoding)
+	}
+	if len(blob.Properties.ContentMD5) > 0 {
+		meta.Set(cloudprovider.META_HEADER_CONTENT_MD5, blob.Properties.ContentMD5)
+	}
+	return meta
+}
+
 func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
 	result := cloudprovider.SListObjectResult{}
 	containers, err := b.GetContainers()
@@ -922,28 +962,6 @@ func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter st
 				}
 				for i := range oResult.Blobs {
 					blob := oResult.Blobs[i]
-					meta := http.Header{}
-					for k, v := range blob.Metadata {
-						meta.Add(k, v)
-					}
-					if len(blob.Properties.CacheControl) > 0 {
-						meta.Set(cloudprovider.META_HEADER_CACHE_CONTROL, blob.Properties.CacheControl)
-					}
-					if len(blob.Properties.ContentType) > 0 {
-						meta.Set(cloudprovider.META_HEADER_CONTENT_TYPE, blob.Properties.ContentType)
-					}
-					if len(blob.Properties.ContentDisposition) > 0 {
-						meta.Set(cloudprovider.META_HEADER_CONTENT_DISPOSITION, blob.Properties.ContentDisposition)
-					}
-					if len(blob.Properties.ContentLanguage) > 0 {
-						meta.Set(cloudprovider.META_HEADER_CONTENT_LANGUAGE, blob.Properties.ContentLanguage)
-					}
-					if len(blob.Properties.ContentEncoding) > 0 {
-						meta.Set(cloudprovider.META_HEADER_CONTENT_ENCODING, blob.Properties.ContentEncoding)
-					}
-					if len(blob.Properties.ContentMD5) > 0 {
-						meta.Set(cloudprovider.META_HEADER_CONTENT_MD5, blob.Properties.ContentMD5)
-					}
 					o := &SObject{
 						container: &container,
 						SBaseCloudObject: cloudprovider.SBaseCloudObject{
@@ -952,7 +970,6 @@ func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter st
 							StorageClass: "",
 							ETag:         blob.Properties.Etag,
 							LastModified: time.Time(blob.Properties.LastModified),
-							Meta:         meta,
 						},
 					}
 					result.Objects = append(result.Objects, o)
@@ -1042,7 +1059,9 @@ func (b *SStorageAccount) NewMultipartUpload(ctx context.Context, key string, ca
 		return "", errors.Wrap(err, "getContainerRef")
 	}
 	blobRef := containerRef.GetBlobReference(blob)
-
+	if meta != nil {
+		setBlobRefMeta(blobRef, meta)
+	}
 	err = blobRef.CreateBlockBlob(&storage.PutBlobOptions{})
 	if err != nil {
 		return "", errors.Wrap(err, "CreateBlockBlob")
@@ -1216,7 +1235,7 @@ func (b *SStorageAccount) CopyObject(ctx context.Context, destKey string, srcBuc
 	}
 	srcBlobRef := srcContRef.GetBlobReference(srcBlob)
 
-	containerName, blob, err := splitKeyAndBlob(destKey)
+	containerName, blobName, err := splitKeyAndBlob(destKey)
 	if err != nil {
 		return errors.Wrap(err, "dest splitKey")
 	}
@@ -1228,8 +1247,10 @@ func (b *SStorageAccount) CopyObject(ctx context.Context, destKey string, srcBuc
 	if err != nil {
 		return errors.Wrap(err, "dest getContainerRef")
 	}
-	blobRef := containerRef.GetBlobReference(blob)
-
+	blobRef := containerRef.GetBlobReference(blobName)
+	if meta != nil {
+		setBlobRefMeta(blobRef, meta)
+	}
 	opts := &storage.CopyOptions{}
 	err = blobRef.Copy(srcBlobRef.GetURL(), opts)
 	if err != nil {

pkg/multicloud/ctyun/provider/provider.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -43,34 +44,36 @@ func (self *SCtyunProviderFactory) IsSupportPrepaidResources() bool {
 	return true
 }
 
-func (self *SCtyunProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SCtyunProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
 	if len(input.Environment) == 0 {
-		return httperrors.NewMissingParameterError("environment")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "environment")
 	}
-	input.Account = input.AccessKeyId
-	input.Secret = input.AccessKeySecret
-	input.AccessUrl = input.Environment
-	return nil
+	output.Account = input.AccessKeyId
+	output.Secret = input.AccessKeySecret
+	output.AccessUrl = input.Environment
+	return output, nil
 }
 
-func (self *SCtyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SCtyunProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.AccessKeyId,
 		Secret:  input.AccessKeySecret,
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SCtyunProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/esxi/provider/provider.go

@@ -23,6 +23,7 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -47,37 +48,39 @@ func (self *SESXiProviderFactory) ValidateChangeBandwidth(instanceId string, ban
 	return fmt.Errorf("Changing %s bandwidth is not supported", esxi.CLOUD_PROVIDER_VMWARE)
 }
 
-func (self *SESXiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SESXiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.Username) == 0 {
-		return httperrors.NewMissingParameterError("username")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "username")
 	}
 	if len(input.Password) == 0 {
-		return httperrors.NewMissingParameterError("password")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "password")
 	}
 	if len(input.Host) == 0 {
-		return httperrors.NewMissingParameterError("host")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "host")
 	}
-	input.AccessUrl = fmt.Sprintf("https://%s:%d/sdk", input.Host, input.Port)
+	output.AccessUrl = fmt.Sprintf("https://%s:%d/sdk", input.Host, input.Port)
 	if input.Port == 0 || input.Port == 443 {
-		input.AccessUrl = fmt.Sprintf("https://%s/sdk", input.Host)
+		output.AccessUrl = fmt.Sprintf("https://%s/sdk", input.Host)
 	}
-	input.Account = input.Username
-	input.Secret = input.Password
-	return nil
+	output.Account = input.Username
+	output.Secret = input.Password
+	return output, nil
 }
 
-func (self *SESXiProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SESXiProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.Username) == 0 {
-		return nil, httperrors.NewMissingParameterError("username")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "username")
 	}
 	if len(input.Password) == 0 {
-		return nil, httperrors.NewMissingParameterError("password")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "password")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.Username,
 		Secret:  input.Password,
 	}
-	return account, nil
+	return output, nil
 }
 
 func parseHostPort(host string, defPort int) (string, int, error) {

pkg/multicloud/google/provider/provider.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -52,7 +53,8 @@ func (self *SGoogleProviderFactory) NeedSyncSkuFromCloud() bool {
 	return false
 }
 
-func (self *SGoogleProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SGoogleProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	for key, value := range map[string]string{
 		"client_email":   input.ClientEmail,
 		"project_id":     input.ProjectId,
@@ -60,15 +62,16 @@ func (self *SGoogleProviderFactory) ValidateCreateCloudaccountData(ctx context.C
 		"private_key":    input.PrivateKey,
 	} {
 		if len(value) == 0 {
-			return httperrors.NewMissingParameterError(key)
+			return output, errors.Wrap(httperrors.ErrMissingParameter, key)
 		}
 	}
-	input.Account = fmt.Sprintf("%s/%s", input.ProjectId, input.ClientEmail)
-	input.Secret = fmt.Sprintf("%s/%s", input.PrivateKeyId, input.PrivateKey)
-	return nil
+	output.Account = fmt.Sprintf("%s/%s", input.ProjectId, input.ClientEmail)
+	output.Secret = fmt.Sprintf("%s/%s", input.PrivateKeyId, input.PrivateKey)
+	return output, nil
 }
 
-func (self *SGoogleProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SGoogleProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	projectID, clientEmail := "", ""
 	accountInfo := strings.Split(cloudaccount, "/")
 	if len(accountInfo) == 2 {
@@ -80,7 +83,7 @@ func (self *SGoogleProviderFactory) ValidateUpdateCloudaccountCredential(ctx con
 		"private_key":    input.PrivateKey,
 	} {
 		if len(value) == 0 {
-			return nil, httperrors.NewMissingParameterError(key)
+			return output, errors.Wrap(httperrors.ErrMissingParameter, key)
 		}
 	}
 	if len(input.ClientEmail) == 0 {
@@ -91,11 +94,11 @@ func (self *SGoogleProviderFactory) ValidateUpdateCloudaccountCredential(ctx con
 		input.ProjectId = projectID
 	}
 
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: fmt.Sprintf("%s/%s", input.ProjectId, input.ClientEmail),
 		Secret:  fmt.Sprintf("%s/%s", input.PrivateKeyId, input.PrivateKey),
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SGoogleProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/huawei/bucket.go

@@ -18,17 +18,17 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"time"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/s3cli"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/multicloud/huawei/obs"
-	"net/http"
-	"yunion.io/x/pkg/utils"
 )
 
 type SBucket struct {
@@ -262,7 +262,7 @@ func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, s
 		}
 		extraMeta := make(map[string]string)
 		for k, v := range meta {
-			if utils.IsInStringArray(k, []string {
+			if utils.IsInStringArray(k, []string{
 				cloudprovider.META_HEADER_CONTENT_TYPE,
 				cloudprovider.META_HEADER_CONTENT_MD5,
 			}) {
@@ -297,7 +297,7 @@ func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl
 		}
 		extraMeta := make(map[string]string)
 		for k, v := range meta {
-			if utils.IsInStringArray(k, []string {
+			if utils.IsInStringArray(k, []string{
 				cloudprovider.META_HEADER_CONTENT_TYPE,
 			}) {
 				continue
@@ -472,6 +472,8 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src
 		return errors.Wrap(err, "GetOBSClient")
 	}
 	input := &obs.CopyObjectInput{}
+	input.Bucket = b.Name
+	input.Key = destKey
 	input.CopySourceBucket = srcBucket
 	input.CopySourceKey = srcKey
 	if len(storageClassStr) > 0 {
@@ -491,7 +493,7 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src
 		}
 		extraMeta := make(map[string]string)
 		for k, v := range meta {
-			if utils.IsInStringArray(k, []string {
+			if utils.IsInStringArray(k, []string{
 				cloudprovider.META_HEADER_CONTENT_TYPE,
 			}) {
 				continue
@@ -501,6 +503,9 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src
 			}
 		}
 		input.Metadata = extraMeta
+		input.MetadataDirective = obs.ReplaceMetadata
+	} else {
+		input.MetadataDirective = obs.CopyMetadata
 	}
 	_, err = obscli.CopyObject(input)
 	if err != nil {

pkg/multicloud/huawei/object.go

@@ -15,12 +15,14 @@
 package huawei
 
 import (
+	"context"
+	"net/http"
+
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud/huawei/obs"
-	"net/http"
 )
 
 type SObject struct {
@@ -90,4 +92,8 @@ func (o *SObject) GetMeta() http.Header {
 	}
 	o.Meta = meta
 	return meta
-}
+}
+
+func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error {
+	return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta)
+}

pkg/multicloud/huawei/provider/provider.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -51,36 +52,38 @@ func (self *SHuaweiProviderFactory) GetMaxCloudEventKeepDays() int {
 	return 7
 }
 
-func (self *SHuaweiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SHuaweiProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
 	if len(input.Environment) == 0 {
-		return httperrors.NewMissingParameterError("environment")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "environment")
 	}
 
-	input.Account = input.AccessKeyId
-	input.Secret = input.AccessKeySecret
-	input.AccessUrl = input.Environment
+	output.Account = input.AccessKeyId
+	output.Secret = input.AccessKeySecret
+	output.AccessUrl = input.Environment
 
-	return nil
+	return output, nil
 }
 
-func (self *SHuaweiProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SHuaweiProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.AccessKeyId,
 		Secret:  input.AccessKeySecret,
 	}
-	return account, nil
+	return output, nil
 }
 
 func parseAccount(account string) (accessKey string, projectId string) {

pkg/multicloud/objectstore/buckets.go

@@ -24,11 +24,11 @@ import (
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
 	"yunion.io/x/s3cli"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
-	"yunion.io/x/pkg/utils"
 )
 
 type SBucket struct {
@@ -142,9 +142,6 @@ func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter strin
 	ret.Objects = make([]cloudprovider.ICloudObject, len(result.Contents))
 	for i := range result.Contents {
 		object := result.Contents[i]
-		if len(object.ContentType) > 0 {
-			object.Metadata.Set(cloudprovider.META_HEADER_CONTENT_TYPE, object.ContentType)
-		}
 		ret.Objects[i] = &SObject{
 			bucket: bucket,
 			SBaseCloudObject: cloudprovider.SBaseCloudObject{
@@ -153,7 +150,6 @@ func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter strin
 				SizeBytes:    object.Size,
 				ETag:         object.ETag,
 				LastModified: object.LastModified,
-				Meta:         object.Metadata,
 			},
 		}
 	}

pkg/multicloud/objectstore/object.go

@@ -15,14 +15,21 @@
 package objectstore
 
 import (
+	"context"
+	"net/http"
 	"strings"
 
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
+	"yunion.io/x/s3cli"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 )
 
+const (
+	META_HEADER = "X-Amz-Meta-"
+)
+
 type SObject struct {
 	bucket *SBucket
 
@@ -53,3 +60,24 @@ func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error {
 	}
 	return nil
 }
+
+func (o *SObject) GetMeta() http.Header {
+	if o.Meta != nil {
+		return o.Meta
+	}
+	cli := o.bucket.client.S3Client()
+	objInfo, err := cli.StatObject(o.bucket.Name, o.Key, s3cli.StatObjectOptions{})
+	if err != nil {
+		log.Errorf("cli.statObject fail %s", err)
+		return nil
+	}
+	if len(objInfo.ContentType) > 0 {
+		objInfo.Metadata.Set(cloudprovider.META_HEADER_CONTENT_TYPE, objInfo.ContentType)
+	}
+	o.Meta = cloudprovider.FetchMetaFromHttpHeader(META_HEADER, objInfo.Metadata)
+	return o.Meta
+}
+
+func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error {
+	return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta)
+}

pkg/multicloud/objectstore/provider/provider.go

@@ -18,6 +18,7 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -46,34 +47,36 @@ func (factory *SObjectStoreProviderFactory) IsSupportComputeEngine() bool {
 	return false
 }
 
-func (self *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
 	if len(input.Endpoint) == 0 {
-		return httperrors.NewMissingParameterError("endpoint")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "endpoint")
 	}
-	input.Account = input.AccessKeyId
-	input.Secret = input.AccessKeySecret
-	input.AccessUrl = input.Endpoint
-	return nil
+	output.Account = input.AccessKeyId
+	output.Secret = input.AccessKeySecret
+	output.AccessUrl = input.Endpoint
+	return output, nil
 }
 
-func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.AccessKeyId,
 		Secret:  input.AccessKeySecret,
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SObjectStoreProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/objectstore/shell.go

@@ -18,10 +18,10 @@ import (
 	"context"
 	"fmt"
 	"io"
-	"os"
-	"time"
 	"net/http"
+	"os"
 	"strings"
+	"time"
 
 	"yunion.io/x/pkg/errors"
 
@@ -42,12 +42,16 @@ type ObjectHeaderOptions struct {
 	Meta []string `help:"header, common seperatored key and value, e.g. max-age:100"`
 }
 
-func objectHeaderOptions2Meta(args ObjectHeaderOptions) http.Header {
+func (args ObjectHeaderOptions) Options2Header() http.Header {
 	meta := http.Header{}
 	for _, kv := range args.Meta {
 		parts := strings.Split(kv, ":")
-		if len(parts) == 2 && len(parts[0]) > 0 && len(parts[1]) > 0 {
-			meta.Add(parts[0], parts[1])
+		if len(parts) == 2 {
+			key := strings.TrimSpace(parts[0])
+			value := strings.TrimSpace(parts[1])
+			if len(key) > 0 && len(value) > 0 {
+				meta.Add(key, value)
+			}
 		}
 	}
 	if len(args.CacheControl) > 0 {
@@ -267,7 +271,7 @@ func S3Shell() {
 		} else {
 			input = os.Stdout
 		}
-		meta := objectHeaderOptions2Meta(args.ObjectHeaderOptions)
+		meta := args.ObjectHeaderOptions.Options2Header()
 		err = cloudprovider.UploadObject(context.Background(), bucket, args.KEY, args.BlockSize*1000*1000, input, fSize, cloudprovider.TBucketACLType(args.Acl), args.StorageClass, meta, true)
 		if err != nil {
 			return err
@@ -426,7 +430,7 @@ func S3Shell() {
 		if err != nil {
 			return err
 		}
-		meta := objectHeaderOptions2Meta(args.ObjectHeaderOptions)
+		meta := args.ObjectHeaderOptions.Options2Header()
 		if args.Native {
 			err = dstBucket.CopyObject(ctx, args.DSTKEY, args.SRC, args.SRCKEY, srcObj.GetAcl(), srcObj.GetStorageClass(), meta)
 			if err != nil {
@@ -477,8 +481,8 @@ func S3Shell() {
 		if err != nil {
 			return err
 		}
-		meta := objectHeaderOptions2Meta(args.ObjectHeaderOptions)
-		err = cloudprovider.ObjectSetMeta(context.Background(), bucket, obj, meta)
+		meta := args.ObjectHeaderOptions.Options2Header()
+		err = obj.SetMeta(context.Background(), meta)
 		if err != nil {
 			return err
 		}

pkg/multicloud/openstack/provider/provider.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -42,43 +43,45 @@ func (self *SOpenStackProviderFactory) GetName() string {
 	return openstack.CLOUD_PROVIDER_OPENSTACK
 }
 
-func (self *SOpenStackProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SOpenStackProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.ProjectName) == 0 {
-		return httperrors.NewMissingParameterError("project_name")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "project_name")
 	}
 	if len(input.Username) == 0 {
-		return httperrors.NewMissingParameterError("username")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "username")
 	}
 	if len(input.Password) == 0 {
-		return httperrors.NewMissingParameterError("password")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "password")
 	}
 	if len(input.AuthUrl) == 0 {
-		return httperrors.NewMissingParameterError("auth_url")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "auth_url")
 	}
 
-	input.Account = fmt.Sprintf("%s/%s", input.ProjectName, input.Username)
+	output.Account = fmt.Sprintf("%s/%s", input.ProjectName, input.Username)
 	if len(input.DomainName) > 0 {
-		input.Account = fmt.Sprintf("%s/%s", input.Account, input.DomainName)
+		output.Account = fmt.Sprintf("%s/%s", output.Account, input.DomainName)
 	}
 
-	input.Secret = input.Password
-	input.AccessUrl = input.AuthUrl
-	return nil
+	output.Secret = input.Password
+	output.AccessUrl = input.AuthUrl
+	return output, nil
 }
 
-func (self *SOpenStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SOpenStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.ProjectName) == 0 {
 		accountInfo := strings.Split(cloudaccount, "/")
 		if len(accountInfo) < 2 {
-			return nil, httperrors.NewMissingParameterError("project_name")
+			return output, errors.Wrap(httperrors.ErrMissingParameter, "project_name")
 		}
 		input.ProjectName = accountInfo[0]
 	}
 	if len(input.Username) == 0 {
-		return nil, httperrors.NewMissingParameterError("username")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "username")
 	}
 	if len(input.Password) == 0 {
-		return nil, httperrors.NewMissingParameterError("password")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "password")
 	}
 
 	_account := fmt.Sprintf("%s/%s", input.ProjectName, input.Username)
@@ -92,11 +95,11 @@ func (self *SOpenStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx
 		_account = fmt.Sprintf("%s/%s", _account, input.DomainName)
 	}
 
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: _account,
 		Secret:  input.Password,
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SOpenStackProviderFactory) GetProvider(providerId, providerName, url, account, password string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/qcloud/bucket.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"net/http"
 	"time"
 
 	"github.com/tencentyun/cos-go-sdk-v5"
@@ -29,7 +30,10 @@ import (
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
-	"net/http"
+)
+
+const (
+	COS_META_HEADER = "X-Cos-Meta-"
 )
 
 type SBucket struct {
@@ -250,7 +254,7 @@ func (b *SBucket) PutObject(ctx context.Context, key string, reader io.Reader, s
 			case cloudprovider.META_HEADER_CONTENT_DISPOSITION:
 				opts.ContentDisposition = v[0]
 			default:
-				extraHdr.Add(k, v[0])
+				extraHdr.Add(fmt.Sprintf("%s%s", COS_META_HEADER, k), v[0])
 			}
 		}
 		if len(extraHdr) > 0 {
@@ -298,7 +302,7 @@ func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl
 			case cloudprovider.META_HEADER_CONTENT_DISPOSITION:
 				opts.ContentDisposition = v[0]
 			default:
-				extraHdr.Add(k, v[0])
+				extraHdr.Add(fmt.Sprintf("%s%s", COS_META_HEADER, k), v[0])
 			}
 		}
 		if len(extraHdr) > 0 {
@@ -419,6 +423,7 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucketName,
 		opts.XCosStorageClass = storageClassStr
 	}
 	if meta != nil {
+		opts.XCosMetadataDirective = "Replaced"
 		extraHdr := http.Header{}
 		for k, v := range meta {
 			if len(v) == 0 || len(v[0]) == 0 {
@@ -434,19 +439,20 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucketName,
 			case cloudprovider.META_HEADER_CONTENT_DISPOSITION:
 				opts.ContentDisposition = v[0]
 			default:
-				extraHdr.Add(k, v[0])
+				extraHdr.Add(fmt.Sprintf("%s%s", COS_META_HEADER, k), v[0])
 			}
 		}
 		if len(extraHdr) > 0 {
 			opts.XCosMetaXXX = &extraHdr
 		}
+	} else {
+		opts.XCosMetadataDirective = "Copy"
 	}
 	srcBucket := SBucket{
 		region: b.region,
 		Name:   srcBucketName,
 	}
 	srcUrl := fmt.Sprintf("%s/%s", srcBucket.getBucketUrlHost(), srcKey)
-	log.Debugf("source url: %s", srcUrl)
 	_, _, err = coscli.Object.Copy(ctx, destKey, srcUrl, opts)
 	if err != nil {
 		return errors.Wrap(err, "coscli.Object.Copy")

pkg/multicloud/qcloud/object.go

@@ -16,6 +16,7 @@ package qcloud
 
 import (
 	"context"
+	"net/http"
 
 	"github.com/tencentyun/cos-go-sdk-v5"
 
@@ -23,7 +24,6 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	"yunion.io/x/onecloud/pkg/cloudprovider"
-	"net/http"
 )
 
 type SObject struct {
@@ -68,5 +68,23 @@ func (o *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error {
 }
 
 func (o *SObject) GetMeta() http.Header {
-	return nil
-}
+	if o.Meta != nil {
+		return o.Meta
+	}
+	coscli, err := o.bucket.region.GetCosClient(o.bucket)
+	if err != nil {
+		log.Errorf("o.bucket.region.GetCosClient fail %s", err)
+		return nil
+	}
+	resp, err := coscli.Object.Head(context.Background(), o.Key, nil)
+	if err != nil {
+		log.Errorf("coscli.Object.Head fail %s", err)
+		return nil
+	}
+	o.Meta = cloudprovider.FetchMetaFromHttpHeader(COS_META_HEADER, resp.Header)
+	return o.Meta
+}
+
+func (o *SObject) SetMeta(ctx context.Context, meta http.Header) error {
+	return cloudprovider.ObjectSetMeta(ctx, o.bucket, o, meta)
+}

pkg/multicloud/qcloud/provider/provider.go

@@ -20,6 +20,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -59,40 +60,42 @@ func (self *SQcloudProviderFactory) ValidateChangeBandwidth(instanceId string, b
 	return nil
 }
 
-func (self *SQcloudProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SQcloudProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AppId) == 0 {
-		return httperrors.NewMissingParameterError("app_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "app_id")
 	}
 	if len(input.SecretId) == 0 {
-		return httperrors.NewMissingParameterError("secret_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "secret_id")
 	}
 	if len(input.SecretKey) == 0 {
-		return httperrors.NewMissingParameterError("secret_key")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "secret_key")
 	}
-	input.Account = fmt.Sprintf("%s/%s", input.SecretId, input.AppId)
-	input.Secret = input.SecretKey
-	return nil
+	output.Account = fmt.Sprintf("%s/%s", input.SecretId, input.AppId)
+	output.Secret = input.SecretKey
+	return output, nil
 }
 
-func (self *SQcloudProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SQcloudProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AppId) == 0 {
 		accountInfo := strings.Split(cloudaccount, "/")
 		if len(accountInfo) < 2 {
-			return nil, httperrors.NewMissingParameterError("app_id")
+			return output, errors.Wrap(httperrors.ErrMissingParameter, "app_id")
 		}
 		input.AppId = accountInfo[1]
 	}
 	if len(input.SecretId) == 0 {
-		return nil, httperrors.NewMissingParameterError("secret_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "secret_id")
 	}
 	if len(input.SecretKey) == 0 {
-		return nil, httperrors.NewMissingParameterError("secret_key")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "secret_key")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: fmt.Sprintf("%s/%s", input.SecretId, input.AppId),
 		Secret:  input.SecretKey,
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SQcloudProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {

pkg/multicloud/ucloud/provider/provider.go

@@ -19,6 +19,7 @@ import (
 	"strings"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -40,30 +41,32 @@ func (self *SUcloudProviderFactory) GetName() string {
 	return ucloud.CLOUD_PROVIDER_UCLOUD_CN
 }
 
-func (self *SUcloudProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SUcloudProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	input.Account = input.AccessKeyId
-	input.Secret = input.AccessKeySecret
-	return nil
+	output.Account = input.AccessKeyId
+	output.Secret = input.AccessKeySecret
+	return output, nil
 }
 
-func (self *SUcloudProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
+func (self *SUcloudProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_id")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id")
 	}
 	if len(input.AccessKeySecret) == 0 {
-		return nil, httperrors.NewMissingParameterError("access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_secret")
 	}
-	account := &cloudprovider.SCloudaccount{
+	output = cloudprovider.SCloudaccount{
 		Account: input.AccessKeyId,
 		Secret:  input.AccessKeySecret,
 	}
-	return account, nil
+	return output, nil
 }
 
 func parseAccount(account string) (accessKey string, projectId string) {

pkg/multicloud/ucloud/ufile.go

@@ -186,6 +186,10 @@ func (self *SFile) GetMeta() http.Header {
 	return nil
 }
 
+func (self *SFile) SetMeta(ctx context.Context, meta http.Header) error {
+	return cloudprovider.ErrNotSupported
+}
+
 func doRequest(req *http.Request) (jsonutils.JSONObject, error) {
 	res, err := httputils.GetDefaultClient().Do(req)
 	if err != nil {

pkg/multicloud/zstack/provider/provider.go

@@ -18,6 +18,7 @@ import (
 	"context"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -42,36 +43,37 @@ func (self *SZStackProviderFactory) GetSupportedBrands() []string {
 	return []string{api.ZSTACK_BRAND_DSTACK}
 }
 
-func (self *SZStackProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCreateInput) error {
+func (self *SZStackProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AuthUrl) == 0 {
-		return httperrors.NewMissingParameterError("auth_url")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "auth_url")
 	}
-	input.AccessUrl = input.AuthUrl
+	output.AccessUrl = input.AuthUrl
 	//为了兼容以前用username的参数，2.12之后尽可能的使用access_key_id参数
 	if len(input.AccessKeyId) > 0 && len(input.AccessKeySecret) > 0 {
-		input.Account = input.AccessKeyId
-		input.Secret = input.AccessKeySecret
+		output.Account = input.AccessKeyId
+		output.Secret = input.AccessKeySecret
 	} else if len(input.Username) > 0 && len(input.Password) > 0 {
-		input.Account = input.Username
-		input.Secret = input.Password
+		output.Account = input.Username
+		output.Secret = input.Password
 	} else {
-		return httperrors.NewMissingParameterError("access_key_id or access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id or access_key_secret")
 	}
-	return nil
+	return output, nil
 }
 
-func (self *SZStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input *api.CloudaccountCredentialInput, cloudaccount string) (*cloudprovider.SCloudaccount, error) {
-	account := &cloudprovider.SCloudaccount{}
+func (self *SZStackProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, userCred mcclient.TokenCredential, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
+	output := cloudprovider.SCloudaccount{}
 	if len(input.AccessKeyId) > 0 && len(input.AccessKeySecret) > 0 {
-		account.Account = input.AccessKeyId
-		account.Secret = input.AccessKeySecret
+		output.Account = input.AccessKeyId
+		output.Secret = input.AccessKeySecret
 	} else if len(input.Username) > 0 && len(input.Password) > 0 {
-		account.Account = input.Username
-		account.Secret = input.Password
+		output.Account = input.Username
+		output.Secret = input.Password
 	} else {
-		return nil, httperrors.NewMissingParameterError("access_key_id or access_key_secret")
+		return output, errors.Wrap(httperrors.ErrMissingParameter, "access_key_id or access_key_secret")
 	}
-	return account, nil
+	return output, nil
 }
 
 func (self *SZStackProviderFactory) GetProvider(providerId, providerName, url, username, password string) (cloudprovider.ICloudProvider, error) {

pkg/s3gateway/handlers/handlers.go

@@ -270,11 +270,7 @@ func downloadObject(ctx context.Context, userCred mcclient.TokenCredential, buck
 	if err != nil {
 		return errors.Wrap(err, "cloudprovider.GetIObject")
 	}
-	hdr := http.Header{}
-	contType := obj.GetContentType()
-	if len(contType) > 0 {
-		hdr.Set("Content-Type", obj.GetContentType())
-	}
+	hdr := cloudprovider.MetaToHttpHeader(cloudprovider.META_HEADER_PREFIX, obj.GetMeta())
 	eTag := obj.GetETag()
 	if len(eTag) > 0 {
 		hdr.Set("ETag", eTag)

pkg/s3gateway/handlers/multipart.go

@@ -36,10 +36,10 @@ func initMultipartUpload(ctx context.Context, userCred mcclient.TokenCredential,
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "bucket.GetIBucket")
 	}
-	contType := hdr.Get(http.CanonicalHeaderKey("content-type"))
+	meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, hdr)
 	aclStr := hdr.Get(http.CanonicalHeaderKey("x-amz-acl"))
 	storageClassStr := hdr.Get(http.CanonicalHeaderKey("x-amz-storage-class"))
-	uploadId, err := iBucket.NewMultipartUpload(ctx, key, contType, cloudprovider.TBucketACLType(aclStr), storageClassStr)
+	uploadId, err := iBucket.NewMultipartUpload(ctx, key, cloudprovider.TBucketACLType(aclStr), storageClassStr, meta)
 	if err != nil {
 		return nil, nil, errors.Wrap(err, "NewMultipartUpload")
 	}

pkg/s3gateway/handlers/object.go

@@ -47,11 +47,10 @@ func headObject(ctx context.Context, userCred mcclient.TokenCredential, bucketNa
 	if err != nil {
 		return nil, errors.Wrap(err, "cloudprovider.GetIObject")
 	}
-	hdr := http.Header{}
+	hdr := cloudprovider.MetaToHttpHeader(cloudprovider.META_HEADER_PREFIX, obj.GetMeta())
 	hdr.Set(http.CanonicalHeaderKey("x-amz-acl"), string(obj.GetAcl()))
 	hdr.Set(http.CanonicalHeaderKey("x-amz-storage-class"), obj.GetStorageClass())
 	hdr.Set(http.CanonicalHeaderKey("content-length"), strconv.FormatInt(obj.GetSizeBytes(), 10))
-	hdr.Set(http.CanonicalHeaderKey("content-type"), obj.GetContentType())
 	hdr.Set(http.CanonicalHeaderKey("etag"), obj.GetETag())
 	hdr.Set(http.CanonicalHeaderKey("last-modified"), obj.GetLastModified().Format(timeutils.RFC2882Format))
 	return hdr, nil
@@ -86,10 +85,10 @@ func uploadObject(ctx context.Context, userCred mcclient.TokenCredential, bucket
 		}
 		respHdr.Set("ETag", etag)
 	} else {
-		contType := header.Get(http.CanonicalHeaderKey("content-type"))
+		meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, header)
 		aclStr := header.Get(http.CanonicalHeaderKey("x-amz-acl"))
 		storageClassStr := header.Get(http.CanonicalHeaderKey("x-amz-storage-class"))
-		err = iBucket.PutObject(ctx, key, body, contLen, contType, cloudprovider.TBucketACLType(aclStr), storageClassStr)
+		err = iBucket.PutObject(ctx, key, body, contLen, cloudprovider.TBucketACLType(aclStr), storageClassStr, meta)
 		if err != nil {
 			return nil, errors.Wrap(err, "iBucket.PutObject")
 		}
@@ -179,13 +178,14 @@ func copyObject(ctx context.Context, userCred mcclient.TokenCredential, bucketNa
 		}
 		return &result, nil, nil
 	} else {
+		meta := cloudprovider.FetchMetaFromHttpHeader(cloudprovider.META_HEADER_PREFIX, hdr)
 		if dstBucket.ManagerId == srcBucket.ManagerId && dstBucket.RegionExternalId == srcBucket.RegionExternalId {
-			err = iDstBucket.CopyObject(ctx, key, iSrcBucket.GetName(), srcKey, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass())
+			err = iDstBucket.CopyObject(ctx, key, iSrcBucket.GetName(), srcKey, srcObj.GetAcl(), srcObj.GetStorageClass(), meta)
 			if err != nil {
 				return nil, nil, errors.Wrap(err, "iDstBucket.CopyObject")
 			}
 		} else {
-			err = cloudprovider.CopyObject(ctx, 0, iDstBucket, key, iSrcBucket, srcKey, false)
+			err = cloudprovider.CopyObject(ctx, 0, iDstBucket, key, iSrcBucket, srcKey, meta, false)
 			if err != nil {
 				return nil, nil, errors.Wrap(err, "cloudprovider.CopyObject")
 			}

pkg/s3gateway/models/buckets.go

@@ -224,7 +224,6 @@ func (bucket *SBucketDelegate) ListObject(ctx context.Context, userCred mcclient
 			ETag:         obj.GetETag(),
 			Size:         obj.GetSizeBytes(),
 			LastModified: obj.GetLastModified(),
-			ContentType:  obj.GetContentType(),
 			StorageClass: obj.GetStorageClass(),
 		}
 	}