Merge pull request #2608 from swordqiu/feature/qj-s3gateway3

feature: 1. ak/sk auth support 2. a full feature s3gateway works with Cyberduck

cmd/climc/shell/buckets.go

@@ -280,4 +280,16 @@ func init() {
 		printObject(result)
 		return nil
 	})
+
+	type BucketAccessInfoOptions struct {
+		ID string `help:"ID or name of bucket" json:"-"`
+	}
+	R(&BucketAccessInfoOptions{}, "bucket-access-info", "Show backend access info of a bucket", func(s *mcclient.ClientSession, args *BucketAccessInfoOptions) error {
+		result, err := modules.Buckets.GetSpecific(s, args.ID, "access-info", nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

cmd/climc/shell/credentials.go

@@ -19,13 +19,15 @@ import (
 
 	"yunion.io/x/jsonutils"
 
+	"time"
 	"yunion.io/x/onecloud/pkg/mcclient"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 )
 
 func init() {
 	type CredentialListOptions struct {
-		Type       string `help:"credential type" choices:"totp|recovery|ec2"`
+		Scope      string `help:"scope" choices:"project|domain|system"`
+		Type       string `help:"credential type" choices:"totp|recovery|aksk"`
 		User       string `help:"filter by user"`
 		UserDomain string `help:"the domain of user"`
 	}
@@ -34,22 +36,14 @@ func init() {
 		if len(args.Type) > 0 {
 			query.Add(jsonutils.NewString(args.Type), "type")
 		}
-		var err error
+		if len(args.Scope) > 0 {
+			query.Add(jsonutils.NewString(args.Scope), "scope")
+		}
 		if len(args.User) > 0 {
-			domainId := "default"
 			if len(args.UserDomain) > 0 {
-				domainId, err = modules.Domains.GetId(s, args.UserDomain, nil)
-				if err != nil {
-					return err
-				}
+				query.Add(jsonutils.NewString(args.UserDomain), "domain_id")
 			}
-			userQuery := jsonutils.NewDict()
-			userQuery.Add(jsonutils.NewString(domainId), "domain_id")
-			userId, err := modules.UsersV3.GetId(s, args.User, userQuery)
-			if err != nil {
-				return err
-			}
-			query.Add(jsonutils.NewString(userId), "user_id")
+			query.Add(jsonutils.NewString(args.User), "user_id")
 		}
 		results, err := modules.Credentials.List(s, query)
 		if err != nil {
@@ -159,4 +153,122 @@ func init() {
 		fmt.Println("success")
 		return nil
 	})
+
+	type CredentialAkSkOptions struct {
+		User          string `help:"User"`
+		UserDomain    string `help:"domain of user"`
+		Project       string `help:"Project"`
+		ProjectDomain string `help:"domain of user"`
+	}
+	R(&CredentialAkSkOptions{}, "credential-create-aksk", "Create AccessKey/Secret credential", func(s *mcclient.ClientSession, args *CredentialAkSkOptions) error {
+		var uid string
+		var pid string
+		var err error
+		if len(args.User) > 0 {
+			uid, err = modules.UsersV3.FetchId(s, args.User, args.UserDomain)
+			if err != nil {
+				return err
+			}
+		}
+		if len(args.Project) > 0 {
+			pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain)
+			if err != nil {
+				return err
+			}
+		}
+		secret, err := modules.Credentials.CreateAccessKeySecret(s, uid, pid, time.Time{})
+		if err != nil {
+			return err
+		}
+		printObject(jsonutils.Marshal(&secret))
+		return nil
+	})
+
+	R(&CredentialAkSkOptions{}, "credential-get-aksk", "Get AccessKey/Secret credential for user and project", func(s *mcclient.ClientSession, args *CredentialAkSkOptions) error {
+		uid, err := modules.UsersV3.FetchId(s, args.User, args.UserDomain)
+		if err != nil {
+			return err
+		}
+		var pid string
+		if len(args.Project) > 0 {
+			pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain)
+			if err != nil {
+				return err
+			}
+		}
+		secrets, err := modules.Credentials.GetAccessKeySecrets(s, uid, pid)
+		if err != nil {
+			return err
+		}
+		result := modules.ListResult{}
+		result.Data = make([]jsonutils.JSONObject, len(secrets))
+		for i := range secrets {
+			result.Data[i] = jsonutils.Marshal(secrets[i])
+			result.Data[i].(*jsonutils.JSONDict).Add(jsonutils.NewString(secrets[i].KeyId), "key_id")
+			result.Data[i].(*jsonutils.JSONDict).Add(jsonutils.NewString(secrets[i].ProjectId), "project_id")
+			result.Data[i].(*jsonutils.JSONDict).Add(jsonutils.NewTimeString(secrets[i].TimeStamp), "time_stamp")
+		}
+		printList(&result, nil)
+		return nil
+	})
+
+	R(&CredentialAkSkOptions{}, "credential-remove-aksk", "Remove AccessKey/Secret credential for user and project", func(s *mcclient.ClientSession, args *CredentialAkSkOptions) error {
+		uid, err := modules.UsersV3.FetchId(s, args.User, args.UserDomain)
+		if err != nil {
+			return err
+		}
+		var pid string
+		if len(args.Project) > 0 {
+			pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain)
+			if err != nil {
+				return err
+			}
+		}
+		err = modules.Credentials.RemoveAccessKeySecrets(s, uid, pid)
+		if err != nil {
+			return err
+		}
+		fmt.Println("success")
+		return nil
+	})
+
+	type CredentialDeleteOptions struct {
+		ID string `help:"ID of credentail"`
+	}
+	R(&CredentialDeleteOptions{}, "credential-delete", "Delete credential", func(s *mcclient.ClientSession, args *CredentialDeleteOptions) error {
+		result, err := modules.Credentials.Delete(s, args.ID, nil)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
+
+	type CredentialUpdateOptions struct {
+		ID      string `help:"ID of credentail"`
+		Enable  bool   `help:"Enable credential"`
+		Disable bool   `help:"Disable credential"`
+		Name    string `help:"new name of credential"`
+		Desc    string `help:"new description of credential"`
+	}
+	R(&CredentialUpdateOptions{}, "credential-update", "Enable/disable credential", func(s *mcclient.ClientSession, args *CredentialUpdateOptions) error {
+		params := jsonutils.NewDict()
+		if args.Enable {
+			params.Add(jsonutils.JSONTrue, "enabled")
+		} else if args.Disable {
+			params.Add(jsonutils.JSONFalse, "enabled")
+		}
+		if args.Name != "" {
+			params.Add(jsonutils.NewString(args.Name), "name")
+		}
+		if args.Desc != "" {
+			params.Add(jsonutils.NewString(args.Desc), "description")
+		}
+		result, err := modules.Credentials.Update(s, args.ID, params)
+		if err != nil {
+			return err
+		}
+		printObject(result)
+		return nil
+	})
 }

docs/bucket/access.yaml

@@ -0,0 +1,11 @@
+get:
+  summary: 获取存储桶的后端访问信息，例如ceph RADOS的endpint/ak/sk信息等
+  parameters:
+    - $ref: '../parameters/bucket.yaml#/bucket_name'
+  responses:
+    200:
+      description: 指定存储桶访问信息
+      schema:
+        $ref: "../schemas/bucket.yaml#/BucketAccessInfoResponse"
+  tags:
+    - buckets

docs/index.yaml

@@ -461,6 +461,8 @@ paths:
     $ref: "./bucket/acl.yaml"
   /buckets/{bucketName}/limit:
     $ref: "./bucket/limit.yaml"
+  /buckets/{bucketName}/access-info:
+    $ref: "./bucket/access.yaml"
 
   /ansibleplaybooks:
     $ref: "./ansibleplaybook/ansibleplaybooks.yaml"

docs/schemas/bucket.yaml

@@ -226,3 +226,10 @@ BucketSetLimitInput:
         object_count:
           type: integer
           description: 对象数量限制，为0则无限制
+
+BucketAccessInfoResponse:
+  type: object
+  properties:
+    bucket:
+      type: object
+      description: 桶的后端访问信息，例如ceph RADOS的endpoint/access_key/secret信息等。具体形式由存储后端决定。

go.mod

@@ -152,7 +152,7 @@ require (
 	yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051
 	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
 	yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224
-	yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e
-	yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba
+	yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd
+	yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f
 	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 )

go.sum

@@ -585,9 +585,9 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRi
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224 h1:dAeUov/CtKcPaOapGVG7+NRqmUF2fr2O3Onb+ID5HS4=
 yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e h1:Jk82txl4vaL/MoVV3+GweEcCmLOcqo5XYF8tlBD/1hY=
-yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba h1:vPCRA59Kq9Hv24ef/G92sCgqmBYmZyLSVP902EUDimw=
-yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
+yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd h1:pwGQ4JDXhuRNjmYI8hQXCs08wppsEgj9+u2udTJJDEo=
+yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
+yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f h1:maHGG78d6vLAyT/lGSOOsXWrylBfjdu+NMCGXFhLuhA=
+yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE=
 yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/identity/aksk.go

@@ -0,0 +1,42 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package identity
+
+import (
+	"time"
+)
+
+const (
+	ACCESS_SECRET_TYPE    = "aksk"
+	TOTP_TYPE             = "totp"
+	RECOVERY_SECRETS_TYPE = "recovery_secret"
+)
+
+type SAccessKeySecretBlob struct {
+	Secret string `json:"secret"`
+	Expire int64  `json:"expire"`
+}
+
+func (info SAccessKeySecretBlob) IsValid() bool {
+	if info.Expire <= 0 || info.Expire > time.Now().Unix() {
+		return true
+	}
+	return false
+}
+
+type SAccessKeySecretInfo struct {
+	AccessKey string
+	SAccessKeySecretBlob
+}

pkg/apis/identity/consts.go

@@ -30,9 +30,10 @@ const (
 
 	AUTH_METHOD_PASSWORD = "password"
 	AUTH_METHOD_TOKEN    = "token"
+	AUTH_METHOD_AKSK     = "aksk"
 
-	AUTH_METHOD_ID_PASSWORD = 1
-	AUTH_METHOD_ID_TOKEN    = 2
+	// AUTH_METHOD_ID_PASSWORD = 1
+	// AUTH_METHOD_ID_TOKEN    = 2
 
 	AUTH_TOKEN_HEADER         = "X-Auth-Token"
 	AUTH_SUBJECT_TOKEN_HEADER = "X-Subject-Token"
@@ -73,7 +74,7 @@ const (
 )
 
 var (
-	AUTH_METHODS = []string{AUTH_METHOD_PASSWORD, AUTH_METHOD_TOKEN}
+	AUTH_METHODS = []string{AUTH_METHOD_PASSWORD, AUTH_METHOD_TOKEN, AUTH_METHOD_AKSK}
 
 	SensitiveDomainConfigMap = map[string]string{
 		"ldap": "password",

pkg/appctx/context.go

@@ -39,6 +39,8 @@ const (
 	APP_CONTEXT_KEY_OBJECT_ID       = AppContextKey("objectid")
 	APP_CONTEXT_KEY_OBJECT_TYPE     = AppContextKey("objecttype")
 	APP_CONTEXT_KEY_START_TIME      = AppContextKey("starttime")
+
+	APP_CONTEXT_KEY_HOST_ID = AppContextKey("hostid")
 )
 
 func AppContextServiceName(ctx context.Context) string {
@@ -149,6 +151,15 @@ func AppContextStartTime(ctx context.Context) time.Time {
 	}
 }
 
+func AppContextHostId(ctx context.Context) string {
+	val := ctx.Value(APP_CONTEXT_KEY_HOST_ID)
+	if val != nil {
+		return val.(string)
+	} else {
+		return ""
+	}
+}
+
 type AppContextData struct {
 	Trace         trace.STrace
 	RequestId     string

pkg/appsrv/appsrv.go

@@ -17,6 +17,8 @@ package appsrv
 import (
 	"bufio"
 	"context"
+	"crypto/sha1"
+	"encoding/base64"
 	"fmt"
 	"math/rand"
 	"net"
@@ -57,6 +59,7 @@ type Application struct {
 	defHandlerInfo    SHandlerInfo
 	cors              *Cors
 	middlewares       []MiddlewareFunc
+	hostId            string
 
 	isExiting       bool
 	idleConnsClosed chan struct{}
@@ -91,6 +94,18 @@ func NewApplication(name string, connMax int, db bool) *Application {
 	app.SetContext(appctx.APP_CONTEXT_KEY_APP, &app)
 	app.SetContext(appctx.APP_CONTEXT_KEY_APPNAME, app.name)
 
+	hm := sha1.New()
+	hm.Write([]byte(name))
+	hostname, _ := os.Hostname()
+	hm.Write([]byte(hostname))
+	outIp := utils.GetOutboundIP()
+	hm.Write([]byte(outIp.String()))
+	hostId := base64.URLEncoding.EncodeToString(hm.Sum(nil))
+
+	log.Infof("App hostId: %s (%s,%s,%s)", hostId, name, hostname, outIp.String())
+	app.hostId = hostId
+	app.SetContext(appctx.APP_CONTEXT_KEY_HOST_ID, hostId)
+
 	// initialize random seed
 	rand.Seed(time.Now().UnixNano())
 
@@ -168,6 +183,7 @@ func (lrw *loggingResponseWriter) Hijack() (rwc net.Conn, buf *bufio.ReadWriter,
 }
 
 func (lrw *loggingResponseWriter) WriteHeader(code int) {
+	log.Debugf("XXXX loggingResponseWriter WriteHeader %d", code)
 	if code < 100 || code >= 600 {
 		log.Errorf("Invalud status code %d, set code to 598", code)
 		code = 598
@@ -190,6 +206,7 @@ func genRequestId(w http.ResponseWriter, r *http.Request) string {
 func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	// log.Printf("defaultHandler %s %s", r.Method, r.URL.Path)
 	rid := genRequestId(w, r)
+	w.Header().Set("X-Request-Host-Id", app.hostId)
 	lrw := &loggingResponseWriter{w, http.StatusOK}
 	start := time.Now()
 	hi, params := app.defaultHandle(lrw, r, rid)
@@ -216,7 +233,7 @@ func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		skipLog = true
 	}
 	if !skipLog {
-		log.Infof("%d %s %s %s (%s) %.2fms", lrw.status, rid, r.Method, r.URL, r.RemoteAddr, duration)
+		log.Infof("%s %d %s %s %s (%s) %.2fms", app.hostId, lrw.status, rid, r.Method, r.URL, r.RemoteAddr, duration)
 	}
 }
 

pkg/appsrv/fetch.go

@@ -18,6 +18,8 @@ import (
 	"io/ioutil"
 	"net/http"
 
+	"encoding/xml"
+	"github.com/pkg/errors"
 	"yunion.io/x/jsonutils"
 )
 
@@ -53,3 +55,15 @@ func FetchJSON(req *http.Request) (jsonutils.JSONObject, error) {
 		return nil, nil
 	}
 }
+
+func FetchXml(req *http.Request, target interface{}) error {
+	b, e := Fetch(req)
+	if e != nil {
+		return errors.Wrap(e, "Fetch")
+	}
+	if len(b) > 0 {
+		return xml.Unmarshal(b, target)
+	} else {
+		return nil
+	}
+}

pkg/appsrv/send.go

@@ -15,36 +15,124 @@
 package appsrv
 
 import (
-	"encoding/json"
+	"encoding/xml"
 	"net/http"
+	"strconv"
 
+	"fmt"
+	"github.com/pkg/errors"
+	"io"
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/gotypes"
 )
 
+func SendNoContent(w http.ResponseWriter) {
+	w.WriteHeader(204)
+	sendBytes(w, []byte{})
+}
+
 func Send(w http.ResponseWriter, text string) {
 	w.Header().Set("Content-Type", "text/plain")
-	w.Write([]byte(text))
+	sendBytes(w, []byte(text))
+}
+
+func sendBytes(w http.ResponseWriter, output []byte) {
+	w.Header().Set("Content-Length", strconv.FormatInt(int64(len(output)), 10))
+	w.Write(output)
 }
 
 func SendStruct(w http.ResponseWriter, obj interface{}) {
-	w.Header().Set("Content-Type", "application/json")
-	if obj != nil {
-		b, e := json.Marshal(obj)
-		if e != nil {
-			log.Errorln("SendStruct json Marshal error: ", e)
-		}
-		w.Write(b)
-	} else {
-		w.Write([]byte{})
-	}
+	jsonObj := jsonutils.Marshal(obj)
+	SendJSON(w, jsonObj)
 }
 
 func SendJSON(w http.ResponseWriter, obj jsonutils.JSONObject) {
+	var output []byte
 	w.Header().Set("Content-Type", "application/json")
 	if obj != nil {
-		w.Write([]byte(obj.String()))
+		output = []byte(obj.String())
+	}
+	sendBytes(w, output)
+}
+
+func SendHeader(w http.ResponseWriter, hdr http.Header) {
+	w.WriteHeader(204)
+	for k, v := range hdr {
+		if len(v) > 0 && len(v[0]) > 0 {
+			w.Header().Set(k, v[0])
+		}
+	}
+	w.Write([]byte{})
+}
+
+func SendXml(w http.ResponseWriter, hdr http.Header, obj interface{}) {
+	if !gotypes.IsNil(obj) {
+		xmlBytes, err := xml.Marshal(obj)
+		if err == nil {
+			for k, v := range hdr {
+				if k != "Content-Type" && k != "Content-Length" {
+					w.Header().Set(k, v[0])
+				}
+			}
+			w.Header().Set("Content-Type", "application/xml")
+			w.Header().Set("Content-Length", strconv.FormatInt(int64(len(xmlBytes)+len(xml.Header)), 10))
+			w.Write([]byte(xml.Header))
+			w.Write(xmlBytes)
+		} else {
+			w.WriteHeader(400)
+			Send(w, err.Error())
+		}
 	} else {
-		w.Write([]byte{})
+		for k, v := range hdr {
+			if k != "Content-Type" && k != "Content-Length" {
+				w.Header().Set(k, v[0])
+			}
+		}
+		SendNoContent(w)
 	}
 }
+
+func SendStream(w http.ResponseWriter, isPartial bool, hdr http.Header, stream io.ReadCloser, sizeBytes int64) error {
+	defer stream.Close()
+	if isPartial {
+		log.Debugf("send partial 206")
+		w.WriteHeader(206)
+	} else {
+		log.Debugf("send full 200")
+		w.WriteHeader(200)
+	}
+	for k, v := range hdr {
+		if k != "Content-Length" {
+			log.Debugf("send %s %s", k, v)
+			w.Header().Set(k, v[0])
+		}
+	}
+	if sizeBytes > 0 {
+		log.Debugf("send content-length %d", sizeBytes)
+		w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10))
+	}
+	offset := 0
+	buf := make([]byte, 4096)
+	for sizeBytes <= 0 || int64(offset) < sizeBytes {
+		n, err := stream.Read(buf)
+		if n > 0 {
+			woff := 0
+			for woff < n {
+				m, err := w.Write(buf[woff:n])
+				if err != nil {
+					return errors.Wrap(err, fmt.Sprintf("w.Write read_offset %d write_offset %d", offset, woff))
+				}
+				woff += m
+			}
+			offset += n
+		}
+		if err != nil {
+			if err == io.EOF {
+				break
+			}
+			return errors.Wrap(err, "stream.Read")
+		}
+	}
+	return nil
+}

pkg/cloudcommon/policy/defaults.go

@@ -204,6 +204,36 @@ var (
 					Action:   PolicyActionDelete,
 					Result:   rbacutils.Allow,
 				},
+				{
+					Service:  "identity",
+					Resource: "credentials",
+					Action:   PolicyActionGet,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "identity",
+					Resource: "credentials",
+					Action:   PolicyActionList,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "identity",
+					Resource: "credentials",
+					Action:   PolicyActionCreate,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "identity",
+					Resource: "credentials",
+					Action:   PolicyActionUpdate,
+					Result:   rbacutils.Allow,
+				},
+				{
+					Service:  "identity",
+					Resource: "credentials",
+					Action:   PolicyActionDelete,
+					Result:   rbacutils.Allow,
+				},
 				{
 					Service:  "yunionconf",
 					Resource: "parameters",

pkg/cloudprovider/consts.go

@@ -26,6 +26,8 @@ const (
 	CloudVMStatusDeploying    = "deploying"
 	CloudVMStatusOther        = "other"
 
+	ErrUnauthenticated     = errors.Error("not authenticated")
+	ErrUnauthorized        = errors.Error("not authorized")
 	ErrNotFound            = errors.Error("id not found")
 	ErrDuplicateId         = errors.Error("duplicate id")
 	ErrInvalidStatus       = errors.Error("invalid status")
@@ -34,4 +36,5 @@ const (
 	ErrNotSupported        = errors.Error("Not supported")
 	ErrInvalidProvider     = errors.Error("Invalid provider")
 	ErrNoBalancePermission = errors.Error("No balance permission")
+	ErrBadRequest          = errors.Error("bad request")
 )

pkg/cloudprovider/objectstore.go

@@ -17,9 +17,12 @@ package cloudprovider
 import (
 	"context"
 	"io"
+	"regexp"
+	"strconv"
 	"strings"
 	"time"
 
+	"fmt"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/s3cli"
@@ -75,6 +78,43 @@ type SListObjectResult struct {
 	IsTruncated    bool
 }
 
+type SGetObjectRange struct {
+	Start int64
+	End   int64
+}
+
+func (r SGetObjectRange) SizeBytes() int64 {
+	return r.End - r.Start + 1
+}
+
+var (
+	rangeExp = regexp.MustCompile(`(bytes=)?(\d*)-(\d*)`)
+)
+
+func ParseRange(rangeStr string) SGetObjectRange {
+	objRange := SGetObjectRange{}
+	if len(rangeStr) > 0 {
+		find := rangeExp.FindAllStringSubmatch(rangeStr, -1)
+		if len(find) > 0 && len(find[0]) > 3 {
+			objRange.Start, _ = strconv.ParseInt(find[0][2], 10, 64)
+			objRange.End, _ = strconv.ParseInt(find[0][3], 10, 64)
+		}
+	}
+	return objRange
+}
+
+func (r SGetObjectRange) String() string {
+	if r.Start > 0 && r.End > 0 {
+		return fmt.Sprintf("bytes=%d-%d", r.Start, r.End)
+	} else if r.Start > 0 && r.End <= 0 {
+		return fmt.Sprintf("bytes=%d-", r.Start)
+	} else if r.Start <= 0 && r.End > 0 {
+		return fmt.Sprintf("bytes=0-%d", r.End)
+	} else {
+		return ""
+	}
+}
+
 type ICloudBucket interface {
 	IVirtualResource
 
@@ -98,12 +138,16 @@ type ICloudBucket interface {
 	ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error)
 	GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error)
 
+	CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl TBucketACLType, storageClassStr string) error
+	GetObject(ctx context.Context, key string, rangeOpt *SGetObjectRange) (io.ReadCloser, error)
+
 	DeleteObject(ctx context.Context, keys string) error
 	GetTempUrl(method string, key string, expire time.Duration) (string, error)
 
 	PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, contType string, cannedAcl TBucketACLType, storageClassStr string) error
 	NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl TBucketACLType, storageClassStr string) (string, error)
 	UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error)
+	CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucketName string, srcKey string, srcOffset int64, srcLength int64) (string, error)
 	CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error
 	AbortMultipartUpload(ctx context.Context, key string, uploadId string) error
 }
@@ -350,3 +394,110 @@ func DeletePrefix(ctx context.Context, bucket ICloudBucket, prefix string) error
 	}
 	return nil
 }
+
+func CopyObject(ctx context.Context, blocksz int64, dstBucket ICloudBucket, dstKey string, srcBucket ICloudBucket, srcKey string, debug bool) error {
+	srcObj, err := GetIObject(srcBucket, srcKey)
+	if err != nil {
+		return errors.Wrap(err, "GetIObject")
+	}
+	if blocksz <= 0 {
+		blocksz = MAX_PUT_OBJECT_SIZEBYTES
+	}
+	sizeBytes := srcObj.GetSizeBytes()
+	if sizeBytes < blocksz {
+		if debug {
+			log.Debugf("too small, copy object in one shot")
+		}
+		srcStream, err := srcBucket.GetObject(ctx, srcKey, nil)
+		if err != nil {
+			return errors.Wrap(err, "srcBucket.GetObject")
+		}
+		defer srcStream.Close()
+		err = dstBucket.PutObject(ctx, dstKey, srcStream, sizeBytes, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass())
+		if err != nil {
+			return errors.Wrap(err, "dstBucket.PutObject")
+		}
+		return nil
+	}
+	partSize := blocksz
+	partCount := sizeBytes / partSize
+	if partCount*partSize < sizeBytes {
+		partCount += 1
+	}
+	if partCount > int64(dstBucket.MaxPartCount()) {
+		partCount = int64(dstBucket.MaxPartCount())
+		partSize = sizeBytes / partCount
+		if partSize*partCount < sizeBytes {
+			partSize += 1
+		}
+		if partSize > dstBucket.MaxPartSizeBytes() {
+			return errors.Error("too larget object")
+		}
+	}
+	if debug {
+		log.Debugf("multipart upload part count %d part size %d", partCount, partSize)
+	}
+	uploadId, err := dstBucket.NewMultipartUpload(ctx, dstKey, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass())
+	if err != nil {
+		return errors.Wrap(err, "bucket.NewMultipartUpload")
+	}
+	etags := make([]string, partCount)
+	// offset := int64(0)
+	for i := 0; i < int(partCount); i += 1 {
+		start := int64(i) * partSize
+		if i == int(partCount)-1 {
+			partSize = sizeBytes - partSize*(partCount-1)
+		}
+		end := start + partSize - 1
+		rangeOpt := SGetObjectRange{
+			Start: start,
+			End:   end,
+		}
+		if debug {
+			log.Debugf("UploadPart %d %d range: %s (%d)", i+1, partSize, rangeOpt.String(), rangeOpt.SizeBytes())
+		}
+		srcStream, err := srcBucket.GetObject(ctx, srcKey, &rangeOpt)
+		if err == nil {
+			defer srcStream.Close()
+			var etag string
+			etag, err = dstBucket.UploadPart(ctx, dstKey, uploadId, i+1, io.LimitReader(srcStream, partSize), partSize)
+			if err == nil {
+				etags[i] = etag
+				continue
+			}
+		}
+		if err != nil {
+			err2 := dstBucket.AbortMultipartUpload(ctx, dstKey, uploadId)
+			if err2 != nil {
+				log.Errorf("bucket.AbortMultipartUpload error %s", err2)
+			}
+			return errors.Wrap(err, "bucket.UploadPart")
+		}
+	}
+	err = dstBucket.CompleteMultipartUpload(ctx, dstKey, uploadId, etags)
+	if err != nil {
+		err2 := dstBucket.AbortMultipartUpload(ctx, dstKey, uploadId)
+		if err2 != nil {
+			log.Errorf("bucket.AbortMultipartUpload error %s", err2)
+		}
+		return errors.Wrap(err, "CompleteMultipartUpload")
+	}
+	return nil
+}
+
+func CopyPart(ctx context.Context,
+	iDstBucket ICloudBucket, dstKey string, uploadId string, partNumber int,
+	iSrcBucket ICloudBucket, srcKey string, rangeOpt *SGetObjectRange,
+) (string, error) {
+	srcReader, err := iSrcBucket.GetObject(ctx, srcKey, rangeOpt)
+	if err != nil {
+		return "", errors.Wrap(err, "iSrcBucket.GetObject")
+	}
+	defer srcReader.Close()
+
+	etag, err := iDstBucket.UploadPart(ctx, dstKey, uploadId, partNumber, io.LimitReader(srcReader, rangeOpt.SizeBytes()), rangeOpt.SizeBytes())
+	if err != nil {
+		return "", errors.Wrap(err, "iDstBucket.UploadPart")
+	}
+	return etag, nil
+}

pkg/cloudprovider/objectstore_test.go

@@ -0,0 +1,52 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cloudprovider
+
+import "testing"
+
+func TestParseRange(t *testing.T) {
+	cases := []struct {
+		in    string
+		start int64
+		end   int64
+	}{
+		{
+			in:    "bytes=0-200",
+			start: 0,
+			end:   200,
+		},
+		{
+			in:    "200-3232300",
+			start: 200,
+			end:   3232300,
+		},
+		{
+			in:    "200-",
+			start: 200,
+			end:   0,
+		},
+		{
+			in:    "-232323",
+			start: 0,
+			end:   232323,
+		},
+	}
+	for _, c := range cases {
+		got := ParseRange(c.in)
+		if got.Start != c.start || got.End != c.end {
+			t.Fatalf("got.start(%d) != want.start(%d) or got.end(%d) != want.end(%d)", got.Start, c.start, got.End, c.end)
+		}
+	}
+}

pkg/compute/models/buckets.go

@@ -1060,3 +1060,29 @@ func (bucket *SBucket) PerformLimit(
 
 	return nil, nil
 }
+
+func (bucket *SBucket) AllowGetDetailsAccessInfo(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) bool {
+	return bucket.IsOwner(userCred)
+}
+
+func (bucket *SBucket) GetDetailsAccessInfo(
+	ctx context.Context,
+	userCred mcclient.TokenCredential,
+	query jsonutils.JSONObject,
+) (jsonutils.JSONObject, error) {
+	manager := bucket.GetCloudprovider()
+	if manager == nil {
+		return nil, httperrors.NewInternalServerError("missing manager?")
+	}
+	info, err := manager.GetDetailsClirc(ctx, userCred, nil)
+	if err != nil {
+		return nil, err
+	}
+	account := manager.GetCloudaccount()
+	info.(*jsonutils.JSONDict).Add(jsonutils.NewString(account.Brand), "PROVIDER")
+	return info, err
+}

pkg/compute/models/managedresource.go

@@ -380,6 +380,7 @@ type SCloudProviderInfo struct {
 	ManagerDomainId  string `json:",omitempty"`
 	Region           string `json:",omitempty"`
 	RegionId         string `json:",omitempty"`
+	RegionExternalId string `json:",omitempty"`
 	RegionExtId      string `json:",omitempty"`
 	Zone             string `json:",omitempty"`
 	ZoneId           string `json:",omitempty"`
@@ -399,6 +400,7 @@ var (
 		"manager_project_id",
 		"region",
 		"region_id",
+		"region_external_id",
 		"region_ext_id",
 		"zone",
 		"zone_id",
@@ -452,6 +454,7 @@ func MakeCloudProviderInfo(region *SCloudregion, zone *SZone, provider *SCloudpr
 		info.CloudEnv = account.getCloudEnv()
 
 		if region != nil {
+			info.RegionExternalId = region.ExternalId
 			info.RegionExtId = fetchExternalId(region.ExternalId)
 			if zone != nil {
 				info.ZoneExtId = fetchExternalId(zone.ExternalId)

pkg/compute/tasks/guest_block_io_throttle_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/compute/tasks/guest_resize_disk_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/compute/tasks/host_guests_migrate_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/compute/tasks/host_maintenance_task.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package tasks
 
 import (

pkg/httperrors/consts.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package httperrors
+
+import (
+	"yunion.io/x/pkg/errors"
+)
+
+const (
+	ErrUnauthenticated = errors.Error("not authenticated")
+	ErrUnauthorized    = errors.Error("not authorized")
+	ErrNotFound        = errors.Error("id not found")
+	ErrDuplicateId     = errors.Error("duplicate id")
+	ErrInvalidStatus   = errors.Error("invalid status")
+	ErrTimeout         = errors.Error("timeout")
+	ErrNotImplemented  = errors.Error("Not implemented")
+	ErrNotSupported    = errors.Error("Not supported")
+	ErrBadRequest      = errors.Error("bad request")
+	ErrOutOfRange      = errors.Error("out of range")
+	ErrForbidden       = errors.Error("not allowed")
+	ErrOutOfLimit      = errors.Error("out of limit")
+)

pkg/keystone/models/credentials.go

@@ -16,15 +16,21 @@ package models
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/tristate"
+	"yunion.io/x/sqlchemy"
 
+	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/keystone/keys"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
 type SCredentialManager struct {
@@ -62,14 +68,16 @@ func init() {
 type SCredential struct {
 	db.SStandaloneResourceBase
 
-	UserId    string `width:"64" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"`
-	ProjectId string `width:"64" charset:"ascii" nullable:"true" list:"admin" create:"admin_required"`
-	Type      string `width:"255" charset:"utf8" nullable:"false" list:"admin" create:"admin_required"`
-	KeyHash   string `width:"64" charset:"ascii" nullable:"false" create:"admin_required"`
+	UserId    string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"required"`
+	ProjectId string `width:"64" charset:"ascii" nullable:"true" list:"user" create:"required"`
+	Type      string `width:"255" charset:"utf8" nullable:"false" list:"user" create:"required"`
+	KeyHash   string `width:"64" charset:"ascii" nullable:"false" create:"required"`
 
 	Extra *jsonutils.JSONDict `nullable:"true" list:"admin"`
 
-	EncryptedBlob string `nullable:"false" create:"admin_required"`
+	EncryptedBlob string `nullable:"false" create:"required"`
+
+	Enabled tristate.TriState `nullable:"false" default:"true" list:"user" update:"user" create:"optional"`
 }
 
 func (manager *SCredentialManager) InitializeData() error {
@@ -99,10 +107,36 @@ func (manager *SCredentialManager) ValidateCreateData(ctx context.Context, userC
 	if !data.Contains("type") {
 		return nil, httperrors.NewInputParameterError("missing input feild type")
 	}
+	userId, _ := data.GetString("user_id")
+	projectId, _ := data.GetString("project_id")
+	if len(userId) == 0 {
+		userId = userCred.GetUserId()
+		data.Set("user_id", jsonutils.NewString(userId))
+	} else {
+		_, err := UserManager.FetchById(userId)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(UserManager.Keyword(), userId)
+			} else {
+				return nil, httperrors.NewGeneralError(err)
+			}
+		}
+	}
+	if len(projectId) == 0 {
+		projectId = userCred.GetProjectId()
+		data.Set("project_id", jsonutils.NewString(projectId))
+	} else {
+		_, err := ProjectManager.FetchById(projectId)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2(ProjectManager.Keyword(), projectId)
+			} else {
+				return nil, httperrors.NewGeneralError(err)
+			}
+		}
+	}
 	if !data.Contains("name") {
 		typeStr, _ := data.GetString("type")
-		userId, _ := data.GetString("user_id")
-		projectId, _ := data.GetString("project_id")
 		data.Add(jsonutils.NewString(fmt.Sprintf("%s-%s-%s", typeStr, projectId, userId)), "name")
 	}
 	blob, _ := data.GetString("blob")
@@ -124,6 +158,7 @@ func (self *SCredential) ValidateDeleteCondition(ctx context.Context) error {
 }
 
 func (self *SCredential) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
+
 	return self.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }
 
@@ -141,8 +176,7 @@ func (self *SCredential) GetExtraDetails(ctx context.Context, userCred mcclient.
 }
 
 func credentialExtra(cred *SCredential, extra *jsonutils.JSONDict) *jsonutils.JSONDict {
-	blob := keys.CredentialKeyManager.Decrypt([]byte(cred.EncryptedBlob), time.Duration(-1))
-	extra.Add(jsonutils.NewString(string(blob)), "blob")
+	extra.Add(jsonutils.NewString(string(cred.getBlob())), "blob")
 
 	usr, _ := UserManager.FetchUserExtended(cred.UserId, "", "", "")
 	if usr != nil {
@@ -152,3 +186,73 @@ func credentialExtra(cred *SCredential, extra *jsonutils.JSONDict) *jsonutils.JS
 	}
 	return extra
 }
+
+func (self *SCredential) getBlob() []byte {
+	return keys.CredentialKeyManager.Decrypt([]byte(self.EncryptedBlob), time.Duration(-1))
+}
+
+func (self *SCredential) GetAccessKeySecret() (*api.SAccessKeySecretBlob, error) {
+	if self.Type == api.ACCESS_SECRET_TYPE {
+		blobJson, err := jsonutils.Parse(self.getBlob())
+		if err != nil {
+			return nil, errors.Wrap(err, "jsonutils.Parse")
+		}
+		akBlob := api.SAccessKeySecretBlob{}
+		err = blobJson.Unmarshal(&akBlob)
+		if err != nil {
+			return nil, errors.Wrap(err, "blobJson.Unmarshal")
+		}
+		return &akBlob, nil
+	}
+	return nil, errors.Error("no an AK/SK credential")
+}
+
+func (manager *SCredentialManager) ResourceScope() rbacutils.TRbacScope {
+	return rbacutils.ScopeUser
+}
+
+func (manager *SCredentialManager) FilterByOwner(q *sqlchemy.SQuery, owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery {
+	if owner != nil {
+		if scope == rbacutils.ScopeUser {
+			if len(owner.GetUserId()) > 0 {
+				q = q.Equals("user_id", owner.GetUserId())
+			}
+		}
+	}
+	return q
+}
+
+func (self *SCredential) GetOwnerId() mcclient.IIdentityProvider {
+	owner := db.SOwnerId{UserId: self.UserId}
+	return &owner
+}
+
+func (manager *SCredentialManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	userStr, key := jsonutils.GetAnyString2(data, []string{"user", "user_id"})
+	if len(userStr) > 0 {
+		domainOwner, err := fetchDomainInfo(data)
+		if err != nil {
+			return nil, err
+		}
+		if domainOwner == nil {
+			domainOwner = &db.SOwnerId{DomainId: api.DEFAULT_DOMAIN_ID}
+		}
+		data.(*jsonutils.JSONDict).Remove(key)
+		usrObj, err := UserManager.FetchByIdOrName(domainOwner, userStr)
+		if err != nil {
+			if err == sql.ErrNoRows {
+				return nil, httperrors.NewResourceNotFoundError2("user", userStr)
+			} else {
+				return nil, httperrors.NewGeneralError(err)
+			}
+		}
+		usr := usrObj.(*SUser)
+		ownerId := db.SOwnerId{
+			UserDomainId: usr.DomainId,
+			UserId:       usr.Id,
+		}
+		data.(*jsonutils.JSONDict).Set("user", jsonutils.NewString(usr.Id))
+		return &ownerId, nil
+	}
+	return nil, nil
+}

pkg/keystone/models/identitybase.go

@@ -142,7 +142,7 @@ func (manager *SIdentityBaseResourceManager) OrderByExtraFields(ctx context.Cont
 	return q, nil
 }
 
-func (manager *SIdentityBaseResourceManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+func fetchDomainInfo(data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
 	domainId, key := jsonutils.GetAnyString2(data, []string{"domain_id", "project_domain", "project_domain_id"})
 	if len(domainId) > 0 {
 		data.(*jsonutils.JSONDict).Remove(key)
@@ -154,11 +154,16 @@ func (manager *SIdentityBaseResourceManager) FetchOwnerId(ctx context.Context, d
 			return nil, httperrors.NewGeneralError(err)
 		}
 		owner := db.SOwnerId{DomainId: domain.Id, Domain: domain.Name}
+		data.(*jsonutils.JSONDict).Set("project_domain", jsonutils.NewString(domain.Id))
 		return &owner, nil
 	}
 	return nil, nil
 }
 
+func (manager *SIdentityBaseResourceManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
+	return fetchDomainInfo(data)
+}
+
 func (manager *SIdentityBaseResourceManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	domain, _ := DomainManager.FetchDomainById(ownerId.GetProjectDomainId())
 	if domain.Enabled.IsFalse() {

pkg/keystone/tokens/auth.go

@@ -25,10 +25,12 @@ import (
 	"yunion.io/x/sqlchemy"
 
 	api "yunion.io/x/onecloud/pkg/apis/identity"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/keystone/driver"
 	"yunion.io/x/onecloud/pkg/keystone/models"
 	"yunion.io/x/onecloud/pkg/keystone/options"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/s3auth"
 )
 
 func authUserByTokenV2(ctx context.Context, input mcclient.SAuthenticationInputV2) (*api.SUserExtended, error) {
@@ -162,26 +164,77 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent
 	return usr, nil
 }
 
+func authUserByAccessKeyV3(ctx context.Context, input mcclient.SAuthenticationInputV3) (*api.SUserExtended, string, api.SAccessKeySecretInfo, error) {
+	var aksk api.SAccessKeySecretInfo
+
+	akskRequest, err := s3auth.Decode(input.Auth.Identity.AccessKeyRequest)
+	if err != nil {
+		return nil, "", aksk, errors.Wrap(err, "s3auth.Decode")
+	}
+	keyId := akskRequest.GetAccessKey()
+	obj, err := models.CredentialManager.FetchById(keyId)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, "", aksk, ErrInvalidAccessKeyId
+		} else {
+			return nil, "", aksk, errors.Wrap(err, "CredentialManager.FetchById")
+		}
+	}
+	credential := obj.(*models.SCredential)
+	if !credential.Enabled.IsTrue() {
+		return nil, "", aksk, errors.Wrap(httperrors.ErrInvalidStatus, "Access Key disabled")
+	}
+	akBlob, err := credential.GetAccessKeySecret()
+	if err != nil {
+		return nil, "", aksk, errors.Wrap(err, "credential.GetAccessKeySecret")
+	}
+	if !akBlob.IsValid() {
+		return nil, "", aksk, ErrExpiredAccessKey
+	}
+	aksk.AccessKey = keyId
+	aksk.Secret = akBlob.Secret
+	aksk.Expire = akBlob.Expire
+
+	err = akskRequest.Verify(akBlob.Secret)
+	if err != nil {
+		return nil, "", aksk, errors.Wrap(err, "Verify")
+	}
+	usrExt, err := models.UserManager.FetchUserExtended(credential.UserId, "", "", "")
+	if err != nil {
+		return nil, "", aksk, errors.Wrap(err, "UserManager.FetchUserExtended")
+	}
+	return usrExt, credential.ProjectId, aksk, nil
+}
+
 func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3) (*mcclient.TokenCredentialV3, error) {
+	var akskInfo api.SAccessKeySecretInfo
 	var user *api.SUserExtended
 	var err error
 	if len(input.Auth.Identity.Methods) != 1 {
 		return nil, ErrInvalidAuthMethod
 	}
 	method := input.Auth.Identity.Methods[0]
-	if method == api.AUTH_METHOD_TOKEN {
+	switch method {
+	case api.AUTH_METHOD_TOKEN:
 		// auth by token
 		user, err = authUserByTokenV3(ctx, input)
 		if err != nil {
 			return nil, errors.Wrap(err, "authUserByTokenV3")
 		}
-	} else {
+	case api.AUTH_METHOD_AKSK:
+		// auth by aksk
+		user, input.Auth.Scope.Project.Id, akskInfo, err = authUserByAccessKeyV3(ctx, input)
+		if err != nil {
+			return nil, errors.Wrap(err, "authUserByAccessKeyV3")
+		}
+	default:
 		// auth by other methods, password, openid, saml, etc...
 		user, err = authUserByIdentityV3(ctx, input)
 		if err != nil {
 			return nil, errors.Wrap(err, "authUserByIdentityV3")
 		}
 	}
+
 	// user not found
 	if user == nil {
 		return nil, ErrUserNotFound
@@ -205,7 +258,7 @@ func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3)
 
 	if len(input.Auth.Scope.Project.Id) == 0 && len(input.Auth.Scope.Project.Name) == 0 && len(input.Auth.Scope.Domain.Id) == 0 && len(input.Auth.Scope.Domain.Name) == 0 {
 		// unscoped auth
-		return token.getTokenV3(ctx, user, nil, nil)
+		return token.getTokenV3(ctx, user, nil, nil, akskInfo)
 	}
 	var projExt *models.SProjectExtended
 	var domain *models.SDomain
@@ -238,7 +291,11 @@ func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3)
 		}
 		token.DomainId = domain.Id
 	}
-	return token.getTokenV3(ctx, user, projExt, domain)
+	tokenV3, err := token.getTokenV3(ctx, user, projExt, domain, akskInfo)
+	if err != nil {
+		return nil, errors.Wrap(err, "getTokenV3")
+	}
+	return tokenV3, nil
 }
 
 func AuthenticateV2(ctx context.Context, input mcclient.SAuthenticationInputV2) (*mcclient.TokenCredentialV2, error) {

pkg/keystone/tokens/errors.go

@@ -27,4 +27,6 @@ const (
 	ErrDomainDisabled     = errors.Error("domain is disabled")
 	ErrEmptyAuth          = errors.Error("empty auth request")
 	ErrUserNotInProject   = errors.Error("user not in project")
+	ErrInvalidAccessKeyId = errors.Error("invalid access key id")
+	ErrExpiredAccessKey   = errors.Error("expired access key")
 )

pkg/keystone/tokens/handlers.go

@@ -94,6 +94,7 @@ func authenticateTokensV3(ctx context.Context, w http.ResponseWriter, r *http.Re
 		return
 	}
 	w.Header().Set(api.AUTH_SUBJECT_TOKEN_HEADER, token.Id)
+
 	appsrv.SendJSON(w, jsonutils.Marshal(token))
 
 	models.UserManager.TraceLoginV3(ctx, token)
@@ -166,7 +167,7 @@ func verifyTokensV3(ctx context.Context, w http.ResponseWriter, r *http.Request)
 		}
 	}
 
-	v3token, err := token.getTokenV3(ctx, user, projExt, domain)
+	v3token, err := token.getTokenV3(ctx, user, projExt, domain, api.SAccessKeySecretInfo{})
 	if err != nil {
 		httperrors.InternalServerError(w, "internal server error %s", err)
 		return

pkg/keystone/tokens/token.go

@@ -243,8 +243,10 @@ func (t *SAuthToken) getTokenV3(
 	user *api.SUserExtended,
 	project *models.SProjectExtended,
 	domain *models.SDomain,
+	akskInfo api.SAccessKeySecretInfo,
 ) (*mcclient.TokenCredentialV3, error) {
 	token := mcclient.TokenCredentialV3{}
+	token.Token.AccessKey = akskInfo
 	token.Token.ExpiresAt = t.ExpiresAt
 	token.Token.IssuedAt = t.ExpiresAt.Add(-time.Duration(options.Options.TokenExpirationSeconds) * time.Second)
 	token.Token.AuditIds = t.AuditIds

pkg/mcclient/aksk.go

@@ -0,0 +1,72 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package mcclient
+
+import (
+	"context"
+	"net/http"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+
+	api "yunion.io/x/onecloud/pkg/apis/identity"
+	"yunion.io/x/onecloud/pkg/util/netutils2"
+	"yunion.io/x/onecloud/pkg/util/s3auth"
+)
+
+type SAkskTokenCredential struct {
+	AccessKeySecret api.SAccessKeySecretInfo
+	Token           TokenCredential
+}
+
+func (this *Client) _verifyKeySecret(aksk s3auth.IAccessKeySecretRequest, aCtx SAuthContext) (*SAkskTokenCredential, error) {
+	input := SAuthenticationInputV3{}
+	input.Auth.Identity.Methods = []string{api.AUTH_METHOD_AKSK}
+	input.Auth.Identity.AccessKeyRequest = aksk.Encode()
+	input.Auth.Context = aCtx
+
+	hdr, rbody, err := this.jsonRequest(context.Background(), this.authUrl, "", "POST", "/auth/tokens", nil, jsonutils.Marshal(&input))
+	if err != nil {
+		return nil, err
+	}
+
+	tokenId := hdr.Get("X-Subject-Token")
+	if len(tokenId) == 0 {
+		return nil, errors.Error("No X-Subject-Token in header")
+	}
+
+	ret := SAkskTokenCredential{}
+	ret.Token, err = this.unmarshalV3Token(rbody, tokenId)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshalV3Token")
+	}
+	ret.AccessKeySecret = ret.Token.(*TokenCredentialV3).Token.AccessKey
+	return &ret, nil
+}
+
+func (this *Client) VerifyRequest(req http.Request, aksk s3auth.IAccessKeySecretRequest, virtualHost bool) (*SAkskTokenCredential, error) {
+	cliIp := netutils2.GetHttpRequestIp(&req)
+	aCtx := SAuthContext{
+		Source: AuthSourceSrv,
+		Ip:     cliIp,
+	}
+
+	token, err := this._verifyKeySecret(aksk, aCtx)
+	if err != nil {
+		return nil, errors.Wrap(err, "this._verifyKeySecret")
+	}
+
+	return token, nil
+}

pkg/mcclient/auth/aksk.go

@@ -0,0 +1,87 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package auth
+
+import (
+	"net/http"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/s3auth"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/cache"
+)
+
+type sAccessKeyCache struct {
+	*cache.LRUCache
+}
+
+type sAkSkCacheItem struct {
+	credential *mcclient.SAkskTokenCredential
+}
+
+func (item *sAkSkCacheItem) Size() int {
+	return 1
+}
+
+func newAccessKeyCache() *sAccessKeyCache {
+	return &sAccessKeyCache{
+		LRUCache: cache.NewLRUCache(defaultCacheCount),
+	}
+}
+
+func (c *sAccessKeyCache) addToken(cred *mcclient.SAkskTokenCredential) {
+	item := &sAkSkCacheItem{cred}
+	c.Set(cred.AccessKeySecret.AccessKey, item)
+}
+
+func (c *sAccessKeyCache) getToken(token string) (*mcclient.SAkskTokenCredential, bool) {
+	item, found := c.Get(token)
+	if !found {
+		return nil, false
+	}
+	return item.(*sAkSkCacheItem).credential, true
+}
+
+func (c *sAccessKeyCache) deleteToken(token string) bool {
+	return c.Delete(token)
+}
+
+func (c *sAccessKeyCache) Verify(cli *mcclient.Client, req http.Request, virtualHost bool) (mcclient.TokenCredential, error) {
+	aksk, err := s3auth.DecodeAccessKeyRequest(req, virtualHost)
+	if err != nil {
+		return nil, errors.Wrap(err, "s3auth.DecodeAccessKeyRequestV2")
+	}
+
+	token, found := c.getToken(aksk.GetAccessKey())
+	if found {
+		if token.Token.IsValid() && token.AccessKeySecret.IsValid() {
+			err = aksk.Verify(token.AccessKeySecret.Secret)
+			if err != nil {
+				return nil, errors.Wrap(err, "aksk.Verify")
+			}
+			return token.Token, nil
+		} else {
+			c.deleteToken(aksk.GetAccessKey())
+		}
+	}
+
+	token, err = cli.VerifyRequest(req, aksk, virtualHost)
+	if err != nil {
+		return nil, errors.Wrap(err, "cli.VerifyRequest")
+	}
+
+	c.addToken(token)
+
+	return token.Token, nil
+}

pkg/mcclient/auth/auth.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/cache"
 
+	"net/http"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -131,6 +132,7 @@ type authManager struct {
 	info             *AuthInfo
 	adminCredential  mcclient.TokenCredential
 	tokenCacheVerify *TokenCacheVerify
+	accessKeyCache   *sAccessKeyCache
 }
 
 func newAuthManager(cli *mcclient.Client, info *AuthInfo) *authManager {
@@ -138,9 +140,21 @@ func newAuthManager(cli *mcclient.Client, info *AuthInfo) *authManager {
 		client:           cli,
 		info:             info,
 		tokenCacheVerify: NewTokenCacheVerify(),
+		accessKeyCache:   newAccessKeyCache(),
 	}
 }
 
+func (a *authManager) verifyRequest(req http.Request, virtualHost bool) (mcclient.TokenCredential, error) {
+	if a.adminCredential == nil {
+		return nil, fmt.Errorf("No valid admin token credential")
+	}
+	cred, err := a.accessKeyCache.Verify(a.client, req, virtualHost)
+	if err != nil {
+		return nil, err
+	}
+	return cred, nil
+}
+
 func (a *authManager) verify(token string) (mcclient.TokenCredential, error) {
 	if a.adminCredential == nil {
 		return nil, fmt.Errorf("No valid admin token credential")
@@ -229,6 +243,10 @@ func Verify(tokenId string) (mcclient.TokenCredential, error) {
 	return manager.verify(tokenId)
 }
 
+func VerifyRequest(req http.Request, virtualHost bool) (mcclient.TokenCredential, error) {
+	return manager.verifyRequest(req, virtualHost)
+}
+
 func GetServiceURL(service, region, zone, endpointType string) (string, error) {
 	return manager.GetServiceURL(service, region, zone, endpointType)
 }

pkg/mcclient/input.go

@@ -58,6 +58,7 @@ type SAuthenticationIdentity struct {
 	Token struct {
 		Id string `json:"id,omitempty"`
 	} `json:"token,omitempty"`
+	AccessKeyRequest string `json:"access_key_secret,omitempty"`
 }
 
 type SAuthenticationInputV3 struct {

pkg/mcclient/modules/mod_credentials.go

@@ -15,11 +15,15 @@
 package modules
 
 import (
+	"encoding/base64"
+	"fmt"
 	"time"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/util/seclib"
 
+	"github.com/pkg/errors"
+	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
@@ -31,8 +35,9 @@ type SCredentialManager struct {
 const (
 	DEFAULT_PROJECT = "default"
 
-	TOTP_TYPE             = "totp"
-	RECOVERY_SECRETS_TYPE = "recovery_secret"
+	ACCESS_SECRET_TYPE    = api.ACCESS_SECRET_TYPE
+	TOTP_TYPE             = api.TOTP_TYPE
+	RECOVERY_SECRETS_TYPE = api.RECOVERY_SECRETS_TYPE
 )
 
 type STotpSecret struct {
@@ -45,15 +50,26 @@ type SRecoverySecret struct {
 	Answer   string
 }
 
+type SAccessKeySecret struct {
+	KeyId     string    `json:"-"`
+	ProjectId string    `json:"-"`
+	TimeStamp time.Time `json:"-"`
+	api.SAccessKeySecretBlob
+}
+
 type SRecoverySecretSet struct {
 	Questions []SRecoverySecret
 	Timestamp int64
 }
 
-func (manager *SCredentialManager) fetchCredentials(s *mcclient.ClientSession, secType string, uid string) ([]jsonutils.JSONObject, error) {
+func (manager *SCredentialManager) fetchCredentials(s *mcclient.ClientSession, secType string, uid string, pid string) ([]jsonutils.JSONObject, error) {
 	query := jsonutils.NewDict()
 	query.Add(jsonutils.NewString(secType), "type")
+	query.Add(jsonutils.NewString("system"), "scope")
 	query.Add(jsonutils.NewString(uid), "user_id")
+	if len(pid) > 0 {
+		query.Add(jsonutils.NewString(pid), "project_id")
+	}
 	results, err := manager.List(s, query)
 	if err != nil {
 		return nil, err
@@ -61,12 +77,16 @@ func (manager *SCredentialManager) fetchCredentials(s *mcclient.ClientSession, s
 	return results.Data, nil
 }
 
+func (manager *SCredentialManager) FetchAccessKeySecrets(s *mcclient.ClientSession, uid string, pid string) ([]jsonutils.JSONObject, error) {
+	return manager.fetchCredentials(s, ACCESS_SECRET_TYPE, uid, pid)
+}
+
 func (manager *SCredentialManager) FetchTotpSecrets(s *mcclient.ClientSession, uid string) ([]jsonutils.JSONObject, error) {
-	return manager.fetchCredentials(s, TOTP_TYPE, uid)
+	return manager.fetchCredentials(s, TOTP_TYPE, uid, "")
 }
 
 func (manager *SCredentialManager) FetchRecoverySecrets(s *mcclient.ClientSession, uid string) ([]jsonutils.JSONObject, error) {
-	return manager.fetchCredentials(s, RECOVERY_SECRETS_TYPE, uid)
+	return manager.fetchCredentials(s, RECOVERY_SECRETS_TYPE, uid, "")
 }
 
 func (manager *SCredentialManager) GetTotpSecret(s *mcclient.ClientSession, uid string) (string, error) {
@@ -119,6 +139,89 @@ func (manager *SCredentialManager) GetRecoverySecrets(s *mcclient.ClientSession,
 	return latestQ.Questions, nil
 }
 
+func DecodeAccessKeySecret(secret jsonutils.JSONObject) (SAccessKeySecret, error) {
+	curr := SAccessKeySecret{}
+	blobStr, err := secret.GetString("blob")
+	if err != nil {
+		return curr, errors.Wrap(err, "secret.GetString")
+	}
+	blobJson, err := jsonutils.ParseString(blobStr)
+	if err != nil {
+		return curr, errors.Wrap(err, "jsonutils.ParseString")
+	}
+	err = blobJson.Unmarshal(&curr)
+	if err != nil {
+		return curr, errors.Wrap(err, "blobJson.Unmarshal")
+	}
+	curr.ProjectId, err = secret.GetString("project_id")
+	if err != nil {
+		return curr, errors.Wrap(err, "secret.GetString('project_id')")
+	}
+	curr.TimeStamp, err = secret.GetTime("created_at")
+	if err != nil {
+		return curr, errors.Wrap(err, "secret.GetTime('created_at')")
+	}
+	curr.KeyId, err = secret.GetString("id")
+	if err != nil {
+		return curr, errors.Wrap(err, "secret.GetString('id')")
+	}
+	return curr, nil
+}
+
+func (manager *SCredentialManager) GetAccessKeySecrets(s *mcclient.ClientSession, uid string, pid string) ([]SAccessKeySecret, error) {
+	secrets, err := manager.FetchAccessKeySecrets(s, uid, pid)
+	if err != nil {
+		return nil, err
+	}
+	aksk := make([]SAccessKeySecret, 0)
+	for i := range secrets {
+		curr, err := DecodeAccessKeySecret(secrets[i])
+		if err != nil {
+			return nil, errors.Wrap(err, "DecodeAccessKeySecret")
+		}
+		aksk = append(aksk, curr)
+	}
+	return aksk, nil
+}
+
+func (manager *SCredentialManager) DoCreateAccessKeySecret(s *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) {
+	key, err := manager.CreateAccessKeySecret(s, "", "", time.Time{})
+	if err != nil {
+		return nil, err
+	}
+	result := jsonutils.Marshal(key)
+	result.(*jsonutils.JSONDict).Add(jsonutils.NewString(key.KeyId), "key_id")
+	return result, nil
+}
+
+func (manager *SCredentialManager) CreateAccessKeySecret(s *mcclient.ClientSession, uid string, pid string, expireAt time.Time) (SAccessKeySecret, error) {
+	aksk := SAccessKeySecret{}
+	aksk.Secret = base64.URLEncoding.EncodeToString([]byte(seclib.RandomPassword(32)))
+	if !expireAt.IsZero() {
+		aksk.Expire = expireAt.Unix()
+	}
+	blobJson := jsonutils.Marshal(&aksk)
+	params := jsonutils.NewDict()
+	name := fmt.Sprintf("%s-%s-%d", uid, pid, time.Now().Unix())
+	if len(pid) > 0 {
+		params.Add(jsonutils.NewString(pid), "project_id")
+	}
+	params.Add(jsonutils.NewString(ACCESS_SECRET_TYPE), "type")
+	if len(uid) > 0 {
+		params.Add(jsonutils.NewString(uid), "user_id")
+	}
+	params.Add(jsonutils.NewString(blobJson.String()), "blob")
+	params.Add(jsonutils.NewString(name), "name")
+	result, err := manager.Create(s, params)
+	if err != nil {
+		return aksk, err
+	}
+	aksk.ProjectId = pid
+	aksk.TimeStamp, _ = result.GetTime("created_at")
+	aksk.KeyId, _ = result.GetString("id")
+	return aksk, nil
+}
+
 func (manager *SCredentialManager) CreateTotpSecret(s *mcclient.ClientSession, uid string) (string, error) {
 	_, err := manager.GetTotpSecret(s, uid)
 	if err == nil {
@@ -163,8 +266,8 @@ func (manager *SCredentialManager) SaveRecoverySecrets(s *mcclient.ClientSession
 	return nil
 }
 
-func (manager *SCredentialManager) removeCredentials(s *mcclient.ClientSession, secType string, uid string) error {
-	secrets, err := manager.fetchCredentials(s, secType, uid)
+func (manager *SCredentialManager) removeCredentials(s *mcclient.ClientSession, secType string, uid string, pid string) error {
+	secrets, err := manager.fetchCredentials(s, secType, uid, pid)
 	if err != nil {
 		return err
 	}
@@ -180,12 +283,16 @@ func (manager *SCredentialManager) removeCredentials(s *mcclient.ClientSession,
 	return nil
 }
 
+func (manager *SCredentialManager) RemoveAccessKeySecrets(s *mcclient.ClientSession, uid string, pid string) error {
+	return manager.removeCredentials(s, ACCESS_SECRET_TYPE, uid, pid)
+}
+
 func (manager *SCredentialManager) RemoveTotpSecrets(s *mcclient.ClientSession, uid string) error {
-	return manager.removeCredentials(s, TOTP_TYPE, uid)
+	return manager.removeCredentials(s, TOTP_TYPE, uid, "")
 }
 
 func (manager *SCredentialManager) RemoveRecoverySecrets(s *mcclient.ClientSession, uid string) error {
-	return manager.removeCredentials(s, RECOVERY_SECRETS_TYPE, uid)
+	return manager.removeCredentials(s, RECOVERY_SECRETS_TYPE, uid, "")
 }
 
 var (
@@ -198,4 +305,6 @@ func init() {
 			[]string{},
 			[]string{"ID", "Type", "user_id", "project_id", "blob"}),
 	}
+
+	register(&Credentials)
 }

pkg/mcclient/modules/mod_projects.go

@@ -266,6 +266,18 @@ func (this *ProjectManagerV3) AddTags(session *mcclient.ClientSession, id string
 	return nil
 }
 
+func (this *ProjectManagerV3) FetchId(s *mcclient.ClientSession, project string, domain string) (string, error) {
+	query := jsonutils.NewDict()
+	if len(domain) > 0 {
+		domainId, err := Domains.GetId(s, domain, nil)
+		if err != nil {
+			return "", err
+		}
+		query.Add(jsonutils.NewString(domainId), "domain_id")
+	}
+	return this.GetId(s, project, query)
+}
+
 func init() {
 	Projects = ProjectManagerV3{NewIdentityV3Manager("project", "projects",
 		[]string{},

pkg/mcclient/token3.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/pkg/utils"
 
+	api "yunion.io/x/onecloud/pkg/apis/identity"
 	"yunion.io/x/onecloud/pkg/util/rbacutils"
 )
 
@@ -81,6 +82,8 @@ type KeystoneTokenV3 struct {
 	User      KeystoneUserV3           `json:"user"`
 	Catalog   KeystoneServiceCatalogV3 `json:"catalog"`
 	Context   SAuthContext             `json:"context"`
+
+	AccessKey api.SAccessKeySecretInfo `json:"access_key"`
 }
 
 type TokenCredentialV3 struct {

pkg/multicloud/aliyun/bucket.go

@@ -360,3 +360,79 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s
 	}
 	return urlStr, nil
 }
+
+func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return errors.Wrap(err, "Bucket")
+	}
+	opts := make([]oss.Option, 0)
+	if len(contType) > 0 {
+		opts = append(opts, oss.ContentType(contType))
+	}
+	if len(cannedAcl) > 0 {
+		acl, err := str2Acl(string(cannedAcl))
+		if err != nil {
+			return errors.Wrap(err, "")
+		}
+		opts = append(opts, oss.ObjectACL(acl))
+	}
+	if len(storageClassStr) > 0 {
+		storageClass, err := str2StorageClass(storageClassStr)
+		if err != nil {
+			return errors.Wrap(err, "str2StorageClass")
+		}
+		opts = append(opts, oss.ObjectStorageClass(storageClass))
+	}
+	_, err = bucket.CopyObjectFrom(srcBucket, srcKey, destKey, opts...)
+	if err != nil {
+		return errors.Wrap(err, "CopyObjectFrom")
+	}
+	return nil
+}
+
+func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return nil, errors.Wrap(err, "Bucket")
+	}
+	opts := make([]oss.Option, 0)
+	if rangeOpt != nil {
+		opts = append(opts, oss.NormalizedRange(rangeOpt.String()))
+	}
+	output, err := bucket.GetObject(key, opts...)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetObject")
+	}
+	return output, nil
+}
+
+func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	osscli, err := b.region.GetOssClient()
+	if err != nil {
+		return "", errors.Wrap(err, "GetOssClient")
+	}
+	bucket, err := osscli.Bucket(b.Name)
+	if err != nil {
+		return "", errors.Wrap(err, "Bucket")
+	}
+	imur := oss.InitiateMultipartUploadResult{
+		Bucket:   b.Name,
+		Key:      key,
+		UploadID: uploadId,
+	}
+	opts := make([]oss.Option, 0)
+	part, err := bucket.UploadPartCopy(imur, srcBucket, srcKey, srcOffset, srcLength, partNumber, opts...)
+	if err != nil {
+		return "", errors.Wrap(err, "bucket.UploadPartCopy")
+	}
+	return part.ETag, nil
+}

pkg/multicloud/aws/bucket.go

@@ -27,6 +27,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/s3cli"
 
+	"net/url"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
@@ -378,3 +379,62 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s
 	}
 	return url, nil
 }
+
+func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	s3cli, err := b.region.GetS3Client()
+	if err != nil {
+		return errors.Wrap(err, "GetS3Client")
+	}
+	log.Debugf("copy from %s/%s to %s/%s", srcBucket, srcKey, b.Name, destKey)
+	input := &s3.CopyObjectInput{}
+	input.SetBucket(b.Name)
+	input.SetKey(destKey)
+	input.SetCopySource(fmt.Sprintf("%s/%s", srcBucket, url.PathEscape(srcKey)))
+	input.SetStorageClass(storageClassStr)
+	input.SetACL(string(cannedAcl))
+	input.SetContentType(contType)
+	_, err = s3cli.CopyObject(input)
+	if err != nil {
+		return errors.Wrap(err, "CopyObject")
+	}
+	return nil
+}
+
+func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	s3cli, err := b.region.GetS3Client()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.GetObjectInput{}
+	input.SetBucket(b.Name)
+	input.SetKey(key)
+	if rangeOpt != nil {
+		input.SetRange(rangeOpt.String())
+	}
+	output, err := s3cli.GetObject(input)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetObject")
+	}
+	return output.Body, nil
+}
+
+func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	s3cli, err := b.region.GetS3Client()
+	if err != nil {
+		return "", errors.Wrap(err, "GetS3Client")
+	}
+	input := &s3.UploadPartCopyInput{}
+	input.SetBucket(b.Name)
+	input.SetKey(key)
+	input.SetUploadId(uploadId)
+	input.SetPartNumber(int64(partNumber))
+	input.SetCopySource(fmt.Sprintf("/%s/%s", srcBucket, url.PathEscape(srcKey)))
+	if srcLength > 0 {
+		input.SetCopySourceRange(fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1))
+	}
+	output, err := s3cli.UploadPartCopy(input)
+	if err != nil {
+		return "", errors.Wrap(err, "s3cli.UploadPartCopy")
+	}
+	return *output.CopyPartResult.ETag, nil
+}

pkg/multicloud/azure/storageaccount.go

@@ -35,6 +35,7 @@ import (
 	"encoding/base64"
 	"strconv"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/multicloud"
 )
 
@@ -927,7 +928,7 @@ func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter st
 func splitKey(key string) (string, string, error) {
 	slashPos := strings.IndexByte(key, '/')
 	if slashPos <= 0 {
-		return "", "", errors.Error("cannot put object to root")
+		return "", "", errors.Wrap(httperrors.ErrForbidden, "cannot put object to root")
 	}
 	containerName := key[:slashPos]
 	key = key[slashPos+1:]
@@ -977,6 +978,10 @@ func (b *SStorageAccount) NewMultipartUpload(ctx context.Context, key string, co
 	return uploadId, nil
 }
 
+func partIndex2BlockId(partIndex int) string {
+	return base64.URLEncoding.EncodeToString([]byte(strconv.FormatInt(int64(partIndex), 10)))
+}
+
 func (b *SStorageAccount) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) {
 	containerName, blob, err := splitKey(key)
 	if err != nil {
@@ -995,7 +1000,7 @@ func (b *SStorageAccount) UploadPart(ctx context.Context, key string, uploadId s
 	opts := &storage.PutBlockOptions{}
 	opts.LeaseID = uploadId
 
-	blockId := base64.URLEncoding.EncodeToString([]byte(strconv.FormatInt(int64(partIndex), 10)))
+	blockId := partIndex2BlockId(partIndex)
 	err = blobRef.PutBlockWithLength(blockId, uint64(partSize), input, opts)
 	if err != nil {
 		return "", errors.Wrap(err, "PutBlockWithLength")
@@ -1113,3 +1118,118 @@ func (b *SStorageAccount) GetTempUrl(method string, key string, expire time.Dura
 	}
 	return container.SignUrl(method, blob, expire)
 }
+
+func (b *SStorageAccount) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	srcIBucket, err := b.region.GetIBucketByName(srcBucket)
+	if err != nil {
+		return errors.Wrap(err, "GetIBucketByName")
+	}
+	srcAccount := srcIBucket.(*SStorageAccount)
+	srcContName, srcBlob, err := splitKey(srcKey)
+	if err != nil {
+		return errors.Wrap(err, "src splitKey")
+	}
+	srcCont, err := srcAccount.getOrCreateContainer(srcContName, false)
+	if err != nil {
+		return errors.Wrap(err, "src getOrCreateContainer")
+	}
+	srcContRef, err := srcCont.getContainerRef()
+	if err != nil {
+		return errors.Wrap(err, "src getContainerRef")
+	}
+	srcBlobRef := srcContRef.GetBlobReference(srcBlob)
+
+	containerName, blob, err := splitKey(destKey)
+	if err != nil {
+		return errors.Wrap(err, "dest splitKey")
+	}
+	container, err := b.getOrCreateContainer(containerName, true)
+	if err != nil {
+		return errors.Wrap(err, "dest getOrCreateContainer")
+	}
+	containerRef, err := container.getContainerRef()
+	if err != nil {
+		return errors.Wrap(err, "dest getContainerRef")
+	}
+	blobRef := containerRef.GetBlobReference(blob)
+
+	opts := &storage.CopyOptions{}
+	err = blobRef.Copy(srcBlobRef.GetURL(), opts)
+	if err != nil {
+		return errors.Wrap(err, "blboRef.Copy")
+	}
+	return nil
+}
+
+func (b *SStorageAccount) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	containerName, blob, err := splitKey(key)
+	if err != nil {
+		return nil, errors.Wrap(err, "splitKey")
+	}
+	container, err := b.getOrCreateContainer(containerName, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "getOrCreateContainer")
+	}
+	containerRef, err := container.getContainerRef()
+	if err != nil {
+		return nil, errors.Wrap(err, "container.getContainerRef")
+	}
+	blobRef := containerRef.GetBlobReference(blob)
+	if rangeOpt != nil {
+		opts := &storage.GetBlobRangeOptions{}
+		opts.Range = &storage.BlobRange{
+			Start: uint64(rangeOpt.Start),
+			End:   uint64(rangeOpt.End),
+		}
+		return blobRef.GetRange(opts)
+	} else {
+		opts := &storage.GetBlobOptions{}
+		return blobRef.Get(opts)
+	}
+}
+
+func (b *SStorageAccount) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	srcIBucket, err := b.region.GetIBucketByName(srcBucket)
+	if err != nil {
+		return "", errors.Wrap(err, "GetIBucketByName")
+	}
+	srcAccount := srcIBucket.(*SStorageAccount)
+	srcContName, srcBlob, err := splitKey(srcKey)
+	if err != nil {
+		return "", errors.Wrap(err, "src splitKey")
+	}
+	srcCont, err := srcAccount.getOrCreateContainer(srcContName, false)
+	if err != nil {
+		return "", errors.Wrap(err, "src getOrCreateContainer")
+	}
+	srcContRef, err := srcCont.getContainerRef()
+	if err != nil {
+		return "", errors.Wrap(err, "src getContainerRef")
+	}
+	srcBlobRef := srcContRef.GetBlobReference(srcBlob)
+
+	containerName, blob, err := splitKey(key)
+	if err != nil {
+		return "", errors.Wrap(err, "splitKey")
+	}
+	container, err := b.getOrCreateContainer(containerName, true)
+	if err != nil {
+		return "", errors.Wrap(err, "getOrCreateContainer")
+	}
+	containerRef, err := container.getContainerRef()
+	if err != nil {
+		return "", errors.Wrap(err, "getContainerRef")
+	}
+	blobRef := containerRef.GetBlobReference(blob)
+
+	opts := &storage.PutBlockFromURLOptions{}
+	opts.LeaseID = uploadId
+
+	blockId := partIndex2BlockId(partIndex)
+	err = blobRef.PutBlockFromURL(blockId, srcBlobRef.GetURL(), srcOffset, uint64(srcLength), opts)
+	if err != nil {
+		return "", errors.Wrap(err, "PutBlockFromUrl")
+	}
+
+	return blockId, nil
+}

pkg/multicloud/huawei/bucket.go

@@ -420,3 +420,70 @@ func (b *SBucket) SetLimit(limit cloudprovider.SBucketStats) error {
 	}
 	return nil
 }
+
+func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	obscli, err := b.region.getOBSClient()
+	if err != nil {
+		return errors.Wrap(err, "GetOBSClient")
+	}
+	input := &obs.CopyObjectInput{}
+	input.CopySourceBucket = srcBucket
+	input.CopySourceKey = srcKey
+	if len(storageClassStr) > 0 {
+		input.StorageClass, err = str2StorageClass(storageClassStr)
+		if err != nil {
+			return err
+		}
+	}
+	if len(cannedAcl) > 0 {
+		input.ACL = obs.AclType(string(cannedAcl))
+	}
+	if len(contType) > 0 {
+		input.ContentType = contType
+	}
+	_, err = obscli.CopyObject(input)
+	if err != nil {
+		return errors.Wrap(err, "obscli.CopyObject")
+	}
+	return nil
+}
+
+func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	obscli, err := b.region.getOBSClient()
+	if err != nil {
+		return nil, errors.Wrap(err, "GetOBSClient")
+	}
+	input := &obs.GetObjectInput{}
+	input.Bucket = b.Name
+	input.Key = key
+	if rangeOpt != nil {
+		input.RangeStart = rangeOpt.Start
+		input.RangeEnd = rangeOpt.End
+	}
+	output, err := obscli.GetObject(input)
+	if err != nil {
+		return nil, errors.Wrap(err, "obscli.GetObject")
+	}
+	return output.Body, nil
+}
+
+func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	obscli, err := b.region.getOBSClient()
+	if err != nil {
+		return "", errors.Wrap(err, "GetOBSClient")
+	}
+	input := &obs.CopyPartInput{}
+	input.Bucket = b.Name
+	input.Key = key
+	input.UploadId = uploadId
+	input.PartNumber = partIndex
+	input.CopySourceBucket = srcBucket
+	input.CopySourceKey = srcKey
+	input.CopySourceRangeStart = srcOffset
+	input.CopySourceRangeEnd = srcOffset + srcLength - 1
+	output, err := obscli.CopyPart(input)
+	if err != nil {
+		return "", errors.Wrap(err, "CopyPart")
+	}
+	return output.ETag, nil
+}

pkg/multicloud/objectstore/buckets.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/s3cli"
 
+	"net/http"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 )
@@ -114,14 +115,38 @@ func (bucket *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
 }
 
 func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) {
-	isRecursive := true
-	if delimiter == "/" {
-		isRecursive = false
+	ret := cloudprovider.SListObjectResult{}
+	result, err := bucket.client.client.ListObjectsQuery(bucket.Name, prefix, marker, delimiter, maxCount)
+	if err != nil {
+		return ret, errors.Wrap(err, "ListObjectsQuery")
 	}
-	result := cloudprovider.SListObjectResult{}
-	var err error
-	result.Objects, err = bucket.GetIObjects(prefix, isRecursive)
-	return result, err
+	ret.NextMarker = result.NextMarker
+	ret.IsTruncated = result.IsTruncated
+	ret.CommonPrefixes = make([]cloudprovider.ICloudObject, len(result.CommonPrefixes))
+	for i := range result.CommonPrefixes {
+		ret.CommonPrefixes[i] = &SObject{
+			bucket: bucket,
+			SBaseCloudObject: cloudprovider.SBaseCloudObject{
+				Key: result.CommonPrefixes[i].Prefix,
+			},
+		}
+	}
+	ret.Objects = make([]cloudprovider.ICloudObject, len(result.Contents))
+	for i := range result.Contents {
+		object := result.Contents[i]
+		ret.Objects[i] = &SObject{
+			bucket: bucket,
+			SBaseCloudObject: cloudprovider.SBaseCloudObject{
+				StorageClass: object.StorageClass,
+				Key:          object.Key,
+				SizeBytes:    object.Size,
+				ETag:         object.ETag,
+				LastModified: object.LastModified,
+				ContentType:  object.ContentType,
+			},
+		}
+	}
+	return ret, nil
 }
 
 func (bucket *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) {
@@ -242,3 +267,51 @@ func (bucket *SBucket) GetTempUrl(method string, key string, expire time.Duratio
 	}
 	return url.String(), nil
 }
+
+func (bucket *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	meta := make(map[string]string)
+	if len(contType) > 0 {
+		meta[http.CanonicalHeaderKey("Content-Type")] = contType
+	}
+	if len(storageClassStr) > 0 {
+		meta[http.CanonicalHeaderKey("x-amz-storage-class")] = storageClassStr
+	}
+	dest, err := s3cli.NewDestinationInfo(bucket.Name, destKey, nil, meta)
+	if err != nil {
+		return errors.Wrap(err, "NewDestinationInfo")
+	}
+	src := s3cli.NewSourceInfo(srcBucket, srcKey, nil)
+	err = bucket.client.client.CopyObject(dest, src)
+	if err != nil {
+		return errors.Wrap(err, "CopyObject")
+	}
+	obj, err := cloudprovider.GetIObject(bucket, destKey)
+	if err != nil {
+		return errors.Wrap(err, "GetIObject")
+	}
+	err = obj.SetAcl(cannedAcl)
+	if err != nil {
+		return errors.Wrap(err, "obj.SetAcl")
+	}
+	return nil
+}
+
+func (bucket *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	opts := s3cli.GetObjectOptions{}
+	if rangeOpt != nil {
+		opts.SetRange(rangeOpt.Start, rangeOpt.End)
+	}
+	output, err := bucket.client.client.GetObject(bucket.Name, key, opts)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetObject")
+	}
+	return output, nil
+}
+
+func (bucket *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	result, err := bucket.client.client.CopyObjectPartDo(ctx, srcBucket, srcKey, bucket.Name, key, uploadId, partNumber, srcOffset, srcLength, nil)
+	if err != nil {
+		return "", errors.Wrap(err, "CopyObjectPartDo")
+	}
+	return result.ETag, nil
+}

pkg/multicloud/objectstore/provider/provider.go

@@ -91,9 +91,9 @@ func (self *SObjectStoreProviderFactory) GetProvider(providerId, providerName, u
 
 func (self *SObjectStoreProviderFactory) GetClientRC(url, account, secret string) (map[string]string, error) {
 	return map[string]string{
-		"OBJECTSTORE_ACCESSKEY": account,
-		"OBJECTSTORE_SECRET":    secret,
-		"OBJECTSTORE_ENDPOINT":  url,
+		"S3_ACCESS_KEY": account,
+		"S3_SECRET":     secret,
+		"S3_ACCESS_URL": url,
 	}, nil
 }
 

pkg/multicloud/objectstore/shell.go

@@ -25,6 +25,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/util/printutils"
 	"yunion.io/x/onecloud/pkg/util/shellutils"
+	"yunion.io/x/onecloud/pkg/util/streamutils"
 )
 
 func S3Shell() {
@@ -280,4 +281,92 @@ func S3Shell() {
 		fmt.Println("Success!")
 		return nil
 	})
+
+	type BucketObjectDownloadOptions struct {
+		BUCKET string `help:"name of bucket"`
+		KEY    string `help:"Key of object"`
+		Output string `help:"target output, default to stdout"`
+		Start  int64  `help:"partial download start"`
+		End    int64  `help:"partial download end"`
+	}
+	shellutils.R(&BucketObjectDownloadOptions{}, "object-download", "Download", func(cli cloudprovider.ICloudRegion, args *BucketObjectDownloadOptions) error {
+		bucket, err := cli.GetIBucketById(args.BUCKET)
+		if err != nil {
+			return err
+		}
+		obj, err := cloudprovider.GetIObject(bucket, args.KEY)
+		if err != nil {
+			return err
+		}
+
+		var rangeOpt *cloudprovider.SGetObjectRange
+		if args.Start != 0 || args.End != 0 {
+			if args.End <= 0 {
+				args.End = obj.GetSizeBytes() - 1
+			}
+			rangeOpt = &cloudprovider.SGetObjectRange{Start: args.Start, End: args.End}
+		}
+		output, err := bucket.GetObject(context.Background(), args.KEY, rangeOpt)
+		if err != nil {
+			return err
+		}
+		defer output.Close()
+		var target io.Writer
+		if len(args.Output) == 0 {
+			target = os.Stdout
+		} else {
+			fp, err := os.Create(args.Output)
+			if err != nil {
+				return err
+			}
+			defer fp.Close()
+			target = fp
+		}
+		prop, err := streamutils.StreamPipe(output, target, false)
+		if err != nil {
+			return err
+		}
+		if len(args.Output) > 0 {
+			fmt.Println("Success:", prop.Size, "written")
+		}
+		return nil
+	})
+
+	type BucketObjectCopyOptions struct {
+		SRC       string `help:"name of source bucket"`
+		SRCKEY    string `help:"Key of source object"`
+		DST       string `help:"name of destination bucket"`
+		DSTKEY    string `help:"key of destination object"`
+		Debug     bool   `help:"show debug info"`
+		BlockSize int64  `help:"block size in MB"`
+		Native    bool   `help:"Use native copy"`
+	}
+	shellutils.R(&BucketObjectCopyOptions{}, "object-copy", "Copy object", func(cli cloudprovider.ICloudRegion, args *BucketObjectCopyOptions) error {
+		ctx := context.Background()
+		dstBucket, err := cli.GetIBucketByName(args.DST)
+		if err != nil {
+			return err
+		}
+		srcBucket, err := cli.GetIBucketByName(args.SRC)
+		if err != nil {
+			return err
+		}
+		srcObj, err := cloudprovider.GetIObject(srcBucket, args.SRCKEY)
+		if err != nil {
+			return err
+		}
+		if args.Native {
+			err = dstBucket.CopyObject(ctx, args.DSTKEY, args.SRC, args.SRCKEY, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass())
+			if err != nil {
+				return err
+			}
+		} else {
+			err = cloudprovider.CopyObject(ctx, args.BlockSize*1000*1000, dstBucket, args.DSTKEY, srcBucket, args.SRCKEY, args.Debug)
+			if err != nil {
+				return err
+			}
+		}
+		fmt.Println("Success!")
+		return nil
+	})
 }

pkg/multicloud/qcloud/bucket.go

@@ -133,8 +133,12 @@ func (b *SBucket) getFullName() string {
 	return fmt.Sprintf("%s-%s", b.Name, b.region.client.AppID)
 }
 
+func (b *SBucket) getBucketUrlHost() string {
+	return fmt.Sprintf("%s.%s", b.getFullName(), b.region.getCosEndpoint())
+}
+
 func (b *SBucket) getBucketUrl() string {
-	return fmt.Sprintf("https://%s.%s", b.getFullName(), b.region.getCosEndpoint())
+	return fmt.Sprintf("https://%s", b.getBucketUrlHost())
 }
 
 func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl {
@@ -351,3 +355,69 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s
 	}
 	return url.String(), nil
 }
+
+func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucketName, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	coscli, err := b.region.GetCosClient(b)
+	if err != nil {
+		return errors.Wrap(err, "GetCosClient")
+	}
+	opts := &cos.ObjectCopyOptions{
+		ObjectCopyHeaderOptions: &cos.ObjectCopyHeaderOptions{},
+		ACLHeaderOptions:        &cos.ACLHeaderOptions{},
+	}
+	if len(cannedAcl) > 0 {
+		opts.XCosACL = string(cannedAcl)
+	}
+	if len(storageClassStr) > 0 {
+		opts.XCosStorageClass = storageClassStr
+	}
+	if len(contType) > 0 {
+		opts.ContentType = contType
+	}
+	srcBucket := SBucket{
+		region: b.region,
+		Name:   srcBucketName,
+	}
+	srcUrl := fmt.Sprintf("%s/%s", srcBucket.getBucketUrlHost(), srcKey)
+	log.Debugf("source url: %s", srcUrl)
+	_, _, err = coscli.Object.Copy(ctx, destKey, srcUrl, opts)
+	if err != nil {
+		return errors.Wrap(err, "coscli.Object.Copy")
+	}
+	return nil
+}
+
+func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	coscli, err := b.region.GetCosClient(b)
+	if err != nil {
+		return nil, errors.Wrap(err, "GetCosClient")
+	}
+	opts := &cos.ObjectGetOptions{}
+	if rangeOpt != nil {
+		opts.Range = rangeOpt.String()
+	}
+	resp, err := coscli.Object.Get(ctx, key, opts)
+	if err != nil {
+		return nil, errors.Wrap(err, "coscli.Object.Get")
+	}
+	return resp.Body, nil
+}
+
+func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucketName string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	coscli, err := b.region.GetCosClient(b)
+	if err != nil {
+		return "", errors.Wrap(err, "GetCosClient")
+	}
+	srcBucket := SBucket{
+		region: b.region,
+		Name:   srcBucketName,
+	}
+	opts := cos.ObjectCopyPartOptions{}
+	opts.XCosCopySource = fmt.Sprintf("%s/%s", srcBucket.getBucketUrlHost(), srcKey)
+	opts.XCosCopySourceRange = fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1)
+	result, _, err := coscli.Object.CopyPart(ctx, key, uploadId, partIndex, &opts)
+	if err != nil {
+		return "", errors.Wrap(err, "coscli.Object.CopyPart")
+	}
+	return result.ETag, nil
+}

pkg/multicloud/ucloud/ufile.go

@@ -374,3 +374,15 @@ func (b *SBucket) DeleteObject(ctx context.Context, key string) error {
 func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) {
 	return "", cloudprovider.ErrNotSupported
 }
+
+func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error {
+	return cloudprovider.ErrNotSupported
+}
+
+func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) {
+	return nil, cloudprovider.ErrNotSupported
+}
+
+func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucketName string, srcKey string, srcOffset int64, srcLength int64) (string, error) {
+	return "", cloudprovider.ErrNotSupported
+}

pkg/notify/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package notify // import "yunion.io/x/onecloud/pkg/notify"

pkg/notify/models/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package models // import "yunion.io/x/onecloud/pkg/notify/models"

pkg/notify/options/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package options // import "yunion.io/x/onecloud/pkg/notify/options"

pkg/notify/utils/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package utils // import "yunion.io/x/onecloud/pkg/notify/utils"

pkg/s3gateway/handlers/bucket.go

@@ -0,0 +1,64 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/s3cli"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+)
+
+func headBucket(ctx context.Context, userCred mcclient.TokenCredential, bucketName string) error {
+	_, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	return nil
+}
+
+func removeBucket(ctx context.Context, userCred mcclient.TokenCredential, bucket string) error {
+	return models.BucketManager.DeleteByName(ctx, userCred, bucket)
+}
+
+func bucketAcl(ctx context.Context, userCred mcclient.TokenCredential, bucketName string) (*s3cli.AccessControlPolicy, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetIBucket")
+	}
+
+	result := str2Acl(userCred, iBucket.GetAcl())
+
+	return result, nil
+}
+
+func listBucketUploads(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, input *s3cli.ListMultipartUploadsInput) (*s3cli.ListMultipartUploadsResult, error) {
+	result := s3cli.ListMultipartUploadsResult{}
+	result.Bucket = bucketName
+	result.Delimiter = input.Delimiter
+	result.MaxUploads = input.MaxUploads
+	result.KeyMarker = input.KeyMarker
+	result.Prefix = input.Prefix
+	result.UploadIDMarker = input.UploadIdMarker
+	result.EncodingType = input.EncodingType
+	return &result, nil
+}

pkg/s3gateway/handlers/errors.go

@@ -0,0 +1,138 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"net/http"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/s3cli"
+
+	"runtime/debug"
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/httperrors"
+)
+
+func generalError(ctx context.Context, statusCode int, errCode string, msg string) s3cli.ErrorResponse {
+	o := fetchObjectRequest(ctx)
+	resp := s3cli.ErrorResponse{}
+	resp.StatusCode = statusCode
+	resp.Code = errCode
+	resp.Message = msg
+	resp.BucketName = o.Bucket
+	resp.Key = o.Key
+	resp.HostID = appctx.AppContextHostId(ctx)
+	resp.RequestID = appctx.AppContextRequestId(ctx)
+	return resp
+}
+
+func BadRequest(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 400, "Bad Request", msg)
+}
+
+func Unauthenticated(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 401, "Unauthenticated", msg)
+}
+
+func Unauthorized(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 403, "Unauthorized", msg)
+}
+
+func Forbidden(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 403, "Forbidden", msg)
+}
+
+func NotFound(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 404, "Not Found", msg)
+}
+
+func NotSupported(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 404, "Not Supported", msg)
+}
+
+func NotImplemented(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 406, "Not Implemented", msg)
+}
+
+func InvalidStatus(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 406, "Invalid Status", msg)
+}
+
+func Conflict(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 409, "Conflict", msg)
+}
+
+func ServerTimeout(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 504, "Server Timeout", msg)
+}
+
+func ServerError(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 500, "Internal Server Error", msg)
+}
+
+func OutOfRangeError(ctx context.Context, msg string) s3cli.ErrorResponse {
+	return generalError(ctx, 416, "Range Not Satisfiable", msg)
+}
+
+func SendGeneralError(ctx context.Context, w http.ResponseWriter, err error) {
+	switch e := err.(type) {
+	case s3cli.ErrorResponse:
+		SendError(w, e)
+	case *s3cli.ErrorResponse:
+		SendError(w, *e)
+	default:
+		var eresp s3cli.ErrorResponse
+		cause := errors.Cause(err)
+		switch cause {
+		case httperrors.ErrUnauthenticated:
+			eresp = Unauthenticated(ctx, err.Error())
+		case httperrors.ErrUnauthorized:
+			eresp = Unauthorized(ctx, err.Error())
+		case httperrors.ErrNotFound:
+			eresp = NotFound(ctx, err.Error())
+		case httperrors.ErrNotSupported:
+			eresp = NotSupported(ctx, err.Error())
+		case httperrors.ErrNotImplemented:
+			eresp = NotImplemented(ctx, err.Error())
+		case httperrors.ErrDuplicateId:
+			eresp = Conflict(ctx, err.Error())
+		case httperrors.ErrTimeout:
+			eresp = ServerTimeout(ctx, err.Error())
+		case httperrors.ErrInvalidStatus:
+			eresp = InvalidStatus(ctx, err.Error())
+		case httperrors.ErrBadRequest:
+			eresp = BadRequest(ctx, err.Error())
+		case httperrors.ErrOutOfRange:
+			eresp = OutOfRangeError(ctx, err.Error())
+		case httperrors.ErrForbidden:
+			eresp = Forbidden(ctx, err.Error())
+		default:
+			eresp = ServerError(ctx, err.Error())
+		}
+		SendError(w, eresp)
+	}
+}
+
+func SendError(w http.ResponseWriter, resp s3cli.ErrorResponse) {
+	w.WriteHeader(resp.StatusCode)
+
+	log.Errorf("SendError: %s", resp)
+	debug.PrintStack()
+
+	appsrv.SendXml(w, nil, resp)
+}

pkg/s3gateway/handlers/handlers.go

@@ -0,0 +1,606 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
+
+	"github.com/minio/minio-go/pkg/s3utils"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/regutils"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/s3cli"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+	"yunion.io/x/onecloud/pkg/s3gateway/options"
+)
+
+func InitHandlers(app *appsrv.Application) {
+	h := app.AddHandler2("HEAD", "", s3authenticate(headHandler), nil, "head", nil)
+	h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo)
+	h = app.AddHandler2("GET", "", s3authenticate(readHandler), nil, "get", nil)
+	h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo)
+	h = app.AddHandler2("PUT", "", s3authenticate(putHandler), nil, "put", nil)
+	h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo)
+	h = app.AddHandler2("POST", "", s3authenticate(postHandler), nil, "post", nil)
+	h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo)
+	h = app.AddHandler2("DELETE", "", s3authenticate(deleteHandler), nil, "delete", nil)
+	h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo)
+}
+
+func s3HandlerTimeoutInfo(info *appsrv.SHandlerInfo, r *http.Request) time.Duration {
+	o, _ := getObjectRequest(r)
+	if len(o.Bucket) > 0 && len(o.Key) > 0 {
+		if r.Method == http.MethodGet && len(r.URL.RawQuery) == 0 {
+			return 2 * time.Hour
+		} else if r.Method == http.MethodPut && (len(r.URL.RawQuery) == 0 || strings.Contains(r.URL.RawQuery, "partNumber=")) {
+			return 2 * time.Hour
+		}
+	}
+	return time.Duration(0)
+}
+
+type SObjectRequest struct {
+	VirtualHost bool
+	Bucket      string
+	Key         string
+}
+
+func (o SObjectRequest) Validate() error {
+	if len(o.Bucket) == 0 {
+		return nil
+	}
+	err := s3utils.CheckValidBucketNameStrict(o.Bucket)
+	if err != nil {
+		return err
+	}
+	if len(o.Key) == 0 {
+		return nil
+	}
+	err = s3utils.CheckValidObjectName(o.Key)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func getObjectRequest(r *http.Request) (SObjectRequest, error) {
+	o := SObjectRequest{}
+	if regutils.MatchIP4Addr(r.Host) || r.Host == options.Options.DomainName {
+		o.VirtualHost = false
+		segs := appsrv.SplitPath(r.URL.Path)
+		if len(segs) > 0 {
+			o.Bucket = segs[0]
+			if len(segs) > 1 {
+				o.Key = strings.Join(segs[1:], "/")
+				if strings.HasSuffix(r.URL.Path, "/") {
+					o.Key += "/"
+				}
+			}
+		}
+	} else if strings.HasSuffix(r.Host, "."+options.Options.DomainName) {
+		o.VirtualHost = true
+		o.Bucket = r.Host[:len(r.Host)-len(options.Options.DomainName)-1]
+		segs := appsrv.SplitPath(r.URL.Path)
+		o.Key = strings.Join(segs, "/")
+		if strings.HasSuffix(r.URL.Path, "/") {
+			o.Key += "/"
+		}
+	} else {
+		return o, errors.Error("invalid S3 request")
+	}
+	var err error
+	o.Key, err = url.PathUnescape(o.Key)
+	if err != nil {
+		return o, errors.Wrap(err, "url.PathUnescape")
+	}
+	return o, o.Validate()
+}
+
+func headHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	o := fetchObjectRequest(ctx)
+	userCred := auth.FetchUserCredential(ctx, nil)
+	if len(o.Bucket) > 0 && len(o.Key) == 0 {
+		// head bucket
+		err := headBucket(ctx, userCred, o.Bucket)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+		} else {
+			appsrv.SendHeader(w, nil)
+		}
+		return
+	} else if len(o.Bucket) > 0 && len(o.Key) > 0 {
+		// head object
+		hdr, err := headObject(ctx, userCred, o.Bucket, o.Key)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+		} else {
+			appsrv.SendHeader(w, hdr)
+		}
+		return
+	} else {
+		// do nothing
+	}
+	SendError(w, NotSupported(ctx, "method not supported"))
+}
+
+func readBucket(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	if query.Contains("accelerate") {
+
+	} else if query.Contains("acl") {
+		resp, err := bucketAcl(ctx, userCred, bucketName)
+		return resp, nil, err
+	} else if query.Contains("analytics") {
+
+	} else if query.Contains("cors") {
+
+	} else if query.Contains("encryption") {
+
+	} else if query.Contains("inventory") {
+
+	} else if query.Contains("lifecycle") {
+
+	} else if query.Contains("location") {
+		result := s3cli.LocationConstraint(bucket.Location)
+		return &result, nil, nil
+	} else if query.Contains("publicAccessBlock") {
+
+	} else if query.Contains("logging") {
+
+	} else if query.Contains("metrics") {
+
+	} else if query.Contains("notification") {
+
+	} else if query.Contains("object-lock") {
+
+	} else if query.Contains("policyStatus") {
+
+	} else if query.Contains("versions") {
+
+	} else if query.Contains("policy") {
+
+	} else if query.Contains("replication") {
+
+	} else if query.Contains("requestPayment") {
+
+	} else if query.Contains("tagging") {
+
+	} else if query.Contains("versioning") {
+		return &s3cli.VersioningConfiguration{}, nil, nil
+	} else if query.Contains("website") {
+
+	} else if query.Contains("uploads") {
+		input := s3cli.ListMultipartUploadsInput{}
+		err := query.Unmarshal(&input)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "query.Unmarshal ListMultipartUploadsInput")
+		}
+		result, err := listBucketUploads(ctx, userCred, bucketName, &input)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "listBucketUploads")
+		}
+		return result, nil, nil
+	} else {
+		// list objects in bucket
+		input := s3cli.ListObjectInput{}
+		err := query.Unmarshal(&input)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "query.Unmarshal")
+		}
+		result, err := bucket.ListObject(ctx, userCred, &input)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "bucket.ListObject")
+		}
+		return result, nil, nil
+	}
+	return nil, nil, NotImplemented(ctx, "not implemented")
+}
+
+func readObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, objKey string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) {
+	if query.Contains("acl") {
+		resp, err := objectAcl(ctx, userCred, bucketName, objKey)
+		return resp, nil, err
+	} else if query.Contains("legal-hold") {
+
+	} else if query.Contains("retention") {
+
+	} else if query.Contains("tagging") {
+
+	} else if query.Contains("torrent") {
+
+	} else {
+		// download object itself, which has been handled
+	}
+	return nil, nil, NotImplemented(ctx, "not implemented")
+}
+
+func getRangeOpt(rangeStr string, sizeBytes int64) (*cloudprovider.SGetObjectRange, error) {
+	if len(rangeStr) > 0 {
+		rangeOptObj := cloudprovider.ParseRange(rangeStr)
+		if rangeOptObj.End == 0 {
+			rangeOptObj.End = sizeBytes - 1
+		}
+		if rangeOptObj.Start >= sizeBytes || rangeOptObj.End >= sizeBytes {
+			return nil, httperrors.ErrOutOfRange
+		}
+		if rangeOptObj.Start > 0 || rangeOptObj.End < sizeBytes-1 {
+			return &rangeOptObj, nil
+		}
+	}
+	return nil, nil
+}
+
+func downloadObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, reqHdr http.Header, w http.ResponseWriter) error {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "bucket.GetIBucket")
+	}
+	obj, err := cloudprovider.GetIObject(iBucket, key)
+	if err != nil {
+		return errors.Wrap(err, "cloudprovider.GetIObject")
+	}
+	hdr := http.Header{}
+	contType := obj.GetContentType()
+	if len(contType) > 0 {
+		hdr.Set("Content-Type", obj.GetContentType())
+	}
+	eTag := obj.GetETag()
+	if len(eTag) > 0 {
+		hdr.Set("ETag", eTag)
+	}
+	lastModified := obj.GetLastModified()
+	if !lastModified.IsZero() {
+		hdr.Set("Last-Modified", lastModified.Format(timeutils.RFC2882Format))
+	}
+	rangeStr := reqHdr.Get(http.CanonicalHeaderKey("range"))
+	rangeOpt, err := getRangeOpt(rangeStr, obj.GetSizeBytes())
+	if err != nil {
+		return errors.Wrap(err, rangeStr)
+	}
+	stream, err := iBucket.GetObject(ctx, key, rangeOpt)
+	if err != nil {
+		return errors.Wrap(err, "iBucket.GetObject")
+	}
+	err = appsrv.SendStream(w, rangeOpt != nil, hdr, stream, obj.GetSizeBytes())
+	if err != nil {
+		return errors.Wrap(err, "appsrv.SendStream")
+	}
+	return nil
+}
+
+func readHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	o := fetchObjectRequest(ctx)
+	userCred := auth.FetchUserCredential(ctx, nil)
+	if len(o.Bucket) == 0 {
+		// service
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		input := s3cli.ListBucketsInput{}
+		err = query.Unmarshal(&input)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+		} else {
+			resp, err := listService(ctx, userCred, input)
+			if err != nil {
+				SendGeneralError(ctx, w, err)
+			} else {
+				appsrv.SendXml(w, nil, resp)
+			}
+		}
+	} else if len(o.Bucket) > 0 && len(o.Key) == 0 {
+		// bucket get
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, respHdr, err := readBucket(ctx, userCred, o.Bucket, query, r)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+			return
+		}
+		appsrv.SendXml(w, respHdr, resp)
+	} else {
+		// object get
+		if len(r.URL.RawQuery) == 0 {
+			// download object
+			err := downloadObject(ctx, userCred, o.Bucket, o.Key, r.Header, w)
+			if err != nil {
+				SendGeneralError(ctx, w, err)
+			}
+			return
+		}
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, respHdr, err := readObject(ctx, userCred, o.Bucket, o.Key, query, r)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+			return
+		}
+		if resp != nil {
+			appsrv.SendXml(w, respHdr, resp)
+		}
+	}
+}
+
+func postObject(ctx context.Context, userCred mcclient.TokenCredential, bucket string, key string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) {
+	if query.Contains("uploads") {
+		// initialize multipart upload
+		return initMultipartUpload(ctx, userCred, r.Header, bucket, key)
+	} else if query.Contains("uploadId") {
+		// complete multipart upload
+		uploadId, err := query.GetString("uploadId")
+		if err != nil || len(uploadId) == 0 {
+			return nil, nil, errors.Wrap(httperrors.ErrBadRequest, "uploadId")
+		}
+		request := s3cli.CompleteMultipartUpload{}
+		err = appsrv.FetchXml(r, &request)
+		if err != nil {
+			return nil, nil, errors.Wrap(httperrors.ErrBadRequest, "FetchXml")
+		}
+		return completeMultipartUpload(ctx, userCred, r.Header, bucket, key, uploadId, &request)
+	} else if query.Contains("select") {
+		// select object
+		return selectObject(ctx, userCred, r.Header, bucket, key)
+	} else {
+		// upload object by form POST
+	}
+	return nil, nil, NotImplemented(ctx, "not implemented")
+}
+
+func postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	o := fetchObjectRequest(ctx)
+	userCred := auth.FetchUserCredential(ctx, nil)
+	if len(o.Bucket) == 0 {
+		// no bucket
+		// do nothing
+	} else if len(o.Bucket) > 0 && len(o.Key) == 0 {
+		// bucket post
+		// do nothing
+	} else {
+		// object post
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, respHdr, err := postObject(ctx, userCred, o.Bucket, o.Key, query, r)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+			return
+		}
+		appsrv.SendXml(w, respHdr, resp)
+		return
+	}
+	SendError(w, NotSupported(ctx, "method not supported"))
+}
+
+func putBucket(ctx context.Context, userCred mcclient.TokenCredential, bucket string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) {
+	if query.Contains("accelerate") {
+
+	} else if query.Contains("acl") {
+
+	} else if query.Contains("analytics") {
+
+	} else if query.Contains("cors") {
+
+	} else if query.Contains("encryption") {
+
+	} else if query.Contains("inventory") {
+
+	} else if query.Contains("lifecycle") {
+
+	} else if query.Contains("publicAccessBlock") {
+
+	} else if query.Contains("logging") {
+
+	} else if query.Contains("metrics") {
+
+	} else if query.Contains("notification") {
+
+	} else if query.Contains("object-lock") {
+
+	} else if query.Contains("policy") {
+
+	} else if query.Contains("replication") {
+
+	} else if query.Contains("requestPayment") {
+
+	} else if query.Contains("tagging") {
+
+	} else if query.Contains("versioning") {
+
+	} else if query.Contains("website") {
+
+	} else {
+		// create bucket
+		return nil, nil, NotSupported(ctx, "Not supported")
+	}
+	return nil, nil, NotImplemented(ctx, "not implemented")
+}
+
+func putObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) {
+	if query.Contains("legal-hold") {
+
+	} else if query.Contains("retention") {
+
+	} else if query.Contains("acl") {
+
+	} else if query.Contains("tagging") {
+
+	} else {
+		// upload object
+		uploadId, _ := query.GetString("uploadId")
+		partNumber, _ := query.Int("partNumber")
+		copySource := r.Header.Get(http.CanonicalHeaderKey("x-amz-copy-source"))
+		if len(copySource) > 0 {
+			return copyObject(ctx, userCred, bucketName, key, copySource, r.Header, uploadId, int(partNumber))
+		} else {
+			hdr, err := uploadObject(ctx, userCred, bucketName, key, r.Header, r.Body, uploadId, int(partNumber))
+			defer r.Body.Close()
+			if err != nil {
+				return nil, nil, err
+			}
+			return nil, hdr, nil
+		}
+	}
+	return nil, nil, NotImplemented(ctx, "not implemented")
+}
+
+func putHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	o := fetchObjectRequest(ctx)
+	userCred := auth.FetchUserCredential(ctx, nil)
+	if len(o.Bucket) == 0 {
+		// no bucket
+	} else if len(o.Bucket) > 0 && len(o.Key) == 0 {
+		// bucket put
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, respHdr, err := putBucket(ctx, userCred, o.Bucket, query, r)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+			return
+		}
+		appsrv.SendXml(w, respHdr, resp)
+		return
+	} else {
+		// object put
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, respHdr, err := putObject(ctx, userCred, o.Bucket, o.Key, query, r)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+			return
+		}
+		appsrv.SendXml(w, respHdr, resp)
+		return
+	}
+	SendError(w, NotSupported(ctx, "method not supported"))
+}
+
+func deleteBucket(ctx context.Context, userCred mcclient.TokenCredential, bucket string, query jsonutils.JSONObject) (interface{}, error) {
+	if query.Contains("analytics") {
+
+	} else if query.Contains("cors") {
+
+	} else if query.Contains("encryption") {
+
+	} else if query.Contains("inventory") {
+
+	} else if query.Contains("lifecycle") {
+
+	} else if query.Contains("publicAccessBlock") {
+
+	} else if query.Contains("metrics") {
+
+	} else if query.Contains("policy") {
+
+	} else if query.Contains("replication") {
+
+	} else if query.Contains("tagging") {
+
+	} else if query.Contains("website") {
+
+	} else {
+		// delete bucket
+		err := removeBucket(ctx, userCred, bucket)
+		if err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+	return nil, NotImplemented(ctx, "not implemented")
+}
+
+func deleteObject(ctx context.Context, userCred mcclient.TokenCredential, bucket string, key string, query jsonutils.JSONObject) (interface{}, error) {
+	if query.Contains("tagging") {
+		return deleteObjectTags(ctx, userCred, bucket, key)
+	} else {
+		// delete object
+		err := removeObject(ctx, userCred, bucket, key)
+		if err != nil {
+			return nil, err
+		}
+		return nil, nil
+	}
+}
+
+func deleteHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+	o := fetchObjectRequest(ctx)
+	userCred := auth.FetchUserCredential(ctx, nil)
+	if len(o.Bucket) == 0 {
+		// no bucket
+	} else if len(o.Bucket) > 0 && len(o.Key) == 0 {
+		// bucket delete
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, err := deleteBucket(ctx, userCred, o.Bucket, query)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+		} else {
+			appsrv.SendXml(w, nil, resp)
+		}
+		return
+	} else {
+		// object delete
+		query, err := jsonutils.ParseQueryString(r.URL.RawQuery)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		resp, err := deleteObject(ctx, userCred, o.Bucket, o.Key, query)
+		if err != nil {
+			SendGeneralError(ctx, w, err)
+		} else {
+			appsrv.SendXml(w, nil, resp)
+		}
+		return
+	}
+	SendError(w, NotSupported(ctx, "method not supported"))
+}

pkg/s3gateway/handlers/middelware.go

@@ -0,0 +1,57 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"net/http"
+
+	"yunion.io/x/pkg/gotypes"
+
+	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+const (
+	S3_OBJECT_REQUEST = appctx.AppContextKey("S3_OBJECT_REQUEST")
+)
+
+func s3authenticate(f appsrv.FilterHandler) appsrv.FilterHandler {
+	return func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
+		o, err := getObjectRequest(r)
+		if err != nil {
+			SendError(w, BadRequest(ctx, err.Error()))
+			return
+		}
+		ctx = context.WithValue(ctx, S3_OBJECT_REQUEST, o)
+		userCred, err := auth.VerifyRequest(*r, o.VirtualHost)
+		if err != nil {
+			SendError(w, Unauthenticated(ctx, err.Error()))
+			return
+		}
+		ctx = context.WithValue(ctx, auth.AUTH_TOKEN, userCred)
+
+		f(ctx, w, r)
+	}
+}
+
+func fetchObjectRequest(ctx context.Context) SObjectRequest {
+	val := ctx.Value(S3_OBJECT_REQUEST)
+	if gotypes.IsNil(val) {
+		return SObjectRequest{}
+	}
+	return val.(SObjectRequest)
+}

pkg/s3gateway/handlers/multipart.go

@@ -0,0 +1,87 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"net/http"
+	"sort"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/s3cli"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+)
+
+func initMultipartUpload(ctx context.Context, userCred mcclient.TokenCredential, hdr http.Header, bucketName string, key string) (*s3cli.InitiateMultipartUploadResult, http.Header, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "bucket.GetIBucket")
+	}
+	contType := hdr.Get(http.CanonicalHeaderKey("content-type"))
+	aclStr := hdr.Get(http.CanonicalHeaderKey("x-amz-acl"))
+	storageClassStr := hdr.Get(http.CanonicalHeaderKey("x-amz-storage-class"))
+	uploadId, err := iBucket.NewMultipartUpload(ctx, key, contType, cloudprovider.TBucketACLType(aclStr), storageClassStr)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "NewMultipartUpload")
+	}
+	result := s3cli.InitiateMultipartUploadResult{}
+	result.Bucket = bucketName
+	result.Key = key
+	result.UploadID = uploadId
+	return &result, nil, nil
+}
+
+type SMultiparts []s3cli.CompletePart
+
+func (a SMultiparts) Len() int           { return len(a) }
+func (a SMultiparts) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a SMultiparts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber }
+
+func completeMultipartUpload(ctx context.Context, userCred mcclient.TokenCredential, hdr http.Header, bucketName string, key string, uploadId string, request *s3cli.CompleteMultipartUpload) (*s3cli.CompleteMultipartUploadResult, http.Header, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "bucket.GetIBucket")
+	}
+	sort.Sort(SMultiparts(request.Parts))
+	partEtags := make([]string, len(request.Parts))
+	for i := range request.Parts {
+		partEtags[i] = request.Parts[i].ETag
+	}
+	err = iBucket.CompleteMultipartUpload(ctx, key, uploadId, partEtags)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "CompleteMultipartUpload")
+	}
+	obj, err := cloudprovider.GetIObject(iBucket, key)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "cloudprovider.GetIObject")
+	}
+	result := s3cli.CompleteMultipartUploadResult{}
+	result.Bucket = bucketName
+	result.Key = key
+	result.ETag = obj.GetETag()
+	result.Location = iBucket.GetLocation()
+	return &result, nil, nil
+}

pkg/s3gateway/handlers/object.go

@@ -0,0 +1,299 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"io"
+	"net/http"
+	"net/url"
+	"strconv"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/timeutils"
+	"yunion.io/x/s3cli"
+
+	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+)
+
+func headObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string) (http.Header, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetIBucket")
+	}
+	obj, err := cloudprovider.GetIObject(iBucket, key)
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudprovider.GetIObject")
+	}
+	hdr := http.Header{}
+	hdr.Set(http.CanonicalHeaderKey("x-amz-acl"), string(obj.GetAcl()))
+	hdr.Set(http.CanonicalHeaderKey("x-amz-storage-class"), obj.GetStorageClass())
+	hdr.Set(http.CanonicalHeaderKey("content-length"), strconv.FormatInt(obj.GetSizeBytes(), 10))
+	hdr.Set(http.CanonicalHeaderKey("content-type"), obj.GetContentType())
+	hdr.Set(http.CanonicalHeaderKey("etag"), obj.GetETag())
+	hdr.Set(http.CanonicalHeaderKey("last-modified"), obj.GetLastModified().Format(timeutils.RFC2882Format))
+	return hdr, nil
+}
+
+func uploadObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, header http.Header, body io.Reader, uploadId string, partNumber int) (http.Header, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+
+	err = bucket.IsOutOfLimit()
+	if err != nil {
+		return nil, errors.Wrap(err, "IsOutOfLimit")
+	}
+
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetIBucket")
+	}
+
+	contLenStr := header.Get(http.CanonicalHeaderKey("Content-Length"))
+	contLen, err := strconv.ParseInt(contLenStr, 10, 64)
+	if err != nil {
+		return nil, errors.Wrap(httperrors.ErrBadRequest, "missing content length")
+	}
+	respHdr := http.Header{}
+	if len(uploadId) > 0 {
+		etag, err := iBucket.UploadPart(ctx, key, uploadId, partNumber, body, contLen)
+		if err != nil {
+			return nil, errors.Wrap(err, "iBucket.UploadPart")
+		}
+		respHdr.Set("ETag", etag)
+	} else {
+		contType := header.Get(http.CanonicalHeaderKey("content-type"))
+		aclStr := header.Get(http.CanonicalHeaderKey("x-amz-acl"))
+		storageClassStr := header.Get(http.CanonicalHeaderKey("x-amz-storage-class"))
+		err = iBucket.PutObject(ctx, key, body, contLen, contType, cloudprovider.TBucketACLType(aclStr), storageClassStr)
+		if err != nil {
+			return nil, errors.Wrap(err, "iBucket.PutObject")
+		}
+		obj, err := cloudprovider.GetIObject(iBucket, key)
+		if err != nil {
+			return nil, errors.Wrap(err, "cloudprovider.GetIObject")
+		}
+		respHdr.Set("ETag", obj.GetETag())
+	}
+
+	bucket.Invalidate()
+
+	return respHdr, nil
+}
+
+const (
+	MIN_PART_BYTES = 1000 * 1000 * 10 // 100 MB
+	MAX_PART_COUNT = 10000
+)
+
+func copyObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, copySource string, hdr http.Header, uploadId string, partNumber int) (interface{}, http.Header, error) {
+	log.Debugf("CopyObject %s => %s/%s %s %d %s", copySource, bucketName, key, uploadId, partNumber, hdr)
+	srcSegs := appsrv.SplitPath(copySource)
+	srcBucketName := srcSegs[0]
+	srcKey := strings.Join(srcSegs[1:], "/")
+	if strings.HasSuffix(copySource, "/") {
+		srcKey += "/"
+	}
+	var err error
+	srcKey, err = url.PathUnescape(srcKey)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "url.PathUnescape")
+	}
+
+	srcBucket, err := models.BucketManager.GetByName(ctx, userCred, srcBucketName)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "source bucket GetByName")
+	}
+	iSrcBucket, err := srcBucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "srcBucket.GetIBucket")
+	}
+	srcObj, err := cloudprovider.GetIObject(iSrcBucket, srcKey)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "src cloudprovider.GetIObject")
+	}
+
+	dstBucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "dest bucket GetByName")
+	}
+
+	err = dstBucket.IsOutOfLimit()
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "IsOutOfLimit")
+	}
+
+	iDstBucket, err := dstBucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, "dstBucket.GetIBucket")
+	}
+
+	sizeBytes := srcObj.GetSizeBytes()
+	rangeStr := hdr.Get(http.CanonicalHeaderKey("x-amz-copy-source-range"))
+	rangeOpt, err := getRangeOpt(rangeStr, sizeBytes)
+	if err != nil {
+		return nil, nil, errors.Wrap(err, rangeStr)
+	}
+
+	if rangeOpt != nil {
+		if len(uploadId) == 0 {
+			return nil, nil, errors.Wrap(httperrors.ErrBadRequest, "range copy must be a multipart upload")
+		}
+		// upload directory
+		var etag string
+		if dstBucket.ManagerId == srcBucket.ManagerId && dstBucket.RegionExternalId == srcBucket.RegionExternalId {
+			etag, err = iDstBucket.CopyPart(ctx, key, uploadId, partNumber, iSrcBucket.GetName(), srcKey, rangeOpt.Start, rangeOpt.SizeBytes())
+		} else {
+			etag, err = cloudprovider.CopyPart(ctx, iDstBucket, key, uploadId, partNumber, iSrcBucket, srcKey, rangeOpt)
+		}
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "copyPart fail")
+		}
+		result := s3cli.CopyPartResult{
+			ETag:         etag,
+			LastModified: srcObj.GetLastModified(),
+		}
+		return &result, nil, nil
+	} else {
+		if dstBucket.ManagerId == srcBucket.ManagerId && dstBucket.RegionExternalId == srcBucket.RegionExternalId {
+			err = iDstBucket.CopyObject(ctx, key, iSrcBucket.GetName(), srcKey, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass())
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "iDstBucket.CopyObject")
+			}
+		} else {
+			err = cloudprovider.CopyObject(ctx, 0, iDstBucket, key, iSrcBucket, srcKey, false)
+			if err != nil {
+				return nil, nil, errors.Wrap(err, "cloudprovider.CopyObject")
+			}
+		}
+
+		dstBucket.Invalidate()
+
+		dstObj, err := cloudprovider.GetIObject(iDstBucket, key)
+		if err != nil {
+			return nil, nil, errors.Wrap(err, "cloudprovider.GetIObject")
+		}
+		result := s3cli.CopyObjectResult{
+			ETag:         dstObj.GetETag(),
+			LastModified: dstObj.GetLastModified(),
+		}
+		return &result, nil, nil
+	}
+}
+
+func deleteObjectTags(ctx context.Context, userCred mcclient.TokenCredential, bucket string, key string) (*s3cli.Tagging, error) {
+	return nil, nil
+}
+
+func removeObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string) error {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return errors.Wrap(err, "bucket.GetIBucket")
+	}
+	err = iBucket.DeleteObject(ctx, key)
+	if err != nil {
+		return errors.Wrap(err, "DeleteObject")
+	}
+
+	bucket.Invalidate()
+
+	return nil
+}
+
+func objectAcl(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, objKey string) (*s3cli.AccessControlPolicy, error) {
+	bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName)
+	if err != nil {
+		return nil, errors.Wrap(err, "models.BucketManager.GetByName")
+	}
+	iBucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.GetIBucket")
+	}
+	obj, err := cloudprovider.GetIObject(iBucket, objKey)
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudprovider.GetIObject")
+	}
+
+	result := str2Acl(userCred, obj.GetAcl())
+
+	return result, nil
+}
+
+func str2Acl(userCred mcclient.TokenCredential, aclStr cloudprovider.TBucketACLType) *s3cli.AccessControlPolicy {
+	result := s3cli.AccessControlPolicy{}
+	result.Owner.DisplayName = userCred.GetProjectName()
+	result.Owner.ID = userCred.GetProjectId()
+
+	fullControl := s3cli.Grant{}
+	fullControl.Permission = s3cli.PERMISSION_FULL_CONTROL
+	fullControl.Grantee.Type = s3cli.GRANTEE_TYPE_USER
+	fullControl.Grantee.ID = userCred.GetProjectId()
+	fullControl.Grantee.DisplayName = userCred.GetProjectName()
+
+	publicRead := s3cli.Grant{}
+	publicRead.Permission = s3cli.PERMISSION_READ
+	publicRead.Grantee.Type = s3cli.GRANTEE_TYPE_GROUP
+	publicRead.Grantee.URI = s3cli.GRANTEE_GROUP_URI_ALL_USERS
+
+	publicWrite := s3cli.Grant{}
+	publicWrite.Permission = s3cli.PERMISSION_WRITE
+	publicWrite.Grantee.Type = s3cli.GRANTEE_TYPE_GROUP
+	publicWrite.Grantee.URI = s3cli.GRANTEE_GROUP_URI_ALL_USERS
+
+	authRead := s3cli.Grant{}
+	authRead.Permission = s3cli.PERMISSION_READ
+	authRead.Grantee.Type = s3cli.GRANTEE_TYPE_GROUP
+	authRead.Grantee.URI = s3cli.GRANTEE_GROUP_URI_AUTH_USERS
+
+	switch aclStr {
+	case cloudprovider.ACLPrivate:
+		result.AccessControlList.Grant = []s3cli.Grant{
+			fullControl,
+		}
+	case cloudprovider.ACLAuthRead:
+		result.AccessControlList.Grant = []s3cli.Grant{
+			fullControl,
+			authRead,
+		}
+	case cloudprovider.ACLPublicRead:
+		result.AccessControlList.Grant = []s3cli.Grant{
+			fullControl,
+			publicRead,
+		}
+	case cloudprovider.ACLPublicReadWrite:
+		result.AccessControlList.Grant = []s3cli.Grant{
+			fullControl,
+			publicRead,
+			publicWrite,
+		}
+	}
+	return &result
+}

pkg/s3gateway/handlers/select.go

@@ -12,26 +12,17 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package models
+package handlers
 
 import (
-	"yunion.io/x/log"
+	"context"
+	"net/http"
 
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/s3cli"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
-func InitDB() error {
-	for _, manager := range []db.IModelManager{
-		/*
-		 * Important!!!
-		 * initialization order matters, do not change the order
-		 */
-	} {
-		err := manager.InitializeData()
-		if err != nil {
-			log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err)
-			// return err skip error table
-		}
-	}
-	return nil
+func selectObject(ctx context.Context, userCred mcclient.TokenCredential, hdr http.Header, bucket string, key string) (*s3cli.InitiateMultipartUploadResult, http.Header, error) {
+	return nil, nil, NotImplemented(ctx, "")
 }

pkg/s3gateway/handlers/service.go

@@ -0,0 +1,50 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package handlers
+
+import (
+	"context"
+	"time"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/s3cli"
+
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/s3gateway/models"
+)
+
+type sBucketInfo struct {
+	Name      string
+	CreatedAt time.Time
+}
+
+func listService(ctx context.Context, userCred mcclient.TokenCredential, query s3cli.ListBucketsInput) (*s3cli.ListAllMyBucketsResult, error) {
+	result, err := models.BucketManager.List(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "models.BucketManager.List")
+	}
+	resp := s3cli.ListAllMyBucketsResult{}
+	resp.Owner.ID = userCred.GetProjectId()
+	resp.Owner.DisplayName = userCred.GetProjectName()
+	resp.Buckets.Bucket = make([]s3cli.BucketInfo, 0)
+	for i := range result {
+		info := result[i]
+		resp.Buckets.Bucket = append(resp.Buckets.Bucket, s3cli.BucketInfo{
+			Name:         info.Name,
+			CreationDate: info.CreatedAt,
+		})
+	}
+	return &resp, nil
+}

pkg/s3gateway/models/base.go

@@ -12,4 +12,15 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package models // import "yunion.io/x/onecloud/pkg/s3gateway/models"
+package models
+
+import "time"
+
+type SBaseModelManagerDelegate struct {
+}
+
+type SBaseModelDelegate struct {
+	Id        string
+	Name      string
+	CreatedAt time.Time
+}

pkg/s3gateway/models/buckets.go

@@ -0,0 +1,243 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"github.com/pkg/errors"
+	"time"
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/s3gateway/session"
+	"yunion.io/x/onecloud/pkg/util/hashcache"
+	"yunion.io/x/pkg/gotypes"
+	"yunion.io/x/s3cli"
+)
+
+type SBucketManagerDelegate struct {
+	buckets *hashcache.Cache
+}
+
+var BucketManager *SBucketManagerDelegate
+
+func init() {
+	BucketManager = &SBucketManagerDelegate{
+		buckets: hashcache.NewCache(2048, time.Minute*15),
+	}
+}
+
+/*
+{
+	"access_urls":[{"description":"bucket domain","primary":true,"url":"https://yunion-billing-reports.s3.cn-northwest-1.amazonaws.com.cn"},{"description":"s3 domain","primary":false,"url":"https://s3.cn-northwest-1.amazonaws.com.cn/yunion-billing-reports"}],
+	"account":"aws-cn",
+	"account_id":"edc90a61-7f8a-4be7-84f1-8f3ac70ef5e6",
+	"acl":"private",
+	"brand":"Aws",
+	"can_delete":false,
+	"can_update":true,
+	"cloud_env":"public",
+	"cloudregion_id":"4cbf92a5-337b-4cc6-82c7-86e5427b69e3",
+	"created_at":"2019-03-11T10:31:26.000000Z",
+	"domain_id":"default",
+	"external_id":"yunion-billing-reports",
+	"id":"056bb8c6-527d-4554-8939-12eb6aa803bd",
+	"is_emulated":false,
+	"is_system":false,
+	"location":"cn-northwest-1",
+	"manager":"aws-cn",
+	"manager_domain":"Default",
+	"manager_domain_id":"default",
+	"manager_id":"d8df39fa-b212-43c1-8d44-aeef897e216d",
+	"manager_project":"system",
+	"manager_project_id":"5d65667d112e47249ae66dbd7bc07030",
+	"name":"yunion-billing-reports",
+	"object_cnt":44,
+	"object_cnt_limit":0,
+	"project_domain":"Default",
+	"project_src":"cloud",
+	"provider":"Aws",
+	"region":"AWS 中国(宁夏)",
+	"region_ext_id":"cn-northwest-1",
+	"region_id":"4cbf92a5-337b-4cc6-82c7-86e5427b69e3",
+	"size_bytes":3332448,
+	"size_bytes_limit":0,
+	"status":"ready",
+	"tenant":"system",
+	"tenant_id":"5d65667d112e47249ae66dbd7bc07030",
+	"update_version":1,
+	"updated_at":"2019-08-18T15:52:42.000000Z",
+}
+*/
+type SBucketDelegate struct {
+	SBaseModelDelegate
+
+	Location  string
+	ManagerId string
+
+	ObjectCnt int
+	SizeBytes int64
+
+	ObjectCntLimit int
+	SizeBytesLimit int64
+
+	RegionExternalId string
+	ExternalId       string
+}
+
+func (manager *SBucketManagerDelegate) List(ctx context.Context, userCred mcclient.TokenCredential) ([]*SBucketDelegate, error) {
+	s := session.GetSession(ctx, userCred)
+	offset := 0
+	total := -1
+	ret := make([]*SBucketDelegate, 0)
+	for total < 0 || offset < total {
+		params := struct {
+			Limit  int
+			Offset int
+		}{}
+		params.Limit = 1000
+		params.Offset = offset
+		result, err := modules.Buckets.List(s, jsonutils.Marshal(params))
+		if err != nil {
+			return nil, errors.Wrap(err, "List")
+		}
+		total = result.Total
+		offset += len(result.Data)
+		for i := range result.Data {
+			bucket := &SBucketDelegate{}
+			err := result.Data[i].Unmarshal(bucket)
+			if err != nil {
+				return nil, errors.Wrap(err, "Unmarshal")
+			}
+			ret = append(ret, bucket)
+			manager.buckets.AtomicSet(bucket.Name, bucket)
+		}
+	}
+	return ret, nil
+}
+
+func (manager *SBucketManagerDelegate) GetByName(ctx context.Context, userCred mcclient.TokenCredential, name string) (*SBucketDelegate, error) {
+	val := manager.buckets.AtomicGet(name)
+	if !gotypes.IsNil(val) {
+		return val.(*SBucketDelegate), nil
+	}
+	s := session.GetSession(ctx, userCred)
+	result, err := modules.Buckets.PerformAction(s, name, "sync", nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "modules.Buckets.Get")
+	}
+	bucket := &SBucketDelegate{}
+	err = result.Unmarshal(bucket)
+	if err != nil {
+		return nil, errors.Wrap(err, "result.Unmarshal")
+	}
+	manager.buckets.AtomicSet(bucket.Name, bucket)
+	return bucket, nil
+}
+
+func (manager *SBucketManagerDelegate) DeleteByName(ctx context.Context, userCred mcclient.TokenCredential, name string) error {
+	s := session.GetSession(ctx, userCred)
+	_, err := modules.Buckets.Delete(s, name, nil)
+	if err != nil {
+		return errors.Wrap(err, "modules.Buckets.Delete")
+	}
+	manager.buckets.AtomicRemove(name)
+	return nil
+}
+
+func (manager *SBucketManagerDelegate) Invalidate(name string) {
+	manager.buckets.AtomicRemove(name)
+}
+
+func (bucket *SBucketDelegate) getManager(ctx context.Context, userCred mcclient.TokenCredential) (*SCloudproviderDelegate, error) {
+	return CloudproviderManager.GetById(ctx, userCred, bucket.ManagerId)
+}
+
+func (bucket *SBucketDelegate) GetIBucket(ctx context.Context, userCred mcclient.TokenCredential) (cloudprovider.ICloudBucket, error) {
+	manager, err := bucket.getManager(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "bucket.getManager")
+	}
+	driver, err := manager.GetProvider()
+	if err != nil {
+		return nil, errors.Wrap(err, "cloudprovider.GetProvider")
+	}
+	var iRegion cloudprovider.ICloudRegion
+	if len(bucket.RegionExternalId) == 0 {
+		iRegion, err = driver.GetOnPremiseIRegion()
+	} else {
+		iRegion, err = driver.GetIRegionById(bucket.RegionExternalId)
+	}
+	if err != nil {
+		return nil, errors.Wrap(err, "driver.GetIRegionById")
+	}
+	iBucket, err := iRegion.GetIBucketById(bucket.ExternalId)
+	if err != nil {
+		return nil, errors.Wrap(err, "iRegion.GetIBucketById")
+	}
+	return iBucket, nil
+}
+
+func (bucket *SBucketDelegate) ListObject(ctx context.Context, userCred mcclient.TokenCredential, input *s3cli.ListObjectInput) (*s3cli.ListBucketResult, error) {
+	ibucket, err := bucket.GetIBucket(ctx, userCred)
+	if err != nil {
+		return nil, errors.Wrap(err, "getIBucket")
+	}
+	result, err := ibucket.ListObjects(input.Prefix, input.Marker, input.Delimiter, int(input.MaxKeys))
+	if err != nil {
+		return nil, errors.Wrap(err, "ibucket.ListObjects")
+	}
+	ret := s3cli.ListBucketResult{}
+	ret.IsTruncated = result.IsTruncated
+	ret.MaxKeys = input.MaxKeys
+	ret.Delimiter = input.Delimiter
+	ret.Prefix = input.Prefix
+	ret.Marker = input.Marker
+	ret.CommonPrefixes = make([]s3cli.CommonPrefix, len(result.CommonPrefixes))
+	for i := range result.CommonPrefixes {
+		ret.CommonPrefixes[i] = s3cli.CommonPrefix{
+			Prefix: result.CommonPrefixes[i].GetKey(),
+		}
+	}
+	ret.Contents = make([]s3cli.ObjectInfo, len(result.Objects))
+	for i := range result.Objects {
+		obj := result.Objects[i]
+		ret.Contents[i] = s3cli.ObjectInfo{
+			Key:          obj.GetKey(),
+			ETag:         obj.GetETag(),
+			Size:         obj.GetSizeBytes(),
+			LastModified: obj.GetLastModified(),
+			ContentType:  obj.GetContentType(),
+			StorageClass: obj.GetStorageClass(),
+		}
+	}
+	return &ret, nil
+}
+
+func (bucket *SBucketDelegate) IsOutOfLimit() error {
+	if bucket.ObjectCntLimit > 0 && bucket.ObjectCnt >= bucket.ObjectCntLimit {
+		return errors.Wrap(httperrors.ErrOutOfLimit, "object_count")
+	}
+	if bucket.SizeBytesLimit > 0 && bucket.SizeBytes >= bucket.SizeBytesLimit {
+		return errors.Wrap(httperrors.ErrOutOfLimit, "size_bytes")
+	}
+	return nil
+}
+
+func (bucket *SBucketDelegate) Invalidate() {
+	BucketManager.Invalidate(bucket.Name)
+}

pkg/s3gateway/models/cloudproviders.go

@@ -0,0 +1,140 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package models
+
+import (
+	"context"
+	"time"
+
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/gotypes"
+
+	"yunion.io/x/onecloud/pkg/cloudprovider"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/s3gateway/session"
+	"yunion.io/x/onecloud/pkg/util/hashcache"
+	"yunion.io/x/pkg/utils"
+)
+
+type SCloudproviderManagerDelegate struct {
+	providers *hashcache.Cache
+}
+
+var CloudproviderManager *SCloudproviderManagerDelegate
+
+func init() {
+	CloudproviderManager = &SCloudproviderManagerDelegate{
+		providers: hashcache.NewCache(2048, time.Minute*15),
+	}
+}
+
+/*
+{
+	"account":"oH7Qrw75AqrI4BXn",
+	"can_delete":false,
+	"can_update":true,
+	"cloudaccount":"testaliyun",
+	"cloudaccount_id":"f1a927b4-a433-486e-86ae-e9aa36e447b1",
+	"created_at":"2019-04-16T13:55:11.000000Z",
+	"domain":"Default",
+	"domain_id":"default",
+	"eip_count":9,
+	"enabled":true,
+	"guest_count":1,
+	"health_status":"normal",
+	"host_count":61,
+	"id":"57c84d93-8f06-4a85-8963-4ce42eabb339",
+	"is_emulated":false,
+	"last_sync":"2019-07-23T15:29:34.000000Z",
+	"last_sync_end_at":"2019-07-23T15:33:34.000000Z",
+	"loadbalancer_count":13,
+	"name":"testaliyun",
+	"project_count":0,
+	"provider":"Aliyun",
+	"secret":"Y5YFmuwVI4frJ8kVgWL0z5Kan/sJ3JMyjyFRxAXwXvsUKd8aNohPp2T/Kr1BqA==",
+	"snapshot_count":6,
+	"status":"connected",
+	"storage_cache_count":20,
+	"storage_count":141,
+	"sync_region_count":20,
+	"sync_status":"idle",
+	"sync_status2":"idle",
+	"tenant":"system",
+	"tenant_id":"5d65667d112e47249ae66dbd7bc07030",
+	"update_version":5003,
+	"updated_at":"2019-08-18T14:47:42.000000Z",
+	"vpc_count":13,
+}
+*/
+
+type SCloudproviderDelegate struct {
+	SBaseModelDelegate
+
+	Enabled    bool
+	Status     string
+	SyncStatus string
+
+	AccessUrl string
+	Account   string
+	Secret    string
+
+	Provider string
+	Brand    string
+}
+
+func (manager *SCloudproviderManagerDelegate) GetById(ctx context.Context, userCred mcclient.TokenCredential, id string) (*SCloudproviderDelegate, error) {
+	val := manager.providers.AtomicGet(id)
+	if !gotypes.IsNil(val) {
+		return val.(*SCloudproviderDelegate), nil
+	}
+	s := session.GetSession(ctx, userCred)
+	result, err := modules.Cloudproviders.Get(s, id, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "modules.Cloudproviders.Get")
+	}
+	provider := &SCloudproviderDelegate{}
+	err = result.Unmarshal(provider)
+	if err != nil {
+		return nil, errors.Wrap(err, "result.Unmarshal")
+	}
+	manager.providers.AtomicSet(provider.Id, provider)
+	return provider, nil
+}
+
+func (provider *SCloudproviderDelegate) getPassword() (string, error) {
+	return utils.DescryptAESBase64(provider.Id, provider.Secret)
+}
+
+func (provider *SCloudproviderDelegate) getAccessUrl() string {
+	return provider.AccessUrl
+}
+
+func (provider *SCloudproviderDelegate) GetProviderFactory() (cloudprovider.ICloudProviderFactory, error) {
+	return cloudprovider.GetProviderFactory(provider.Provider)
+}
+
+func (provider *SCloudproviderDelegate) GetProvider() (cloudprovider.ICloudProvider, error) {
+	if !provider.Enabled {
+		return nil, errors.Error("Cloud provider is not enabled")
+	}
+
+	accessUrl := provider.getAccessUrl()
+	passwd, err := provider.getPassword()
+	if err != nil {
+		return nil, err
+	}
+	return cloudprovider.GetProvider(provider.Id, provider.Name, accessUrl, provider.Account, passwd, provider.Provider)
+}

pkg/s3gateway/options/options.go

@@ -21,7 +21,7 @@ import (
 type SS3GatewayOptions struct {
 	common_options.CommonOptions
 
-	common_options.DBOptions
+	DomainName string `help:"s3 domain name"`
 }
 
 var (

pkg/s3gateway/service/handlers.go

@@ -1,48 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package service
-
-import (
-	"yunion.io/x/onecloud/pkg/appsrv"
-	"yunion.io/x/onecloud/pkg/appsrv/dispatcher"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
-	// "yunion.io/x/onecloud/pkg/s3gateway/models"
-)
-
-func initHandlers(app *appsrv.Application) {
-	db.InitAllManagers()
-
-	// quotas.AddQuotaHandler(models.QuotaManager, API_VERSION, app)
-	// usages.AddUsageHandler(API_VERSION, app)
-	taskman.AddTaskHandler("", app)
-
-	for _, manager := range []db.IModelManager{
-		taskman.TaskManager,
-		taskman.SubTaskManager,
-		taskman.TaskObjectManager,
-		db.Metadata,
-	} {
-		db.RegisterModelManager(manager)
-	}
-
-	for _, manager := range []db.IModelManager{
-		db.OpsLog,
-	} {
-		db.RegisterModelManager(manager)
-		handler := db.NewModelHandler(manager)
-		dispatcher.AddModelDispatcher("", app, handler)
-	}
-}

pkg/s3gateway/service/service.go

@@ -22,40 +22,31 @@ import (
 	api "yunion.io/x/onecloud/pkg/apis/s3gateway"
 	"yunion.io/x/onecloud/pkg/cloudcommon"
 	app_common "yunion.io/x/onecloud/pkg/cloudcommon/app"
-	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
-	"yunion.io/x/onecloud/pkg/s3gateway/models"
+	"yunion.io/x/onecloud/pkg/s3gateway/handlers"
 	"yunion.io/x/onecloud/pkg/s3gateway/options"
+
+	_ "yunion.io/x/onecloud/pkg/multicloud/aliyun/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/aws/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/azure/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/huawei/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/qcloud/provider"
+	_ "yunion.io/x/onecloud/pkg/multicloud/ucloud/provider"
 )
 
 func StartService() {
 	opts := &options.Options
 	commonOpts := &opts.CommonOptions
 	baseOpts := &opts.BaseOptions
-	dbOpts := &opts.DBOptions
 	common_options.ParseOptions(opts, os.Args, "s3gateway.conf", api.SERVICE_TYPE)
 
 	app_common.InitAuth(commonOpts, func() {
 		log.Infof("Auth complete!!")
 	})
 
-	cloudcommon.InitDB(dbOpts)
-
-	app := app_common.InitApp(&opts.BaseOptions, true)
-	initHandlers(app)
-
-	cloudcommon.InitDB(&opts.DBOptions)
-
-	if !db.CheckSync(opts.AutoSyncTable) {
-		log.Fatalf("database schema not in sync!")
-	}
-
-	models.InitDB()
-
-	if opts.ExitAfterDBInit {
-		log.Infof("Exiting after db initialization ...")
-		os.Exit(0)
-	}
+	app := app_common.InitApp(&opts.BaseOptions, false)
+	handlers.InitHandlers(app)
 
 	/*if !opts.IsSlaveNode {
 		cron := cronman.GetCronJobManager(true)

pkg/s3gateway/session/session.go

@@ -0,0 +1,30 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package session
+
+import (
+	"context"
+	"yunion.io/x/onecloud/pkg/image/options"
+	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+)
+
+func GetSession(ctx context.Context, token mcclient.TokenCredential) *mcclient.ClientSession {
+	return auth.GetSession(ctx, token, options.Options.Region, "")
+}
+
+func GetAdminSession(ctx context.Context) *mcclient.ClientSession {
+	return auth.GetAdminSession(ctx, options.Options.Region, "")
+}

pkg/util/bitmap/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package bitmap // import "yunion.io/x/onecloud/pkg/util/bitmap"

pkg/util/hashcache/cache.go

@@ -18,6 +18,7 @@ import (
 	"crypto/md5"
 	"crypto/sha1"
 	"crypto/sha256"
+	"crypto/sha512"
 	"sync"
 	"time"
 )
@@ -54,6 +55,7 @@ const (
 	HASH_ALG_MD5 int = iota
 	HASH_ALG_SHA1
 	HASH_ALG_SHA256
+	HASH_ALG_SHA512
 )
 
 func bytes2int(b []byte) uint32 {
@@ -72,6 +74,9 @@ func checksum(alg int, key string) uint32 {
 	case HASH_ALG_SHA256:
 		v := sha256.Sum224([]byte(key))
 		hash = bytes2int(v[0:4])
+	case HASH_ALG_SHA512:
+		v := sha512.Sum512([]byte(key))
+		hash = bytes2int(v[0:4])
 	}
 	return hash
 }
@@ -79,7 +84,7 @@ func checksum(alg int, key string) uint32 {
 func (c *Cache) find(key string) (bool, uint32) {
 	var idx uint32
 	now := time.Now()
-	for _, alg := range []int{HASH_ALG_MD5, HASH_ALG_SHA1, HASH_ALG_SHA256} {
+	for _, alg := range []int{HASH_ALG_MD5, HASH_ALG_SHA1, HASH_ALG_SHA256, HASH_ALG_SHA512} {
 		idx = checksum(alg, key) % c.size
 		if c.table[idx].key == key {
 			if c.table[idx].expire.IsZero() || c.table[idx].expire.After(now) {
@@ -126,6 +131,20 @@ func (c *Cache) AtomicSet(key string, val interface{}) {
 	c.Set(key, val)
 }
 
+func (c *Cache) Remove(key string) {
+	find, idx := c.find(key)
+	if !find {
+		return
+	}
+	c.table[idx].reset()
+}
+
+func (c *Cache) AtomicRemove(key string) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	c.Remove(key)
+}
+
 func (c *Cache) Invalidate() {
 	c.lock.Lock()
 	defer c.lock.Unlock()

pkg/util/s3auth/interface.go

@@ -0,0 +1,118 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3auth
+
+import (
+	"net/http"
+	"strings"
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+)
+
+type IAccessKeySecretRequest interface {
+	GetAccessKey() string
+	Validate() error
+	ParseRequest(req http.Request, virtualHost bool) error
+	Verify(secret string) error
+	Encode() string
+}
+
+type SAccessKeyRequest struct {
+	Algorithm string `json:"algorithm,omitempty"`
+	AccessKey string `json:"access_key,omitempty"`
+	Signature string `json:"signature,omitempty"`
+	Request   string `json:"request,omitempty"`
+}
+
+func (aksk SAccessKeyRequest) GetAccessKey() string {
+	return aksk.AccessKey
+}
+
+func (aksk SAccessKeyRequest) Validate() error {
+	if len(aksk.AccessKey) == 0 {
+		return errors.Error("Missing AWSAccessKeyId")
+	}
+	if len(aksk.Signature) == 0 {
+		return errors.Error("Missing Signature")
+	}
+	return nil
+}
+
+func decodeAuthHeader(authHeader string) (IAccessKeySecretRequest, error) {
+	pos := strings.IndexByte(authHeader, ' ')
+	if pos <= 0 {
+		return nil, errors.Error("illegal authorization header")
+	}
+	algo := authHeader[:pos]
+	switch algo {
+	case signV2Algorithm:
+		req, err := decodeAuthHeaderV2(authHeader[pos+1:])
+		if err != nil {
+			return nil, errors.Wrap(err, "decodeAuthHeaderV2")
+		}
+		return req, nil
+	case signV4Algorithm:
+		req, err := decodeAuthHeaderV4(authHeader[pos+1:])
+		if err != nil {
+			return nil, errors.Wrap(err, "decodeAuthHeaderV4")
+		}
+		return req, nil
+	default:
+		return nil, errors.Error("unsupported signing algorithm")
+	}
+}
+
+func DecodeAccessKeyRequest(req http.Request, virtualHost bool) (IAccessKeySecretRequest, error) {
+	authHeader := req.Header.Get("Authorization")
+	if len(authHeader) == 0 {
+		return nil, errors.Error("missing authorization header")
+	}
+	akskReq, err := decodeAuthHeader(authHeader)
+	if err != nil {
+		return nil, errors.Wrap(err, "decodeAuthHeader")
+	}
+	err = akskReq.ParseRequest(req, virtualHost)
+	if err != nil {
+		return nil, errors.Wrap(err, "akskReq.ParseRequest")
+	}
+
+	return akskReq, akskReq.Validate()
+}
+
+func Decode(reqStr string) (IAccessKeySecretRequest, error) {
+	rawReq := SAccessKeyRequest{}
+	reqJson, err := jsonutils.ParseString(reqStr)
+	if err != nil {
+		return nil, errors.Wrap(err, "jsonutils.ParseString")
+	}
+	err = reqJson.Unmarshal(&rawReq)
+	if err != nil {
+		return nil, errors.Wrap(err, "reqJson.Unmarshal rawReq")
+	}
+	var ret IAccessKeySecretRequest
+	switch rawReq.Algorithm {
+	case signV2Algorithm:
+		ret = &SAccessKeyRequestV2{}
+	case signV4Algorithm:
+		ret = &SAccessKeyRequestV4{}
+	default:
+		return nil, errors.Error("unsupported sign algorithm")
+	}
+	err = reqJson.Unmarshal(ret)
+	if err != nil {
+		return nil, errors.Wrap(err, "reqJson.Unmarshal")
+	}
+	return ret, nil
+}

pkg/util/s3auth/utils.go

@@ -0,0 +1,89 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3auth
+
+import (
+	"encoding/hex"
+	"net/http"
+	"regexp"
+	"strings"
+	"unicode/utf8"
+)
+
+// if object matches reserved string, no need to encode them
+var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$")
+
+// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences
+//
+// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8
+// non english characters cannot be parsed due to the nature in which url.Encode() is written
+//
+// This function on the other hand is a direct replacement for url.Encode() technique to support
+// pretty much every UTF-8 character.
+func encodePath(pathName string) string {
+	if reservedObjectNames.MatchString(pathName) {
+		return pathName
+	}
+	var encodedPathname string
+	for _, s := range pathName {
+		if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark)
+			encodedPathname = encodedPathname + string(s)
+			continue
+		}
+		switch s {
+		case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark)
+			encodedPathname = encodedPathname + string(s)
+			continue
+		default:
+			len := utf8.RuneLen(s)
+			if len < 0 {
+				// if utf8 cannot convert return the same string as is
+				return pathName
+			}
+			u := make([]byte, len)
+			utf8.EncodeRune(u, s)
+			for _, r := range u {
+				hex := hex.EncodeToString([]byte{r})
+				encodedPathname = encodedPathname + "%" + strings.ToUpper(hex)
+			}
+		}
+	}
+	return encodedPathname
+}
+
+// getHostAddr returns host header if available, otherwise returns host from URL
+func getHostAddr(req http.Request) string {
+	if req.Host != "" {
+		return req.Host
+	}
+	return req.URL.Host
+}
+
+// Encode input URL path to URL encoded path.
+func encodeURL2Path(req http.Request, virtualHost bool) (path string) {
+	if virtualHost {
+		reqHost := getHostAddr(req)
+		dotPos := strings.Index(reqHost, ".")
+		if dotPos > -1 {
+			bucketName := reqHost[:dotPos]
+			path = "/" + bucketName
+			path += req.URL.Path
+			path = encodePath(path)
+			return
+		}
+	}
+	path = encodePath(req.URL.Path)
+	return
+}

pkg/util/s3auth/v2.go

@@ -0,0 +1,244 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3auth
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha1"
+	"encoding/base64"
+	"net/http"
+	"net/url"
+	"sort"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+)
+
+// Signature and API related constants.
+const (
+	signV2Algorithm = "AWS"
+)
+
+// From the Amazon docs:
+//
+// StringToSign = HTTP-Verb + "\n" +
+// 	 Content-Md5 + "\n" +
+//	 Content-Type + "\n" +
+//	 Date + "\n" +
+//	 CanonicalizedProtocolHeaders +
+//	 CanonicalizedResource;
+func stringToSignV2(req http.Request, virtualHost bool) string {
+	buf := new(bytes.Buffer)
+	// Write standard headers.
+	writeSignV2Headers(buf, req)
+	// Write canonicalized protocol headers if any.
+	writeCanonicalizedHeaders(buf, req)
+	// Write canonicalized Query resources if any.
+	writeCanonicalizedResource(buf, req, virtualHost)
+	return buf.String()
+}
+
+// writeSignV2Headers - write signV2 required headers.
+func writeSignV2Headers(buf *bytes.Buffer, req http.Request) {
+	buf.WriteString(req.Method + "\n")
+	buf.WriteString(req.Header.Get("Content-Md5") + "\n")
+	buf.WriteString(req.Header.Get("Content-Type") + "\n")
+	buf.WriteString(req.Header.Get("Date") + "\n")
+}
+
+// writeCanonicalizedHeaders - write canonicalized headers.
+func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) {
+	var protoHeaders []string
+	vals := make(map[string][]string)
+	for k, vv := range req.Header {
+		// All the AMZ headers should be lowercase
+		lk := strings.ToLower(k)
+		if strings.HasPrefix(lk, "x-amz") {
+			protoHeaders = append(protoHeaders, lk)
+			vals[lk] = vv
+		}
+	}
+	sort.Strings(protoHeaders)
+	for _, k := range protoHeaders {
+		buf.WriteString(k)
+		buf.WriteByte(':')
+		for idx, v := range vals[k] {
+			if idx > 0 {
+				buf.WriteByte(',')
+			}
+			if strings.Contains(v, "\n") {
+				// TODO: "Unfold" long headers that
+				// span multiple lines (as allowed by
+				// RFC 2616, section 4.2) by replacing
+				// the folding white-space (including
+				// new-line) by a single space.
+				buf.WriteString(v)
+			} else {
+				buf.WriteString(v)
+			}
+		}
+		buf.WriteByte('\n')
+	}
+}
+
+// AWS S3 Signature V2 calculation rule is give here:
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign
+
+// Whitelist resource list that will be used in query string for signature-V2 calculation.
+// The list should be alphabetically sorted
+var resourceList = []string{
+	"acl",
+	"delete",
+	"lifecycle",
+	"location",
+	"logging",
+	"notification",
+	"partNumber",
+	"policy",
+	"requestPayment",
+	"response-cache-control",
+	"response-content-disposition",
+	"response-content-encoding",
+	"response-content-language",
+	"response-content-type",
+	"response-expires",
+	"torrent",
+	"uploadId",
+	"uploads",
+	"versionId",
+	"versioning",
+	"versions",
+	"website",
+}
+
+// From the Amazon docs:
+//
+// CanonicalizedResource = [ "/" + Bucket ] +
+// 	  <HTTP-Request-URI, from the protocol name up to the query string> +
+// 	  [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
+func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) {
+	// Save request URL.
+	requestURL := req.URL
+	// Get encoded URL path.
+	buf.WriteString(encodeURL2Path(req, virtualHost))
+	if requestURL.RawQuery != "" {
+		var n int
+		vals, _ := url.ParseQuery(requestURL.RawQuery)
+		// Verify if any sub resource queries are present, if yes
+		// canonicallize them.
+		for _, resource := range resourceList {
+			if vv, ok := vals[resource]; ok && len(vv) > 0 {
+				n++
+				// First element
+				switch n {
+				case 1:
+					buf.WriteByte('?')
+					// The rest
+				default:
+					buf.WriteByte('&')
+				}
+				buf.WriteString(resource)
+				// Request parameters
+				if len(vv[0]) > 0 {
+					buf.WriteByte('=')
+					buf.WriteString(vv[0])
+				}
+			}
+		}
+	}
+}
+
+// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature;
+// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) );
+//
+// StringToSign = HTTP-Verb + "\n" +
+//  	Content-Md5 + "\n" +
+//  	Content-Type + "\n" +
+//  	Date + "\n" +
+//  	CanonicalizedProtocolHeaders +
+//  	CanonicalizedResource;
+//
+// CanonicalizedResource = [ "/" + Bucket ] +
+//  	<HTTP-Request-URI, from the protocol name up to the query string> +
+//  	[ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"];
+//
+// CanonicalizedProtocolHeaders = <described below>
+// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}.
+/*func verifyV2(ctx context.Context, req http.Request, virtualHost bool) error {
+	aksk, err := DecodeAccessKeyRequestV2(req, virtualHost)
+	if err != nil {
+		return errors.Wrap(err, "DecodeAccessKeyRequestV2")
+	}
+
+	authSession := session.GetAdminSession(ctx)
+	result, err := modules.Credentials.Get(authSession, aksk.AccessKey, nil)
+	if err != nil {
+		return errors.Wrap(err, "modules.Credentials.Get")
+	}
+	secret, err := modules.DecodeAccessKeySecret(result)
+	if err != nil {
+		return errors.Wrap(err, "modules.DecodeAccessKeySecret")
+	}
+
+	hm := hmac.New(sha1.New, []byte(secret.Secret))
+	hm.Write([]byte(aksk.RequestString))
+
+	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
+
+	if aksk.Signature != signature {
+		return errors.Error("signature mismatch")
+	}
+
+	return nil
+}*/
+
+type SAccessKeyRequestV2 struct {
+	SAccessKeyRequest
+}
+
+func (aksk *SAccessKeyRequestV2) ParseRequest(req http.Request, virtualHost bool) error {
+	aksk.Request = stringToSignV2(req, virtualHost)
+	return nil
+}
+
+func (aksk SAccessKeyRequestV2) Verify(secret string) error {
+	hm := hmac.New(sha1.New, []byte(secret))
+	hm.Write([]byte(aksk.Request))
+
+	signature := base64.StdEncoding.EncodeToString(hm.Sum(nil))
+	if signature != aksk.Signature {
+		return errors.Error("signature mismatch")
+	}
+
+	return nil
+}
+
+func (aksk SAccessKeyRequestV2) Encode() string {
+	return jsonutils.Marshal(aksk).String()
+}
+
+func decodeAuthHeaderV2(authStr string) (*SAccessKeyRequestV2, error) {
+	akskReq := SAccessKeyRequestV2{}
+	akskReq.Algorithm = signV2Algorithm
+	pos := strings.IndexByte(authStr, ':')
+	if pos <= 0 {
+		return nil, errors.Error("illegal authorization header")
+	}
+	akskReq.AccessKey = authStr[:pos]
+	akskReq.Signature = authStr[pos+1:]
+	return &akskReq, nil
+}

pkg/util/s3auth/v4.go

@@ -0,0 +1,270 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package s3auth
+
+import (
+	"bytes"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/hex"
+	"net/http"
+	"sort"
+	"strings"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+)
+
+// Signature and API related constants.
+const (
+	signV4Algorithm   = "AWS4-HMAC-SHA256"
+	iso8601DateFormat = "20060102T150405Z"
+	yyyymmdd          = "20060102"
+
+	unsignedPayload = "UNSIGNED-PAYLOAD"
+)
+
+// getScope generate a string of a specific date, an AWS region, and a
+// service.
+func getScope(location string, t time.Time) string {
+	scope := strings.Join([]string{
+		t.Format(yyyymmdd),
+		location,
+		"s3",
+		"aws4_request",
+	}, "/")
+	return scope
+}
+
+// sum256 calculate sha256 sum for an input byte array.
+func sum256(data []byte) []byte {
+	hash := sha256.New()
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// getStringToSign a string based on selected query values.
+func getStringToSignV4(t time.Time, location, canonicalRequest string) string {
+	stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n"
+	stringToSign += getScope(location, t) + "\n"
+	stringToSign += hex.EncodeToString(sum256([]byte(canonicalRequest)))
+	return stringToSign
+}
+
+///
+/// Excerpts from @lsegal -
+/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258.
+///
+///  User-Agent:
+///
+///      This is ignored from signing because signing this causes
+///      problems with generating pre-signed URLs (that are executed
+///      by other agents) or when customers pass requests through
+///      proxies, which may modify the user-agent.
+///
+///  Content-Length:
+///
+///      This is ignored from signing because generating a pre-signed
+///      URL should not provide a content-length constraint,
+///      specifically when vending a S3 pre-signed PUT URL. The
+///      corollary to this is that when sending regular requests
+///      (non-pre-signed), the signature contains a checksum of the
+///      body, which implicitly validates the payload length (since
+///      changing the number of bytes would change the checksum)
+///      and therefore this header is not valuable in the signature.
+///
+///  Content-Type:
+///
+///      Signing this header causes quite a number of problems in
+///      browser environments, where browsers like to modify and
+///      normalize the content-type header in different ways. There is
+///      more information on this in https://goo.gl/2E9gyy. Avoiding
+///      this field simplifies logic and reduces the possibility of
+///      future bugs.
+///
+///  Authorization:
+///
+///      Is skipped for obvious reasons
+///
+var v4IgnoredHeaders = map[string]bool{
+	"Authorization":  true,
+	"Content-Type":   true,
+	"Content-Length": true,
+	"User-Agent":     true,
+}
+
+// sumHMAC calculate hmac between two input byte array.
+func sumHMAC(key []byte, data []byte) []byte {
+	hash := hmac.New(sha256.New, key)
+	hash.Write(data)
+	return hash.Sum(nil)
+}
+
+// getSigningKey hmac seed to calculate final signature.
+func getSigningKey(secret, loc string, t time.Time) []byte {
+	date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd)))
+	location := sumHMAC(date, []byte(loc))
+	service := sumHMAC(location, []byte("s3"))
+	signingKey := sumHMAC(service, []byte("aws4_request"))
+	return signingKey
+}
+
+// getSignature final signature in hexadecimal form.
+func getSignature(signingKey []byte, stringToSign string) string {
+	return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign)))
+}
+
+// getCanonicalRequest generate a canonical request of style.
+//
+// canonicalRequest =
+//  <HTTPMethod>\n
+//  <CanonicalURI>\n
+//  <CanonicalQueryString>\n
+//  <CanonicalHeaders>\n
+//  <SignedHeaders>\n
+//  <HashedPayload>
+func getCanonicalRequest(req http.Request, signedHeaders []string) string {
+	req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1)
+	canonicalRequest := strings.Join([]string{
+		req.Method,
+		encodePath(req.URL.Path),
+		req.URL.RawQuery,
+		getCanonicalHeaders(req, signedHeaders),
+		strings.Join(signedHeaders, ";"),
+		getHashedPayload(req),
+	}, "\n")
+	return canonicalRequest
+}
+
+// Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall()
+// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+func signV4TrimAll(input string) string {
+	// Compress adjacent spaces (a space is determined by
+	// unicode.IsSpace() internally here) to one space and return
+	return strings.Join(strings.Fields(input), " ")
+}
+
+// getCanonicalHeaders generate a list of request headers for
+// signature.
+func getCanonicalHeaders(req http.Request, signedHeaders []string) string {
+	var buf bytes.Buffer
+	// Save all the headers in canonical form <header>:<value> newline
+	// separated for each header.
+	for _, k := range signedHeaders {
+		buf.WriteString(k)
+		buf.WriteByte(':')
+		switch {
+		case k == "host":
+			buf.WriteString(getHostAddr(req))
+			fallthrough
+		default:
+			for idx, v := range req.Header[http.CanonicalHeaderKey(k)] {
+				if idx > 0 {
+					buf.WriteByte(',')
+				}
+				buf.WriteString(signV4TrimAll(v))
+			}
+			buf.WriteByte('\n')
+		}
+	}
+	return buf.String()
+}
+
+// getSignedHeaders generate all signed request headers.
+// i.e lexically sorted, semicolon-separated list of lowercase
+// request header names.
+func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) []string {
+	var headers []string
+	for k := range req.Header {
+		if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok {
+			continue // Ignored header found continue.
+		}
+		headers = append(headers, strings.ToLower(k))
+	}
+	headers = append(headers, "host")
+	sort.Strings(headers)
+	return headers
+}
+
+// getHashedPayload get the hexadecimal value of the SHA256 hash of
+// the request payload.
+func getHashedPayload(req http.Request) string {
+	hashedPayload := req.Header.Get("X-Amz-Content-Sha256")
+	if hashedPayload == "" {
+		// Presign does not have a payload, use S3 recommended value.
+		hashedPayload = unsignedPayload
+	}
+	return hashedPayload
+}
+
+type SAccessKeyRequestV4 struct {
+	SAccessKeyRequest
+	Location      string
+	SignedHeaders []string
+	SignDate      time.Time
+}
+
+// AWS4-HMAC-SHA256
+// Credential=xxxx/20190824/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=27a135c6f51cc
+func decodeAuthHeaderV4(authStr string) (*SAccessKeyRequestV4, error) {
+	req := SAccessKeyRequestV4{}
+	req.Algorithm = signV4Algorithm
+	parts := strings.Split(authStr, ",")
+	if len(parts) != 3 ||
+		!strings.HasPrefix(parts[0], "Credential=") ||
+		!strings.HasPrefix(parts[1], "SignedHeaders=") ||
+		!strings.HasPrefix(parts[2], "Signature=") {
+		return nil, errors.Error("illegal v4 auth header")
+	}
+	credParts := strings.Split(parts[0][len("Credential="):], "/")
+	if len(credParts) != 5 {
+		return nil, errors.Error("illegal v4 auth header Credential")
+	}
+	req.AccessKey = credParts[0]
+	req.Location = credParts[2]
+	req.SignedHeaders = strings.Split(parts[1][len("SignedHeaders="):], ";")
+	sort.Strings(req.SignedHeaders)
+	req.Signature = parts[2][len("Signature="):]
+	return &req, nil
+}
+
+func (aksk *SAccessKeyRequestV4) ParseRequest(req http.Request, virtualHost bool) error {
+	dateStr := req.Header.Get(http.CanonicalHeaderKey("x-amz-date"))
+	if len(dateStr) == 0 {
+		return errors.Error("missing x-amz-date")
+	}
+	dateSign, err := time.Parse(iso8601DateFormat, dateStr)
+	if err != nil {
+		return errors.Wrap(err, "time.Parse")
+	}
+	canonicalReq := getCanonicalRequest(req, aksk.SignedHeaders)
+	aksk.SignDate = dateSign
+	aksk.Request = getStringToSignV4(dateSign, aksk.Location, canonicalReq)
+	return nil
+}
+
+func (aksk SAccessKeyRequestV4) Verify(secret string) error {
+	signingKey := getSigningKey(secret, aksk.Location, aksk.SignDate)
+	signature := getSignature(signingKey, aksk.Request)
+	if signature != aksk.Signature {
+		return errors.Error("signature mismatch")
+	}
+	return nil
+}
+
+func (aksk SAccessKeyRequestV4) Encode() string {
+	return jsonutils.Marshal(aksk).String()
+}

vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go

@@ -197,6 +197,47 @@ func (b *Blob) PutBlockWithLength(blockID string, size uint64, blob io.Reader, o
 	return b.respondCreation(resp, BlobTypeBlock)
 }
 
+// PutBlockFromURLOptions includes the options for a put block from URL operation
+type PutBlockFromURLOptions struct {
+	PutBlockOptions
+
+	SourceContentMD5   string `header:"x-ms-source-content-md5"`
+	SourceContentCRC64 string `header:"x-ms-source-content-crc64"`
+}
+
+// PutBlockFromURL copy data of exactly specified size from specified URL to
+// the block blob with given ID. It is an alternative to PutBlocks where data
+// comes from a remote URL and the offset and length is known in advance.
+//
+// The API rejects requests with size > 100 MiB (but this limit is not
+// checked by the SDK).
+//
+// See https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url
+func (b *Blob) PutBlockFromURL(blockID string, blobURL string, offset int64, size uint64, options *PutBlockFromURLOptions) error {
+	query := url.Values{
+		"comp":    {"block"},
+		"blockid": {blockID},
+	}
+	headers := b.Container.bsc.client.getStandardHeaders()
+	// The value of this header must be set to zero.
+	// When the length is not zero, the operation will fail with the status code 400 (Bad Request).
+	headers["Content-Length"] = "0"
+	headers["x-ms-copy-source"] = blobURL
+	headers["x-ms-copy-source-range"] = fmt.Sprintf("bytes=%d-%d", offset, uint64(offset)+size-1)
+
+	if options != nil {
+		query = addTimeout(query, options.Timeout)
+		headers = mergeHeaders(headers, headersFromStruct(*options))
+	}
+	uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), query)
+
+	resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth)
+	if err != nil {
+		return err
+	}
+	return b.respondCreation(resp, BlobTypeBlock)
+}
+
 // PutBlockListOptions includes the options for a put block list operation
 type PutBlockListOptions struct {
 	Timeout           uint

vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"time"
 )
 
 // InitiateMultipartUploadOptions is the option of InitateMultipartUpload
@@ -189,3 +190,48 @@ func (s *ObjectService) AbortMultipartUpload(ctx context.Context, name, uploadID
 	resp, err := s.client.send(ctx, &sendOpt)
 	return resp, err
 }
+
+// ObjectCopyPartOptions is the options of copy-part
+type ObjectCopyPartOptions struct {
+	XCosCopySource                  string `header:"x-cos-copy-source" url:"-"`
+	XCosCopySourceRange             string `header:"x-cos-copy-source-range" url:"-"`
+	XCosCopySourceIfModifiedSince   string `header:"x-cos-copy-source-If-Modified-Since" url:"-"`
+	XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since" url:"-"`
+	XCosCopySourceIfMatch           string `help:"x-cos-copy-source-If-Match" url:"-"`
+	XCosCopySourceIfNoneMatch       string `help:"x-cos-copy-source-If-None-Match" url:"-"`
+}
+
+// CopyPartResult is the result CopyPart
+type CopyPartResult struct {
+	XMLName      xml.Name `xml:"CopyPartResult"`
+	ETag         string
+	LastModified time.Time
+}
+
+// CopyPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。
+// 在每次请求Upload Part时候，需要携带partNumber和uploadID，partNumber为块的编号，支持乱序上传。
+// ObjectCopyPartOptions的XCosCopySource为必填参数，格式为<bucket-name>-<app-id>.cos.<region-id>.myqcloud.com/<object-key>
+// ObjectCopyPartOptions的XCosCopySourceRange指定源的Range，格式为bytes=<start>-<end>
+//
+// 当传入uploadID和partNumber都相同的时候，后传入的块将覆盖之前传入的块。当uploadID不存在时会返回404错误，NoSuchUpload.
+//
+// https://www.qcloud.com/document/product/436/7750
+func (s *ObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, opt *ObjectCopyPartOptions) (*CopyPartResult, *Response, error) {
+	u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID)
+	var res CopyPartResult
+	sendOpt := sendOptions{
+		baseURL:   s.client.BaseURL.BucketURL,
+		uri:       u,
+		method:    http.MethodPut,
+		optHeader: opt,
+		result:    &res,
+	}
+	resp, err := s.client.send(ctx, &sendOpt)
+	// If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error.
+	if err == nil && resp.StatusCode == 200 {
+		if res.ETag == "" {
+			return &res, resp, errors.New("response 200 OK, but body contains an error")
+		}
+	}
+	return &res, resp, err
+}

vendor/modules.txt

@@ -777,6 +777,7 @@ yunion.io/x/pkg/util/timeutils
 yunion.io/x/pkg/util/sets
 yunion.io/x/pkg/errors
 yunion.io/x/pkg/trace
+yunion.io/x/pkg/gotypes
 yunion.io/x/pkg/util/errors
 yunion.io/x/pkg/util/netutils
 yunion.io/x/pkg/util/seclib
@@ -785,7 +786,6 @@ yunion.io/x/pkg/tristate
 yunion.io/x/pkg/util/stringutils
 yunion.io/x/pkg/util/fileutils
 yunion.io/x/pkg/util/osprofile
-yunion.io/x/pkg/gotypes
 yunion.io/x/pkg/util/filterclause
 yunion.io/x/pkg/util/reflectutils
 yunion.io/x/pkg/util/secrules
@@ -798,9 +798,9 @@ yunion.io/x/pkg/util/prometheus
 yunion.io/x/pkg/prettytable
 yunion.io/x/pkg/util/runtime
 yunion.io/x/pkg/util/clock
-# yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e
+# yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba
+# yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f
 yunion.io/x/sqlchemy
 # yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
 yunion.io/x/structarg

vendor/yunion.io/x/s3cli/api-compose-object.go

@@ -247,7 +247,7 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck
 		return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject)
 	}
 
-	cpObjRes := copyObjectResult{}
+	cpObjRes := CopyObjectResult{}
 	err = xmlDecoder(resp.Body, &cpObjRes)
 	if err != nil {
 		return ObjectInfo{}, err
@@ -261,7 +261,7 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck
 	return objInfo, nil
 }
 
-func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string,
+func (c Client) CopyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string,
 	partID int, startOffset int64, length int64, metadata map[string]string) (p CompletePart, err error) {
 
 	headers := make(http.Header)
@@ -302,7 +302,7 @@ func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, dest
 	}
 
 	// Decode copy-part response on success.
-	cpObjRes := copyObjectResult{}
+	cpObjRes := CopyObjectResult{}
 	err = xmlDecoder(resp.Body, &cpObjRes)
 	if err != nil {
 		return p, err
@@ -340,7 +340,7 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str
 	}
 
 	// Decode copy-part response on success.
-	cpObjRes := copyObjectResult{}
+	cpObjRes := CopyObjectResult{}
 	err = xmlDecoder(resp.Body, &cpObjRes)
 	if err != nil {
 		return p, err

vendor/yunion.io/x/s3cli/api-list.go

@@ -28,6 +28,16 @@ import (
 	"github.com/minio/minio-go/v6/pkg/s3utils"
 )
 
+type ListBucketsInput struct {
+}
+
+type ListObjectInput struct {
+	Prefix    string
+	Marker    string
+	Delimiter string
+	MaxKeys   int64
+}
+
 // ListBuckets list all buckets owned by this authenticated user.
 //
 // This call requires explicit authentication, no anonymous requests are
@@ -307,7 +317,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
 		var marker string
 		for {
 			// Get list of objects a maximum of 1000 per request.
-			result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000)
+			result, err := c.ListObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000)
 			if err != nil {
 				objectStatCh <- ObjectInfo{
 					Err: err,
@@ -366,7 +376,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don
 // ?delimiter - A delimiter is a character you use to group keys.
 // ?prefix - Limits the response to keys that begin with the specified prefix.
 // ?max-keys - Sets the maximum number of keys returned in the response body.
-func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) {
+func (c Client) ListObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) {
 	// Validate bucket name.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {
 		return ListBucketResult{}, err

vendor/yunion.io/x/s3cli/api-put-bucket.go

@@ -70,7 +70,7 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) {
 
 	// If location is not 'us-east-1' create bucket location config.
 	if location != "us-east-1" && location != "" {
-		createBucketConfig := createBucketConfiguration{}
+		createBucketConfig := CreateBucketConfiguration{}
 		createBucketConfig.Location = location
 		var createBucketConfigBytes []byte
 		createBucketConfigBytes, err = xml.Marshal(createBucketConfig)

vendor/yunion.io/x/s3cli/api-s3-datatypes.go

@@ -103,6 +103,15 @@ type ListBucketResult struct {
 	Prefix     string
 }
 
+type ListMultipartUploadsInput struct {
+	Delimiter      string
+	MaxUploads     int64
+	KeyMarker      string
+	Prefix         string
+	UploadIdMarker string
+	EncodingType   string
+}
+
 // ListMultipartUploadsResult container for ListMultipartUploads response
 type ListMultipartUploadsResult struct {
 	Bucket             string
@@ -127,7 +136,13 @@ type initiator struct {
 }
 
 // copyObjectResult container for copy object response.
-type copyObjectResult struct {
+type CopyObjectResult struct {
+	ETag         string
+	LastModified time.Time // time string format "2006-01-02T15:04:05.000Z"
+}
+
+// copyPartResult container for copy part response
+type CopyPartResult struct {
 	ETag         string
 	LastModified time.Time // time string format "2006-01-02T15:04:05.000Z"
 }
@@ -207,11 +222,14 @@ type CompleteMultipartUpload struct {
 }
 
 // createBucketConfiguration container for bucket configuration.
-type createBucketConfiguration struct {
+type CreateBucketConfiguration struct {
 	XMLName  xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"`
 	Location string   `xml:"LocationConstraint"`
 }
 
+// LocationConstraint
+type LocationConstraint string
+
 // deleteObject container for Delete element in MultiObjects Delete XML request
 type deleteObject struct {
 	Key       string
@@ -247,3 +265,9 @@ type deleteMultiObjectsResult struct {
 	DeletedObjects   []deletedObject    `xml:"Deleted"`
 	UnDeletedObjects []nonDeletedObject `xml:"Error"`
 }
+
+type VersioningConfiguration struct {
+	XMLName   xml.Name `xmlns:"http://s3.amazonaws.com/doc/2006-03-01/" xml:"VersioningConfiguration"`
+	Status    string   `xml:"Status,omitempty"`
+	MfaDelete string   `xml:"MfaDelete,omitempty"`
+}

vendor/yunion.io/x/s3cli/tags.go

@@ -0,0 +1,10 @@
+package s3cli
+
+type Tag struct {
+	Key   string
+	Value string
+}
+
+type Tagging struct {
+	TagSet []Tag
+}

vendor/yunion.io/x/sqlchemy/column.go

@@ -22,7 +22,6 @@ import (
 	"strings"
 	"time"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/regutils"
@@ -310,8 +309,8 @@ func (c *SBooleanColumn) IsZero(val interface{}) bool {
 func NewBooleanColumn(name string, tagmap map[string]string, isPointer bool) SBooleanColumn {
 	bc := SBooleanColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "TINYINT", tagmap, isPointer)}
 	if !bc.IsPointer() && len(bc.Default()) > 0 && bc.ConvertFromString(bc.Default()) == "1" {
-		log.Warningf("Non-pointer boolean type should not set default value: %s(%s)", name, tagmap)
-		// bc.defaultString = ""
+		msg := fmt.Sprintf("Non-pointer boolean column should not default true: %s(%s)", name, tagmap)
+		panic(msg)
 	}
 	return bc
 }