diff --git a/cmd/climc/shell/buckets.go b/cmd/climc/shell/buckets.go index 5e59008bc8..8771780e02 100644 --- a/cmd/climc/shell/buckets.go +++ b/cmd/climc/shell/buckets.go @@ -280,4 +280,16 @@ func init() { printObject(result) return nil }) + + type BucketAccessInfoOptions struct { + ID string `help:"ID or name of bucket" json:"-"` + } + R(&BucketAccessInfoOptions{}, "bucket-access-info", "Show backend access info of a bucket", func(s *mcclient.ClientSession, args *BucketAccessInfoOptions) error { + result, err := modules.Buckets.GetSpecific(s, args.ID, "access-info", nil) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/cmd/climc/shell/credentials.go b/cmd/climc/shell/credentials.go index 39724e0ad9..0006eda478 100644 --- a/cmd/climc/shell/credentials.go +++ b/cmd/climc/shell/credentials.go @@ -19,13 +19,15 @@ import ( "yunion.io/x/jsonutils" + "time" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" ) func init() { type CredentialListOptions struct { - Type string `help:"credential type" choices:"totp|recovery|ec2"` + Scope string `help:"scope" choices:"project|domain|system"` + Type string `help:"credential type" choices:"totp|recovery|aksk"` User string `help:"filter by user"` UserDomain string `help:"the domain of user"` } @@ -34,22 +36,14 @@ func init() { if len(args.Type) > 0 { query.Add(jsonutils.NewString(args.Type), "type") } - var err error + if len(args.Scope) > 0 { + query.Add(jsonutils.NewString(args.Scope), "scope") + } if len(args.User) > 0 { - domainId := "default" if len(args.UserDomain) > 0 { - domainId, err = modules.Domains.GetId(s, args.UserDomain, nil) - if err != nil { - return err - } + query.Add(jsonutils.NewString(args.UserDomain), "domain_id") } - userQuery := jsonutils.NewDict() - userQuery.Add(jsonutils.NewString(domainId), "domain_id") - userId, err := modules.UsersV3.GetId(s, args.User, userQuery) - if err != nil { - return err - } - query.Add(jsonutils.NewString(userId), "user_id") + query.Add(jsonutils.NewString(args.User), "user_id") } results, err := modules.Credentials.List(s, query) if err != nil { @@ -159,4 +153,122 @@ func init() { fmt.Println("success") return nil }) + + type CredentialAkSkOptions struct { + User string `help:"User"` + UserDomain string `help:"domain of user"` + Project string `help:"Project"` + ProjectDomain string `help:"domain of user"` + } + R(&CredentialAkSkOptions{}, "credential-create-aksk", "Create AccessKey/Secret credential", func(s *mcclient.ClientSession, args *CredentialAkSkOptions) error { + var uid string + var pid string + var err error + if len(args.User) > 0 { + uid, err = modules.UsersV3.FetchId(s, args.User, args.UserDomain) + if err != nil { + return err + } + } + if len(args.Project) > 0 { + pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain) + if err != nil { + return err + } + } + secret, err := modules.Credentials.CreateAccessKeySecret(s, uid, pid, time.Time{}) + if err != nil { + return err + } + printObject(jsonutils.Marshal(&secret)) + return nil + }) + + R(&CredentialAkSkOptions{}, "credential-get-aksk", "Get AccessKey/Secret credential for user and project", func(s *mcclient.ClientSession, args *CredentialAkSkOptions) error { + uid, err := modules.UsersV3.FetchId(s, args.User, args.UserDomain) + if err != nil { + return err + } + var pid string + if len(args.Project) > 0 { + pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain) + if err != nil { + return err + } + } + secrets, err := modules.Credentials.GetAccessKeySecrets(s, uid, pid) + if err != nil { + return err + } + result := modules.ListResult{} + result.Data = make([]jsonutils.JSONObject, len(secrets)) + for i := range secrets { + result.Data[i] = jsonutils.Marshal(secrets[i]) + result.Data[i].(*jsonutils.JSONDict).Add(jsonutils.NewString(secrets[i].KeyId), "key_id") + result.Data[i].(*jsonutils.JSONDict).Add(jsonutils.NewString(secrets[i].ProjectId), "project_id") + result.Data[i].(*jsonutils.JSONDict).Add(jsonutils.NewTimeString(secrets[i].TimeStamp), "time_stamp") + } + printList(&result, nil) + return nil + }) + + R(&CredentialAkSkOptions{}, "credential-remove-aksk", "Remove AccessKey/Secret credential for user and project", func(s *mcclient.ClientSession, args *CredentialAkSkOptions) error { + uid, err := modules.UsersV3.FetchId(s, args.User, args.UserDomain) + if err != nil { + return err + } + var pid string + if len(args.Project) > 0 { + pid, err = modules.Projects.FetchId(s, args.Project, args.ProjectDomain) + if err != nil { + return err + } + } + err = modules.Credentials.RemoveAccessKeySecrets(s, uid, pid) + if err != nil { + return err + } + fmt.Println("success") + return nil + }) + + type CredentialDeleteOptions struct { + ID string `help:"ID of credentail"` + } + R(&CredentialDeleteOptions{}, "credential-delete", "Delete credential", func(s *mcclient.ClientSession, args *CredentialDeleteOptions) error { + result, err := modules.Credentials.Delete(s, args.ID, nil) + if err != nil { + return err + } + printObject(result) + return nil + }) + + type CredentialUpdateOptions struct { + ID string `help:"ID of credentail"` + Enable bool `help:"Enable credential"` + Disable bool `help:"Disable credential"` + Name string `help:"new name of credential"` + Desc string `help:"new description of credential"` + } + R(&CredentialUpdateOptions{}, "credential-update", "Enable/disable credential", func(s *mcclient.ClientSession, args *CredentialUpdateOptions) error { + params := jsonutils.NewDict() + if args.Enable { + params.Add(jsonutils.JSONTrue, "enabled") + } else if args.Disable { + params.Add(jsonutils.JSONFalse, "enabled") + } + if args.Name != "" { + params.Add(jsonutils.NewString(args.Name), "name") + } + if args.Desc != "" { + params.Add(jsonutils.NewString(args.Desc), "description") + } + result, err := modules.Credentials.Update(s, args.ID, params) + if err != nil { + return err + } + printObject(result) + return nil + }) } diff --git a/docs/bucket/access.yaml b/docs/bucket/access.yaml new file mode 100644 index 0000000000..8e1976ed8d --- /dev/null +++ b/docs/bucket/access.yaml @@ -0,0 +1,11 @@ +get: + summary: 获取存储桶的后端访问信息，例如ceph RADOS的endpint/ak/sk信息等 + parameters: + - $ref: '../parameters/bucket.yaml#/bucket_name' + responses: + 200: + description: 指定存储桶访问信息 + schema: + $ref: "../schemas/bucket.yaml#/BucketAccessInfoResponse" + tags: + - buckets diff --git a/docs/index.yaml b/docs/index.yaml index 26e6421421..9476555378 100644 --- a/docs/index.yaml +++ b/docs/index.yaml @@ -461,6 +461,8 @@ paths: $ref: "./bucket/acl.yaml" /buckets/{bucketName}/limit: $ref: "./bucket/limit.yaml" + /buckets/{bucketName}/access-info: + $ref: "./bucket/access.yaml" /ansibleplaybooks: $ref: "./ansibleplaybook/ansibleplaybooks.yaml" diff --git a/docs/schemas/bucket.yaml b/docs/schemas/bucket.yaml index 586f2c236a..c2319d55cf 100644 --- a/docs/schemas/bucket.yaml +++ b/docs/schemas/bucket.yaml @@ -226,3 +226,10 @@ BucketSetLimitInput: object_count: type: integer description: 对象数量限制，为0则无限制 + +BucketAccessInfoResponse: + type: object + properties: + bucket: + type: object + description: 桶的后端访问信息，例如ceph RADOS的endpoint/access_key/secret信息等。具体形式由存储后端决定。 diff --git a/go.mod b/go.mod index 767b3a2e7a..f3cff7e7d9 100644 --- a/go.mod +++ b/go.mod @@ -152,7 +152,7 @@ require ( yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224 - yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e - yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba + yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd + yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 ) diff --git a/go.sum b/go.sum index 08d24a0f2d..1d0518c3a4 100644 --- a/go.sum +++ b/go.sum @@ -585,9 +585,9 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRi yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224 h1:dAeUov/CtKcPaOapGVG7+NRqmUF2fr2O3Onb+ID5HS4= yunion.io/x/pkg v0.0.0-20190726033806-b564cfdcc224/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e h1:Jk82txl4vaL/MoVV3+GweEcCmLOcqo5XYF8tlBD/1hY= -yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba h1:vPCRA59Kq9Hv24ef/G92sCgqmBYmZyLSVP902EUDimw= -yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd h1:pwGQ4JDXhuRNjmYI8hQXCs08wppsEgj9+u2udTJJDEo= +yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= +yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f h1:maHGG78d6vLAyT/lGSOOsXWrylBfjdu+NMCGXFhLuhA= +yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 h1:bfC8EhXYvyGYldRWlzxiCM39Zfj3s3+zham9mW2h2LE= yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/identity/aksk.go b/pkg/apis/identity/aksk.go new file mode 100644 index 0000000000..23d6e70044 --- /dev/null +++ b/pkg/apis/identity/aksk.go @@ -0,0 +1,42 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package identity + +import ( + "time" +) + +const ( + ACCESS_SECRET_TYPE = "aksk" + TOTP_TYPE = "totp" + RECOVERY_SECRETS_TYPE = "recovery_secret" +) + +type SAccessKeySecretBlob struct { + Secret string `json:"secret"` + Expire int64 `json:"expire"` +} + +func (info SAccessKeySecretBlob) IsValid() bool { + if info.Expire <= 0 || info.Expire > time.Now().Unix() { + return true + } + return false +} + +type SAccessKeySecretInfo struct { + AccessKey string + SAccessKeySecretBlob +} diff --git a/pkg/apis/identity/consts.go b/pkg/apis/identity/consts.go index 36469e9436..fa17635da9 100644 --- a/pkg/apis/identity/consts.go +++ b/pkg/apis/identity/consts.go @@ -30,9 +30,10 @@ const ( AUTH_METHOD_PASSWORD = "password" AUTH_METHOD_TOKEN = "token" + AUTH_METHOD_AKSK = "aksk" - AUTH_METHOD_ID_PASSWORD = 1 - AUTH_METHOD_ID_TOKEN = 2 + // AUTH_METHOD_ID_PASSWORD = 1 + // AUTH_METHOD_ID_TOKEN = 2 AUTH_TOKEN_HEADER = "X-Auth-Token" AUTH_SUBJECT_TOKEN_HEADER = "X-Subject-Token" @@ -73,7 +74,7 @@ const ( ) var ( - AUTH_METHODS = []string{AUTH_METHOD_PASSWORD, AUTH_METHOD_TOKEN} + AUTH_METHODS = []string{AUTH_METHOD_PASSWORD, AUTH_METHOD_TOKEN, AUTH_METHOD_AKSK} SensitiveDomainConfigMap = map[string]string{ "ldap": "password", diff --git a/pkg/appctx/context.go b/pkg/appctx/context.go index 3297398051..3cfa1e10f9 100644 --- a/pkg/appctx/context.go +++ b/pkg/appctx/context.go @@ -39,6 +39,8 @@ const ( APP_CONTEXT_KEY_OBJECT_ID = AppContextKey("objectid") APP_CONTEXT_KEY_OBJECT_TYPE = AppContextKey("objecttype") APP_CONTEXT_KEY_START_TIME = AppContextKey("starttime") + + APP_CONTEXT_KEY_HOST_ID = AppContextKey("hostid") ) func AppContextServiceName(ctx context.Context) string { @@ -149,6 +151,15 @@ func AppContextStartTime(ctx context.Context) time.Time { } } +func AppContextHostId(ctx context.Context) string { + val := ctx.Value(APP_CONTEXT_KEY_HOST_ID) + if val != nil { + return val.(string) + } else { + return "" + } +} + type AppContextData struct { Trace trace.STrace RequestId string diff --git a/pkg/appsrv/appsrv.go b/pkg/appsrv/appsrv.go index 5848cd9d6f..74c27495be 100644 --- a/pkg/appsrv/appsrv.go +++ b/pkg/appsrv/appsrv.go @@ -17,6 +17,8 @@ package appsrv import ( "bufio" "context" + "crypto/sha1" + "encoding/base64" "fmt" "math/rand" "net" @@ -57,6 +59,7 @@ type Application struct { defHandlerInfo SHandlerInfo cors *Cors middlewares []MiddlewareFunc + hostId string isExiting bool idleConnsClosed chan struct{} @@ -91,6 +94,18 @@ func NewApplication(name string, connMax int, db bool) *Application { app.SetContext(appctx.APP_CONTEXT_KEY_APP, &app) app.SetContext(appctx.APP_CONTEXT_KEY_APPNAME, app.name) + hm := sha1.New() + hm.Write([]byte(name)) + hostname, _ := os.Hostname() + hm.Write([]byte(hostname)) + outIp := utils.GetOutboundIP() + hm.Write([]byte(outIp.String())) + hostId := base64.URLEncoding.EncodeToString(hm.Sum(nil)) + + log.Infof("App hostId: %s (%s,%s,%s)", hostId, name, hostname, outIp.String()) + app.hostId = hostId + app.SetContext(appctx.APP_CONTEXT_KEY_HOST_ID, hostId) + // initialize random seed rand.Seed(time.Now().UnixNano()) @@ -168,6 +183,7 @@ func (lrw *loggingResponseWriter) Hijack() (rwc net.Conn, buf *bufio.ReadWriter, } func (lrw *loggingResponseWriter) WriteHeader(code int) { + log.Debugf("XXXX loggingResponseWriter WriteHeader %d", code) if code < 100 || code >= 600 { log.Errorf("Invalud status code %d, set code to 598", code) code = 598 @@ -190,6 +206,7 @@ func genRequestId(w http.ResponseWriter, r *http.Request) string { func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) { // log.Printf("defaultHandler %s %s", r.Method, r.URL.Path) rid := genRequestId(w, r) + w.Header().Set("X-Request-Host-Id", app.hostId) lrw := &loggingResponseWriter{w, http.StatusOK} start := time.Now() hi, params := app.defaultHandle(lrw, r, rid) @@ -216,7 +233,7 @@ func (app *Application) ServeHTTP(w http.ResponseWriter, r *http.Request) { skipLog = true } if !skipLog { - log.Infof("%d %s %s %s (%s) %.2fms", lrw.status, rid, r.Method, r.URL, r.RemoteAddr, duration) + log.Infof("%s %d %s %s %s (%s) %.2fms", app.hostId, lrw.status, rid, r.Method, r.URL, r.RemoteAddr, duration) } } diff --git a/pkg/appsrv/fetch.go b/pkg/appsrv/fetch.go index e98c7cd2b2..fc45d11fcd 100644 --- a/pkg/appsrv/fetch.go +++ b/pkg/appsrv/fetch.go @@ -18,6 +18,8 @@ import ( "io/ioutil" "net/http" + "encoding/xml" + "github.com/pkg/errors" "yunion.io/x/jsonutils" ) @@ -53,3 +55,15 @@ func FetchJSON(req *http.Request) (jsonutils.JSONObject, error) { return nil, nil } } + +func FetchXml(req *http.Request, target interface{}) error { + b, e := Fetch(req) + if e != nil { + return errors.Wrap(e, "Fetch") + } + if len(b) > 0 { + return xml.Unmarshal(b, target) + } else { + return nil + } +} diff --git a/pkg/appsrv/send.go b/pkg/appsrv/send.go index a6880245f7..3ee66b8695 100644 --- a/pkg/appsrv/send.go +++ b/pkg/appsrv/send.go @@ -15,36 +15,124 @@ package appsrv import ( - "encoding/json" + "encoding/xml" "net/http" + "strconv" + "fmt" + "github.com/pkg/errors" + "io" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" ) +func SendNoContent(w http.ResponseWriter) { + w.WriteHeader(204) + sendBytes(w, []byte{}) +} + func Send(w http.ResponseWriter, text string) { w.Header().Set("Content-Type", "text/plain") - w.Write([]byte(text)) + sendBytes(w, []byte(text)) +} + +func sendBytes(w http.ResponseWriter, output []byte) { + w.Header().Set("Content-Length", strconv.FormatInt(int64(len(output)), 10)) + w.Write(output) } func SendStruct(w http.ResponseWriter, obj interface{}) { - w.Header().Set("Content-Type", "application/json") - if obj != nil { - b, e := json.Marshal(obj) - if e != nil { - log.Errorln("SendStruct json Marshal error: ", e) - } - w.Write(b) - } else { - w.Write([]byte{}) - } + jsonObj := jsonutils.Marshal(obj) + SendJSON(w, jsonObj) } func SendJSON(w http.ResponseWriter, obj jsonutils.JSONObject) { + var output []byte w.Header().Set("Content-Type", "application/json") if obj != nil { - w.Write([]byte(obj.String())) + output = []byte(obj.String()) + } + sendBytes(w, output) +} + +func SendHeader(w http.ResponseWriter, hdr http.Header) { + w.WriteHeader(204) + for k, v := range hdr { + if len(v) > 0 && len(v[0]) > 0 { + w.Header().Set(k, v[0]) + } + } + w.Write([]byte{}) +} + +func SendXml(w http.ResponseWriter, hdr http.Header, obj interface{}) { + if !gotypes.IsNil(obj) { + xmlBytes, err := xml.Marshal(obj) + if err == nil { + for k, v := range hdr { + if k != "Content-Type" && k != "Content-Length" { + w.Header().Set(k, v[0]) + } + } + w.Header().Set("Content-Type", "application/xml") + w.Header().Set("Content-Length", strconv.FormatInt(int64(len(xmlBytes)+len(xml.Header)), 10)) + w.Write([]byte(xml.Header)) + w.Write(xmlBytes) + } else { + w.WriteHeader(400) + Send(w, err.Error()) + } } else { - w.Write([]byte{}) + for k, v := range hdr { + if k != "Content-Type" && k != "Content-Length" { + w.Header().Set(k, v[0]) + } + } + SendNoContent(w) } } + +func SendStream(w http.ResponseWriter, isPartial bool, hdr http.Header, stream io.ReadCloser, sizeBytes int64) error { + defer stream.Close() + if isPartial { + log.Debugf("send partial 206") + w.WriteHeader(206) + } else { + log.Debugf("send full 200") + w.WriteHeader(200) + } + for k, v := range hdr { + if k != "Content-Length" { + log.Debugf("send %s %s", k, v) + w.Header().Set(k, v[0]) + } + } + if sizeBytes > 0 { + log.Debugf("send content-length %d", sizeBytes) + w.Header().Set("Content-Length", strconv.FormatInt(sizeBytes, 10)) + } + offset := 0 + buf := make([]byte, 4096) + for sizeBytes <= 0 || int64(offset) < sizeBytes { + n, err := stream.Read(buf) + if n > 0 { + woff := 0 + for woff < n { + m, err := w.Write(buf[woff:n]) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("w.Write read_offset %d write_offset %d", offset, woff)) + } + woff += m + } + offset += n + } + if err != nil { + if err == io.EOF { + break + } + return errors.Wrap(err, "stream.Read") + } + } + return nil +} diff --git a/pkg/cloudcommon/policy/defaults.go b/pkg/cloudcommon/policy/defaults.go index 84b07d77d9..7223c7c9f3 100644 --- a/pkg/cloudcommon/policy/defaults.go +++ b/pkg/cloudcommon/policy/defaults.go @@ -204,6 +204,36 @@ var ( Action: PolicyActionDelete, Result: rbacutils.Allow, }, + { + Service: "identity", + Resource: "credentials", + Action: PolicyActionGet, + Result: rbacutils.Allow, + }, + { + Service: "identity", + Resource: "credentials", + Action: PolicyActionList, + Result: rbacutils.Allow, + }, + { + Service: "identity", + Resource: "credentials", + Action: PolicyActionCreate, + Result: rbacutils.Allow, + }, + { + Service: "identity", + Resource: "credentials", + Action: PolicyActionUpdate, + Result: rbacutils.Allow, + }, + { + Service: "identity", + Resource: "credentials", + Action: PolicyActionDelete, + Result: rbacutils.Allow, + }, { Service: "yunionconf", Resource: "parameters", diff --git a/pkg/cloudprovider/consts.go b/pkg/cloudprovider/consts.go index 86c283f2fa..8c29405cb4 100644 --- a/pkg/cloudprovider/consts.go +++ b/pkg/cloudprovider/consts.go @@ -26,6 +26,8 @@ const ( CloudVMStatusDeploying = "deploying" CloudVMStatusOther = "other" + ErrUnauthenticated = errors.Error("not authenticated") + ErrUnauthorized = errors.Error("not authorized") ErrNotFound = errors.Error("id not found") ErrDuplicateId = errors.Error("duplicate id") ErrInvalidStatus = errors.Error("invalid status") @@ -34,4 +36,5 @@ const ( ErrNotSupported = errors.Error("Not supported") ErrInvalidProvider = errors.Error("Invalid provider") ErrNoBalancePermission = errors.Error("No balance permission") + ErrBadRequest = errors.Error("bad request") ) diff --git a/pkg/cloudprovider/objectstore.go b/pkg/cloudprovider/objectstore.go index f240f23bd6..83891defe8 100644 --- a/pkg/cloudprovider/objectstore.go +++ b/pkg/cloudprovider/objectstore.go @@ -17,9 +17,12 @@ package cloudprovider import ( "context" "io" + "regexp" + "strconv" "strings" "time" + "fmt" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/s3cli" @@ -75,6 +78,43 @@ type SListObjectResult struct { IsTruncated bool } +type SGetObjectRange struct { + Start int64 + End int64 +} + +func (r SGetObjectRange) SizeBytes() int64 { + return r.End - r.Start + 1 +} + +var ( + rangeExp = regexp.MustCompile(`(bytes=)?(\d*)-(\d*)`) +) + +func ParseRange(rangeStr string) SGetObjectRange { + objRange := SGetObjectRange{} + if len(rangeStr) > 0 { + find := rangeExp.FindAllStringSubmatch(rangeStr, -1) + if len(find) > 0 && len(find[0]) > 3 { + objRange.Start, _ = strconv.ParseInt(find[0][2], 10, 64) + objRange.End, _ = strconv.ParseInt(find[0][3], 10, 64) + } + } + return objRange +} + +func (r SGetObjectRange) String() string { + if r.Start > 0 && r.End > 0 { + return fmt.Sprintf("bytes=%d-%d", r.Start, r.End) + } else if r.Start > 0 && r.End <= 0 { + return fmt.Sprintf("bytes=%d-", r.Start) + } else if r.Start <= 0 && r.End > 0 { + return fmt.Sprintf("bytes=0-%d", r.End) + } else { + return "" + } +} + type ICloudBucket interface { IVirtualResource @@ -98,12 +138,16 @@ type ICloudBucket interface { ListObjects(prefix string, marker string, delimiter string, maxCount int) (SListObjectResult, error) GetIObjects(prefix string, isRecursive bool) ([]ICloudObject, error) + CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl TBucketACLType, storageClassStr string) error + GetObject(ctx context.Context, key string, rangeOpt *SGetObjectRange) (io.ReadCloser, error) + DeleteObject(ctx context.Context, keys string) error GetTempUrl(method string, key string, expire time.Duration) (string, error) PutObject(ctx context.Context, key string, input io.Reader, sizeBytes int64, contType string, cannedAcl TBucketACLType, storageClassStr string) error NewMultipartUpload(ctx context.Context, key string, contType string, cannedAcl TBucketACLType, storageClassStr string) (string, error) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) + CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucketName string, srcKey string, srcOffset int64, srcLength int64) (string, error) CompleteMultipartUpload(ctx context.Context, key string, uploadId string, partEtags []string) error AbortMultipartUpload(ctx context.Context, key string, uploadId string) error } @@ -350,3 +394,110 @@ func DeletePrefix(ctx context.Context, bucket ICloudBucket, prefix string) error } return nil } + +func CopyObject(ctx context.Context, blocksz int64, dstBucket ICloudBucket, dstKey string, srcBucket ICloudBucket, srcKey string, debug bool) error { + srcObj, err := GetIObject(srcBucket, srcKey) + if err != nil { + return errors.Wrap(err, "GetIObject") + } + if blocksz <= 0 { + blocksz = MAX_PUT_OBJECT_SIZEBYTES + } + sizeBytes := srcObj.GetSizeBytes() + if sizeBytes < blocksz { + if debug { + log.Debugf("too small, copy object in one shot") + } + srcStream, err := srcBucket.GetObject(ctx, srcKey, nil) + if err != nil { + return errors.Wrap(err, "srcBucket.GetObject") + } + defer srcStream.Close() + err = dstBucket.PutObject(ctx, dstKey, srcStream, sizeBytes, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass()) + if err != nil { + return errors.Wrap(err, "dstBucket.PutObject") + } + return nil + } + partSize := blocksz + partCount := sizeBytes / partSize + if partCount*partSize < sizeBytes { + partCount += 1 + } + if partCount > int64(dstBucket.MaxPartCount()) { + partCount = int64(dstBucket.MaxPartCount()) + partSize = sizeBytes / partCount + if partSize*partCount < sizeBytes { + partSize += 1 + } + if partSize > dstBucket.MaxPartSizeBytes() { + return errors.Error("too larget object") + } + } + if debug { + log.Debugf("multipart upload part count %d part size %d", partCount, partSize) + } + uploadId, err := dstBucket.NewMultipartUpload(ctx, dstKey, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass()) + if err != nil { + return errors.Wrap(err, "bucket.NewMultipartUpload") + } + etags := make([]string, partCount) + // offset := int64(0) + for i := 0; i < int(partCount); i += 1 { + start := int64(i) * partSize + if i == int(partCount)-1 { + partSize = sizeBytes - partSize*(partCount-1) + } + end := start + partSize - 1 + rangeOpt := SGetObjectRange{ + Start: start, + End: end, + } + if debug { + log.Debugf("UploadPart %d %d range: %s (%d)", i+1, partSize, rangeOpt.String(), rangeOpt.SizeBytes()) + } + srcStream, err := srcBucket.GetObject(ctx, srcKey, &rangeOpt) + if err == nil { + defer srcStream.Close() + var etag string + etag, err = dstBucket.UploadPart(ctx, dstKey, uploadId, i+1, io.LimitReader(srcStream, partSize), partSize) + if err == nil { + etags[i] = etag + continue + } + } + if err != nil { + err2 := dstBucket.AbortMultipartUpload(ctx, dstKey, uploadId) + if err2 != nil { + log.Errorf("bucket.AbortMultipartUpload error %s", err2) + } + return errors.Wrap(err, "bucket.UploadPart") + } + } + err = dstBucket.CompleteMultipartUpload(ctx, dstKey, uploadId, etags) + if err != nil { + err2 := dstBucket.AbortMultipartUpload(ctx, dstKey, uploadId) + if err2 != nil { + log.Errorf("bucket.AbortMultipartUpload error %s", err2) + } + return errors.Wrap(err, "CompleteMultipartUpload") + } + return nil +} + +func CopyPart(ctx context.Context, + iDstBucket ICloudBucket, dstKey string, uploadId string, partNumber int, + iSrcBucket ICloudBucket, srcKey string, rangeOpt *SGetObjectRange, +) (string, error) { + srcReader, err := iSrcBucket.GetObject(ctx, srcKey, rangeOpt) + if err != nil { + return "", errors.Wrap(err, "iSrcBucket.GetObject") + } + defer srcReader.Close() + + etag, err := iDstBucket.UploadPart(ctx, dstKey, uploadId, partNumber, io.LimitReader(srcReader, rangeOpt.SizeBytes()), rangeOpt.SizeBytes()) + if err != nil { + return "", errors.Wrap(err, "iDstBucket.UploadPart") + } + return etag, nil +} diff --git a/pkg/cloudprovider/objectstore_test.go b/pkg/cloudprovider/objectstore_test.go new file mode 100644 index 0000000000..8d5eafed39 --- /dev/null +++ b/pkg/cloudprovider/objectstore_test.go @@ -0,0 +1,52 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cloudprovider + +import "testing" + +func TestParseRange(t *testing.T) { + cases := []struct { + in string + start int64 + end int64 + }{ + { + in: "bytes=0-200", + start: 0, + end: 200, + }, + { + in: "200-3232300", + start: 200, + end: 3232300, + }, + { + in: "200-", + start: 200, + end: 0, + }, + { + in: "-232323", + start: 0, + end: 232323, + }, + } + for _, c := range cases { + got := ParseRange(c.in) + if got.Start != c.start || got.End != c.end { + t.Fatalf("got.start(%d) != want.start(%d) or got.end(%d) != want.end(%d)", got.Start, c.start, got.End, c.end) + } + } +} diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index 0fd5bfdd10..a07dc50292 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -1060,3 +1060,29 @@ func (bucket *SBucket) PerformLimit( return nil, nil } + +func (bucket *SBucket) AllowGetDetailsAccessInfo( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, +) bool { + return bucket.IsOwner(userCred) +} + +func (bucket *SBucket) GetDetailsAccessInfo( + ctx context.Context, + userCred mcclient.TokenCredential, + query jsonutils.JSONObject, +) (jsonutils.JSONObject, error) { + manager := bucket.GetCloudprovider() + if manager == nil { + return nil, httperrors.NewInternalServerError("missing manager?") + } + info, err := manager.GetDetailsClirc(ctx, userCred, nil) + if err != nil { + return nil, err + } + account := manager.GetCloudaccount() + info.(*jsonutils.JSONDict).Add(jsonutils.NewString(account.Brand), "PROVIDER") + return info, err +} diff --git a/pkg/compute/models/managedresource.go b/pkg/compute/models/managedresource.go index e053379d56..da36bab173 100644 --- a/pkg/compute/models/managedresource.go +++ b/pkg/compute/models/managedresource.go @@ -380,6 +380,7 @@ type SCloudProviderInfo struct { ManagerDomainId string `json:",omitempty"` Region string `json:",omitempty"` RegionId string `json:",omitempty"` + RegionExternalId string `json:",omitempty"` RegionExtId string `json:",omitempty"` Zone string `json:",omitempty"` ZoneId string `json:",omitempty"` @@ -399,6 +400,7 @@ var ( "manager_project_id", "region", "region_id", + "region_external_id", "region_ext_id", "zone", "zone_id", @@ -452,6 +454,7 @@ func MakeCloudProviderInfo(region *SCloudregion, zone *SZone, provider *SCloudpr info.CloudEnv = account.getCloudEnv() if region != nil { + info.RegionExternalId = region.ExternalId info.RegionExtId = fetchExternalId(region.ExternalId) if zone != nil { info.ZoneExtId = fetchExternalId(zone.ExternalId) diff --git a/pkg/compute/tasks/guest_block_io_throttle_task.go b/pkg/compute/tasks/guest_block_io_throttle_task.go index b2328f386f..5bc2f78ae9 100644 --- a/pkg/compute/tasks/guest_block_io_throttle_task.go +++ b/pkg/compute/tasks/guest_block_io_throttle_task.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( diff --git a/pkg/compute/tasks/guest_resize_disk_task.go b/pkg/compute/tasks/guest_resize_disk_task.go index 7de8bda4fa..96c4be0620 100644 --- a/pkg/compute/tasks/guest_resize_disk_task.go +++ b/pkg/compute/tasks/guest_resize_disk_task.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( diff --git a/pkg/compute/tasks/host_guests_migrate_task.go b/pkg/compute/tasks/host_guests_migrate_task.go index e7d0ac7aaa..d28827b27e 100644 --- a/pkg/compute/tasks/host_guests_migrate_task.go +++ b/pkg/compute/tasks/host_guests_migrate_task.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( diff --git a/pkg/compute/tasks/host_maintenance_task.go b/pkg/compute/tasks/host_maintenance_task.go index 993cdde4db..b8ff7c0b6f 100644 --- a/pkg/compute/tasks/host_maintenance_task.go +++ b/pkg/compute/tasks/host_maintenance_task.go @@ -1,3 +1,17 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package tasks import ( diff --git a/pkg/httperrors/consts.go b/pkg/httperrors/consts.go new file mode 100644 index 0000000000..e9d2ef95f8 --- /dev/null +++ b/pkg/httperrors/consts.go @@ -0,0 +1,34 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package httperrors + +import ( + "yunion.io/x/pkg/errors" +) + +const ( + ErrUnauthenticated = errors.Error("not authenticated") + ErrUnauthorized = errors.Error("not authorized") + ErrNotFound = errors.Error("id not found") + ErrDuplicateId = errors.Error("duplicate id") + ErrInvalidStatus = errors.Error("invalid status") + ErrTimeout = errors.Error("timeout") + ErrNotImplemented = errors.Error("Not implemented") + ErrNotSupported = errors.Error("Not supported") + ErrBadRequest = errors.Error("bad request") + ErrOutOfRange = errors.Error("out of range") + ErrForbidden = errors.Error("not allowed") + ErrOutOfLimit = errors.Error("out of limit") +) diff --git a/pkg/keystone/models/credentials.go b/pkg/keystone/models/credentials.go index 1735d48f97..085a54a328 100644 --- a/pkg/keystone/models/credentials.go +++ b/pkg/keystone/models/credentials.go @@ -16,15 +16,21 @@ package models import ( "context" + "database/sql" "fmt" "time" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" + "yunion.io/x/sqlchemy" + api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/keystone/keys" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/rbacutils" ) type SCredentialManager struct { @@ -62,14 +68,16 @@ func init() { type SCredential struct { db.SStandaloneResourceBase - UserId string `width:"64" charset:"ascii" nullable:"false" list:"admin" create:"admin_required"` - ProjectId string `width:"64" charset:"ascii" nullable:"true" list:"admin" create:"admin_required"` - Type string `width:"255" charset:"utf8" nullable:"false" list:"admin" create:"admin_required"` - KeyHash string `width:"64" charset:"ascii" nullable:"false" create:"admin_required"` + UserId string `width:"64" charset:"ascii" nullable:"false" list:"user" create:"required"` + ProjectId string `width:"64" charset:"ascii" nullable:"true" list:"user" create:"required"` + Type string `width:"255" charset:"utf8" nullable:"false" list:"user" create:"required"` + KeyHash string `width:"64" charset:"ascii" nullable:"false" create:"required"` Extra *jsonutils.JSONDict `nullable:"true" list:"admin"` - EncryptedBlob string `nullable:"false" create:"admin_required"` + EncryptedBlob string `nullable:"false" create:"required"` + + Enabled tristate.TriState `nullable:"false" default:"true" list:"user" update:"user" create:"optional"` } func (manager *SCredentialManager) InitializeData() error { @@ -99,10 +107,36 @@ func (manager *SCredentialManager) ValidateCreateData(ctx context.Context, userC if !data.Contains("type") { return nil, httperrors.NewInputParameterError("missing input feild type") } + userId, _ := data.GetString("user_id") + projectId, _ := data.GetString("project_id") + if len(userId) == 0 { + userId = userCred.GetUserId() + data.Set("user_id", jsonutils.NewString(userId)) + } else { + _, err := UserManager.FetchById(userId) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(UserManager.Keyword(), userId) + } else { + return nil, httperrors.NewGeneralError(err) + } + } + } + if len(projectId) == 0 { + projectId = userCred.GetProjectId() + data.Set("project_id", jsonutils.NewString(projectId)) + } else { + _, err := ProjectManager.FetchById(projectId) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2(ProjectManager.Keyword(), projectId) + } else { + return nil, httperrors.NewGeneralError(err) + } + } + } if !data.Contains("name") { typeStr, _ := data.GetString("type") - userId, _ := data.GetString("user_id") - projectId, _ := data.GetString("project_id") data.Add(jsonutils.NewString(fmt.Sprintf("%s-%s-%s", typeStr, projectId, userId)), "name") } blob, _ := data.GetString("blob") @@ -124,6 +158,7 @@ func (self *SCredential) ValidateDeleteCondition(ctx context.Context) error { } func (self *SCredential) ValidateUpdateData(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { + return self.SStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data) } @@ -141,8 +176,7 @@ func (self *SCredential) GetExtraDetails(ctx context.Context, userCred mcclient. } func credentialExtra(cred *SCredential, extra *jsonutils.JSONDict) *jsonutils.JSONDict { - blob := keys.CredentialKeyManager.Decrypt([]byte(cred.EncryptedBlob), time.Duration(-1)) - extra.Add(jsonutils.NewString(string(blob)), "blob") + extra.Add(jsonutils.NewString(string(cred.getBlob())), "blob") usr, _ := UserManager.FetchUserExtended(cred.UserId, "", "", "") if usr != nil { @@ -152,3 +186,73 @@ func credentialExtra(cred *SCredential, extra *jsonutils.JSONDict) *jsonutils.JS } return extra } + +func (self *SCredential) getBlob() []byte { + return keys.CredentialKeyManager.Decrypt([]byte(self.EncryptedBlob), time.Duration(-1)) +} + +func (self *SCredential) GetAccessKeySecret() (*api.SAccessKeySecretBlob, error) { + if self.Type == api.ACCESS_SECRET_TYPE { + blobJson, err := jsonutils.Parse(self.getBlob()) + if err != nil { + return nil, errors.Wrap(err, "jsonutils.Parse") + } + akBlob := api.SAccessKeySecretBlob{} + err = blobJson.Unmarshal(&akBlob) + if err != nil { + return nil, errors.Wrap(err, "blobJson.Unmarshal") + } + return &akBlob, nil + } + return nil, errors.Error("no an AK/SK credential") +} + +func (manager *SCredentialManager) ResourceScope() rbacutils.TRbacScope { + return rbacutils.ScopeUser +} + +func (manager *SCredentialManager) FilterByOwner(q *sqlchemy.SQuery, owner mcclient.IIdentityProvider, scope rbacutils.TRbacScope) *sqlchemy.SQuery { + if owner != nil { + if scope == rbacutils.ScopeUser { + if len(owner.GetUserId()) > 0 { + q = q.Equals("user_id", owner.GetUserId()) + } + } + } + return q +} + +func (self *SCredential) GetOwnerId() mcclient.IIdentityProvider { + owner := db.SOwnerId{UserId: self.UserId} + return &owner +} + +func (manager *SCredentialManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) { + userStr, key := jsonutils.GetAnyString2(data, []string{"user", "user_id"}) + if len(userStr) > 0 { + domainOwner, err := fetchDomainInfo(data) + if err != nil { + return nil, err + } + if domainOwner == nil { + domainOwner = &db.SOwnerId{DomainId: api.DEFAULT_DOMAIN_ID} + } + data.(*jsonutils.JSONDict).Remove(key) + usrObj, err := UserManager.FetchByIdOrName(domainOwner, userStr) + if err != nil { + if err == sql.ErrNoRows { + return nil, httperrors.NewResourceNotFoundError2("user", userStr) + } else { + return nil, httperrors.NewGeneralError(err) + } + } + usr := usrObj.(*SUser) + ownerId := db.SOwnerId{ + UserDomainId: usr.DomainId, + UserId: usr.Id, + } + data.(*jsonutils.JSONDict).Set("user", jsonutils.NewString(usr.Id)) + return &ownerId, nil + } + return nil, nil +} diff --git a/pkg/keystone/models/identitybase.go b/pkg/keystone/models/identitybase.go index 0ff11018d1..a02ae063cc 100644 --- a/pkg/keystone/models/identitybase.go +++ b/pkg/keystone/models/identitybase.go @@ -142,7 +142,7 @@ func (manager *SIdentityBaseResourceManager) OrderByExtraFields(ctx context.Cont return q, nil } -func (manager *SIdentityBaseResourceManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) { +func fetchDomainInfo(data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) { domainId, key := jsonutils.GetAnyString2(data, []string{"domain_id", "project_domain", "project_domain_id"}) if len(domainId) > 0 { data.(*jsonutils.JSONDict).Remove(key) @@ -154,11 +154,16 @@ func (manager *SIdentityBaseResourceManager) FetchOwnerId(ctx context.Context, d return nil, httperrors.NewGeneralError(err) } owner := db.SOwnerId{DomainId: domain.Id, Domain: domain.Name} + data.(*jsonutils.JSONDict).Set("project_domain", jsonutils.NewString(domain.Id)) return &owner, nil } return nil, nil } +func (manager *SIdentityBaseResourceManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) { + return fetchDomainInfo(data) +} + func (manager *SIdentityBaseResourceManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) { domain, _ := DomainManager.FetchDomainById(ownerId.GetProjectDomainId()) if domain.Enabled.IsFalse() { diff --git a/pkg/keystone/tokens/auth.go b/pkg/keystone/tokens/auth.go index fc12acc3b5..aa6baa59d1 100644 --- a/pkg/keystone/tokens/auth.go +++ b/pkg/keystone/tokens/auth.go @@ -25,10 +25,12 @@ import ( "yunion.io/x/sqlchemy" api "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/keystone/driver" "yunion.io/x/onecloud/pkg/keystone/models" "yunion.io/x/onecloud/pkg/keystone/options" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/s3auth" ) func authUserByTokenV2(ctx context.Context, input mcclient.SAuthenticationInputV2) (*api.SUserExtended, error) { @@ -162,26 +164,77 @@ func authUserByIdentity(ctx context.Context, ident mcclient.SAuthenticationIdent return usr, nil } +func authUserByAccessKeyV3(ctx context.Context, input mcclient.SAuthenticationInputV3) (*api.SUserExtended, string, api.SAccessKeySecretInfo, error) { + var aksk api.SAccessKeySecretInfo + + akskRequest, err := s3auth.Decode(input.Auth.Identity.AccessKeyRequest) + if err != nil { + return nil, "", aksk, errors.Wrap(err, "s3auth.Decode") + } + keyId := akskRequest.GetAccessKey() + obj, err := models.CredentialManager.FetchById(keyId) + if err != nil { + if err == sql.ErrNoRows { + return nil, "", aksk, ErrInvalidAccessKeyId + } else { + return nil, "", aksk, errors.Wrap(err, "CredentialManager.FetchById") + } + } + credential := obj.(*models.SCredential) + if !credential.Enabled.IsTrue() { + return nil, "", aksk, errors.Wrap(httperrors.ErrInvalidStatus, "Access Key disabled") + } + akBlob, err := credential.GetAccessKeySecret() + if err != nil { + return nil, "", aksk, errors.Wrap(err, "credential.GetAccessKeySecret") + } + if !akBlob.IsValid() { + return nil, "", aksk, ErrExpiredAccessKey + } + aksk.AccessKey = keyId + aksk.Secret = akBlob.Secret + aksk.Expire = akBlob.Expire + + err = akskRequest.Verify(akBlob.Secret) + if err != nil { + return nil, "", aksk, errors.Wrap(err, "Verify") + } + usrExt, err := models.UserManager.FetchUserExtended(credential.UserId, "", "", "") + if err != nil { + return nil, "", aksk, errors.Wrap(err, "UserManager.FetchUserExtended") + } + return usrExt, credential.ProjectId, aksk, nil +} + func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3) (*mcclient.TokenCredentialV3, error) { + var akskInfo api.SAccessKeySecretInfo var user *api.SUserExtended var err error if len(input.Auth.Identity.Methods) != 1 { return nil, ErrInvalidAuthMethod } method := input.Auth.Identity.Methods[0] - if method == api.AUTH_METHOD_TOKEN { + switch method { + case api.AUTH_METHOD_TOKEN: // auth by token user, err = authUserByTokenV3(ctx, input) if err != nil { return nil, errors.Wrap(err, "authUserByTokenV3") } - } else { + case api.AUTH_METHOD_AKSK: + // auth by aksk + user, input.Auth.Scope.Project.Id, akskInfo, err = authUserByAccessKeyV3(ctx, input) + if err != nil { + return nil, errors.Wrap(err, "authUserByAccessKeyV3") + } + default: // auth by other methods, password, openid, saml, etc... user, err = authUserByIdentityV3(ctx, input) if err != nil { return nil, errors.Wrap(err, "authUserByIdentityV3") } } + // user not found if user == nil { return nil, ErrUserNotFound @@ -205,7 +258,7 @@ func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3) if len(input.Auth.Scope.Project.Id) == 0 && len(input.Auth.Scope.Project.Name) == 0 && len(input.Auth.Scope.Domain.Id) == 0 && len(input.Auth.Scope.Domain.Name) == 0 { // unscoped auth - return token.getTokenV3(ctx, user, nil, nil) + return token.getTokenV3(ctx, user, nil, nil, akskInfo) } var projExt *models.SProjectExtended var domain *models.SDomain @@ -238,7 +291,11 @@ func AuthenticateV3(ctx context.Context, input mcclient.SAuthenticationInputV3) } token.DomainId = domain.Id } - return token.getTokenV3(ctx, user, projExt, domain) + tokenV3, err := token.getTokenV3(ctx, user, projExt, domain, akskInfo) + if err != nil { + return nil, errors.Wrap(err, "getTokenV3") + } + return tokenV3, nil } func AuthenticateV2(ctx context.Context, input mcclient.SAuthenticationInputV2) (*mcclient.TokenCredentialV2, error) { diff --git a/pkg/keystone/tokens/errors.go b/pkg/keystone/tokens/errors.go index 82811995c3..802815a1f7 100644 --- a/pkg/keystone/tokens/errors.go +++ b/pkg/keystone/tokens/errors.go @@ -27,4 +27,6 @@ const ( ErrDomainDisabled = errors.Error("domain is disabled") ErrEmptyAuth = errors.Error("empty auth request") ErrUserNotInProject = errors.Error("user not in project") + ErrInvalidAccessKeyId = errors.Error("invalid access key id") + ErrExpiredAccessKey = errors.Error("expired access key") ) diff --git a/pkg/keystone/tokens/handlers.go b/pkg/keystone/tokens/handlers.go index ed9c2a63b7..520a5267dc 100644 --- a/pkg/keystone/tokens/handlers.go +++ b/pkg/keystone/tokens/handlers.go @@ -94,6 +94,7 @@ func authenticateTokensV3(ctx context.Context, w http.ResponseWriter, r *http.Re return } w.Header().Set(api.AUTH_SUBJECT_TOKEN_HEADER, token.Id) + appsrv.SendJSON(w, jsonutils.Marshal(token)) models.UserManager.TraceLoginV3(ctx, token) @@ -166,7 +167,7 @@ func verifyTokensV3(ctx context.Context, w http.ResponseWriter, r *http.Request) } } - v3token, err := token.getTokenV3(ctx, user, projExt, domain) + v3token, err := token.getTokenV3(ctx, user, projExt, domain, api.SAccessKeySecretInfo{}) if err != nil { httperrors.InternalServerError(w, "internal server error %s", err) return diff --git a/pkg/keystone/tokens/token.go b/pkg/keystone/tokens/token.go index a3cdc33c0f..8a1dbb3bc2 100644 --- a/pkg/keystone/tokens/token.go +++ b/pkg/keystone/tokens/token.go @@ -243,8 +243,10 @@ func (t *SAuthToken) getTokenV3( user *api.SUserExtended, project *models.SProjectExtended, domain *models.SDomain, + akskInfo api.SAccessKeySecretInfo, ) (*mcclient.TokenCredentialV3, error) { token := mcclient.TokenCredentialV3{} + token.Token.AccessKey = akskInfo token.Token.ExpiresAt = t.ExpiresAt token.Token.IssuedAt = t.ExpiresAt.Add(-time.Duration(options.Options.TokenExpirationSeconds) * time.Second) token.Token.AuditIds = t.AuditIds diff --git a/pkg/mcclient/aksk.go b/pkg/mcclient/aksk.go new file mode 100644 index 0000000000..a222750440 --- /dev/null +++ b/pkg/mcclient/aksk.go @@ -0,0 +1,72 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mcclient + +import ( + "context" + "net/http" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + + api "yunion.io/x/onecloud/pkg/apis/identity" + "yunion.io/x/onecloud/pkg/util/netutils2" + "yunion.io/x/onecloud/pkg/util/s3auth" +) + +type SAkskTokenCredential struct { + AccessKeySecret api.SAccessKeySecretInfo + Token TokenCredential +} + +func (this *Client) _verifyKeySecret(aksk s3auth.IAccessKeySecretRequest, aCtx SAuthContext) (*SAkskTokenCredential, error) { + input := SAuthenticationInputV3{} + input.Auth.Identity.Methods = []string{api.AUTH_METHOD_AKSK} + input.Auth.Identity.AccessKeyRequest = aksk.Encode() + input.Auth.Context = aCtx + + hdr, rbody, err := this.jsonRequest(context.Background(), this.authUrl, "", "POST", "/auth/tokens", nil, jsonutils.Marshal(&input)) + if err != nil { + return nil, err + } + + tokenId := hdr.Get("X-Subject-Token") + if len(tokenId) == 0 { + return nil, errors.Error("No X-Subject-Token in header") + } + + ret := SAkskTokenCredential{} + ret.Token, err = this.unmarshalV3Token(rbody, tokenId) + if err != nil { + return nil, errors.Wrap(err, "unmarshalV3Token") + } + ret.AccessKeySecret = ret.Token.(*TokenCredentialV3).Token.AccessKey + return &ret, nil +} + +func (this *Client) VerifyRequest(req http.Request, aksk s3auth.IAccessKeySecretRequest, virtualHost bool) (*SAkskTokenCredential, error) { + cliIp := netutils2.GetHttpRequestIp(&req) + aCtx := SAuthContext{ + Source: AuthSourceSrv, + Ip: cliIp, + } + + token, err := this._verifyKeySecret(aksk, aCtx) + if err != nil { + return nil, errors.Wrap(err, "this._verifyKeySecret") + } + + return token, nil +} diff --git a/pkg/mcclient/auth/aksk.go b/pkg/mcclient/auth/aksk.go new file mode 100644 index 0000000000..80fcd67346 --- /dev/null +++ b/pkg/mcclient/auth/aksk.go @@ -0,0 +1,87 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package auth + +import ( + "net/http" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/s3auth" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/cache" +) + +type sAccessKeyCache struct { + *cache.LRUCache +} + +type sAkSkCacheItem struct { + credential *mcclient.SAkskTokenCredential +} + +func (item *sAkSkCacheItem) Size() int { + return 1 +} + +func newAccessKeyCache() *sAccessKeyCache { + return &sAccessKeyCache{ + LRUCache: cache.NewLRUCache(defaultCacheCount), + } +} + +func (c *sAccessKeyCache) addToken(cred *mcclient.SAkskTokenCredential) { + item := &sAkSkCacheItem{cred} + c.Set(cred.AccessKeySecret.AccessKey, item) +} + +func (c *sAccessKeyCache) getToken(token string) (*mcclient.SAkskTokenCredential, bool) { + item, found := c.Get(token) + if !found { + return nil, false + } + return item.(*sAkSkCacheItem).credential, true +} + +func (c *sAccessKeyCache) deleteToken(token string) bool { + return c.Delete(token) +} + +func (c *sAccessKeyCache) Verify(cli *mcclient.Client, req http.Request, virtualHost bool) (mcclient.TokenCredential, error) { + aksk, err := s3auth.DecodeAccessKeyRequest(req, virtualHost) + if err != nil { + return nil, errors.Wrap(err, "s3auth.DecodeAccessKeyRequestV2") + } + + token, found := c.getToken(aksk.GetAccessKey()) + if found { + if token.Token.IsValid() && token.AccessKeySecret.IsValid() { + err = aksk.Verify(token.AccessKeySecret.Secret) + if err != nil { + return nil, errors.Wrap(err, "aksk.Verify") + } + return token.Token, nil + } else { + c.deleteToken(aksk.GetAccessKey()) + } + } + + token, err = cli.VerifyRequest(req, aksk, virtualHost) + if err != nil { + return nil, errors.Wrap(err, "cli.VerifyRequest") + } + + c.addToken(token) + + return token.Token, nil +} diff --git a/pkg/mcclient/auth/auth.go b/pkg/mcclient/auth/auth.go index 577e9cf6df..41c90b5f80 100644 --- a/pkg/mcclient/auth/auth.go +++ b/pkg/mcclient/auth/auth.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/cache" + "net/http" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -131,6 +132,7 @@ type authManager struct { info *AuthInfo adminCredential mcclient.TokenCredential tokenCacheVerify *TokenCacheVerify + accessKeyCache *sAccessKeyCache } func newAuthManager(cli *mcclient.Client, info *AuthInfo) *authManager { @@ -138,9 +140,21 @@ func newAuthManager(cli *mcclient.Client, info *AuthInfo) *authManager { client: cli, info: info, tokenCacheVerify: NewTokenCacheVerify(), + accessKeyCache: newAccessKeyCache(), } } +func (a *authManager) verifyRequest(req http.Request, virtualHost bool) (mcclient.TokenCredential, error) { + if a.adminCredential == nil { + return nil, fmt.Errorf("No valid admin token credential") + } + cred, err := a.accessKeyCache.Verify(a.client, req, virtualHost) + if err != nil { + return nil, err + } + return cred, nil +} + func (a *authManager) verify(token string) (mcclient.TokenCredential, error) { if a.adminCredential == nil { return nil, fmt.Errorf("No valid admin token credential") @@ -229,6 +243,10 @@ func Verify(tokenId string) (mcclient.TokenCredential, error) { return manager.verify(tokenId) } +func VerifyRequest(req http.Request, virtualHost bool) (mcclient.TokenCredential, error) { + return manager.verifyRequest(req, virtualHost) +} + func GetServiceURL(service, region, zone, endpointType string) (string, error) { return manager.GetServiceURL(service, region, zone, endpointType) } diff --git a/pkg/mcclient/input.go b/pkg/mcclient/input.go index 4a85abc805..672c18baaa 100644 --- a/pkg/mcclient/input.go +++ b/pkg/mcclient/input.go @@ -58,6 +58,7 @@ type SAuthenticationIdentity struct { Token struct { Id string `json:"id,omitempty"` } `json:"token,omitempty"` + AccessKeyRequest string `json:"access_key_secret,omitempty"` } type SAuthenticationInputV3 struct { diff --git a/pkg/mcclient/modules/mod_credentials.go b/pkg/mcclient/modules/mod_credentials.go index 156e30cce0..fdcfcf7fe4 100644 --- a/pkg/mcclient/modules/mod_credentials.go +++ b/pkg/mcclient/modules/mod_credentials.go @@ -15,11 +15,15 @@ package modules import ( + "encoding/base64" + "fmt" "time" "yunion.io/x/jsonutils" "yunion.io/x/pkg/util/seclib" + "github.com/pkg/errors" + api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" ) @@ -31,8 +35,9 @@ type SCredentialManager struct { const ( DEFAULT_PROJECT = "default" - TOTP_TYPE = "totp" - RECOVERY_SECRETS_TYPE = "recovery_secret" + ACCESS_SECRET_TYPE = api.ACCESS_SECRET_TYPE + TOTP_TYPE = api.TOTP_TYPE + RECOVERY_SECRETS_TYPE = api.RECOVERY_SECRETS_TYPE ) type STotpSecret struct { @@ -45,15 +50,26 @@ type SRecoverySecret struct { Answer string } +type SAccessKeySecret struct { + KeyId string `json:"-"` + ProjectId string `json:"-"` + TimeStamp time.Time `json:"-"` + api.SAccessKeySecretBlob +} + type SRecoverySecretSet struct { Questions []SRecoverySecret Timestamp int64 } -func (manager *SCredentialManager) fetchCredentials(s *mcclient.ClientSession, secType string, uid string) ([]jsonutils.JSONObject, error) { +func (manager *SCredentialManager) fetchCredentials(s *mcclient.ClientSession, secType string, uid string, pid string) ([]jsonutils.JSONObject, error) { query := jsonutils.NewDict() query.Add(jsonutils.NewString(secType), "type") + query.Add(jsonutils.NewString("system"), "scope") query.Add(jsonutils.NewString(uid), "user_id") + if len(pid) > 0 { + query.Add(jsonutils.NewString(pid), "project_id") + } results, err := manager.List(s, query) if err != nil { return nil, err @@ -61,12 +77,16 @@ func (manager *SCredentialManager) fetchCredentials(s *mcclient.ClientSession, s return results.Data, nil } +func (manager *SCredentialManager) FetchAccessKeySecrets(s *mcclient.ClientSession, uid string, pid string) ([]jsonutils.JSONObject, error) { + return manager.fetchCredentials(s, ACCESS_SECRET_TYPE, uid, pid) +} + func (manager *SCredentialManager) FetchTotpSecrets(s *mcclient.ClientSession, uid string) ([]jsonutils.JSONObject, error) { - return manager.fetchCredentials(s, TOTP_TYPE, uid) + return manager.fetchCredentials(s, TOTP_TYPE, uid, "") } func (manager *SCredentialManager) FetchRecoverySecrets(s *mcclient.ClientSession, uid string) ([]jsonutils.JSONObject, error) { - return manager.fetchCredentials(s, RECOVERY_SECRETS_TYPE, uid) + return manager.fetchCredentials(s, RECOVERY_SECRETS_TYPE, uid, "") } func (manager *SCredentialManager) GetTotpSecret(s *mcclient.ClientSession, uid string) (string, error) { @@ -119,6 +139,89 @@ func (manager *SCredentialManager) GetRecoverySecrets(s *mcclient.ClientSession, return latestQ.Questions, nil } +func DecodeAccessKeySecret(secret jsonutils.JSONObject) (SAccessKeySecret, error) { + curr := SAccessKeySecret{} + blobStr, err := secret.GetString("blob") + if err != nil { + return curr, errors.Wrap(err, "secret.GetString") + } + blobJson, err := jsonutils.ParseString(blobStr) + if err != nil { + return curr, errors.Wrap(err, "jsonutils.ParseString") + } + err = blobJson.Unmarshal(&curr) + if err != nil { + return curr, errors.Wrap(err, "blobJson.Unmarshal") + } + curr.ProjectId, err = secret.GetString("project_id") + if err != nil { + return curr, errors.Wrap(err, "secret.GetString('project_id')") + } + curr.TimeStamp, err = secret.GetTime("created_at") + if err != nil { + return curr, errors.Wrap(err, "secret.GetTime('created_at')") + } + curr.KeyId, err = secret.GetString("id") + if err != nil { + return curr, errors.Wrap(err, "secret.GetString('id')") + } + return curr, nil +} + +func (manager *SCredentialManager) GetAccessKeySecrets(s *mcclient.ClientSession, uid string, pid string) ([]SAccessKeySecret, error) { + secrets, err := manager.FetchAccessKeySecrets(s, uid, pid) + if err != nil { + return nil, err + } + aksk := make([]SAccessKeySecret, 0) + for i := range secrets { + curr, err := DecodeAccessKeySecret(secrets[i]) + if err != nil { + return nil, errors.Wrap(err, "DecodeAccessKeySecret") + } + aksk = append(aksk, curr) + } + return aksk, nil +} + +func (manager *SCredentialManager) DoCreateAccessKeySecret(s *mcclient.ClientSession, params jsonutils.JSONObject) (jsonutils.JSONObject, error) { + key, err := manager.CreateAccessKeySecret(s, "", "", time.Time{}) + if err != nil { + return nil, err + } + result := jsonutils.Marshal(key) + result.(*jsonutils.JSONDict).Add(jsonutils.NewString(key.KeyId), "key_id") + return result, nil +} + +func (manager *SCredentialManager) CreateAccessKeySecret(s *mcclient.ClientSession, uid string, pid string, expireAt time.Time) (SAccessKeySecret, error) { + aksk := SAccessKeySecret{} + aksk.Secret = base64.URLEncoding.EncodeToString([]byte(seclib.RandomPassword(32))) + if !expireAt.IsZero() { + aksk.Expire = expireAt.Unix() + } + blobJson := jsonutils.Marshal(&aksk) + params := jsonutils.NewDict() + name := fmt.Sprintf("%s-%s-%d", uid, pid, time.Now().Unix()) + if len(pid) > 0 { + params.Add(jsonutils.NewString(pid), "project_id") + } + params.Add(jsonutils.NewString(ACCESS_SECRET_TYPE), "type") + if len(uid) > 0 { + params.Add(jsonutils.NewString(uid), "user_id") + } + params.Add(jsonutils.NewString(blobJson.String()), "blob") + params.Add(jsonutils.NewString(name), "name") + result, err := manager.Create(s, params) + if err != nil { + return aksk, err + } + aksk.ProjectId = pid + aksk.TimeStamp, _ = result.GetTime("created_at") + aksk.KeyId, _ = result.GetString("id") + return aksk, nil +} + func (manager *SCredentialManager) CreateTotpSecret(s *mcclient.ClientSession, uid string) (string, error) { _, err := manager.GetTotpSecret(s, uid) if err == nil { @@ -163,8 +266,8 @@ func (manager *SCredentialManager) SaveRecoverySecrets(s *mcclient.ClientSession return nil } -func (manager *SCredentialManager) removeCredentials(s *mcclient.ClientSession, secType string, uid string) error { - secrets, err := manager.fetchCredentials(s, secType, uid) +func (manager *SCredentialManager) removeCredentials(s *mcclient.ClientSession, secType string, uid string, pid string) error { + secrets, err := manager.fetchCredentials(s, secType, uid, pid) if err != nil { return err } @@ -180,12 +283,16 @@ func (manager *SCredentialManager) removeCredentials(s *mcclient.ClientSession, return nil } +func (manager *SCredentialManager) RemoveAccessKeySecrets(s *mcclient.ClientSession, uid string, pid string) error { + return manager.removeCredentials(s, ACCESS_SECRET_TYPE, uid, pid) +} + func (manager *SCredentialManager) RemoveTotpSecrets(s *mcclient.ClientSession, uid string) error { - return manager.removeCredentials(s, TOTP_TYPE, uid) + return manager.removeCredentials(s, TOTP_TYPE, uid, "") } func (manager *SCredentialManager) RemoveRecoverySecrets(s *mcclient.ClientSession, uid string) error { - return manager.removeCredentials(s, RECOVERY_SECRETS_TYPE, uid) + return manager.removeCredentials(s, RECOVERY_SECRETS_TYPE, uid, "") } var ( @@ -198,4 +305,6 @@ func init() { []string{}, []string{"ID", "Type", "user_id", "project_id", "blob"}), } + + register(&Credentials) } diff --git a/pkg/mcclient/modules/mod_projects.go b/pkg/mcclient/modules/mod_projects.go index 4f0cb6a1d6..ce36c94e06 100644 --- a/pkg/mcclient/modules/mod_projects.go +++ b/pkg/mcclient/modules/mod_projects.go @@ -266,6 +266,18 @@ func (this *ProjectManagerV3) AddTags(session *mcclient.ClientSession, id string return nil } +func (this *ProjectManagerV3) FetchId(s *mcclient.ClientSession, project string, domain string) (string, error) { + query := jsonutils.NewDict() + if len(domain) > 0 { + domainId, err := Domains.GetId(s, domain, nil) + if err != nil { + return "", err + } + query.Add(jsonutils.NewString(domainId), "domain_id") + } + return this.GetId(s, project, query) +} + func init() { Projects = ProjectManagerV3{NewIdentityV3Manager("project", "projects", []string{}, diff --git a/pkg/mcclient/token3.go b/pkg/mcclient/token3.go index 784fe872f4..fd5570863b 100644 --- a/pkg/mcclient/token3.go +++ b/pkg/mcclient/token3.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/utils" + api "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/util/rbacutils" ) @@ -81,6 +82,8 @@ type KeystoneTokenV3 struct { User KeystoneUserV3 `json:"user"` Catalog KeystoneServiceCatalogV3 `json:"catalog"` Context SAuthContext `json:"context"` + + AccessKey api.SAccessKeySecretInfo `json:"access_key"` } type TokenCredentialV3 struct { diff --git a/pkg/multicloud/aliyun/bucket.go b/pkg/multicloud/aliyun/bucket.go index a6577c3b79..6aa8eb8d4d 100644 --- a/pkg/multicloud/aliyun/bucket.go +++ b/pkg/multicloud/aliyun/bucket.go @@ -360,3 +360,79 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s } return urlStr, nil } + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + osscli, err := b.region.GetOssClient() + if err != nil { + return errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return errors.Wrap(err, "Bucket") + } + opts := make([]oss.Option, 0) + if len(contType) > 0 { + opts = append(opts, oss.ContentType(contType)) + } + if len(cannedAcl) > 0 { + acl, err := str2Acl(string(cannedAcl)) + if err != nil { + return errors.Wrap(err, "") + } + opts = append(opts, oss.ObjectACL(acl)) + } + if len(storageClassStr) > 0 { + storageClass, err := str2StorageClass(storageClassStr) + if err != nil { + return errors.Wrap(err, "str2StorageClass") + } + opts = append(opts, oss.ObjectStorageClass(storageClass)) + } + _, err = bucket.CopyObjectFrom(srcBucket, srcKey, destKey, opts...) + if err != nil { + return errors.Wrap(err, "CopyObjectFrom") + } + return nil +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + osscli, err := b.region.GetOssClient() + if err != nil { + return nil, errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return nil, errors.Wrap(err, "Bucket") + } + opts := make([]oss.Option, 0) + if rangeOpt != nil { + opts = append(opts, oss.NormalizedRange(rangeOpt.String())) + } + output, err := bucket.GetObject(key, opts...) + if err != nil { + return nil, errors.Wrap(err, "bucket.GetObject") + } + return output, nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + osscli, err := b.region.GetOssClient() + if err != nil { + return "", errors.Wrap(err, "GetOssClient") + } + bucket, err := osscli.Bucket(b.Name) + if err != nil { + return "", errors.Wrap(err, "Bucket") + } + imur := oss.InitiateMultipartUploadResult{ + Bucket: b.Name, + Key: key, + UploadID: uploadId, + } + opts := make([]oss.Option, 0) + part, err := bucket.UploadPartCopy(imur, srcBucket, srcKey, srcOffset, srcLength, partNumber, opts...) + if err != nil { + return "", errors.Wrap(err, "bucket.UploadPartCopy") + } + return part.ETag, nil +} diff --git a/pkg/multicloud/aws/bucket.go b/pkg/multicloud/aws/bucket.go index 1634a38584..970d906bd9 100644 --- a/pkg/multicloud/aws/bucket.go +++ b/pkg/multicloud/aws/bucket.go @@ -27,6 +27,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/s3cli" + "net/url" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" "yunion.io/x/onecloud/pkg/util/fileutils2" @@ -378,3 +379,62 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s } return url, nil } + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + s3cli, err := b.region.GetS3Client() + if err != nil { + return errors.Wrap(err, "GetS3Client") + } + log.Debugf("copy from %s/%s to %s/%s", srcBucket, srcKey, b.Name, destKey) + input := &s3.CopyObjectInput{} + input.SetBucket(b.Name) + input.SetKey(destKey) + input.SetCopySource(fmt.Sprintf("%s/%s", srcBucket, url.PathEscape(srcKey))) + input.SetStorageClass(storageClassStr) + input.SetACL(string(cannedAcl)) + input.SetContentType(contType) + _, err = s3cli.CopyObject(input) + if err != nil { + return errors.Wrap(err, "CopyObject") + } + return nil +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return nil, errors.Wrap(err, "GetS3Client") + } + input := &s3.GetObjectInput{} + input.SetBucket(b.Name) + input.SetKey(key) + if rangeOpt != nil { + input.SetRange(rangeOpt.String()) + } + output, err := s3cli.GetObject(input) + if err != nil { + return nil, errors.Wrap(err, "GetObject") + } + return output.Body, nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + s3cli, err := b.region.GetS3Client() + if err != nil { + return "", errors.Wrap(err, "GetS3Client") + } + input := &s3.UploadPartCopyInput{} + input.SetBucket(b.Name) + input.SetKey(key) + input.SetUploadId(uploadId) + input.SetPartNumber(int64(partNumber)) + input.SetCopySource(fmt.Sprintf("/%s/%s", srcBucket, url.PathEscape(srcKey))) + if srcLength > 0 { + input.SetCopySourceRange(fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1)) + } + output, err := s3cli.UploadPartCopy(input) + if err != nil { + return "", errors.Wrap(err, "s3cli.UploadPartCopy") + } + return *output.CopyPartResult.ETag, nil +} diff --git a/pkg/multicloud/azure/storageaccount.go b/pkg/multicloud/azure/storageaccount.go index 5de0a7dec1..33fd93229b 100644 --- a/pkg/multicloud/azure/storageaccount.go +++ b/pkg/multicloud/azure/storageaccount.go @@ -35,6 +35,7 @@ import ( "encoding/base64" "strconv" "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/multicloud" ) @@ -927,7 +928,7 @@ func (b *SStorageAccount) ListObjects(prefix string, marker string, delimiter st func splitKey(key string) (string, string, error) { slashPos := strings.IndexByte(key, '/') if slashPos <= 0 { - return "", "", errors.Error("cannot put object to root") + return "", "", errors.Wrap(httperrors.ErrForbidden, "cannot put object to root") } containerName := key[:slashPos] key = key[slashPos+1:] @@ -977,6 +978,10 @@ func (b *SStorageAccount) NewMultipartUpload(ctx context.Context, key string, co return uploadId, nil } +func partIndex2BlockId(partIndex int) string { + return base64.URLEncoding.EncodeToString([]byte(strconv.FormatInt(int64(partIndex), 10))) +} + func (b *SStorageAccount) UploadPart(ctx context.Context, key string, uploadId string, partIndex int, input io.Reader, partSize int64) (string, error) { containerName, blob, err := splitKey(key) if err != nil { @@ -995,7 +1000,7 @@ func (b *SStorageAccount) UploadPart(ctx context.Context, key string, uploadId s opts := &storage.PutBlockOptions{} opts.LeaseID = uploadId - blockId := base64.URLEncoding.EncodeToString([]byte(strconv.FormatInt(int64(partIndex), 10))) + blockId := partIndex2BlockId(partIndex) err = blobRef.PutBlockWithLength(blockId, uint64(partSize), input, opts) if err != nil { return "", errors.Wrap(err, "PutBlockWithLength") @@ -1113,3 +1118,118 @@ func (b *SStorageAccount) GetTempUrl(method string, key string, expire time.Dura } return container.SignUrl(method, blob, expire) } + +func (b *SStorageAccount) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + srcIBucket, err := b.region.GetIBucketByName(srcBucket) + if err != nil { + return errors.Wrap(err, "GetIBucketByName") + } + srcAccount := srcIBucket.(*SStorageAccount) + srcContName, srcBlob, err := splitKey(srcKey) + if err != nil { + return errors.Wrap(err, "src splitKey") + } + srcCont, err := srcAccount.getOrCreateContainer(srcContName, false) + if err != nil { + return errors.Wrap(err, "src getOrCreateContainer") + } + srcContRef, err := srcCont.getContainerRef() + if err != nil { + return errors.Wrap(err, "src getContainerRef") + } + srcBlobRef := srcContRef.GetBlobReference(srcBlob) + + containerName, blob, err := splitKey(destKey) + if err != nil { + return errors.Wrap(err, "dest splitKey") + } + container, err := b.getOrCreateContainer(containerName, true) + if err != nil { + return errors.Wrap(err, "dest getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return errors.Wrap(err, "dest getContainerRef") + } + blobRef := containerRef.GetBlobReference(blob) + + opts := &storage.CopyOptions{} + err = blobRef.Copy(srcBlobRef.GetURL(), opts) + if err != nil { + return errors.Wrap(err, "blboRef.Copy") + } + return nil +} + +func (b *SStorageAccount) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + containerName, blob, err := splitKey(key) + if err != nil { + return nil, errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, false) + if err != nil { + return nil, errors.Wrap(err, "getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return nil, errors.Wrap(err, "container.getContainerRef") + } + blobRef := containerRef.GetBlobReference(blob) + if rangeOpt != nil { + opts := &storage.GetBlobRangeOptions{} + opts.Range = &storage.BlobRange{ + Start: uint64(rangeOpt.Start), + End: uint64(rangeOpt.End), + } + return blobRef.GetRange(opts) + } else { + opts := &storage.GetBlobOptions{} + return blobRef.Get(opts) + } +} + +func (b *SStorageAccount) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + srcIBucket, err := b.region.GetIBucketByName(srcBucket) + if err != nil { + return "", errors.Wrap(err, "GetIBucketByName") + } + srcAccount := srcIBucket.(*SStorageAccount) + srcContName, srcBlob, err := splitKey(srcKey) + if err != nil { + return "", errors.Wrap(err, "src splitKey") + } + srcCont, err := srcAccount.getOrCreateContainer(srcContName, false) + if err != nil { + return "", errors.Wrap(err, "src getOrCreateContainer") + } + srcContRef, err := srcCont.getContainerRef() + if err != nil { + return "", errors.Wrap(err, "src getContainerRef") + } + srcBlobRef := srcContRef.GetBlobReference(srcBlob) + + containerName, blob, err := splitKey(key) + if err != nil { + return "", errors.Wrap(err, "splitKey") + } + container, err := b.getOrCreateContainer(containerName, true) + if err != nil { + return "", errors.Wrap(err, "getOrCreateContainer") + } + containerRef, err := container.getContainerRef() + if err != nil { + return "", errors.Wrap(err, "getContainerRef") + } + blobRef := containerRef.GetBlobReference(blob) + + opts := &storage.PutBlockFromURLOptions{} + opts.LeaseID = uploadId + + blockId := partIndex2BlockId(partIndex) + err = blobRef.PutBlockFromURL(blockId, srcBlobRef.GetURL(), srcOffset, uint64(srcLength), opts) + if err != nil { + return "", errors.Wrap(err, "PutBlockFromUrl") + } + + return blockId, nil +} diff --git a/pkg/multicloud/huawei/bucket.go b/pkg/multicloud/huawei/bucket.go index a0194c7581..0be0e17cdf 100644 --- a/pkg/multicloud/huawei/bucket.go +++ b/pkg/multicloud/huawei/bucket.go @@ -420,3 +420,70 @@ func (b *SBucket) SetLimit(limit cloudprovider.SBucketStats) error { } return nil } + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + obscli, err := b.region.getOBSClient() + if err != nil { + return errors.Wrap(err, "GetOBSClient") + } + input := &obs.CopyObjectInput{} + input.CopySourceBucket = srcBucket + input.CopySourceKey = srcKey + if len(storageClassStr) > 0 { + input.StorageClass, err = str2StorageClass(storageClassStr) + if err != nil { + return err + } + } + if len(cannedAcl) > 0 { + input.ACL = obs.AclType(string(cannedAcl)) + } + if len(contType) > 0 { + input.ContentType = contType + } + _, err = obscli.CopyObject(input) + if err != nil { + return errors.Wrap(err, "obscli.CopyObject") + } + return nil +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return nil, errors.Wrap(err, "GetOBSClient") + } + input := &obs.GetObjectInput{} + input.Bucket = b.Name + input.Key = key + if rangeOpt != nil { + input.RangeStart = rangeOpt.Start + input.RangeEnd = rangeOpt.End + } + output, err := obscli.GetObject(input) + if err != nil { + return nil, errors.Wrap(err, "obscli.GetObject") + } + return output.Body, nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + obscli, err := b.region.getOBSClient() + if err != nil { + return "", errors.Wrap(err, "GetOBSClient") + } + input := &obs.CopyPartInput{} + input.Bucket = b.Name + input.Key = key + input.UploadId = uploadId + input.PartNumber = partIndex + input.CopySourceBucket = srcBucket + input.CopySourceKey = srcKey + input.CopySourceRangeStart = srcOffset + input.CopySourceRangeEnd = srcOffset + srcLength - 1 + output, err := obscli.CopyPart(input) + if err != nil { + return "", errors.Wrap(err, "CopyPart") + } + return output.ETag, nil +} diff --git a/pkg/multicloud/objectstore/buckets.go b/pkg/multicloud/objectstore/buckets.go index 6f3820995c..2f8016c1e2 100644 --- a/pkg/multicloud/objectstore/buckets.go +++ b/pkg/multicloud/objectstore/buckets.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/s3cli" + "net/http" "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/multicloud" ) @@ -114,14 +115,38 @@ func (bucket *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { } func (bucket *SBucket) ListObjects(prefix string, marker string, delimiter string, maxCount int) (cloudprovider.SListObjectResult, error) { - isRecursive := true - if delimiter == "/" { - isRecursive = false + ret := cloudprovider.SListObjectResult{} + result, err := bucket.client.client.ListObjectsQuery(bucket.Name, prefix, marker, delimiter, maxCount) + if err != nil { + return ret, errors.Wrap(err, "ListObjectsQuery") } - result := cloudprovider.SListObjectResult{} - var err error - result.Objects, err = bucket.GetIObjects(prefix, isRecursive) - return result, err + ret.NextMarker = result.NextMarker + ret.IsTruncated = result.IsTruncated + ret.CommonPrefixes = make([]cloudprovider.ICloudObject, len(result.CommonPrefixes)) + for i := range result.CommonPrefixes { + ret.CommonPrefixes[i] = &SObject{ + bucket: bucket, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + Key: result.CommonPrefixes[i].Prefix, + }, + } + } + ret.Objects = make([]cloudprovider.ICloudObject, len(result.Contents)) + for i := range result.Contents { + object := result.Contents[i] + ret.Objects[i] = &SObject{ + bucket: bucket, + SBaseCloudObject: cloudprovider.SBaseCloudObject{ + StorageClass: object.StorageClass, + Key: object.Key, + SizeBytes: object.Size, + ETag: object.ETag, + LastModified: object.LastModified, + ContentType: object.ContentType, + }, + } + } + return ret, nil } func (bucket *SBucket) GetIObjects(prefix string, isRecursive bool) ([]cloudprovider.ICloudObject, error) { @@ -242,3 +267,51 @@ func (bucket *SBucket) GetTempUrl(method string, key string, expire time.Duratio } return url.String(), nil } + +func (bucket *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + meta := make(map[string]string) + if len(contType) > 0 { + meta[http.CanonicalHeaderKey("Content-Type")] = contType + } + if len(storageClassStr) > 0 { + meta[http.CanonicalHeaderKey("x-amz-storage-class")] = storageClassStr + } + dest, err := s3cli.NewDestinationInfo(bucket.Name, destKey, nil, meta) + if err != nil { + return errors.Wrap(err, "NewDestinationInfo") + } + src := s3cli.NewSourceInfo(srcBucket, srcKey, nil) + err = bucket.client.client.CopyObject(dest, src) + if err != nil { + return errors.Wrap(err, "CopyObject") + } + obj, err := cloudprovider.GetIObject(bucket, destKey) + if err != nil { + return errors.Wrap(err, "GetIObject") + } + err = obj.SetAcl(cannedAcl) + if err != nil { + return errors.Wrap(err, "obj.SetAcl") + } + return nil +} + +func (bucket *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + opts := s3cli.GetObjectOptions{} + if rangeOpt != nil { + opts.SetRange(rangeOpt.Start, rangeOpt.End) + } + output, err := bucket.client.client.GetObject(bucket.Name, key, opts) + if err != nil { + return nil, errors.Wrap(err, "GetObject") + } + return output, nil +} + +func (bucket *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partNumber int, srcBucket string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + result, err := bucket.client.client.CopyObjectPartDo(ctx, srcBucket, srcKey, bucket.Name, key, uploadId, partNumber, srcOffset, srcLength, nil) + if err != nil { + return "", errors.Wrap(err, "CopyObjectPartDo") + } + return result.ETag, nil +} diff --git a/pkg/multicloud/objectstore/provider/provider.go b/pkg/multicloud/objectstore/provider/provider.go index c8aa37da63..b7709fe5e5 100644 --- a/pkg/multicloud/objectstore/provider/provider.go +++ b/pkg/multicloud/objectstore/provider/provider.go @@ -91,9 +91,9 @@ func (self *SObjectStoreProviderFactory) GetProvider(providerId, providerName, u func (self *SObjectStoreProviderFactory) GetClientRC(url, account, secret string) (map[string]string, error) { return map[string]string{ - "OBJECTSTORE_ACCESSKEY": account, - "OBJECTSTORE_SECRET": secret, - "OBJECTSTORE_ENDPOINT": url, + "S3_ACCESS_KEY": account, + "S3_SECRET": secret, + "S3_ACCESS_URL": url, }, nil } diff --git a/pkg/multicloud/objectstore/shell.go b/pkg/multicloud/objectstore/shell.go index 643a6d11c8..7751ea28ad 100644 --- a/pkg/multicloud/objectstore/shell.go +++ b/pkg/multicloud/objectstore/shell.go @@ -25,6 +25,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" "yunion.io/x/onecloud/pkg/util/printutils" "yunion.io/x/onecloud/pkg/util/shellutils" + "yunion.io/x/onecloud/pkg/util/streamutils" ) func S3Shell() { @@ -280,4 +281,92 @@ func S3Shell() { fmt.Println("Success!") return nil }) + + type BucketObjectDownloadOptions struct { + BUCKET string `help:"name of bucket"` + KEY string `help:"Key of object"` + Output string `help:"target output, default to stdout"` + Start int64 `help:"partial download start"` + End int64 `help:"partial download end"` + } + shellutils.R(&BucketObjectDownloadOptions{}, "object-download", "Download", func(cli cloudprovider.ICloudRegion, args *BucketObjectDownloadOptions) error { + bucket, err := cli.GetIBucketById(args.BUCKET) + if err != nil { + return err + } + obj, err := cloudprovider.GetIObject(bucket, args.KEY) + if err != nil { + return err + } + + var rangeOpt *cloudprovider.SGetObjectRange + if args.Start != 0 || args.End != 0 { + if args.End <= 0 { + args.End = obj.GetSizeBytes() - 1 + } + rangeOpt = &cloudprovider.SGetObjectRange{Start: args.Start, End: args.End} + } + output, err := bucket.GetObject(context.Background(), args.KEY, rangeOpt) + if err != nil { + return err + } + defer output.Close() + var target io.Writer + if len(args.Output) == 0 { + target = os.Stdout + } else { + fp, err := os.Create(args.Output) + if err != nil { + return err + } + defer fp.Close() + target = fp + } + prop, err := streamutils.StreamPipe(output, target, false) + if err != nil { + return err + } + if len(args.Output) > 0 { + fmt.Println("Success:", prop.Size, "written") + } + return nil + }) + + type BucketObjectCopyOptions struct { + SRC string `help:"name of source bucket"` + SRCKEY string `help:"Key of source object"` + DST string `help:"name of destination bucket"` + DSTKEY string `help:"key of destination object"` + Debug bool `help:"show debug info"` + BlockSize int64 `help:"block size in MB"` + Native bool `help:"Use native copy"` + } + shellutils.R(&BucketObjectCopyOptions{}, "object-copy", "Copy object", func(cli cloudprovider.ICloudRegion, args *BucketObjectCopyOptions) error { + ctx := context.Background() + dstBucket, err := cli.GetIBucketByName(args.DST) + if err != nil { + return err + } + srcBucket, err := cli.GetIBucketByName(args.SRC) + if err != nil { + return err + } + srcObj, err := cloudprovider.GetIObject(srcBucket, args.SRCKEY) + if err != nil { + return err + } + if args.Native { + err = dstBucket.CopyObject(ctx, args.DSTKEY, args.SRC, args.SRCKEY, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass()) + if err != nil { + return err + } + } else { + err = cloudprovider.CopyObject(ctx, args.BlockSize*1000*1000, dstBucket, args.DSTKEY, srcBucket, args.SRCKEY, args.Debug) + if err != nil { + return err + } + } + fmt.Println("Success!") + return nil + }) } diff --git a/pkg/multicloud/qcloud/bucket.go b/pkg/multicloud/qcloud/bucket.go index 8369562934..e9bb6999db 100644 --- a/pkg/multicloud/qcloud/bucket.go +++ b/pkg/multicloud/qcloud/bucket.go @@ -133,8 +133,12 @@ func (b *SBucket) getFullName() string { return fmt.Sprintf("%s-%s", b.Name, b.region.client.AppID) } +func (b *SBucket) getBucketUrlHost() string { + return fmt.Sprintf("%s.%s", b.getFullName(), b.region.getCosEndpoint()) +} + func (b *SBucket) getBucketUrl() string { - return fmt.Sprintf("https://%s.%s", b.getFullName(), b.region.getCosEndpoint()) + return fmt.Sprintf("https://%s", b.getBucketUrlHost()) } func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { @@ -351,3 +355,69 @@ func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (s } return url.String(), nil } + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucketName, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return errors.Wrap(err, "GetCosClient") + } + opts := &cos.ObjectCopyOptions{ + ObjectCopyHeaderOptions: &cos.ObjectCopyHeaderOptions{}, + ACLHeaderOptions: &cos.ACLHeaderOptions{}, + } + if len(cannedAcl) > 0 { + opts.XCosACL = string(cannedAcl) + } + if len(storageClassStr) > 0 { + opts.XCosStorageClass = storageClassStr + } + if len(contType) > 0 { + opts.ContentType = contType + } + srcBucket := SBucket{ + region: b.region, + Name: srcBucketName, + } + srcUrl := fmt.Sprintf("%s/%s", srcBucket.getBucketUrlHost(), srcKey) + log.Debugf("source url: %s", srcUrl) + _, _, err = coscli.Object.Copy(ctx, destKey, srcUrl, opts) + if err != nil { + return errors.Wrap(err, "coscli.Object.Copy") + } + return nil +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return nil, errors.Wrap(err, "GetCosClient") + } + opts := &cos.ObjectGetOptions{} + if rangeOpt != nil { + opts.Range = rangeOpt.String() + } + resp, err := coscli.Object.Get(ctx, key, opts) + if err != nil { + return nil, errors.Wrap(err, "coscli.Object.Get") + } + return resp.Body, nil +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucketName string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + coscli, err := b.region.GetCosClient(b) + if err != nil { + return "", errors.Wrap(err, "GetCosClient") + } + srcBucket := SBucket{ + region: b.region, + Name: srcBucketName, + } + opts := cos.ObjectCopyPartOptions{} + opts.XCosCopySource = fmt.Sprintf("%s/%s", srcBucket.getBucketUrlHost(), srcKey) + opts.XCosCopySourceRange = fmt.Sprintf("bytes=%d-%d", srcOffset, srcOffset+srcLength-1) + result, _, err := coscli.Object.CopyPart(ctx, key, uploadId, partIndex, &opts) + if err != nil { + return "", errors.Wrap(err, "coscli.Object.CopyPart") + } + return result.ETag, nil +} diff --git a/pkg/multicloud/ucloud/ufile.go b/pkg/multicloud/ucloud/ufile.go index 9a1249805b..529ac72d83 100644 --- a/pkg/multicloud/ucloud/ufile.go +++ b/pkg/multicloud/ucloud/ufile.go @@ -374,3 +374,15 @@ func (b *SBucket) DeleteObject(ctx context.Context, key string) error { func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { return "", cloudprovider.ErrNotSupported } + +func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, srcKey string, contType string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string) error { + return cloudprovider.ErrNotSupported +} + +func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprovider.SGetObjectRange) (io.ReadCloser, error) { + return nil, cloudprovider.ErrNotSupported +} + +func (b *SBucket) CopyPart(ctx context.Context, key string, uploadId string, partIndex int, srcBucketName string, srcKey string, srcOffset int64, srcLength int64) (string, error) { + return "", cloudprovider.ErrNotSupported +} diff --git a/pkg/notify/doc.go b/pkg/notify/doc.go index 835cf70868..3aff52f84f 100644 --- a/pkg/notify/doc.go +++ b/pkg/notify/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package notify // import "yunion.io/x/onecloud/pkg/notify" diff --git a/pkg/notify/models/doc.go b/pkg/notify/models/doc.go index 016928b87e..2ae2d251c5 100644 --- a/pkg/notify/models/doc.go +++ b/pkg/notify/models/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package models // import "yunion.io/x/onecloud/pkg/notify/models" diff --git a/pkg/notify/options/doc.go b/pkg/notify/options/doc.go index 4b4bf7a27b..1d7d4a8e66 100644 --- a/pkg/notify/options/doc.go +++ b/pkg/notify/options/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package options // import "yunion.io/x/onecloud/pkg/notify/options" diff --git a/pkg/notify/utils/doc.go b/pkg/notify/utils/doc.go index 171885b326..37fa0628ed 100644 --- a/pkg/notify/utils/doc.go +++ b/pkg/notify/utils/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package utils // import "yunion.io/x/onecloud/pkg/notify/utils" diff --git a/pkg/s3gateway/handlers/bucket.go b/pkg/s3gateway/handlers/bucket.go new file mode 100644 index 0000000000..8c925858e3 --- /dev/null +++ b/pkg/s3gateway/handlers/bucket.go @@ -0,0 +1,64 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + + "yunion.io/x/pkg/errors" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/s3gateway/models" +) + +func headBucket(ctx context.Context, userCred mcclient.TokenCredential, bucketName string) error { + _, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return errors.Wrap(err, "models.BucketManager.GetByName") + } + return nil +} + +func removeBucket(ctx context.Context, userCred mcclient.TokenCredential, bucket string) error { + return models.BucketManager.DeleteByName(ctx, userCred, bucket) +} + +func bucketAcl(ctx context.Context, userCred mcclient.TokenCredential, bucketName string) (*s3cli.AccessControlPolicy, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "bucket.GetIBucket") + } + + result := str2Acl(userCred, iBucket.GetAcl()) + + return result, nil +} + +func listBucketUploads(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, input *s3cli.ListMultipartUploadsInput) (*s3cli.ListMultipartUploadsResult, error) { + result := s3cli.ListMultipartUploadsResult{} + result.Bucket = bucketName + result.Delimiter = input.Delimiter + result.MaxUploads = input.MaxUploads + result.KeyMarker = input.KeyMarker + result.Prefix = input.Prefix + result.UploadIDMarker = input.UploadIdMarker + result.EncodingType = input.EncodingType + return &result, nil +} diff --git a/pkg/s3gateway/handlers/errors.go b/pkg/s3gateway/handlers/errors.go new file mode 100644 index 0000000000..3585544973 --- /dev/null +++ b/pkg/s3gateway/handlers/errors.go @@ -0,0 +1,138 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/s3cli" + + "runtime/debug" + "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/httperrors" +) + +func generalError(ctx context.Context, statusCode int, errCode string, msg string) s3cli.ErrorResponse { + o := fetchObjectRequest(ctx) + resp := s3cli.ErrorResponse{} + resp.StatusCode = statusCode + resp.Code = errCode + resp.Message = msg + resp.BucketName = o.Bucket + resp.Key = o.Key + resp.HostID = appctx.AppContextHostId(ctx) + resp.RequestID = appctx.AppContextRequestId(ctx) + return resp +} + +func BadRequest(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 400, "Bad Request", msg) +} + +func Unauthenticated(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 401, "Unauthenticated", msg) +} + +func Unauthorized(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 403, "Unauthorized", msg) +} + +func Forbidden(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 403, "Forbidden", msg) +} + +func NotFound(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 404, "Not Found", msg) +} + +func NotSupported(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 404, "Not Supported", msg) +} + +func NotImplemented(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 406, "Not Implemented", msg) +} + +func InvalidStatus(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 406, "Invalid Status", msg) +} + +func Conflict(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 409, "Conflict", msg) +} + +func ServerTimeout(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 504, "Server Timeout", msg) +} + +func ServerError(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 500, "Internal Server Error", msg) +} + +func OutOfRangeError(ctx context.Context, msg string) s3cli.ErrorResponse { + return generalError(ctx, 416, "Range Not Satisfiable", msg) +} + +func SendGeneralError(ctx context.Context, w http.ResponseWriter, err error) { + switch e := err.(type) { + case s3cli.ErrorResponse: + SendError(w, e) + case *s3cli.ErrorResponse: + SendError(w, *e) + default: + var eresp s3cli.ErrorResponse + cause := errors.Cause(err) + switch cause { + case httperrors.ErrUnauthenticated: + eresp = Unauthenticated(ctx, err.Error()) + case httperrors.ErrUnauthorized: + eresp = Unauthorized(ctx, err.Error()) + case httperrors.ErrNotFound: + eresp = NotFound(ctx, err.Error()) + case httperrors.ErrNotSupported: + eresp = NotSupported(ctx, err.Error()) + case httperrors.ErrNotImplemented: + eresp = NotImplemented(ctx, err.Error()) + case httperrors.ErrDuplicateId: + eresp = Conflict(ctx, err.Error()) + case httperrors.ErrTimeout: + eresp = ServerTimeout(ctx, err.Error()) + case httperrors.ErrInvalidStatus: + eresp = InvalidStatus(ctx, err.Error()) + case httperrors.ErrBadRequest: + eresp = BadRequest(ctx, err.Error()) + case httperrors.ErrOutOfRange: + eresp = OutOfRangeError(ctx, err.Error()) + case httperrors.ErrForbidden: + eresp = Forbidden(ctx, err.Error()) + default: + eresp = ServerError(ctx, err.Error()) + } + SendError(w, eresp) + } +} + +func SendError(w http.ResponseWriter, resp s3cli.ErrorResponse) { + w.WriteHeader(resp.StatusCode) + + log.Errorf("SendError: %s", resp) + debug.PrintStack() + + appsrv.SendXml(w, nil, resp) +} diff --git a/pkg/s3gateway/handlers/handlers.go b/pkg/s3gateway/handlers/handlers.go new file mode 100644 index 0000000000..85fd4ef392 --- /dev/null +++ b/pkg/s3gateway/handlers/handlers.go @@ -0,0 +1,606 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + "net/url" + "strings" + "time" + + "github.com/minio/minio-go/pkg/s3utils" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" + "yunion.io/x/onecloud/pkg/s3gateway/models" + "yunion.io/x/onecloud/pkg/s3gateway/options" +) + +func InitHandlers(app *appsrv.Application) { + h := app.AddHandler2("HEAD", "", s3authenticate(headHandler), nil, "head", nil) + h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo) + h = app.AddHandler2("GET", "", s3authenticate(readHandler), nil, "get", nil) + h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo) + h = app.AddHandler2("PUT", "", s3authenticate(putHandler), nil, "put", nil) + h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo) + h = app.AddHandler2("POST", "", s3authenticate(postHandler), nil, "post", nil) + h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo) + h = app.AddHandler2("DELETE", "", s3authenticate(deleteHandler), nil, "delete", nil) + h.SetProcessTimeoutCallback(s3HandlerTimeoutInfo) +} + +func s3HandlerTimeoutInfo(info *appsrv.SHandlerInfo, r *http.Request) time.Duration { + o, _ := getObjectRequest(r) + if len(o.Bucket) > 0 && len(o.Key) > 0 { + if r.Method == http.MethodGet && len(r.URL.RawQuery) == 0 { + return 2 * time.Hour + } else if r.Method == http.MethodPut && (len(r.URL.RawQuery) == 0 || strings.Contains(r.URL.RawQuery, "partNumber=")) { + return 2 * time.Hour + } + } + return time.Duration(0) +} + +type SObjectRequest struct { + VirtualHost bool + Bucket string + Key string +} + +func (o SObjectRequest) Validate() error { + if len(o.Bucket) == 0 { + return nil + } + err := s3utils.CheckValidBucketNameStrict(o.Bucket) + if err != nil { + return err + } + if len(o.Key) == 0 { + return nil + } + err = s3utils.CheckValidObjectName(o.Key) + if err != nil { + return err + } + return nil +} + +func getObjectRequest(r *http.Request) (SObjectRequest, error) { + o := SObjectRequest{} + if regutils.MatchIP4Addr(r.Host) || r.Host == options.Options.DomainName { + o.VirtualHost = false + segs := appsrv.SplitPath(r.URL.Path) + if len(segs) > 0 { + o.Bucket = segs[0] + if len(segs) > 1 { + o.Key = strings.Join(segs[1:], "/") + if strings.HasSuffix(r.URL.Path, "/") { + o.Key += "/" + } + } + } + } else if strings.HasSuffix(r.Host, "."+options.Options.DomainName) { + o.VirtualHost = true + o.Bucket = r.Host[:len(r.Host)-len(options.Options.DomainName)-1] + segs := appsrv.SplitPath(r.URL.Path) + o.Key = strings.Join(segs, "/") + if strings.HasSuffix(r.URL.Path, "/") { + o.Key += "/" + } + } else { + return o, errors.Error("invalid S3 request") + } + var err error + o.Key, err = url.PathUnescape(o.Key) + if err != nil { + return o, errors.Wrap(err, "url.PathUnescape") + } + return o, o.Validate() +} + +func headHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + o := fetchObjectRequest(ctx) + userCred := auth.FetchUserCredential(ctx, nil) + if len(o.Bucket) > 0 && len(o.Key) == 0 { + // head bucket + err := headBucket(ctx, userCred, o.Bucket) + if err != nil { + SendGeneralError(ctx, w, err) + } else { + appsrv.SendHeader(w, nil) + } + return + } else if len(o.Bucket) > 0 && len(o.Key) > 0 { + // head object + hdr, err := headObject(ctx, userCred, o.Bucket, o.Key) + if err != nil { + SendGeneralError(ctx, w, err) + } else { + appsrv.SendHeader(w, hdr) + } + return + } else { + // do nothing + } + SendError(w, NotSupported(ctx, "method not supported")) +} + +func readBucket(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + if query.Contains("accelerate") { + + } else if query.Contains("acl") { + resp, err := bucketAcl(ctx, userCred, bucketName) + return resp, nil, err + } else if query.Contains("analytics") { + + } else if query.Contains("cors") { + + } else if query.Contains("encryption") { + + } else if query.Contains("inventory") { + + } else if query.Contains("lifecycle") { + + } else if query.Contains("location") { + result := s3cli.LocationConstraint(bucket.Location) + return &result, nil, nil + } else if query.Contains("publicAccessBlock") { + + } else if query.Contains("logging") { + + } else if query.Contains("metrics") { + + } else if query.Contains("notification") { + + } else if query.Contains("object-lock") { + + } else if query.Contains("policyStatus") { + + } else if query.Contains("versions") { + + } else if query.Contains("policy") { + + } else if query.Contains("replication") { + + } else if query.Contains("requestPayment") { + + } else if query.Contains("tagging") { + + } else if query.Contains("versioning") { + return &s3cli.VersioningConfiguration{}, nil, nil + } else if query.Contains("website") { + + } else if query.Contains("uploads") { + input := s3cli.ListMultipartUploadsInput{} + err := query.Unmarshal(&input) + if err != nil { + return nil, nil, errors.Wrap(err, "query.Unmarshal ListMultipartUploadsInput") + } + result, err := listBucketUploads(ctx, userCred, bucketName, &input) + if err != nil { + return nil, nil, errors.Wrap(err, "listBucketUploads") + } + return result, nil, nil + } else { + // list objects in bucket + input := s3cli.ListObjectInput{} + err := query.Unmarshal(&input) + if err != nil { + return nil, nil, errors.Wrap(err, "query.Unmarshal") + } + result, err := bucket.ListObject(ctx, userCred, &input) + if err != nil { + return nil, nil, errors.Wrap(err, "bucket.ListObject") + } + return result, nil, nil + } + return nil, nil, NotImplemented(ctx, "not implemented") +} + +func readObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, objKey string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) { + if query.Contains("acl") { + resp, err := objectAcl(ctx, userCred, bucketName, objKey) + return resp, nil, err + } else if query.Contains("legal-hold") { + + } else if query.Contains("retention") { + + } else if query.Contains("tagging") { + + } else if query.Contains("torrent") { + + } else { + // download object itself, which has been handled + } + return nil, nil, NotImplemented(ctx, "not implemented") +} + +func getRangeOpt(rangeStr string, sizeBytes int64) (*cloudprovider.SGetObjectRange, error) { + if len(rangeStr) > 0 { + rangeOptObj := cloudprovider.ParseRange(rangeStr) + if rangeOptObj.End == 0 { + rangeOptObj.End = sizeBytes - 1 + } + if rangeOptObj.Start >= sizeBytes || rangeOptObj.End >= sizeBytes { + return nil, httperrors.ErrOutOfRange + } + if rangeOptObj.Start > 0 || rangeOptObj.End < sizeBytes-1 { + return &rangeOptObj, nil + } + } + return nil, nil +} + +func downloadObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, reqHdr http.Header, w http.ResponseWriter) error { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return errors.Wrap(err, "bucket.GetIBucket") + } + obj, err := cloudprovider.GetIObject(iBucket, key) + if err != nil { + return errors.Wrap(err, "cloudprovider.GetIObject") + } + hdr := http.Header{} + contType := obj.GetContentType() + if len(contType) > 0 { + hdr.Set("Content-Type", obj.GetContentType()) + } + eTag := obj.GetETag() + if len(eTag) > 0 { + hdr.Set("ETag", eTag) + } + lastModified := obj.GetLastModified() + if !lastModified.IsZero() { + hdr.Set("Last-Modified", lastModified.Format(timeutils.RFC2882Format)) + } + rangeStr := reqHdr.Get(http.CanonicalHeaderKey("range")) + rangeOpt, err := getRangeOpt(rangeStr, obj.GetSizeBytes()) + if err != nil { + return errors.Wrap(err, rangeStr) + } + stream, err := iBucket.GetObject(ctx, key, rangeOpt) + if err != nil { + return errors.Wrap(err, "iBucket.GetObject") + } + err = appsrv.SendStream(w, rangeOpt != nil, hdr, stream, obj.GetSizeBytes()) + if err != nil { + return errors.Wrap(err, "appsrv.SendStream") + } + return nil +} + +func readHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + o := fetchObjectRequest(ctx) + userCred := auth.FetchUserCredential(ctx, nil) + if len(o.Bucket) == 0 { + // service + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + input := s3cli.ListBucketsInput{} + err = query.Unmarshal(&input) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + } else { + resp, err := listService(ctx, userCred, input) + if err != nil { + SendGeneralError(ctx, w, err) + } else { + appsrv.SendXml(w, nil, resp) + } + } + } else if len(o.Bucket) > 0 && len(o.Key) == 0 { + // bucket get + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, respHdr, err := readBucket(ctx, userCred, o.Bucket, query, r) + if err != nil { + SendGeneralError(ctx, w, err) + return + } + appsrv.SendXml(w, respHdr, resp) + } else { + // object get + if len(r.URL.RawQuery) == 0 { + // download object + err := downloadObject(ctx, userCred, o.Bucket, o.Key, r.Header, w) + if err != nil { + SendGeneralError(ctx, w, err) + } + return + } + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, respHdr, err := readObject(ctx, userCred, o.Bucket, o.Key, query, r) + if err != nil { + SendGeneralError(ctx, w, err) + return + } + if resp != nil { + appsrv.SendXml(w, respHdr, resp) + } + } +} + +func postObject(ctx context.Context, userCred mcclient.TokenCredential, bucket string, key string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) { + if query.Contains("uploads") { + // initialize multipart upload + return initMultipartUpload(ctx, userCred, r.Header, bucket, key) + } else if query.Contains("uploadId") { + // complete multipart upload + uploadId, err := query.GetString("uploadId") + if err != nil || len(uploadId) == 0 { + return nil, nil, errors.Wrap(httperrors.ErrBadRequest, "uploadId") + } + request := s3cli.CompleteMultipartUpload{} + err = appsrv.FetchXml(r, &request) + if err != nil { + return nil, nil, errors.Wrap(httperrors.ErrBadRequest, "FetchXml") + } + return completeMultipartUpload(ctx, userCred, r.Header, bucket, key, uploadId, &request) + } else if query.Contains("select") { + // select object + return selectObject(ctx, userCred, r.Header, bucket, key) + } else { + // upload object by form POST + } + return nil, nil, NotImplemented(ctx, "not implemented") +} + +func postHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + o := fetchObjectRequest(ctx) + userCred := auth.FetchUserCredential(ctx, nil) + if len(o.Bucket) == 0 { + // no bucket + // do nothing + } else if len(o.Bucket) > 0 && len(o.Key) == 0 { + // bucket post + // do nothing + } else { + // object post + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, respHdr, err := postObject(ctx, userCred, o.Bucket, o.Key, query, r) + if err != nil { + SendGeneralError(ctx, w, err) + return + } + appsrv.SendXml(w, respHdr, resp) + return + } + SendError(w, NotSupported(ctx, "method not supported")) +} + +func putBucket(ctx context.Context, userCred mcclient.TokenCredential, bucket string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) { + if query.Contains("accelerate") { + + } else if query.Contains("acl") { + + } else if query.Contains("analytics") { + + } else if query.Contains("cors") { + + } else if query.Contains("encryption") { + + } else if query.Contains("inventory") { + + } else if query.Contains("lifecycle") { + + } else if query.Contains("publicAccessBlock") { + + } else if query.Contains("logging") { + + } else if query.Contains("metrics") { + + } else if query.Contains("notification") { + + } else if query.Contains("object-lock") { + + } else if query.Contains("policy") { + + } else if query.Contains("replication") { + + } else if query.Contains("requestPayment") { + + } else if query.Contains("tagging") { + + } else if query.Contains("versioning") { + + } else if query.Contains("website") { + + } else { + // create bucket + return nil, nil, NotSupported(ctx, "Not supported") + } + return nil, nil, NotImplemented(ctx, "not implemented") +} + +func putObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, query jsonutils.JSONObject, r *http.Request) (interface{}, http.Header, error) { + if query.Contains("legal-hold") { + + } else if query.Contains("retention") { + + } else if query.Contains("acl") { + + } else if query.Contains("tagging") { + + } else { + // upload object + uploadId, _ := query.GetString("uploadId") + partNumber, _ := query.Int("partNumber") + copySource := r.Header.Get(http.CanonicalHeaderKey("x-amz-copy-source")) + if len(copySource) > 0 { + return copyObject(ctx, userCred, bucketName, key, copySource, r.Header, uploadId, int(partNumber)) + } else { + hdr, err := uploadObject(ctx, userCred, bucketName, key, r.Header, r.Body, uploadId, int(partNumber)) + defer r.Body.Close() + if err != nil { + return nil, nil, err + } + return nil, hdr, nil + } + } + return nil, nil, NotImplemented(ctx, "not implemented") +} + +func putHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + o := fetchObjectRequest(ctx) + userCred := auth.FetchUserCredential(ctx, nil) + if len(o.Bucket) == 0 { + // no bucket + } else if len(o.Bucket) > 0 && len(o.Key) == 0 { + // bucket put + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, respHdr, err := putBucket(ctx, userCred, o.Bucket, query, r) + if err != nil { + SendGeneralError(ctx, w, err) + return + } + appsrv.SendXml(w, respHdr, resp) + return + } else { + // object put + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, respHdr, err := putObject(ctx, userCred, o.Bucket, o.Key, query, r) + if err != nil { + SendGeneralError(ctx, w, err) + return + } + appsrv.SendXml(w, respHdr, resp) + return + } + SendError(w, NotSupported(ctx, "method not supported")) +} + +func deleteBucket(ctx context.Context, userCred mcclient.TokenCredential, bucket string, query jsonutils.JSONObject) (interface{}, error) { + if query.Contains("analytics") { + + } else if query.Contains("cors") { + + } else if query.Contains("encryption") { + + } else if query.Contains("inventory") { + + } else if query.Contains("lifecycle") { + + } else if query.Contains("publicAccessBlock") { + + } else if query.Contains("metrics") { + + } else if query.Contains("policy") { + + } else if query.Contains("replication") { + + } else if query.Contains("tagging") { + + } else if query.Contains("website") { + + } else { + // delete bucket + err := removeBucket(ctx, userCred, bucket) + if err != nil { + return nil, err + } + return nil, nil + } + return nil, NotImplemented(ctx, "not implemented") +} + +func deleteObject(ctx context.Context, userCred mcclient.TokenCredential, bucket string, key string, query jsonutils.JSONObject) (interface{}, error) { + if query.Contains("tagging") { + return deleteObjectTags(ctx, userCred, bucket, key) + } else { + // delete object + err := removeObject(ctx, userCred, bucket, key) + if err != nil { + return nil, err + } + return nil, nil + } +} + +func deleteHandler(ctx context.Context, w http.ResponseWriter, r *http.Request) { + o := fetchObjectRequest(ctx) + userCred := auth.FetchUserCredential(ctx, nil) + if len(o.Bucket) == 0 { + // no bucket + } else if len(o.Bucket) > 0 && len(o.Key) == 0 { + // bucket delete + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, err := deleteBucket(ctx, userCred, o.Bucket, query) + if err != nil { + SendGeneralError(ctx, w, err) + } else { + appsrv.SendXml(w, nil, resp) + } + return + } else { + // object delete + query, err := jsonutils.ParseQueryString(r.URL.RawQuery) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + resp, err := deleteObject(ctx, userCred, o.Bucket, o.Key, query) + if err != nil { + SendGeneralError(ctx, w, err) + } else { + appsrv.SendXml(w, nil, resp) + } + return + } + SendError(w, NotSupported(ctx, "method not supported")) +} diff --git a/pkg/s3gateway/handlers/middelware.go b/pkg/s3gateway/handlers/middelware.go new file mode 100644 index 0000000000..e8e4e97c62 --- /dev/null +++ b/pkg/s3gateway/handlers/middelware.go @@ -0,0 +1,57 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/appctx" + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +const ( + S3_OBJECT_REQUEST = appctx.AppContextKey("S3_OBJECT_REQUEST") +) + +func s3authenticate(f appsrv.FilterHandler) appsrv.FilterHandler { + return func(ctx context.Context, w http.ResponseWriter, r *http.Request) { + o, err := getObjectRequest(r) + if err != nil { + SendError(w, BadRequest(ctx, err.Error())) + return + } + ctx = context.WithValue(ctx, S3_OBJECT_REQUEST, o) + userCred, err := auth.VerifyRequest(*r, o.VirtualHost) + if err != nil { + SendError(w, Unauthenticated(ctx, err.Error())) + return + } + ctx = context.WithValue(ctx, auth.AUTH_TOKEN, userCred) + + f(ctx, w, r) + } +} + +func fetchObjectRequest(ctx context.Context) SObjectRequest { + val := ctx.Value(S3_OBJECT_REQUEST) + if gotypes.IsNil(val) { + return SObjectRequest{} + } + return val.(SObjectRequest) +} diff --git a/pkg/s3gateway/handlers/multipart.go b/pkg/s3gateway/handlers/multipart.go new file mode 100644 index 0000000000..dd8c3f5b5d --- /dev/null +++ b/pkg/s3gateway/handlers/multipart.go @@ -0,0 +1,87 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "net/http" + "sort" + + "yunion.io/x/pkg/errors" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/s3gateway/models" +) + +func initMultipartUpload(ctx context.Context, userCred mcclient.TokenCredential, hdr http.Header, bucketName string, key string) (*s3cli.InitiateMultipartUploadResult, http.Header, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, nil, errors.Wrap(err, "bucket.GetIBucket") + } + contType := hdr.Get(http.CanonicalHeaderKey("content-type")) + aclStr := hdr.Get(http.CanonicalHeaderKey("x-amz-acl")) + storageClassStr := hdr.Get(http.CanonicalHeaderKey("x-amz-storage-class")) + uploadId, err := iBucket.NewMultipartUpload(ctx, key, contType, cloudprovider.TBucketACLType(aclStr), storageClassStr) + if err != nil { + return nil, nil, errors.Wrap(err, "NewMultipartUpload") + } + result := s3cli.InitiateMultipartUploadResult{} + result.Bucket = bucketName + result.Key = key + result.UploadID = uploadId + return &result, nil, nil +} + +type SMultiparts []s3cli.CompletePart + +func (a SMultiparts) Len() int { return len(a) } +func (a SMultiparts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a SMultiparts) Less(i, j int) bool { return a[i].PartNumber < a[j].PartNumber } + +func completeMultipartUpload(ctx context.Context, userCred mcclient.TokenCredential, hdr http.Header, bucketName string, key string, uploadId string, request *s3cli.CompleteMultipartUpload) (*s3cli.CompleteMultipartUploadResult, http.Header, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, nil, errors.Wrap(err, "bucket.GetIBucket") + } + sort.Sort(SMultiparts(request.Parts)) + partEtags := make([]string, len(request.Parts)) + for i := range request.Parts { + partEtags[i] = request.Parts[i].ETag + } + err = iBucket.CompleteMultipartUpload(ctx, key, uploadId, partEtags) + if err != nil { + return nil, nil, errors.Wrap(err, "CompleteMultipartUpload") + } + obj, err := cloudprovider.GetIObject(iBucket, key) + if err != nil { + return nil, nil, errors.Wrap(err, "cloudprovider.GetIObject") + } + result := s3cli.CompleteMultipartUploadResult{} + result.Bucket = bucketName + result.Key = key + result.ETag = obj.GetETag() + result.Location = iBucket.GetLocation() + return &result, nil, nil +} diff --git a/pkg/s3gateway/handlers/object.go b/pkg/s3gateway/handlers/object.go new file mode 100644 index 0000000000..f2bfaed0f9 --- /dev/null +++ b/pkg/s3gateway/handlers/object.go @@ -0,0 +1,299 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "io" + "net/http" + "net/url" + "strconv" + "strings" + + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/timeutils" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/appsrv" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/s3gateway/models" +) + +func headObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string) (http.Header, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "bucket.GetIBucket") + } + obj, err := cloudprovider.GetIObject(iBucket, key) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.GetIObject") + } + hdr := http.Header{} + hdr.Set(http.CanonicalHeaderKey("x-amz-acl"), string(obj.GetAcl())) + hdr.Set(http.CanonicalHeaderKey("x-amz-storage-class"), obj.GetStorageClass()) + hdr.Set(http.CanonicalHeaderKey("content-length"), strconv.FormatInt(obj.GetSizeBytes(), 10)) + hdr.Set(http.CanonicalHeaderKey("content-type"), obj.GetContentType()) + hdr.Set(http.CanonicalHeaderKey("etag"), obj.GetETag()) + hdr.Set(http.CanonicalHeaderKey("last-modified"), obj.GetLastModified().Format(timeutils.RFC2882Format)) + return hdr, nil +} + +func uploadObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, header http.Header, body io.Reader, uploadId string, partNumber int) (http.Header, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + + err = bucket.IsOutOfLimit() + if err != nil { + return nil, errors.Wrap(err, "IsOutOfLimit") + } + + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "bucket.GetIBucket") + } + + contLenStr := header.Get(http.CanonicalHeaderKey("Content-Length")) + contLen, err := strconv.ParseInt(contLenStr, 10, 64) + if err != nil { + return nil, errors.Wrap(httperrors.ErrBadRequest, "missing content length") + } + respHdr := http.Header{} + if len(uploadId) > 0 { + etag, err := iBucket.UploadPart(ctx, key, uploadId, partNumber, body, contLen) + if err != nil { + return nil, errors.Wrap(err, "iBucket.UploadPart") + } + respHdr.Set("ETag", etag) + } else { + contType := header.Get(http.CanonicalHeaderKey("content-type")) + aclStr := header.Get(http.CanonicalHeaderKey("x-amz-acl")) + storageClassStr := header.Get(http.CanonicalHeaderKey("x-amz-storage-class")) + err = iBucket.PutObject(ctx, key, body, contLen, contType, cloudprovider.TBucketACLType(aclStr), storageClassStr) + if err != nil { + return nil, errors.Wrap(err, "iBucket.PutObject") + } + obj, err := cloudprovider.GetIObject(iBucket, key) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.GetIObject") + } + respHdr.Set("ETag", obj.GetETag()) + } + + bucket.Invalidate() + + return respHdr, nil +} + +const ( + MIN_PART_BYTES = 1000 * 1000 * 10 // 100 MB + MAX_PART_COUNT = 10000 +) + +func copyObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string, copySource string, hdr http.Header, uploadId string, partNumber int) (interface{}, http.Header, error) { + log.Debugf("CopyObject %s => %s/%s %s %d %s", copySource, bucketName, key, uploadId, partNumber, hdr) + srcSegs := appsrv.SplitPath(copySource) + srcBucketName := srcSegs[0] + srcKey := strings.Join(srcSegs[1:], "/") + if strings.HasSuffix(copySource, "/") { + srcKey += "/" + } + var err error + srcKey, err = url.PathUnescape(srcKey) + if err != nil { + return nil, nil, errors.Wrap(err, "url.PathUnescape") + } + + srcBucket, err := models.BucketManager.GetByName(ctx, userCred, srcBucketName) + if err != nil { + return nil, nil, errors.Wrap(err, "source bucket GetByName") + } + iSrcBucket, err := srcBucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, nil, errors.Wrap(err, "srcBucket.GetIBucket") + } + srcObj, err := cloudprovider.GetIObject(iSrcBucket, srcKey) + if err != nil { + return nil, nil, errors.Wrap(err, "src cloudprovider.GetIObject") + } + + dstBucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, nil, errors.Wrap(err, "dest bucket GetByName") + } + + err = dstBucket.IsOutOfLimit() + if err != nil { + return nil, nil, errors.Wrap(err, "IsOutOfLimit") + } + + iDstBucket, err := dstBucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, nil, errors.Wrap(err, "dstBucket.GetIBucket") + } + + sizeBytes := srcObj.GetSizeBytes() + rangeStr := hdr.Get(http.CanonicalHeaderKey("x-amz-copy-source-range")) + rangeOpt, err := getRangeOpt(rangeStr, sizeBytes) + if err != nil { + return nil, nil, errors.Wrap(err, rangeStr) + } + + if rangeOpt != nil { + if len(uploadId) == 0 { + return nil, nil, errors.Wrap(httperrors.ErrBadRequest, "range copy must be a multipart upload") + } + // upload directory + var etag string + if dstBucket.ManagerId == srcBucket.ManagerId && dstBucket.RegionExternalId == srcBucket.RegionExternalId { + etag, err = iDstBucket.CopyPart(ctx, key, uploadId, partNumber, iSrcBucket.GetName(), srcKey, rangeOpt.Start, rangeOpt.SizeBytes()) + } else { + etag, err = cloudprovider.CopyPart(ctx, iDstBucket, key, uploadId, partNumber, iSrcBucket, srcKey, rangeOpt) + } + if err != nil { + return nil, nil, errors.Wrap(err, "copyPart fail") + } + result := s3cli.CopyPartResult{ + ETag: etag, + LastModified: srcObj.GetLastModified(), + } + return &result, nil, nil + } else { + if dstBucket.ManagerId == srcBucket.ManagerId && dstBucket.RegionExternalId == srcBucket.RegionExternalId { + err = iDstBucket.CopyObject(ctx, key, iSrcBucket.GetName(), srcKey, srcObj.GetContentType(), srcObj.GetAcl(), srcObj.GetStorageClass()) + if err != nil { + return nil, nil, errors.Wrap(err, "iDstBucket.CopyObject") + } + } else { + err = cloudprovider.CopyObject(ctx, 0, iDstBucket, key, iSrcBucket, srcKey, false) + if err != nil { + return nil, nil, errors.Wrap(err, "cloudprovider.CopyObject") + } + } + + dstBucket.Invalidate() + + dstObj, err := cloudprovider.GetIObject(iDstBucket, key) + if err != nil { + return nil, nil, errors.Wrap(err, "cloudprovider.GetIObject") + } + result := s3cli.CopyObjectResult{ + ETag: dstObj.GetETag(), + LastModified: dstObj.GetLastModified(), + } + return &result, nil, nil + } +} + +func deleteObjectTags(ctx context.Context, userCred mcclient.TokenCredential, bucket string, key string) (*s3cli.Tagging, error) { + return nil, nil +} + +func removeObject(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, key string) error { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return errors.Wrap(err, "bucket.GetIBucket") + } + err = iBucket.DeleteObject(ctx, key) + if err != nil { + return errors.Wrap(err, "DeleteObject") + } + + bucket.Invalidate() + + return nil +} + +func objectAcl(ctx context.Context, userCred mcclient.TokenCredential, bucketName string, objKey string) (*s3cli.AccessControlPolicy, error) { + bucket, err := models.BucketManager.GetByName(ctx, userCred, bucketName) + if err != nil { + return nil, errors.Wrap(err, "models.BucketManager.GetByName") + } + iBucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "bucket.GetIBucket") + } + obj, err := cloudprovider.GetIObject(iBucket, objKey) + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.GetIObject") + } + + result := str2Acl(userCred, obj.GetAcl()) + + return result, nil +} + +func str2Acl(userCred mcclient.TokenCredential, aclStr cloudprovider.TBucketACLType) *s3cli.AccessControlPolicy { + result := s3cli.AccessControlPolicy{} + result.Owner.DisplayName = userCred.GetProjectName() + result.Owner.ID = userCred.GetProjectId() + + fullControl := s3cli.Grant{} + fullControl.Permission = s3cli.PERMISSION_FULL_CONTROL + fullControl.Grantee.Type = s3cli.GRANTEE_TYPE_USER + fullControl.Grantee.ID = userCred.GetProjectId() + fullControl.Grantee.DisplayName = userCred.GetProjectName() + + publicRead := s3cli.Grant{} + publicRead.Permission = s3cli.PERMISSION_READ + publicRead.Grantee.Type = s3cli.GRANTEE_TYPE_GROUP + publicRead.Grantee.URI = s3cli.GRANTEE_GROUP_URI_ALL_USERS + + publicWrite := s3cli.Grant{} + publicWrite.Permission = s3cli.PERMISSION_WRITE + publicWrite.Grantee.Type = s3cli.GRANTEE_TYPE_GROUP + publicWrite.Grantee.URI = s3cli.GRANTEE_GROUP_URI_ALL_USERS + + authRead := s3cli.Grant{} + authRead.Permission = s3cli.PERMISSION_READ + authRead.Grantee.Type = s3cli.GRANTEE_TYPE_GROUP + authRead.Grantee.URI = s3cli.GRANTEE_GROUP_URI_AUTH_USERS + + switch aclStr { + case cloudprovider.ACLPrivate: + result.AccessControlList.Grant = []s3cli.Grant{ + fullControl, + } + case cloudprovider.ACLAuthRead: + result.AccessControlList.Grant = []s3cli.Grant{ + fullControl, + authRead, + } + case cloudprovider.ACLPublicRead: + result.AccessControlList.Grant = []s3cli.Grant{ + fullControl, + publicRead, + } + case cloudprovider.ACLPublicReadWrite: + result.AccessControlList.Grant = []s3cli.Grant{ + fullControl, + publicRead, + publicWrite, + } + } + return &result +} diff --git a/pkg/s3gateway/models/initdb.go b/pkg/s3gateway/handlers/select.go similarity index 58% rename from pkg/s3gateway/models/initdb.go rename to pkg/s3gateway/handlers/select.go index 358ff3535b..8df0653ff9 100644 --- a/pkg/s3gateway/models/initdb.go +++ b/pkg/s3gateway/handlers/select.go @@ -12,26 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package models +package handlers import ( - "yunion.io/x/log" + "context" + "net/http" - "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/mcclient" ) -func InitDB() error { - for _, manager := range []db.IModelManager{ - /* - * Important!!! - * initialization order matters, do not change the order - */ - } { - err := manager.InitializeData() - if err != nil { - log.Errorf("Manager %s initializeData fail %s", manager.Keyword(), err) - // return err skip error table - } - } - return nil +func selectObject(ctx context.Context, userCred mcclient.TokenCredential, hdr http.Header, bucket string, key string) (*s3cli.InitiateMultipartUploadResult, http.Header, error) { + return nil, nil, NotImplemented(ctx, "") } diff --git a/pkg/s3gateway/handlers/service.go b/pkg/s3gateway/handlers/service.go new file mode 100644 index 0000000000..51121ef4c0 --- /dev/null +++ b/pkg/s3gateway/handlers/service.go @@ -0,0 +1,50 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package handlers + +import ( + "context" + "time" + + "yunion.io/x/pkg/errors" + "yunion.io/x/s3cli" + + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/s3gateway/models" +) + +type sBucketInfo struct { + Name string + CreatedAt time.Time +} + +func listService(ctx context.Context, userCred mcclient.TokenCredential, query s3cli.ListBucketsInput) (*s3cli.ListAllMyBucketsResult, error) { + result, err := models.BucketManager.List(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "models.BucketManager.List") + } + resp := s3cli.ListAllMyBucketsResult{} + resp.Owner.ID = userCred.GetProjectId() + resp.Owner.DisplayName = userCred.GetProjectName() + resp.Buckets.Bucket = make([]s3cli.BucketInfo, 0) + for i := range result { + info := result[i] + resp.Buckets.Bucket = append(resp.Buckets.Bucket, s3cli.BucketInfo{ + Name: info.Name, + CreationDate: info.CreatedAt, + }) + } + return &resp, nil +} diff --git a/pkg/s3gateway/models/doc.go b/pkg/s3gateway/models/base.go similarity index 77% rename from pkg/s3gateway/models/doc.go rename to pkg/s3gateway/models/base.go index ebefc3d149..7d99121161 100644 --- a/pkg/s3gateway/models/doc.go +++ b/pkg/s3gateway/models/base.go @@ -12,4 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package models // import "yunion.io/x/onecloud/pkg/s3gateway/models" +package models + +import "time" + +type SBaseModelManagerDelegate struct { +} + +type SBaseModelDelegate struct { + Id string + Name string + CreatedAt time.Time +} diff --git a/pkg/s3gateway/models/buckets.go b/pkg/s3gateway/models/buckets.go new file mode 100644 index 0000000000..e21332547c --- /dev/null +++ b/pkg/s3gateway/models/buckets.go @@ -0,0 +1,243 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "github.com/pkg/errors" + "time" + "yunion.io/x/jsonutils" + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/httperrors" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/s3gateway/session" + "yunion.io/x/onecloud/pkg/util/hashcache" + "yunion.io/x/pkg/gotypes" + "yunion.io/x/s3cli" +) + +type SBucketManagerDelegate struct { + buckets *hashcache.Cache +} + +var BucketManager *SBucketManagerDelegate + +func init() { + BucketManager = &SBucketManagerDelegate{ + buckets: hashcache.NewCache(2048, time.Minute*15), + } +} + +/* +{ + "access_urls":[{"description":"bucket domain","primary":true,"url":"https://yunion-billing-reports.s3.cn-northwest-1.amazonaws.com.cn"},{"description":"s3 domain","primary":false,"url":"https://s3.cn-northwest-1.amazonaws.com.cn/yunion-billing-reports"}], + "account":"aws-cn", + "account_id":"edc90a61-7f8a-4be7-84f1-8f3ac70ef5e6", + "acl":"private", + "brand":"Aws", + "can_delete":false, + "can_update":true, + "cloud_env":"public", + "cloudregion_id":"4cbf92a5-337b-4cc6-82c7-86e5427b69e3", + "created_at":"2019-03-11T10:31:26.000000Z", + "domain_id":"default", + "external_id":"yunion-billing-reports", + "id":"056bb8c6-527d-4554-8939-12eb6aa803bd", + "is_emulated":false, + "is_system":false, + "location":"cn-northwest-1", + "manager":"aws-cn", + "manager_domain":"Default", + "manager_domain_id":"default", + "manager_id":"d8df39fa-b212-43c1-8d44-aeef897e216d", + "manager_project":"system", + "manager_project_id":"5d65667d112e47249ae66dbd7bc07030", + "name":"yunion-billing-reports", + "object_cnt":44, + "object_cnt_limit":0, + "project_domain":"Default", + "project_src":"cloud", + "provider":"Aws", + "region":"AWS 中国(宁夏)", + "region_ext_id":"cn-northwest-1", + "region_id":"4cbf92a5-337b-4cc6-82c7-86e5427b69e3", + "size_bytes":3332448, + "size_bytes_limit":0, + "status":"ready", + "tenant":"system", + "tenant_id":"5d65667d112e47249ae66dbd7bc07030", + "update_version":1, + "updated_at":"2019-08-18T15:52:42.000000Z", +} +*/ +type SBucketDelegate struct { + SBaseModelDelegate + + Location string + ManagerId string + + ObjectCnt int + SizeBytes int64 + + ObjectCntLimit int + SizeBytesLimit int64 + + RegionExternalId string + ExternalId string +} + +func (manager *SBucketManagerDelegate) List(ctx context.Context, userCred mcclient.TokenCredential) ([]*SBucketDelegate, error) { + s := session.GetSession(ctx, userCred) + offset := 0 + total := -1 + ret := make([]*SBucketDelegate, 0) + for total < 0 || offset < total { + params := struct { + Limit int + Offset int + }{} + params.Limit = 1000 + params.Offset = offset + result, err := modules.Buckets.List(s, jsonutils.Marshal(params)) + if err != nil { + return nil, errors.Wrap(err, "List") + } + total = result.Total + offset += len(result.Data) + for i := range result.Data { + bucket := &SBucketDelegate{} + err := result.Data[i].Unmarshal(bucket) + if err != nil { + return nil, errors.Wrap(err, "Unmarshal") + } + ret = append(ret, bucket) + manager.buckets.AtomicSet(bucket.Name, bucket) + } + } + return ret, nil +} + +func (manager *SBucketManagerDelegate) GetByName(ctx context.Context, userCred mcclient.TokenCredential, name string) (*SBucketDelegate, error) { + val := manager.buckets.AtomicGet(name) + if !gotypes.IsNil(val) { + return val.(*SBucketDelegate), nil + } + s := session.GetSession(ctx, userCred) + result, err := modules.Buckets.PerformAction(s, name, "sync", nil) + if err != nil { + return nil, errors.Wrap(err, "modules.Buckets.Get") + } + bucket := &SBucketDelegate{} + err = result.Unmarshal(bucket) + if err != nil { + return nil, errors.Wrap(err, "result.Unmarshal") + } + manager.buckets.AtomicSet(bucket.Name, bucket) + return bucket, nil +} + +func (manager *SBucketManagerDelegate) DeleteByName(ctx context.Context, userCred mcclient.TokenCredential, name string) error { + s := session.GetSession(ctx, userCred) + _, err := modules.Buckets.Delete(s, name, nil) + if err != nil { + return errors.Wrap(err, "modules.Buckets.Delete") + } + manager.buckets.AtomicRemove(name) + return nil +} + +func (manager *SBucketManagerDelegate) Invalidate(name string) { + manager.buckets.AtomicRemove(name) +} + +func (bucket *SBucketDelegate) getManager(ctx context.Context, userCred mcclient.TokenCredential) (*SCloudproviderDelegate, error) { + return CloudproviderManager.GetById(ctx, userCred, bucket.ManagerId) +} + +func (bucket *SBucketDelegate) GetIBucket(ctx context.Context, userCred mcclient.TokenCredential) (cloudprovider.ICloudBucket, error) { + manager, err := bucket.getManager(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "bucket.getManager") + } + driver, err := manager.GetProvider() + if err != nil { + return nil, errors.Wrap(err, "cloudprovider.GetProvider") + } + var iRegion cloudprovider.ICloudRegion + if len(bucket.RegionExternalId) == 0 { + iRegion, err = driver.GetOnPremiseIRegion() + } else { + iRegion, err = driver.GetIRegionById(bucket.RegionExternalId) + } + if err != nil { + return nil, errors.Wrap(err, "driver.GetIRegionById") + } + iBucket, err := iRegion.GetIBucketById(bucket.ExternalId) + if err != nil { + return nil, errors.Wrap(err, "iRegion.GetIBucketById") + } + return iBucket, nil +} + +func (bucket *SBucketDelegate) ListObject(ctx context.Context, userCred mcclient.TokenCredential, input *s3cli.ListObjectInput) (*s3cli.ListBucketResult, error) { + ibucket, err := bucket.GetIBucket(ctx, userCred) + if err != nil { + return nil, errors.Wrap(err, "getIBucket") + } + result, err := ibucket.ListObjects(input.Prefix, input.Marker, input.Delimiter, int(input.MaxKeys)) + if err != nil { + return nil, errors.Wrap(err, "ibucket.ListObjects") + } + ret := s3cli.ListBucketResult{} + ret.IsTruncated = result.IsTruncated + ret.MaxKeys = input.MaxKeys + ret.Delimiter = input.Delimiter + ret.Prefix = input.Prefix + ret.Marker = input.Marker + ret.CommonPrefixes = make([]s3cli.CommonPrefix, len(result.CommonPrefixes)) + for i := range result.CommonPrefixes { + ret.CommonPrefixes[i] = s3cli.CommonPrefix{ + Prefix: result.CommonPrefixes[i].GetKey(), + } + } + ret.Contents = make([]s3cli.ObjectInfo, len(result.Objects)) + for i := range result.Objects { + obj := result.Objects[i] + ret.Contents[i] = s3cli.ObjectInfo{ + Key: obj.GetKey(), + ETag: obj.GetETag(), + Size: obj.GetSizeBytes(), + LastModified: obj.GetLastModified(), + ContentType: obj.GetContentType(), + StorageClass: obj.GetStorageClass(), + } + } + return &ret, nil +} + +func (bucket *SBucketDelegate) IsOutOfLimit() error { + if bucket.ObjectCntLimit > 0 && bucket.ObjectCnt >= bucket.ObjectCntLimit { + return errors.Wrap(httperrors.ErrOutOfLimit, "object_count") + } + if bucket.SizeBytesLimit > 0 && bucket.SizeBytes >= bucket.SizeBytesLimit { + return errors.Wrap(httperrors.ErrOutOfLimit, "size_bytes") + } + return nil +} + +func (bucket *SBucketDelegate) Invalidate() { + BucketManager.Invalidate(bucket.Name) +} diff --git a/pkg/s3gateway/models/cloudproviders.go b/pkg/s3gateway/models/cloudproviders.go new file mode 100644 index 0000000000..9605c166ad --- /dev/null +++ b/pkg/s3gateway/models/cloudproviders.go @@ -0,0 +1,140 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package models + +import ( + "context" + "time" + + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" + + "yunion.io/x/onecloud/pkg/cloudprovider" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/modules" + "yunion.io/x/onecloud/pkg/s3gateway/session" + "yunion.io/x/onecloud/pkg/util/hashcache" + "yunion.io/x/pkg/utils" +) + +type SCloudproviderManagerDelegate struct { + providers *hashcache.Cache +} + +var CloudproviderManager *SCloudproviderManagerDelegate + +func init() { + CloudproviderManager = &SCloudproviderManagerDelegate{ + providers: hashcache.NewCache(2048, time.Minute*15), + } +} + +/* +{ + "account":"oH7Qrw75AqrI4BXn", + "can_delete":false, + "can_update":true, + "cloudaccount":"testaliyun", + "cloudaccount_id":"f1a927b4-a433-486e-86ae-e9aa36e447b1", + "created_at":"2019-04-16T13:55:11.000000Z", + "domain":"Default", + "domain_id":"default", + "eip_count":9, + "enabled":true, + "guest_count":1, + "health_status":"normal", + "host_count":61, + "id":"57c84d93-8f06-4a85-8963-4ce42eabb339", + "is_emulated":false, + "last_sync":"2019-07-23T15:29:34.000000Z", + "last_sync_end_at":"2019-07-23T15:33:34.000000Z", + "loadbalancer_count":13, + "name":"testaliyun", + "project_count":0, + "provider":"Aliyun", + "secret":"Y5YFmuwVI4frJ8kVgWL0z5Kan/sJ3JMyjyFRxAXwXvsUKd8aNohPp2T/Kr1BqA==", + "snapshot_count":6, + "status":"connected", + "storage_cache_count":20, + "storage_count":141, + "sync_region_count":20, + "sync_status":"idle", + "sync_status2":"idle", + "tenant":"system", + "tenant_id":"5d65667d112e47249ae66dbd7bc07030", + "update_version":5003, + "updated_at":"2019-08-18T14:47:42.000000Z", + "vpc_count":13, +} +*/ + +type SCloudproviderDelegate struct { + SBaseModelDelegate + + Enabled bool + Status string + SyncStatus string + + AccessUrl string + Account string + Secret string + + Provider string + Brand string +} + +func (manager *SCloudproviderManagerDelegate) GetById(ctx context.Context, userCred mcclient.TokenCredential, id string) (*SCloudproviderDelegate, error) { + val := manager.providers.AtomicGet(id) + if !gotypes.IsNil(val) { + return val.(*SCloudproviderDelegate), nil + } + s := session.GetSession(ctx, userCred) + result, err := modules.Cloudproviders.Get(s, id, nil) + if err != nil { + return nil, errors.Wrap(err, "modules.Cloudproviders.Get") + } + provider := &SCloudproviderDelegate{} + err = result.Unmarshal(provider) + if err != nil { + return nil, errors.Wrap(err, "result.Unmarshal") + } + manager.providers.AtomicSet(provider.Id, provider) + return provider, nil +} + +func (provider *SCloudproviderDelegate) getPassword() (string, error) { + return utils.DescryptAESBase64(provider.Id, provider.Secret) +} + +func (provider *SCloudproviderDelegate) getAccessUrl() string { + return provider.AccessUrl +} + +func (provider *SCloudproviderDelegate) GetProviderFactory() (cloudprovider.ICloudProviderFactory, error) { + return cloudprovider.GetProviderFactory(provider.Provider) +} + +func (provider *SCloudproviderDelegate) GetProvider() (cloudprovider.ICloudProvider, error) { + if !provider.Enabled { + return nil, errors.Error("Cloud provider is not enabled") + } + + accessUrl := provider.getAccessUrl() + passwd, err := provider.getPassword() + if err != nil { + return nil, err + } + return cloudprovider.GetProvider(provider.Id, provider.Name, accessUrl, provider.Account, passwd, provider.Provider) +} diff --git a/pkg/s3gateway/options/options.go b/pkg/s3gateway/options/options.go index b1b5513deb..2a60f80fbb 100644 --- a/pkg/s3gateway/options/options.go +++ b/pkg/s3gateway/options/options.go @@ -21,7 +21,7 @@ import ( type SS3GatewayOptions struct { common_options.CommonOptions - common_options.DBOptions + DomainName string `help:"s3 domain name"` } var ( diff --git a/pkg/s3gateway/service/handlers.go b/pkg/s3gateway/service/handlers.go deleted file mode 100644 index c09a7c2fbd..0000000000 --- a/pkg/s3gateway/service/handlers.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2019 Yunion -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package service - -import ( - "yunion.io/x/onecloud/pkg/appsrv" - "yunion.io/x/onecloud/pkg/appsrv/dispatcher" - "yunion.io/x/onecloud/pkg/cloudcommon/db" - "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" - // "yunion.io/x/onecloud/pkg/s3gateway/models" -) - -func initHandlers(app *appsrv.Application) { - db.InitAllManagers() - - // quotas.AddQuotaHandler(models.QuotaManager, API_VERSION, app) - // usages.AddUsageHandler(API_VERSION, app) - taskman.AddTaskHandler("", app) - - for _, manager := range []db.IModelManager{ - taskman.TaskManager, - taskman.SubTaskManager, - taskman.TaskObjectManager, - db.Metadata, - } { - db.RegisterModelManager(manager) - } - - for _, manager := range []db.IModelManager{ - db.OpsLog, - } { - db.RegisterModelManager(manager) - handler := db.NewModelHandler(manager) - dispatcher.AddModelDispatcher("", app, handler) - } -} diff --git a/pkg/s3gateway/service/service.go b/pkg/s3gateway/service/service.go index 9f34e8f194..dc2982e11a 100644 --- a/pkg/s3gateway/service/service.go +++ b/pkg/s3gateway/service/service.go @@ -22,40 +22,31 @@ import ( api "yunion.io/x/onecloud/pkg/apis/s3gateway" "yunion.io/x/onecloud/pkg/cloudcommon" app_common "yunion.io/x/onecloud/pkg/cloudcommon/app" - "yunion.io/x/onecloud/pkg/cloudcommon/db" common_options "yunion.io/x/onecloud/pkg/cloudcommon/options" - "yunion.io/x/onecloud/pkg/s3gateway/models" + "yunion.io/x/onecloud/pkg/s3gateway/handlers" "yunion.io/x/onecloud/pkg/s3gateway/options" + + _ "yunion.io/x/onecloud/pkg/multicloud/aliyun/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/aws/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/azure/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/huawei/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/qcloud/provider" + _ "yunion.io/x/onecloud/pkg/multicloud/ucloud/provider" ) func StartService() { opts := &options.Options commonOpts := &opts.CommonOptions baseOpts := &opts.BaseOptions - dbOpts := &opts.DBOptions common_options.ParseOptions(opts, os.Args, "s3gateway.conf", api.SERVICE_TYPE) app_common.InitAuth(commonOpts, func() { log.Infof("Auth complete!!") }) - cloudcommon.InitDB(dbOpts) - - app := app_common.InitApp(&opts.BaseOptions, true) - initHandlers(app) - - cloudcommon.InitDB(&opts.DBOptions) - - if !db.CheckSync(opts.AutoSyncTable) { - log.Fatalf("database schema not in sync!") - } - - models.InitDB() - - if opts.ExitAfterDBInit { - log.Infof("Exiting after db initialization ...") - os.Exit(0) - } + app := app_common.InitApp(&opts.BaseOptions, false) + handlers.InitHandlers(app) /*if !opts.IsSlaveNode { cron := cronman.GetCronJobManager(true) diff --git a/pkg/s3gateway/session/session.go b/pkg/s3gateway/session/session.go new file mode 100644 index 0000000000..29215c2b79 --- /dev/null +++ b/pkg/s3gateway/session/session.go @@ -0,0 +1,30 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package session + +import ( + "context" + "yunion.io/x/onecloud/pkg/image/options" + "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/mcclient/auth" +) + +func GetSession(ctx context.Context, token mcclient.TokenCredential) *mcclient.ClientSession { + return auth.GetSession(ctx, token, options.Options.Region, "") +} + +func GetAdminSession(ctx context.Context) *mcclient.ClientSession { + return auth.GetAdminSession(ctx, options.Options.Region, "") +} diff --git a/pkg/util/bitmap/doc.go b/pkg/util/bitmap/doc.go index a6a4ba08d2..f2a7ff3bec 100644 --- a/pkg/util/bitmap/doc.go +++ b/pkg/util/bitmap/doc.go @@ -1 +1,15 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package bitmap // import "yunion.io/x/onecloud/pkg/util/bitmap" diff --git a/pkg/util/hashcache/cache.go b/pkg/util/hashcache/cache.go index 576b7fcba7..97f26f15a3 100644 --- a/pkg/util/hashcache/cache.go +++ b/pkg/util/hashcache/cache.go @@ -18,6 +18,7 @@ import ( "crypto/md5" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "sync" "time" ) @@ -54,6 +55,7 @@ const ( HASH_ALG_MD5 int = iota HASH_ALG_SHA1 HASH_ALG_SHA256 + HASH_ALG_SHA512 ) func bytes2int(b []byte) uint32 { @@ -72,6 +74,9 @@ func checksum(alg int, key string) uint32 { case HASH_ALG_SHA256: v := sha256.Sum224([]byte(key)) hash = bytes2int(v[0:4]) + case HASH_ALG_SHA512: + v := sha512.Sum512([]byte(key)) + hash = bytes2int(v[0:4]) } return hash } @@ -79,7 +84,7 @@ func checksum(alg int, key string) uint32 { func (c *Cache) find(key string) (bool, uint32) { var idx uint32 now := time.Now() - for _, alg := range []int{HASH_ALG_MD5, HASH_ALG_SHA1, HASH_ALG_SHA256} { + for _, alg := range []int{HASH_ALG_MD5, HASH_ALG_SHA1, HASH_ALG_SHA256, HASH_ALG_SHA512} { idx = checksum(alg, key) % c.size if c.table[idx].key == key { if c.table[idx].expire.IsZero() || c.table[idx].expire.After(now) { @@ -126,6 +131,20 @@ func (c *Cache) AtomicSet(key string, val interface{}) { c.Set(key, val) } +func (c *Cache) Remove(key string) { + find, idx := c.find(key) + if !find { + return + } + c.table[idx].reset() +} + +func (c *Cache) AtomicRemove(key string) { + c.lock.Lock() + defer c.lock.Unlock() + c.Remove(key) +} + func (c *Cache) Invalidate() { c.lock.Lock() defer c.lock.Unlock() diff --git a/pkg/util/s3auth/interface.go b/pkg/util/s3auth/interface.go new file mode 100644 index 0000000000..1e67dd2307 --- /dev/null +++ b/pkg/util/s3auth/interface.go @@ -0,0 +1,118 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3auth + +import ( + "net/http" + "strings" + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +type IAccessKeySecretRequest interface { + GetAccessKey() string + Validate() error + ParseRequest(req http.Request, virtualHost bool) error + Verify(secret string) error + Encode() string +} + +type SAccessKeyRequest struct { + Algorithm string `json:"algorithm,omitempty"` + AccessKey string `json:"access_key,omitempty"` + Signature string `json:"signature,omitempty"` + Request string `json:"request,omitempty"` +} + +func (aksk SAccessKeyRequest) GetAccessKey() string { + return aksk.AccessKey +} + +func (aksk SAccessKeyRequest) Validate() error { + if len(aksk.AccessKey) == 0 { + return errors.Error("Missing AWSAccessKeyId") + } + if len(aksk.Signature) == 0 { + return errors.Error("Missing Signature") + } + return nil +} + +func decodeAuthHeader(authHeader string) (IAccessKeySecretRequest, error) { + pos := strings.IndexByte(authHeader, ' ') + if pos <= 0 { + return nil, errors.Error("illegal authorization header") + } + algo := authHeader[:pos] + switch algo { + case signV2Algorithm: + req, err := decodeAuthHeaderV2(authHeader[pos+1:]) + if err != nil { + return nil, errors.Wrap(err, "decodeAuthHeaderV2") + } + return req, nil + case signV4Algorithm: + req, err := decodeAuthHeaderV4(authHeader[pos+1:]) + if err != nil { + return nil, errors.Wrap(err, "decodeAuthHeaderV4") + } + return req, nil + default: + return nil, errors.Error("unsupported signing algorithm") + } +} + +func DecodeAccessKeyRequest(req http.Request, virtualHost bool) (IAccessKeySecretRequest, error) { + authHeader := req.Header.Get("Authorization") + if len(authHeader) == 0 { + return nil, errors.Error("missing authorization header") + } + akskReq, err := decodeAuthHeader(authHeader) + if err != nil { + return nil, errors.Wrap(err, "decodeAuthHeader") + } + err = akskReq.ParseRequest(req, virtualHost) + if err != nil { + return nil, errors.Wrap(err, "akskReq.ParseRequest") + } + + return akskReq, akskReq.Validate() +} + +func Decode(reqStr string) (IAccessKeySecretRequest, error) { + rawReq := SAccessKeyRequest{} + reqJson, err := jsonutils.ParseString(reqStr) + if err != nil { + return nil, errors.Wrap(err, "jsonutils.ParseString") + } + err = reqJson.Unmarshal(&rawReq) + if err != nil { + return nil, errors.Wrap(err, "reqJson.Unmarshal rawReq") + } + var ret IAccessKeySecretRequest + switch rawReq.Algorithm { + case signV2Algorithm: + ret = &SAccessKeyRequestV2{} + case signV4Algorithm: + ret = &SAccessKeyRequestV4{} + default: + return nil, errors.Error("unsupported sign algorithm") + } + err = reqJson.Unmarshal(ret) + if err != nil { + return nil, errors.Wrap(err, "reqJson.Unmarshal") + } + return ret, nil +} diff --git a/pkg/util/s3auth/utils.go b/pkg/util/s3auth/utils.go new file mode 100644 index 0000000000..2533674046 --- /dev/null +++ b/pkg/util/s3auth/utils.go @@ -0,0 +1,89 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3auth + +import ( + "encoding/hex" + "net/http" + "regexp" + "strings" + "unicode/utf8" +) + +// if object matches reserved string, no need to encode them +var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + +// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +func encodePath(pathName string) string { + if reservedObjectNames.MatchString(pathName) { + return pathName + } + var encodedPathname string + for _, s := range pathName { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + default: + len := utf8.RuneLen(s) + if len < 0 { + // if utf8 cannot convert return the same string as is + return pathName + } + u := make([]byte, len) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + } + } + } + return encodedPathname +} + +// getHostAddr returns host header if available, otherwise returns host from URL +func getHostAddr(req http.Request) string { + if req.Host != "" { + return req.Host + } + return req.URL.Host +} + +// Encode input URL path to URL encoded path. +func encodeURL2Path(req http.Request, virtualHost bool) (path string) { + if virtualHost { + reqHost := getHostAddr(req) + dotPos := strings.Index(reqHost, ".") + if dotPos > -1 { + bucketName := reqHost[:dotPos] + path = "/" + bucketName + path += req.URL.Path + path = encodePath(path) + return + } + } + path = encodePath(req.URL.Path) + return +} diff --git a/pkg/util/s3auth/v2.go b/pkg/util/s3auth/v2.go new file mode 100644 index 0000000000..99522df343 --- /dev/null +++ b/pkg/util/s3auth/v2.go @@ -0,0 +1,244 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3auth + +import ( + "bytes" + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "net/http" + "net/url" + "sort" + "strings" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +// Signature and API related constants. +const ( + signV2Algorithm = "AWS" +) + +// From the Amazon docs: +// +// StringToSign = HTTP-Verb + "\n" + +// Content-Md5 + "\n" + +// Content-Type + "\n" + +// Date + "\n" + +// CanonicalizedProtocolHeaders + +// CanonicalizedResource; +func stringToSignV2(req http.Request, virtualHost bool) string { + buf := new(bytes.Buffer) + // Write standard headers. + writeSignV2Headers(buf, req) + // Write canonicalized protocol headers if any. + writeCanonicalizedHeaders(buf, req) + // Write canonicalized Query resources if any. + writeCanonicalizedResource(buf, req, virtualHost) + return buf.String() +} + +// writeSignV2Headers - write signV2 required headers. +func writeSignV2Headers(buf *bytes.Buffer, req http.Request) { + buf.WriteString(req.Method + "\n") + buf.WriteString(req.Header.Get("Content-Md5") + "\n") + buf.WriteString(req.Header.Get("Content-Type") + "\n") + buf.WriteString(req.Header.Get("Date") + "\n") +} + +// writeCanonicalizedHeaders - write canonicalized headers. +func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) { + var protoHeaders []string + vals := make(map[string][]string) + for k, vv := range req.Header { + // All the AMZ headers should be lowercase + lk := strings.ToLower(k) + if strings.HasPrefix(lk, "x-amz") { + protoHeaders = append(protoHeaders, lk) + vals[lk] = vv + } + } + sort.Strings(protoHeaders) + for _, k := range protoHeaders { + buf.WriteString(k) + buf.WriteByte(':') + for idx, v := range vals[k] { + if idx > 0 { + buf.WriteByte(',') + } + if strings.Contains(v, "\n") { + // TODO: "Unfold" long headers that + // span multiple lines (as allowed by + // RFC 2616, section 4.2) by replacing + // the folding white-space (including + // new-line) by a single space. + buf.WriteString(v) + } else { + buf.WriteString(v) + } + } + buf.WriteByte('\n') + } +} + +// AWS S3 Signature V2 calculation rule is give here: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign + +// Whitelist resource list that will be used in query string for signature-V2 calculation. +// The list should be alphabetically sorted +var resourceList = []string{ + "acl", + "delete", + "lifecycle", + "location", + "logging", + "notification", + "partNumber", + "policy", + "requestPayment", + "response-cache-control", + "response-content-disposition", + "response-content-encoding", + "response-content-language", + "response-content-type", + "response-expires", + "torrent", + "uploadId", + "uploads", + "versionId", + "versioning", + "versions", + "website", +} + +// From the Amazon docs: +// +// CanonicalizedResource = [ "/" + Bucket ] + +// + +// [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; +func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, virtualHost bool) { + // Save request URL. + requestURL := req.URL + // Get encoded URL path. + buf.WriteString(encodeURL2Path(req, virtualHost)) + if requestURL.RawQuery != "" { + var n int + vals, _ := url.ParseQuery(requestURL.RawQuery) + // Verify if any sub resource queries are present, if yes + // canonicallize them. + for _, resource := range resourceList { + if vv, ok := vals[resource]; ok && len(vv) > 0 { + n++ + // First element + switch n { + case 1: + buf.WriteByte('?') + // The rest + default: + buf.WriteByte('&') + } + buf.WriteString(resource) + // Request parameters + if len(vv[0]) > 0 { + buf.WriteByte('=') + buf.WriteString(vv[0]) + } + } + } + } +} + +// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; +// Signature = Base64( HMAC-SHA1( YourSecretAccessKeyID, UTF-8-Encoding-Of( StringToSign ) ) ); +// +// StringToSign = HTTP-Verb + "\n" + +// Content-Md5 + "\n" + +// Content-Type + "\n" + +// Date + "\n" + +// CanonicalizedProtocolHeaders + +// CanonicalizedResource; +// +// CanonicalizedResource = [ "/" + Bucket ] + +// + +// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; +// +// CanonicalizedProtocolHeaders = +// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}. +/*func verifyV2(ctx context.Context, req http.Request, virtualHost bool) error { + aksk, err := DecodeAccessKeyRequestV2(req, virtualHost) + if err != nil { + return errors.Wrap(err, "DecodeAccessKeyRequestV2") + } + + authSession := session.GetAdminSession(ctx) + result, err := modules.Credentials.Get(authSession, aksk.AccessKey, nil) + if err != nil { + return errors.Wrap(err, "modules.Credentials.Get") + } + secret, err := modules.DecodeAccessKeySecret(result) + if err != nil { + return errors.Wrap(err, "modules.DecodeAccessKeySecret") + } + + hm := hmac.New(sha1.New, []byte(secret.Secret)) + hm.Write([]byte(aksk.RequestString)) + + signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) + + if aksk.Signature != signature { + return errors.Error("signature mismatch") + } + + return nil +}*/ + +type SAccessKeyRequestV2 struct { + SAccessKeyRequest +} + +func (aksk *SAccessKeyRequestV2) ParseRequest(req http.Request, virtualHost bool) error { + aksk.Request = stringToSignV2(req, virtualHost) + return nil +} + +func (aksk SAccessKeyRequestV2) Verify(secret string) error { + hm := hmac.New(sha1.New, []byte(secret)) + hm.Write([]byte(aksk.Request)) + + signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) + if signature != aksk.Signature { + return errors.Error("signature mismatch") + } + + return nil +} + +func (aksk SAccessKeyRequestV2) Encode() string { + return jsonutils.Marshal(aksk).String() +} + +func decodeAuthHeaderV2(authStr string) (*SAccessKeyRequestV2, error) { + akskReq := SAccessKeyRequestV2{} + akskReq.Algorithm = signV2Algorithm + pos := strings.IndexByte(authStr, ':') + if pos <= 0 { + return nil, errors.Error("illegal authorization header") + } + akskReq.AccessKey = authStr[:pos] + akskReq.Signature = authStr[pos+1:] + return &akskReq, nil +} diff --git a/pkg/util/s3auth/v4.go b/pkg/util/s3auth/v4.go new file mode 100644 index 0000000000..4bd88b3851 --- /dev/null +++ b/pkg/util/s3auth/v4.go @@ -0,0 +1,270 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3auth + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "sort" + "strings" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" +) + +// Signature and API related constants. +const ( + signV4Algorithm = "AWS4-HMAC-SHA256" + iso8601DateFormat = "20060102T150405Z" + yyyymmdd = "20060102" + + unsignedPayload = "UNSIGNED-PAYLOAD" +) + +// getScope generate a string of a specific date, an AWS region, and a +// service. +func getScope(location string, t time.Time) string { + scope := strings.Join([]string{ + t.Format(yyyymmdd), + location, + "s3", + "aws4_request", + }, "/") + return scope +} + +// sum256 calculate sha256 sum for an input byte array. +func sum256(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +// getStringToSign a string based on selected query values. +func getStringToSignV4(t time.Time, location, canonicalRequest string) string { + stringToSign := signV4Algorithm + "\n" + t.Format(iso8601DateFormat) + "\n" + stringToSign += getScope(location, t) + "\n" + stringToSign += hex.EncodeToString(sum256([]byte(canonicalRequest))) + return stringToSign +} + +/// +/// Excerpts from @lsegal - +/// https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258. +/// +/// User-Agent: +/// +/// This is ignored from signing because signing this causes +/// problems with generating pre-signed URLs (that are executed +/// by other agents) or when customers pass requests through +/// proxies, which may modify the user-agent. +/// +/// Content-Length: +/// +/// This is ignored from signing because generating a pre-signed +/// URL should not provide a content-length constraint, +/// specifically when vending a S3 pre-signed PUT URL. The +/// corollary to this is that when sending regular requests +/// (non-pre-signed), the signature contains a checksum of the +/// body, which implicitly validates the payload length (since +/// changing the number of bytes would change the checksum) +/// and therefore this header is not valuable in the signature. +/// +/// Content-Type: +/// +/// Signing this header causes quite a number of problems in +/// browser environments, where browsers like to modify and +/// normalize the content-type header in different ways. There is +/// more information on this in https://goo.gl/2E9gyy. Avoiding +/// this field simplifies logic and reduces the possibility of +/// future bugs. +/// +/// Authorization: +/// +/// Is skipped for obvious reasons +/// +var v4IgnoredHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Length": true, + "User-Agent": true, +} + +// sumHMAC calculate hmac between two input byte array. +func sumHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +// getSigningKey hmac seed to calculate final signature. +func getSigningKey(secret, loc string, t time.Time) []byte { + date := sumHMAC([]byte("AWS4"+secret), []byte(t.Format(yyyymmdd))) + location := sumHMAC(date, []byte(loc)) + service := sumHMAC(location, []byte("s3")) + signingKey := sumHMAC(service, []byte("aws4_request")) + return signingKey +} + +// getSignature final signature in hexadecimal form. +func getSignature(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) +} + +// getCanonicalRequest generate a canonical request of style. +// +// canonicalRequest = +// \n +// \n +// \n +// \n +// \n +// +func getCanonicalRequest(req http.Request, signedHeaders []string) string { + req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) + canonicalRequest := strings.Join([]string{ + req.Method, + encodePath(req.URL.Path), + req.URL.RawQuery, + getCanonicalHeaders(req, signedHeaders), + strings.Join(signedHeaders, ";"), + getHashedPayload(req), + }, "\n") + return canonicalRequest +} + +// Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall() +// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html +func signV4TrimAll(input string) string { + // Compress adjacent spaces (a space is determined by + // unicode.IsSpace() internally here) to one space and return + return strings.Join(strings.Fields(input), " ") +} + +// getCanonicalHeaders generate a list of request headers for +// signature. +func getCanonicalHeaders(req http.Request, signedHeaders []string) string { + var buf bytes.Buffer + // Save all the headers in canonical form
: newline + // separated for each header. + for _, k := range signedHeaders { + buf.WriteString(k) + buf.WriteByte(':') + switch { + case k == "host": + buf.WriteString(getHostAddr(req)) + fallthrough + default: + for idx, v := range req.Header[http.CanonicalHeaderKey(k)] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(signV4TrimAll(v)) + } + buf.WriteByte('\n') + } + } + return buf.String() +} + +// getSignedHeaders generate all signed request headers. +// i.e lexically sorted, semicolon-separated list of lowercase +// request header names. +func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) []string { + var headers []string + for k := range req.Header { + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { + continue // Ignored header found continue. + } + headers = append(headers, strings.ToLower(k)) + } + headers = append(headers, "host") + sort.Strings(headers) + return headers +} + +// getHashedPayload get the hexadecimal value of the SHA256 hash of +// the request payload. +func getHashedPayload(req http.Request) string { + hashedPayload := req.Header.Get("X-Amz-Content-Sha256") + if hashedPayload == "" { + // Presign does not have a payload, use S3 recommended value. + hashedPayload = unsignedPayload + } + return hashedPayload +} + +type SAccessKeyRequestV4 struct { + SAccessKeyRequest + Location string + SignedHeaders []string + SignDate time.Time +} + +// AWS4-HMAC-SHA256 +// Credential=xxxx/20190824/us-east-1/s3/aws4_request,SignedHeaders=date;host;x-amz-content-sha256;x-amz-date,Signature=27a135c6f51cc +func decodeAuthHeaderV4(authStr string) (*SAccessKeyRequestV4, error) { + req := SAccessKeyRequestV4{} + req.Algorithm = signV4Algorithm + parts := strings.Split(authStr, ",") + if len(parts) != 3 || + !strings.HasPrefix(parts[0], "Credential=") || + !strings.HasPrefix(parts[1], "SignedHeaders=") || + !strings.HasPrefix(parts[2], "Signature=") { + return nil, errors.Error("illegal v4 auth header") + } + credParts := strings.Split(parts[0][len("Credential="):], "/") + if len(credParts) != 5 { + return nil, errors.Error("illegal v4 auth header Credential") + } + req.AccessKey = credParts[0] + req.Location = credParts[2] + req.SignedHeaders = strings.Split(parts[1][len("SignedHeaders="):], ";") + sort.Strings(req.SignedHeaders) + req.Signature = parts[2][len("Signature="):] + return &req, nil +} + +func (aksk *SAccessKeyRequestV4) ParseRequest(req http.Request, virtualHost bool) error { + dateStr := req.Header.Get(http.CanonicalHeaderKey("x-amz-date")) + if len(dateStr) == 0 { + return errors.Error("missing x-amz-date") + } + dateSign, err := time.Parse(iso8601DateFormat, dateStr) + if err != nil { + return errors.Wrap(err, "time.Parse") + } + canonicalReq := getCanonicalRequest(req, aksk.SignedHeaders) + aksk.SignDate = dateSign + aksk.Request = getStringToSignV4(dateSign, aksk.Location, canonicalReq) + return nil +} + +func (aksk SAccessKeyRequestV4) Verify(secret string) error { + signingKey := getSigningKey(secret, aksk.Location, aksk.SignDate) + signature := getSignature(signingKey, aksk.Request) + if signature != aksk.Signature { + return errors.Error("signature mismatch") + } + return nil +} + +func (aksk SAccessKeyRequestV4) Encode() string { + return jsonutils.Marshal(aksk).String() +} diff --git a/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go b/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go index c9c62d799a..17897942db 100644 --- a/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go +++ b/vendor/github.com/Azure/azure-sdk-for-go/storage/blockblob.go @@ -197,6 +197,47 @@ func (b *Blob) PutBlockWithLength(blockID string, size uint64, blob io.Reader, o return b.respondCreation(resp, BlobTypeBlock) } +// PutBlockFromURLOptions includes the options for a put block from URL operation +type PutBlockFromURLOptions struct { + PutBlockOptions + + SourceContentMD5 string `header:"x-ms-source-content-md5"` + SourceContentCRC64 string `header:"x-ms-source-content-crc64"` +} + +// PutBlockFromURL copy data of exactly specified size from specified URL to +// the block blob with given ID. It is an alternative to PutBlocks where data +// comes from a remote URL and the offset and length is known in advance. +// +// The API rejects requests with size > 100 MiB (but this limit is not +// checked by the SDK). +// +// See https://docs.microsoft.com/en-us/rest/api/storageservices/put-block-from-url +func (b *Blob) PutBlockFromURL(blockID string, blobURL string, offset int64, size uint64, options *PutBlockFromURLOptions) error { + query := url.Values{ + "comp": {"block"}, + "blockid": {blockID}, + } + headers := b.Container.bsc.client.getStandardHeaders() + // The value of this header must be set to zero. + // When the length is not zero, the operation will fail with the status code 400 (Bad Request). + headers["Content-Length"] = "0" + headers["x-ms-copy-source"] = blobURL + headers["x-ms-copy-source-range"] = fmt.Sprintf("bytes=%d-%d", offset, uint64(offset)+size-1) + + if options != nil { + query = addTimeout(query, options.Timeout) + headers = mergeHeaders(headers, headersFromStruct(*options)) + } + uri := b.Container.bsc.client.getEndpoint(blobServiceName, b.buildPath(), query) + + resp, err := b.Container.bsc.client.exec(http.MethodPut, uri, headers, nil, b.Container.bsc.auth) + if err != nil { + return err + } + return b.respondCreation(resp, BlobTypeBlock) +} + // PutBlockListOptions includes the options for a put block list operation type PutBlockListOptions struct { Timeout uint diff --git a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go index 6ef5773563..386e3d0177 100644 --- a/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go +++ b/vendor/github.com/tencentyun/cos-go-sdk-v5/object_part.go @@ -7,6 +7,7 @@ import ( "fmt" "io" "net/http" + "time" ) // InitiateMultipartUploadOptions is the option of InitateMultipartUpload @@ -189,3 +190,48 @@ func (s *ObjectService) AbortMultipartUpload(ctx context.Context, name, uploadID resp, err := s.client.send(ctx, &sendOpt) return resp, err } + +// ObjectCopyPartOptions is the options of copy-part +type ObjectCopyPartOptions struct { + XCosCopySource string `header:"x-cos-copy-source" url:"-"` + XCosCopySourceRange string `header:"x-cos-copy-source-range" url:"-"` + XCosCopySourceIfModifiedSince string `header:"x-cos-copy-source-If-Modified-Since" url:"-"` + XCosCopySourceIfUnmodifiedSince string `header:"x-cos-copy-source-If-Unmodified-Since" url:"-"` + XCosCopySourceIfMatch string `help:"x-cos-copy-source-If-Match" url:"-"` + XCosCopySourceIfNoneMatch string `help:"x-cos-copy-source-If-None-Match" url:"-"` +} + +// CopyPartResult is the result CopyPart +type CopyPartResult struct { + XMLName xml.Name `xml:"CopyPartResult"` + ETag string + LastModified time.Time +} + +// CopyPart 请求实现在初始化以后的分块上传，支持的块的数量为1到10000，块的大小为1 MB 到5 GB。 +// 在每次请求Upload Part时候，需要携带partNumber和uploadID，partNumber为块的编号，支持乱序上传。 +// ObjectCopyPartOptions的XCosCopySource为必填参数，格式为-.cos..myqcloud.com/ +// ObjectCopyPartOptions的XCosCopySourceRange指定源的Range，格式为bytes=- +// +// 当传入uploadID和partNumber都相同的时候，后传入的块将覆盖之前传入的块。当uploadID不存在时会返回404错误，NoSuchUpload. +// +// https://www.qcloud.com/document/product/436/7750 +func (s *ObjectService) CopyPart(ctx context.Context, name, uploadID string, partNumber int, opt *ObjectCopyPartOptions) (*CopyPartResult, *Response, error) { + u := fmt.Sprintf("/%s?partNumber=%d&uploadId=%s", encodeURIComponent(name), partNumber, uploadID) + var res CopyPartResult + sendOpt := sendOptions{ + baseURL: s.client.BaseURL.BucketURL, + uri: u, + method: http.MethodPut, + optHeader: opt, + result: &res, + } + resp, err := s.client.send(ctx, &sendOpt) + // If the error occurs during the copy operation, the error response is embedded in the 200 OK response. This means that a 200 OK response can contain either a success or an error. + if err == nil && resp.StatusCode == 200 { + if res.ETag == "" { + return &res, resp, errors.New("response 200 OK, but body contains an error") + } + } + return &res, resp, err +} diff --git a/vendor/modules.txt b/vendor/modules.txt index d9488ed116..02eebe69db 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -777,6 +777,7 @@ yunion.io/x/pkg/util/timeutils yunion.io/x/pkg/util/sets yunion.io/x/pkg/errors yunion.io/x/pkg/trace +yunion.io/x/pkg/gotypes yunion.io/x/pkg/util/errors yunion.io/x/pkg/util/netutils yunion.io/x/pkg/util/seclib @@ -785,7 +786,6 @@ yunion.io/x/pkg/tristate yunion.io/x/pkg/util/stringutils yunion.io/x/pkg/util/fileutils yunion.io/x/pkg/util/osprofile -yunion.io/x/pkg/gotypes yunion.io/x/pkg/util/filterclause yunion.io/x/pkg/util/reflectutils yunion.io/x/pkg/util/secrules @@ -798,9 +798,9 @@ yunion.io/x/pkg/util/prometheus yunion.io/x/pkg/prettytable yunion.io/x/pkg/util/runtime yunion.io/x/pkg/util/clock -# yunion.io/x/s3cli v0.0.0-20190812034537-1e65a6651a3e +# yunion.io/x/s3cli v0.0.0-20190829081311-29d39b4e88bd yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20190817082003-905772542cba +# yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f yunion.io/x/sqlchemy # yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3 yunion.io/x/structarg diff --git a/vendor/yunion.io/x/s3cli/api-compose-object.go b/vendor/yunion.io/x/s3cli/api-compose-object.go index 395d211f28..6347605ebb 100644 --- a/vendor/yunion.io/x/s3cli/api-compose-object.go +++ b/vendor/yunion.io/x/s3cli/api-compose-object.go @@ -247,7 +247,7 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck return ObjectInfo{}, httpRespToErrorResponse(resp, srcBucket, srcObject) } - cpObjRes := copyObjectResult{} + cpObjRes := CopyObjectResult{} err = xmlDecoder(resp.Body, &cpObjRes) if err != nil { return ObjectInfo{}, err @@ -261,7 +261,7 @@ func (c Client) copyObjectDo(ctx context.Context, srcBucket, srcObject, destBuck return objInfo, nil } -func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, +func (c Client) CopyObjectPartDo(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, metadata map[string]string) (p CompletePart, err error) { headers := make(http.Header) @@ -302,7 +302,7 @@ func (c Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, dest } // Decode copy-part response on success. - cpObjRes := copyObjectResult{} + cpObjRes := CopyObjectResult{} err = xmlDecoder(resp.Body, &cpObjRes) if err != nil { return p, err @@ -340,7 +340,7 @@ func (c Client) uploadPartCopy(ctx context.Context, bucket, object, uploadID str } // Decode copy-part response on success. - cpObjRes := copyObjectResult{} + cpObjRes := CopyObjectResult{} err = xmlDecoder(resp.Body, &cpObjRes) if err != nil { return p, err diff --git a/vendor/yunion.io/x/s3cli/api-list.go b/vendor/yunion.io/x/s3cli/api-list.go index d3100edb01..1d413b549b 100644 --- a/vendor/yunion.io/x/s3cli/api-list.go +++ b/vendor/yunion.io/x/s3cli/api-list.go @@ -28,6 +28,16 @@ import ( "github.com/minio/minio-go/v6/pkg/s3utils" ) +type ListBucketsInput struct { +} + +type ListObjectInput struct { + Prefix string + Marker string + Delimiter string + MaxKeys int64 +} + // ListBuckets list all buckets owned by this authenticated user. // // This call requires explicit authentication, no anonymous requests are @@ -307,7 +317,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don var marker string for { // Get list of objects a maximum of 1000 per request. - result, err := c.listObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) + result, err := c.ListObjectsQuery(bucketName, objectPrefix, marker, delimiter, 1000) if err != nil { objectStatCh <- ObjectInfo{ Err: err, @@ -366,7 +376,7 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { +func (c Client) ListObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { // Validate bucket name. if err := s3utils.CheckValidBucketName(bucketName); err != nil { return ListBucketResult{}, err diff --git a/vendor/yunion.io/x/s3cli/api-put-bucket.go b/vendor/yunion.io/x/s3cli/api-put-bucket.go index 280abd2091..b3d2b85455 100644 --- a/vendor/yunion.io/x/s3cli/api-put-bucket.go +++ b/vendor/yunion.io/x/s3cli/api-put-bucket.go @@ -70,7 +70,7 @@ func (c Client) MakeBucket(bucketName string, location string) (err error) { // If location is not 'us-east-1' create bucket location config. if location != "us-east-1" && location != "" { - createBucketConfig := createBucketConfiguration{} + createBucketConfig := CreateBucketConfiguration{} createBucketConfig.Location = location var createBucketConfigBytes []byte createBucketConfigBytes, err = xml.Marshal(createBucketConfig) diff --git a/vendor/yunion.io/x/s3cli/api-s3-datatypes.go b/vendor/yunion.io/x/s3cli/api-s3-datatypes.go index 5fbcaacc20..b9a7e7afd6 100644 --- a/vendor/yunion.io/x/s3cli/api-s3-datatypes.go +++ b/vendor/yunion.io/x/s3cli/api-s3-datatypes.go @@ -103,6 +103,15 @@ type ListBucketResult struct { Prefix string } +type ListMultipartUploadsInput struct { + Delimiter string + MaxUploads int64 + KeyMarker string + Prefix string + UploadIdMarker string + EncodingType string +} + // ListMultipartUploadsResult container for ListMultipartUploads response type ListMultipartUploadsResult struct { Bucket string @@ -127,7 +136,13 @@ type initiator struct { } // copyObjectResult container for copy object response. -type copyObjectResult struct { +type CopyObjectResult struct { + ETag string + LastModified time.Time // time string format "2006-01-02T15:04:05.000Z" +} + +// copyPartResult container for copy part response +type CopyPartResult struct { ETag string LastModified time.Time // time string format "2006-01-02T15:04:05.000Z" } @@ -207,11 +222,14 @@ type CompleteMultipartUpload struct { } // createBucketConfiguration container for bucket configuration. -type createBucketConfiguration struct { +type CreateBucketConfiguration struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"` Location string `xml:"LocationConstraint"` } +// LocationConstraint +type LocationConstraint string + // deleteObject container for Delete element in MultiObjects Delete XML request type deleteObject struct { Key string @@ -247,3 +265,9 @@ type deleteMultiObjectsResult struct { DeletedObjects []deletedObject `xml:"Deleted"` UnDeletedObjects []nonDeletedObject `xml:"Error"` } + +type VersioningConfiguration struct { + XMLName xml.Name `xmlns:"http://s3.amazonaws.com/doc/2006-03-01/" xml:"VersioningConfiguration"` + Status string `xml:"Status,omitempty"` + MfaDelete string `xml:"MfaDelete,omitempty"` +} diff --git a/vendor/yunion.io/x/s3cli/tags.go b/vendor/yunion.io/x/s3cli/tags.go new file mode 100644 index 0000000000..985b871727 --- /dev/null +++ b/vendor/yunion.io/x/s3cli/tags.go @@ -0,0 +1,10 @@ +package s3cli + +type Tag struct { + Key string + Value string +} + +type Tagging struct { + TagSet []Tag +} diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index 625db12088..3034e24468 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -22,7 +22,6 @@ import ( "strings" "time" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/regutils" @@ -310,8 +309,8 @@ func (c *SBooleanColumn) IsZero(val interface{}) bool { func NewBooleanColumn(name string, tagmap map[string]string, isPointer bool) SBooleanColumn { bc := SBooleanColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "TINYINT", tagmap, isPointer)} if !bc.IsPointer() && len(bc.Default()) > 0 && bc.ConvertFromString(bc.Default()) == "1" { - log.Warningf("Non-pointer boolean type should not set default value: %s(%s)", name, tagmap) - // bc.defaultString = "" + msg := fmt.Sprintf("Non-pointer boolean column should not default true: %s(%s)", name, tagmap) + panic(msg) } return bc }