mirror of
https://github.com/yunionio/cloudpods.git
synced 2026-08-31 01:35:56 +08:00
Merge pull request #2965 from swordqiu/feature/qj-xsky-object-store
feature: support xsky object storage
This commit is contained in:
yunion-ci-robot
@@ -200,6 +200,17 @@ func init() {
|
||||
return nil
|
||||
})
|
||||
|
||||
R(&options.SS3CloudAccountCreateOptions{}, "cloud-account-create-xsky", "Create a xsky object storage account", func(s *mcclient.ClientSession, args *options.SS3CloudAccountCreateOptions) error {
|
||||
params := jsonutils.Marshal(args)
|
||||
params.(*jsonutils.JSONDict).Add(jsonutils.NewString("Xsky"), "provider")
|
||||
result, err := modules.Cloudaccounts.Create(s, params)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
printObject(result)
|
||||
return nil
|
||||
})
|
||||
|
||||
type CloudaccountUpdateOptions struct {
|
||||
ID string `help:"ID or Name of cloud account"`
|
||||
Name string `help:"New name to update"`
|
||||
|
||||
@@ -25,6 +25,7 @@ import (
|
||||
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
|
||||
"yunion.io/x/onecloud/pkg/multicloud/objectstore/ceph"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/shell"
|
||||
"yunion.io/x/onecloud/pkg/multicloud/objectstore/xsky"
|
||||
"yunion.io/x/onecloud/pkg/util/shellutils"
|
||||
)
|
||||
|
||||
@@ -94,6 +95,8 @@ func newClient(options *BaseOptions) (cloudprovider.ICloudRegion, error) {
|
||||
|
||||
if options.Backend == api.CLOUD_PROVIDER_CEPH {
|
||||
return ceph.NewCephRados("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
|
||||
} else if options.Backend == api.CLOUD_PROVIDER_XSKY {
|
||||
return xsky.NewXskyClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
|
||||
}
|
||||
return objectstore.NewObjectStoreClient("", "", options.AccessUrl, options.AccessKey, options.Secret, options.Debug)
|
||||
}
|
||||
|
||||
@@ -151,7 +151,7 @@ require (
|
||||
sigs.k8s.io/yaml v1.1.0 // indirect
|
||||
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051
|
||||
yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
|
||||
yunion.io/x/pkg v0.0.0-20190902093114-59ba154a6861
|
||||
yunion.io/x/pkg v0.0.0-20190917154624-e89986e4e4d8
|
||||
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
|
||||
yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f
|
||||
yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
|
||||
|
||||
@@ -583,8 +583,8 @@ yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/
|
||||
yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
|
||||
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf h1:OsKC+2ghZHwp+Ztm/MwKlLKKRiE7QcPG8eTp0GmsHbg=
|
||||
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
|
||||
yunion.io/x/pkg v0.0.0-20190902093114-59ba154a6861 h1:4TDb45M/HGZ3fWvts253g+U3Rb80jgEiNf8R8Ex7HjQ=
|
||||
yunion.io/x/pkg v0.0.0-20190902093114-59ba154a6861/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
|
||||
yunion.io/x/pkg v0.0.0-20190917154624-e89986e4e4d8 h1:wutgIFVln8xNRH9WvrzINQuT1feByaUp0abXq9b7eL4=
|
||||
yunion.io/x/pkg v0.0.0-20190917154624-e89986e4e4d8/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
|
||||
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
|
||||
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
|
||||
yunion.io/x/sqlchemy v0.0.0-20190823062008-bb710661356f h1:maHGG78d6vLAyT/lGSOOsXWrylBfjdu+NMCGXFhLuhA=
|
||||
|
||||
@@ -42,6 +42,7 @@ const (
|
||||
|
||||
CLOUD_PROVIDER_GENERICS3 = "S3"
|
||||
CLOUD_PROVIDER_CEPH = "Ceph"
|
||||
CLOUD_PROVIDER_XSKY = "Xsky"
|
||||
|
||||
CLOUD_PROVIDER_HEALTH_NORMAL = "normal" // 远端处于健康状态
|
||||
CLOUD_PROVIDER_HEALTH_INSUFFICIENT = "insufficient" // 不足按需资源余额
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package baremetal
|
||||
|
||||
import (
|
||||
|
||||
@@ -23,7 +23,6 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/pkg/s3utils"
|
||||
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/log"
|
||||
@@ -42,6 +41,7 @@ import (
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/mcclient"
|
||||
"yunion.io/x/onecloud/pkg/mcclient/auth"
|
||||
"yunion.io/x/onecloud/pkg/mcclient/modulebase"
|
||||
"yunion.io/x/onecloud/pkg/util/logclient"
|
||||
"yunion.io/x/onecloud/pkg/util/rbacutils"
|
||||
)
|
||||
@@ -587,6 +587,9 @@ func (bucket *SBucket) GetDetailsObjects(
|
||||
userCred mcclient.TokenCredential,
|
||||
query jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
iBucket, err := bucket.GetIBucket()
|
||||
if err != nil {
|
||||
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
|
||||
@@ -620,6 +623,10 @@ func (bucket *SBucket) PerformTempUrl(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
method, _ := data.GetString("method")
|
||||
key, _ := data.GetString("key")
|
||||
expire, _ := data.Int("expire_seconds")
|
||||
@@ -661,6 +668,10 @@ func (bucket *SBucket) PerformMakedir(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
key, _ := data.GetString("key")
|
||||
key = strings.Trim(key, " /")
|
||||
if len(key) == 0 {
|
||||
@@ -728,6 +739,10 @@ func (bucket *SBucket) PerformDelete(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
keys, _ := data.Get("keys")
|
||||
if keys == nil {
|
||||
return nil, httperrors.NewInputParameterError("missing keys")
|
||||
@@ -777,6 +792,10 @@ func (bucket *SBucket) PerformUpload(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
appParams := appsrv.AppContextGetParams(ctx)
|
||||
|
||||
key := appParams.Request.Header.Get(api.BUCKET_UPLOAD_OBJECT_KEY_HEADER)
|
||||
@@ -881,6 +900,10 @@ func (bucket *SBucket) PerformAcl(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
aclStr, _ := data.GetString("acl")
|
||||
switch cloudprovider.TBucketACLType(aclStr) {
|
||||
case cloudprovider.ACLPrivate, cloudprovider.ACLAuthRead, cloudprovider.ACLPublicRead, cloudprovider.ACLPublicReadWrite:
|
||||
@@ -959,6 +982,10 @@ func (bucket *SBucket) PerformSync(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
statsOnly := jsonutils.QueryBoolean(data, "stats_only", false)
|
||||
|
||||
iBucket, err := bucket.GetIBucket()
|
||||
@@ -998,6 +1025,9 @@ func (bucket *SBucket) GetDetailsAcl(
|
||||
userCred mcclient.TokenCredential,
|
||||
query jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
iBucket, err := bucket.GetIBucket()
|
||||
if err != nil {
|
||||
return nil, httperrors.NewInternalServerError("fail to find external bucket: %s", err)
|
||||
@@ -1107,6 +1137,10 @@ func (bucket *SBucket) PerformLimit(
|
||||
query jsonutils.JSONObject,
|
||||
data jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
|
||||
limit := cloudprovider.SBucketStats{}
|
||||
err := data.Unmarshal(&limit, "limit")
|
||||
if err != nil {
|
||||
@@ -1154,6 +1188,9 @@ func (bucket *SBucket) GetDetailsAccessInfo(
|
||||
userCred mcclient.TokenCredential,
|
||||
query jsonutils.JSONObject,
|
||||
) (jsonutils.JSONObject, error) {
|
||||
if len(bucket.ExternalId) == 0 {
|
||||
return nil, httperrors.NewInvalidStatusError("no external bucket")
|
||||
}
|
||||
manager := bucket.GetCloudprovider()
|
||||
if manager == nil {
|
||||
return nil, httperrors.NewInternalServerError("missing manager?")
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package tasks
|
||||
|
||||
import (
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package cas // import "yunion.io/x/onecloud/pkg/keystone/driver/cas"
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package modulebase // import "yunion.io/x/onecloud/pkg/mcclient/modulebase"
|
||||
|
||||
@@ -1,3 +1,17 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package modulebase
|
||||
|
||||
import (
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package loader // import "yunion.io/x/onecloud/pkg/multicloud/loader"
|
||||
|
||||
@@ -35,6 +35,7 @@ import (
|
||||
// object storages
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/ceph/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider"
|
||||
_ "yunion.io/x/onecloud/pkg/multicloud/objectstore/xsky/provider"
|
||||
)
|
||||
|
||||
func init() {
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package ceph // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/ceph"
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package provider // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/ceph/provider"
|
||||
|
||||
@@ -152,6 +152,15 @@ func (cli *SObjectStoreClient) S3Client() *s3cli.Client {
|
||||
return cli.client
|
||||
}
|
||||
|
||||
func (cli *SObjectStoreClient) GetClientRC() map[string]string {
|
||||
return map[string]string{
|
||||
"S3_ACCESS_KEY": cli.accessKey,
|
||||
"S3_SECRET": cli.secret,
|
||||
"S3_ACCESS_URL": cli.endpoint,
|
||||
"S3_BACKEND": api.CLOUD_PROVIDER_GENERICS3,
|
||||
}
|
||||
}
|
||||
|
||||
///////////////////////////////// fake impletementations //////////////////////
|
||||
|
||||
func (cli *SObjectStoreClient) GetIZones() ([]cloudprovider.ICloudZone, error) {
|
||||
|
||||
@@ -0,0 +1,518 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package xsky
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/rand"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/cloudprovider"
|
||||
"yunion.io/x/onecloud/pkg/httperrors"
|
||||
"yunion.io/x/onecloud/pkg/util/httputils"
|
||||
)
|
||||
|
||||
type SXskyAdminApi struct {
|
||||
endpoint string
|
||||
username string
|
||||
password string
|
||||
token *sLoginResponse
|
||||
client *http.Client
|
||||
debug bool
|
||||
}
|
||||
|
||||
func newXskyAdminApi(user, passwd, ep string, debug bool) *SXskyAdminApi {
|
||||
return &SXskyAdminApi{
|
||||
endpoint: ep,
|
||||
username: user,
|
||||
password: passwd,
|
||||
client: httputils.GetDefaultClient(),
|
||||
debug: debug,
|
||||
}
|
||||
}
|
||||
|
||||
func getJsonBodyReader(body jsonutils.JSONObject) io.Reader {
|
||||
var reqBody io.Reader
|
||||
if body != nil {
|
||||
reqBody = strings.NewReader(body.String())
|
||||
}
|
||||
return reqBody
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) jsonRequest(ctx context.Context, method httputils.THttpMethod, path string, hdr http.Header, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) {
|
||||
urlStr := strings.TrimRight(api.endpoint, "/") + "/" + strings.TrimLeft(path, "/")
|
||||
req, err := http.NewRequest(string(method), urlStr, getJsonBodyReader(body))
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "http.NewRequest")
|
||||
}
|
||||
if hdr != nil {
|
||||
for k, vs := range hdr {
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if api.isValidToken() {
|
||||
req.Header.Set("xms-auth-token", api.token.Token.Uuid)
|
||||
}
|
||||
|
||||
if api.debug {
|
||||
log.Debugf("request: %s %s %s %s", method, urlStr, req.Header, body)
|
||||
}
|
||||
resp, err := api.client.Do(req)
|
||||
|
||||
return httputils.ParseJSONResponse(resp, err, api.debug)
|
||||
}
|
||||
|
||||
type sLoginResponse struct {
|
||||
Token struct {
|
||||
Create time.Time
|
||||
Expires time.Time
|
||||
Roles []string
|
||||
User struct {
|
||||
Create time.Time
|
||||
Name string
|
||||
Email string
|
||||
Enabled bool
|
||||
Id int
|
||||
PasswordLastUpdate time.Time
|
||||
}
|
||||
Uuid string
|
||||
Valid bool
|
||||
}
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) isValidToken() bool {
|
||||
if api.token != nil && len(api.token.Token.Uuid) > 0 && api.token.Token.Expires.After(time.Now()) {
|
||||
return true
|
||||
} else {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) auth(ctx context.Context) (*sLoginResponse, error) {
|
||||
input := STokenCreateReq{}
|
||||
input.Auth.Identity.Password.User.Name = api.username
|
||||
input.Auth.Identity.Password.User.Password = api.password
|
||||
|
||||
_, resp, err := api.jsonRequest(ctx, httputils.POST, "/api/v1/auth/tokens", nil, jsonutils.Marshal(input))
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "api.jsonRequest")
|
||||
}
|
||||
output := sLoginResponse{}
|
||||
err = resp.Unmarshal(&output)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "resp.Unmarshal")
|
||||
}
|
||||
return &output, err
|
||||
}
|
||||
|
||||
type STokenCreateReq struct {
|
||||
Auth STokenCreateReqAuth `json:"auth"`
|
||||
}
|
||||
|
||||
type STokenCreateReqAuth struct {
|
||||
Identity STokenCreateReqAuthIdentity `json:"identity"`
|
||||
}
|
||||
|
||||
type STokenCreateReqAuthIdentity struct {
|
||||
// password for auth
|
||||
Password SAuthPasswordReq `json:"password,omitempty"`
|
||||
// token for auth
|
||||
Token SAuthTokenReq `json:"token,omitempty"`
|
||||
}
|
||||
|
||||
type SAuthPasswordReq struct {
|
||||
User SAuthPasswordReqUser `json:"user"`
|
||||
}
|
||||
|
||||
type SAuthPasswordReqUser struct {
|
||||
// user email for auth
|
||||
Email string `json:"email,omitempty"`
|
||||
// user id for auth
|
||||
Id int64 `json:"id,omitzero"`
|
||||
// user name or email for auth
|
||||
Name string `json:"name,omitempty"`
|
||||
// password for auth
|
||||
Password string `json:"password"`
|
||||
}
|
||||
|
||||
type SAuthTokenReq struct {
|
||||
// uuid of authorized token
|
||||
Uuid string `json:"uuid"`
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) authRequest(ctx context.Context, method httputils.THttpMethod, path string, hdr http.Header, body jsonutils.JSONObject) (http.Header, jsonutils.JSONObject, error) {
|
||||
if !api.isValidToken() {
|
||||
loginResp, err := api.auth(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "api.auth")
|
||||
}
|
||||
api.token = loginResp
|
||||
}
|
||||
return api.jsonRequest(ctx, method, path, hdr, body)
|
||||
}
|
||||
|
||||
type sUser struct {
|
||||
BucketNum int
|
||||
BucketQuotaMaxObjects int
|
||||
BucketQuotaMaxSize int64
|
||||
Create time.Time
|
||||
DisplayName string
|
||||
Email string
|
||||
Id int
|
||||
MaxBuckets int
|
||||
Name string
|
||||
OpMask string
|
||||
Status string
|
||||
Suspended bool
|
||||
Update time.Time
|
||||
UserQuotaMaxObjects int
|
||||
UserQuotaMaxSize int64
|
||||
samples []sSample
|
||||
Keys []sKey
|
||||
}
|
||||
|
||||
func (u sUser) getMinKey() string {
|
||||
minKey := ""
|
||||
for i := range u.Keys {
|
||||
if len(minKey) == 0 || minKey > u.Keys[i].AccessKey {
|
||||
minKey = u.Keys[i].AccessKey
|
||||
}
|
||||
}
|
||||
return minKey
|
||||
}
|
||||
|
||||
type sKey struct {
|
||||
AccessKey string
|
||||
Create time.Time
|
||||
Id int
|
||||
Reserved bool
|
||||
SecretKey string
|
||||
Status string
|
||||
Type string
|
||||
Update time.Time
|
||||
User struct {
|
||||
Id int
|
||||
Name string
|
||||
}
|
||||
}
|
||||
|
||||
type sSample struct {
|
||||
AllocatedObjects int
|
||||
AllocatedSize int64
|
||||
Create time.Time
|
||||
DelOpsPm int
|
||||
RxBandwidthKbyte int
|
||||
RxOpsPm int
|
||||
TxBandwidthKbyte int
|
||||
TxOpsPm int
|
||||
TotalDelOps int
|
||||
TotalDelSuccessOps int
|
||||
TotalRxBytes int64
|
||||
TotalRxOps int
|
||||
TotalRxSuccessOps int
|
||||
TotalTxBytes int64
|
||||
TotalTxOps int
|
||||
TotalTxSuccessKbyte int
|
||||
}
|
||||
|
||||
type sPaging struct {
|
||||
Count int
|
||||
Limit int
|
||||
Offset int
|
||||
TotalCount int
|
||||
}
|
||||
|
||||
type sUsersResponse struct {
|
||||
OsUsers []sUser
|
||||
Paging sPaging
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) getUsers(ctx context.Context) ([]sUser, error) {
|
||||
usrs := make([]sUser, 0)
|
||||
totalCount := 0
|
||||
for totalCount <= 0 || len(usrs) < totalCount {
|
||||
_, resp, err := api.authRequest(ctx, httputils.GET, fmt.Sprintf("/api/v1/os-users/?limit=1000&offset=%d", len(usrs)), nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "api.authRequest")
|
||||
}
|
||||
output := sUsersResponse{}
|
||||
err = resp.Unmarshal(&output)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "resp.Unmarshal")
|
||||
}
|
||||
usrs = append(usrs, output.OsUsers...)
|
||||
totalCount = output.Paging.TotalCount
|
||||
}
|
||||
return usrs, nil
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) findUserByAccessKey(ctx context.Context, accessKey string) (*sUser, *sKey, error) {
|
||||
usrs, err := api.getUsers(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "api.getUsers")
|
||||
}
|
||||
for i := range usrs {
|
||||
for j := range usrs[i].Keys {
|
||||
if usrs[i].Keys[j].AccessKey == accessKey {
|
||||
return &usrs[i], &usrs[i].Keys[j], nil
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil, nil, httperrors.ErrNotFound
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) findFirstUserWithAccessKey(ctx context.Context) (*sUser, *sKey, error) {
|
||||
usrs, err := api.getUsers(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, errors.Wrap(err, "api.getUsers")
|
||||
}
|
||||
for i := range usrs {
|
||||
if len(usrs[i].Keys) > 0 {
|
||||
return &usrs[i], &usrs[i].Keys[0], nil
|
||||
}
|
||||
}
|
||||
return nil, nil, httperrors.ErrNotFound
|
||||
}
|
||||
|
||||
type sBucket struct {
|
||||
ActionStatus string
|
||||
AllUserPermission string
|
||||
AuthUserPermission string
|
||||
BucketPolicy string
|
||||
Create time.Time
|
||||
Flag struct {
|
||||
Versioned bool
|
||||
VersionsSuspended bool
|
||||
Worm bool
|
||||
}
|
||||
Id int
|
||||
// LifeCycle
|
||||
MetadataSearchEnabled bool
|
||||
Name string
|
||||
NfsClientNum int
|
||||
OsReplicationPathNum int
|
||||
OsReplicationZoneNum int
|
||||
// osZone
|
||||
OsZoneUuid string
|
||||
Owner struct {
|
||||
Id string
|
||||
Name string
|
||||
}
|
||||
OwnerPermission string
|
||||
Policy sPolicy
|
||||
PolicyEnabled bool
|
||||
QuotaMaxObjects int
|
||||
QuotaMaxSize int64
|
||||
// RemteClusters
|
||||
ReplicationUuid string
|
||||
Samples []sSample
|
||||
Shards int
|
||||
Status string
|
||||
Update time.Time
|
||||
Virtual bool
|
||||
// NfsGatewayMaps
|
||||
}
|
||||
|
||||
type sPolicy struct {
|
||||
BucketNum int
|
||||
Compress bool
|
||||
Create time.Time
|
||||
Crypto bool
|
||||
DataPool struct {
|
||||
Id int
|
||||
Name string
|
||||
}
|
||||
Default bool
|
||||
Description string
|
||||
Id int
|
||||
IndexPool struct {
|
||||
Id int
|
||||
Name string
|
||||
}
|
||||
Name string
|
||||
ObjectSizeThreshold int64
|
||||
PolicyName string
|
||||
Status string
|
||||
Update time.Time
|
||||
}
|
||||
|
||||
type sBucketsResponse struct {
|
||||
OsBuckets []sBucket
|
||||
Paging sPaging
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) getBuckets(ctx context.Context) ([]sBucket, error) {
|
||||
buckets := make([]sBucket, 0)
|
||||
totalCount := 0
|
||||
for totalCount <= 0 || len(buckets) < totalCount {
|
||||
_, resp, err := api.authRequest(ctx, httputils.GET, fmt.Sprintf("/api/v1/os-buckets/?limit=1000&offset=%d", len(buckets)), nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "api.authRequest")
|
||||
}
|
||||
output := sBucketsResponse{}
|
||||
err = resp.Unmarshal(&output)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "resp.Unmarshal")
|
||||
}
|
||||
buckets = append(buckets, output.OsBuckets...)
|
||||
totalCount = output.Paging.TotalCount
|
||||
}
|
||||
return buckets, nil
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) getBucketByName(ctx context.Context, name string) (*sBucket, error) {
|
||||
buckets, err := api.getBuckets(ctx)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "api.getBuckets")
|
||||
}
|
||||
for i := range buckets {
|
||||
if buckets[i].Name == name {
|
||||
return &buckets[i], nil
|
||||
}
|
||||
}
|
||||
return nil, cloudprovider.ErrNotFound
|
||||
}
|
||||
|
||||
type sBucketQuotaInput struct {
|
||||
OsBucket struct {
|
||||
QuotaMaxSize int64
|
||||
QuotaMaxObjects int
|
||||
}
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) setBucketQuota(ctx context.Context, bucketId int, input sBucketQuotaInput) error {
|
||||
_, _, err := api.authRequest(ctx, httputils.PATCH, fmt.Sprintf("/api/v1/os-buckets/%d", bucketId), nil, jsonutils.Marshal(&input))
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "api.authRequest")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type sS3LbGroup struct {
|
||||
ActionStatus string
|
||||
Create time.Time
|
||||
Description string
|
||||
HttpsPort int
|
||||
Id int
|
||||
Name string
|
||||
Port int
|
||||
Roles []string
|
||||
SearchHttpsPort int
|
||||
SearchPort int
|
||||
Status string
|
||||
SyncPort int
|
||||
Update time.Time
|
||||
S3LoadBalancers []sS3LoadBalancer `json:"s3_load_balancers"`
|
||||
}
|
||||
|
||||
type sS3LoadBalancer struct {
|
||||
Create time.Time
|
||||
Description string
|
||||
Group struct {
|
||||
Id int
|
||||
Name string
|
||||
Status string
|
||||
}
|
||||
Host struct {
|
||||
AdminIp string
|
||||
Id int
|
||||
Name string
|
||||
}
|
||||
HttpsPort int
|
||||
Id int
|
||||
InterfaceName string
|
||||
Ip string
|
||||
Name string
|
||||
Port int
|
||||
Roles []string
|
||||
Samples []struct {
|
||||
ActiveAconnects int
|
||||
CpuUtil float64
|
||||
Create time.Time
|
||||
DownBandwidthKbytes int64
|
||||
FailureRequests int
|
||||
MemUsagePercent float64
|
||||
SuccessRequests int
|
||||
UpBandwidthKbyte int64
|
||||
}
|
||||
SearchHttpsPort int
|
||||
SearchPort int
|
||||
SslCertificate interface{}
|
||||
Status string
|
||||
SyncPort int
|
||||
Update time.Time
|
||||
Vip string
|
||||
VipMask int
|
||||
Vips string
|
||||
}
|
||||
|
||||
func (lb sS3LoadBalancer) GetGatewayEndpoint() string {
|
||||
if lb.SslCertificate == nil {
|
||||
return fmt.Sprintf("http://%s:%d", lb.Vip, lb.Port)
|
||||
} else {
|
||||
return fmt.Sprintf("https://%s:%d", lb.Vip, lb.HttpsPort)
|
||||
}
|
||||
}
|
||||
|
||||
type sS3LbGroupResponse struct {
|
||||
S3LoadBalancerGroups []sS3LbGroup `json:"s3_load_balancer_groups"`
|
||||
Paging sPaging
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) getS3LbGroup(ctx context.Context) ([]sS3LbGroup, error) {
|
||||
lbGroups := make([]sS3LbGroup, 0)
|
||||
totalCount := 0
|
||||
for totalCount <= 0 || len(lbGroups) < totalCount {
|
||||
_, resp, err := api.authRequest(ctx, httputils.GET, fmt.Sprintf("/api/v1/s3-load-balancer-groups/?limit=1000&offset=%d", len(lbGroups)), nil, nil)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "api.authRequest")
|
||||
}
|
||||
output := sS3LbGroupResponse{}
|
||||
err = resp.Unmarshal(&output)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "resp.Unmarshal")
|
||||
}
|
||||
lbGroups = append(lbGroups, output.S3LoadBalancerGroups...)
|
||||
totalCount = output.Paging.TotalCount
|
||||
}
|
||||
return lbGroups, nil
|
||||
}
|
||||
|
||||
func (api *SXskyAdminApi) getS3GatewayEndpoint(ctx context.Context) (string, error) {
|
||||
s3LbGrps, err := api.getS3LbGroup(ctx)
|
||||
if err != nil {
|
||||
return "", errors.Wrap(err, "api.getS3LbGroup")
|
||||
}
|
||||
lbs := make([]sS3LoadBalancer, 0)
|
||||
for i := range s3LbGrps {
|
||||
lbs = append(lbs, s3LbGrps[i].S3LoadBalancers...)
|
||||
}
|
||||
if len(lbs) == 0 {
|
||||
return "", errors.Wrap(httperrors.ErrNotFound, "empty S3 Lb group")
|
||||
}
|
||||
lb := lbs[rand.Intn(len(lbs))]
|
||||
return lb.GetGatewayEndpoint(), nil
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package xsky
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"yunion.io/x/log"
|
||||
"yunion.io/x/pkg/errors"
|
||||
|
||||
"yunion.io/x/onecloud/pkg/cloudprovider"
|
||||
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
|
||||
)
|
||||
|
||||
type SXskyBucket struct {
|
||||
*objectstore.SBucket
|
||||
|
||||
client *SXskyClient
|
||||
}
|
||||
|
||||
func (b *SXskyBucket) GetStats() cloudprovider.SBucketStats {
|
||||
_, hdr, _ := b.GetIBucketProvider().S3Client().BucketExists(b.Name)
|
||||
if hdr != nil {
|
||||
sizeBytesStr := hdr.Get("X-Rgw-Bytes-Used")
|
||||
sizeBytes, _ := strconv.ParseInt(sizeBytesStr, 10, 64)
|
||||
objCntStr := hdr.Get("X-Rgw-Object-Count")
|
||||
objCnt, _ := strconv.ParseInt(objCntStr, 10, 64)
|
||||
return cloudprovider.SBucketStats{
|
||||
SizeBytes: sizeBytes,
|
||||
ObjectCount: int(objCnt),
|
||||
}
|
||||
}
|
||||
return b.SBucket.GetStats()
|
||||
}
|
||||
|
||||
func (b *SXskyBucket) GetLimit() cloudprovider.SBucketStats {
|
||||
limit := cloudprovider.SBucketStats{}
|
||||
bucket, err := b.client.adminApi.getBucketByName(context.Background(), b.Name)
|
||||
if err != nil {
|
||||
log.Errorf("b.client.adminApi.getBucketByName error %s", err)
|
||||
} else {
|
||||
limit.SizeBytes = bucket.QuotaMaxSize
|
||||
limit.ObjectCount = bucket.QuotaMaxObjects
|
||||
}
|
||||
return limit
|
||||
}
|
||||
|
||||
func (b *SXskyBucket) SetLimit(limit cloudprovider.SBucketStats) error {
|
||||
bucket, err := b.client.adminApi.getBucketByName(context.Background(), b.Name)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "b.client.adminApi.getBucketByName")
|
||||
}
|
||||
input := sBucketQuotaInput{}
|
||||
input.OsBucket.QuotaMaxObjects = limit.ObjectCount
|
||||
input.OsBucket.QuotaMaxSize = limit.SizeBytes
|
||||
err = b.client.adminApi.setBucketQuota(context.Background(), bucket.Id, input)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "b.client.adminApi.setBucketQuota")
|
||||
}
|
||||
cloudprovider.Wait(time.Second, 30*time.Second, func() (bool, error) {
|
||||
target := b.GetLimit()
|
||||
if target.SizeBytes == limit.SizeBytes && target.ObjectCount == limit.ObjectCount {
|
||||
return true, nil
|
||||
}
|
||||
return false, nil
|
||||
})
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package xsky // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/xsky"
|
||||
@@ -0,0 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package provider // import "yunion.io/x/onecloud/pkg/multicloud/objectstore/xsky/provider"
|
||||
@@ -0,0 +1,55 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package provider
|
||||
|
||||
import (
|
||||
api "yunion.io/x/onecloud/pkg/apis/compute"
|
||||
"yunion.io/x/onecloud/pkg/cloudprovider"
|
||||
s3provider "yunion.io/x/onecloud/pkg/multicloud/objectstore/provider"
|
||||
"yunion.io/x/onecloud/pkg/multicloud/objectstore/xsky"
|
||||
)
|
||||
|
||||
type SXskyProviderFactory struct {
|
||||
s3provider.SObjectStoreProviderFactory
|
||||
}
|
||||
|
||||
func (self *SXskyProviderFactory) GetId() string {
|
||||
return api.CLOUD_PROVIDER_XSKY
|
||||
}
|
||||
|
||||
func (self *SXskyProviderFactory) GetName() string {
|
||||
return api.CLOUD_PROVIDER_XSKY
|
||||
}
|
||||
|
||||
func (self *SXskyProviderFactory) GetProvider(providerId, providerName, url, account, secret string) (cloudprovider.ICloudProvider, error) {
|
||||
client, err := xsky.NewXskyClient(providerId, providerName, url, account, secret, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s3provider.NewObjectStoreProvider(self, client), nil
|
||||
}
|
||||
|
||||
func (self *SXskyProviderFactory) GetClientRC(url, account, secret string) (map[string]string, error) {
|
||||
client, err := xsky.NewXskyClient("", "", url, account, secret, false)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return client.GetClientRC(), nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
factory := SXskyProviderFactory{}
|
||||
cloudprovider.RegisterFactory(&factory)
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package xsky
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/jsonutils"
|
||||
"yunion.io/x/pkg/errors"
|
||||
"yunion.io/x/s3cli"
|
||||
|
||||
api "yunion.io/x/onecloud/pkg/apis/compute"
|
||||
"yunion.io/x/onecloud/pkg/cloudprovider"
|
||||
"yunion.io/x/onecloud/pkg/multicloud/objectstore"
|
||||
)
|
||||
|
||||
type SXskyClient struct {
|
||||
*objectstore.SObjectStoreClient
|
||||
|
||||
adminApi *SXskyAdminApi
|
||||
|
||||
adminUser *sUser
|
||||
initAccount string
|
||||
}
|
||||
|
||||
func parseAccount(account string) (user string, accessKey string) {
|
||||
accountInfo := strings.Split(account, "/")
|
||||
user = accountInfo[0]
|
||||
if len(accountInfo) > 1 {
|
||||
accessKey = strings.Join(accountInfo[1:], "/")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func NewXskyClient(providerId string, providerName string, endpoint string, account string, password string, isDebug bool) (*SXskyClient, error) {
|
||||
usrname, accessKey := parseAccount(account)
|
||||
adminApi := newXskyAdminApi(usrname, password, endpoint, isDebug)
|
||||
gwEp, err := adminApi.getS3GatewayEndpoint(context.Background())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "adminApi.getS3GatewayIP")
|
||||
}
|
||||
|
||||
var usr *sUser
|
||||
var key *sKey
|
||||
if len(accessKey) > 0 {
|
||||
usr, key, err = adminApi.findUserByAccessKey(context.Background(), accessKey)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "adminApi.findUserByAccessKey")
|
||||
}
|
||||
} else {
|
||||
usr, key, err = adminApi.findFirstUserWithAccessKey(context.Background())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "adminApi.findFirstUserWithAccessKey")
|
||||
}
|
||||
}
|
||||
|
||||
s3store, err := objectstore.NewObjectStoreClientAndFetch(providerId, providerName, gwEp, accessKey, key.SecretKey, isDebug, false)
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "NewObjectStoreClient")
|
||||
}
|
||||
|
||||
client := SXskyClient{
|
||||
SObjectStoreClient: s3store,
|
||||
adminApi: adminApi,
|
||||
adminUser: usr,
|
||||
}
|
||||
|
||||
if len(accessKey) > 0 {
|
||||
client.initAccount = account
|
||||
}
|
||||
|
||||
client.SetVirtualObject(&client)
|
||||
|
||||
err = client.FetchBuckets()
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "fetchBuckets")
|
||||
}
|
||||
|
||||
return &client, nil
|
||||
}
|
||||
|
||||
func (cli *SXskyClient) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
|
||||
if len(cli.initAccount) > 0 {
|
||||
return []cloudprovider.SSubAccount{
|
||||
{
|
||||
Account: cli.initAccount,
|
||||
Name: cli.adminUser.Name,
|
||||
HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
usrs, err := cli.adminApi.getUsers(context.Background())
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "api.getUsers")
|
||||
}
|
||||
subAccounts := make([]cloudprovider.SSubAccount, 0)
|
||||
for i := range usrs {
|
||||
ak := usrs[i].getMinKey()
|
||||
if len(ak) > 0 {
|
||||
subAccount := cloudprovider.SSubAccount{
|
||||
Account: fmt.Sprintf("%s/%s", cli.adminApi.username, ak),
|
||||
Name: usrs[i].Name,
|
||||
HealthStatus: api.CLOUD_PROVIDER_HEALTH_NORMAL,
|
||||
}
|
||||
subAccounts = append(subAccounts, subAccount)
|
||||
}
|
||||
}
|
||||
return subAccounts, nil
|
||||
}
|
||||
|
||||
func (cli *SXskyClient) GetAccountId() string {
|
||||
return cli.adminApi.username
|
||||
}
|
||||
|
||||
func (cli *SXskyClient) GetVersion() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (cli *SXskyClient) About() jsonutils.JSONObject {
|
||||
about := jsonutils.NewDict()
|
||||
if cli.adminUser != nil {
|
||||
about.Add(jsonutils.Marshal(cli.adminUser), "admin_user")
|
||||
}
|
||||
return about
|
||||
}
|
||||
|
||||
func (cli *SXskyClient) GetProvider() string {
|
||||
return api.CLOUD_PROVIDER_XSKY
|
||||
}
|
||||
|
||||
func (cli *SXskyClient) NewBucket(bucket s3cli.BucketInfo) cloudprovider.ICloudBucket {
|
||||
if cli.SObjectStoreClient == nil {
|
||||
return nil
|
||||
}
|
||||
generalBucket := cli.SObjectStoreClient.NewBucket(bucket)
|
||||
return &SXskyBucket{
|
||||
SBucket: generalBucket.(*objectstore.SBucket),
|
||||
client: cli,
|
||||
}
|
||||
}
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package handlers // import "yunion.io/x/onecloud/pkg/s3gateway/handlers"
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package models // import "yunion.io/x/onecloud/pkg/s3gateway/models"
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package session // import "yunion.io/x/onecloud/pkg/s3gateway/session"
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package s3auth // import "yunion.io/x/onecloud/pkg/util/s3auth"
|
||||
|
||||
@@ -1 +1,15 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package validate // import "yunion.io/x/onecloud/pkg/util/validate"
|
||||
|
||||
Vendored
+1
-1
@@ -768,7 +768,7 @@ yunion.io/x/jsonutils
|
||||
# yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
|
||||
yunion.io/x/log
|
||||
yunion.io/x/log/hooks
|
||||
# yunion.io/x/pkg v0.0.0-20190902093114-59ba154a6861
|
||||
# yunion.io/x/pkg v0.0.0-20190917154624-e89986e4e4d8
|
||||
yunion.io/x/pkg/util/version
|
||||
yunion.io/x/pkg/utils
|
||||
yunion.io/x/pkg/util/regutils
|
||||
|
||||
+199
@@ -0,0 +1,199 @@
|
||||
// Copyright 2019 Yunion
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package errors
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// MessageCountMap contains occurance for each error message.
|
||||
type MessageCountMap map[string]int
|
||||
|
||||
// Aggregate represents an object that contains multiple errors, but does not
|
||||
// necessarily have singular semantic meaning.
|
||||
type Aggregate interface {
|
||||
error
|
||||
Errors() []error
|
||||
}
|
||||
|
||||
// NewAggregate converts a slice of errors into an Aggregate interface, which
|
||||
// is itself an implementation of the error interface. If the slice is empty,
|
||||
// this returns nil.
|
||||
// It will check if any of the element of input error list is nil, to avoid
|
||||
// nil pointer panic when call Error().
|
||||
func NewAggregate(errlist []error) Aggregate {
|
||||
if len(errlist) == 0 {
|
||||
return nil
|
||||
}
|
||||
// In case of input error list contains nil
|
||||
var errs []error
|
||||
for _, e := range errlist {
|
||||
if e != nil {
|
||||
errs = append(errs, e)
|
||||
}
|
||||
}
|
||||
if len(errs) == 0 {
|
||||
return nil
|
||||
}
|
||||
return aggregate(errs)
|
||||
}
|
||||
|
||||
// This helper implements the error and Errors interfaces. Keeping it private
|
||||
// prevents people from making an aggregate of 0 errors, which is not
|
||||
// an error, but does satisfy the error interface.
|
||||
type aggregate []error
|
||||
|
||||
// Error is part of the error interface.
|
||||
func (agg aggregate) Error() string {
|
||||
if len(agg) == 0 {
|
||||
// This should never happen, really.
|
||||
return ""
|
||||
}
|
||||
if len(agg) == 1 {
|
||||
return agg[0].Error()
|
||||
}
|
||||
result := fmt.Sprintf("[%s", agg[0].Error())
|
||||
for i := 1; i < len(agg); i++ {
|
||||
result += fmt.Sprintf(", %s", agg[i].Error())
|
||||
}
|
||||
result += "]"
|
||||
return result
|
||||
}
|
||||
|
||||
// Errors is part of the Aggregate interface.
|
||||
func (agg aggregate) Errors() []error {
|
||||
return []error(agg)
|
||||
}
|
||||
|
||||
// Matcher is used to match errors. Returns true if the error matches.
|
||||
type Matcher func(error) bool
|
||||
|
||||
// FilterOut removes all errors that match any of the matchers from the input
|
||||
// error. If the input is a singular error, only that error is tested. If the
|
||||
// input implements the Aggregate interface, the list of errors will be
|
||||
// processed recursively.
|
||||
//
|
||||
// This can be used, for example, to remove known-OK errors (such as io.EOF or
|
||||
// os.PathNotFound) from a list of errors.
|
||||
func FilterOut(err error, fns ...Matcher) error {
|
||||
if err == nil {
|
||||
return nil
|
||||
}
|
||||
if agg, ok := err.(Aggregate); ok {
|
||||
return NewAggregate(filterErrors(agg.Errors(), fns...))
|
||||
}
|
||||
if !matchesError(err, fns...) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// matchesError returns true if any Matcher returns true
|
||||
func matchesError(err error, fns ...Matcher) bool {
|
||||
for _, fn := range fns {
|
||||
if fn(err) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// filterErrors returns any errors (or nested errors, if the list contains
|
||||
// nested Errors) for which all fns return false. If no errors
|
||||
// remain a nil list is returned. The resulting silec will have all
|
||||
// nested slices flattened as a side effect.
|
||||
func filterErrors(list []error, fns ...Matcher) []error {
|
||||
result := []error{}
|
||||
for _, err := range list {
|
||||
r := FilterOut(err, fns...)
|
||||
if r != nil {
|
||||
result = append(result, r)
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Flatten takes an Aggregate, which may hold other Aggregates in arbitrary
|
||||
// nesting, and flattens them all into a single Aggregate, recursively.
|
||||
func Flatten(agg Aggregate) Aggregate {
|
||||
result := []error{}
|
||||
if agg == nil {
|
||||
return nil
|
||||
}
|
||||
for _, err := range agg.Errors() {
|
||||
if a, ok := err.(Aggregate); ok {
|
||||
r := Flatten(a)
|
||||
if r != nil {
|
||||
result = append(result, r.Errors()...)
|
||||
}
|
||||
} else {
|
||||
if err != nil {
|
||||
result = append(result, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
return NewAggregate(result)
|
||||
}
|
||||
|
||||
// CreateAggregateFromMessageCountMap converts MessageCountMap Aggregate
|
||||
func CreateAggregateFromMessageCountMap(m MessageCountMap) Aggregate {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
result := make([]error, 0, len(m))
|
||||
for errStr, count := range m {
|
||||
var countStr string
|
||||
if count > 1 {
|
||||
countStr = fmt.Sprintf(" (repeated %v times)", count)
|
||||
}
|
||||
result = append(result, fmt.Errorf("%v%v", errStr, countStr))
|
||||
}
|
||||
return NewAggregate(result)
|
||||
}
|
||||
|
||||
// Reduce will return err or, if err is an Aggregate and only has one item,
|
||||
// the first item in the aggregate.
|
||||
func Reduce(err error) error {
|
||||
if agg, ok := err.(Aggregate); ok && err != nil {
|
||||
switch len(agg.Errors()) {
|
||||
case 1:
|
||||
return agg.Errors()[0]
|
||||
case 0:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
// AggregateGoroutines runs the provided functions in parallel, stuffing all
|
||||
// non-nil errors into the returned Aggregate.
|
||||
// Returns nil if all the functions complete successfully.
|
||||
func AggregateGoroutines(funcs ...func() error) Aggregate {
|
||||
errChan := make(chan error, len(funcs))
|
||||
for _, f := range funcs {
|
||||
go func(f func() error) { errChan <- f() }(f)
|
||||
}
|
||||
errs := make([]error, 0)
|
||||
for i := 0; i < cap(errChan); i++ {
|
||||
if err := <-errChan; err != nil {
|
||||
errs = append(errs, err)
|
||||
}
|
||||
}
|
||||
return NewAggregate(errs)
|
||||
}
|
||||
|
||||
// ErrPreconditionViolated is returned when the precondition is violated
|
||||
var ErrPreconditionViolated = errors.New("precondition is violated")
|
||||
+3
-3
@@ -65,9 +65,9 @@ func init() {
|
||||
MONTH_REG = regexp.MustCompile(`^\d{4}-\d{2}$`)
|
||||
DATE_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}$`)
|
||||
DATE_COMPACT_REG = regexp.MustCompile(`^\d{8}$`)
|
||||
ISO_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z$`)
|
||||
ISO_NO_SECOND_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}Z$`)
|
||||
FULLISO_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$`)
|
||||
ISO_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(Z|\+\d{2}:\d{2})$`)
|
||||
ISO_NO_SECOND_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(Z|\+\d{2}:\d{2})$`)
|
||||
FULLISO_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3,9}(Z|\+\d{2}:\d{2})$`)
|
||||
COMPACT_TIME_REG = regexp.MustCompile(`^\d{14}$`)
|
||||
ZSTACK_TIME_REG = regexp.MustCompile(`^\w+ \d{1,2}, \d{4} \d{1,2}:\d{1,2}:\d{1,2} (AM|PM)$`) //ZStack time format "Apr 1, 2019 3:23:17 PM"
|
||||
MYSQL_TIME_REG = regexp.MustCompile(`^\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}$`)
|
||||
|
||||
+26
-4
@@ -17,6 +17,7 @@ package timeutils
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
"strings"
|
||||
|
||||
"yunion.io/x/pkg/util/regutils"
|
||||
)
|
||||
@@ -42,9 +43,10 @@ func Localify(now time.Time) time.Time {
|
||||
}
|
||||
|
||||
const (
|
||||
IsoTimeFormat = "2006-01-02T15:04:05Z"
|
||||
IsoNoSecondTimeFormat = "2006-01-02T15:04Z"
|
||||
FullIsoTimeFormat = "2006-01-02T15:04:05.000000Z"
|
||||
IsoTimeFormat = "2006-01-02T15:04:05Z07:00"
|
||||
IsoNoSecondTimeFormat = "2006-01-02T15:04Z07:00"
|
||||
FullIsoTimeFormat = "2006-01-02T15:04:05.000000Z07:00"
|
||||
FullIsoNanoTimeFormat = "2006-01-02T15:04:05.000000000Z07:00"
|
||||
MysqlTimeFormat = "2006-01-02 15:04:05"
|
||||
NormalTimeFormat = "2006-01-02T15:04:05"
|
||||
FullNormalTimeFormat = "2006-01-02T15:04:05.000000"
|
||||
@@ -68,6 +70,10 @@ func FullIsoTime(now time.Time) string {
|
||||
return Utcify(now).Format(FullIsoTimeFormat)
|
||||
}
|
||||
|
||||
func FullIsoNanoTime(now time.Time) string {
|
||||
return Utcify(now).Format(FullIsoNanoTimeFormat)
|
||||
}
|
||||
|
||||
func MysqlTime(now time.Time) string {
|
||||
return Utcify(now).Format(MysqlTimeFormat)
|
||||
}
|
||||
@@ -100,8 +106,23 @@ func ParseIsoNoSecondTime(str string) (time.Time, error) {
|
||||
return time.Parse(IsoNoSecondTimeFormat, str)
|
||||
}
|
||||
|
||||
func toFullIsoNanoTimeFormat(str string) string {
|
||||
// 2019-09-17T20:50:17.66667134+08:00
|
||||
subsecStr := str[20:]
|
||||
pos := strings.IndexByte(subsecStr, 'Z')
|
||||
if pos < 0 {
|
||||
pos = strings.IndexByte(subsecStr, '+')
|
||||
}
|
||||
leftOver := subsecStr[pos:]
|
||||
subsecStr = subsecStr[:pos]
|
||||
for len(subsecStr) < 9 {
|
||||
subsecStr += "0"
|
||||
}
|
||||
return str[:20] + subsecStr + leftOver
|
||||
}
|
||||
|
||||
func ParseFullIsoTime(str string) (time.Time, error) {
|
||||
return time.Parse(FullIsoTimeFormat, str)
|
||||
return time.Parse(FullIsoNanoTimeFormat, toFullIsoNanoTimeFormat(str))
|
||||
}
|
||||
|
||||
func ParseMysqlTime(str string) (time.Time, error) {
|
||||
@@ -137,6 +158,7 @@ func ParseZStackDate(str string) (time.Time, error) {
|
||||
}
|
||||
|
||||
func ParseTimeStr(str string) (time.Time, error) {
|
||||
str = strings.TrimSpace(str)
|
||||
if regutils.MatchFullISOTime(str) {
|
||||
return ParseFullIsoTime(str)
|
||||
} else if regutils.MatchISOTime(str) {
|
||||
|
||||
Reference in New Issue
Block a user