opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-01 15:07:17 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: backup storage support sign version (#21913)

Browse Source 
Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian Qiu
2025-01-01 10:11:26 +08:00
committed by GitHub
parent efddf8e684
commit d5b26cb86a
16 changed files with 166 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+4 -4
View File
@@ -88,13 +88,13 @@ require (
k8s.io/cluster-bootstrap v0.19.3
k8s.io/cri-api v0.22.17
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241125120206-f98ea0037b11
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241231161259-655e77daa48e
yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256
yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
yunion.io/x/pkg v1.10.2
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
)
go.sum
+8 -10
View File
@@ -793,8 +793,6 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg=
github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zexi/influxql-to-metricsql v0.0.9 h1:IVT4w4OvYvUJvj5c1TvH0G/ueM4ZZsfHGMoXtzC5rQE=
github.com/zexi/influxql-to-metricsql v0.0.9/go.mod h1:JlC5FY+6De9ZPxG47G5DOgva8P9X1VaKS4ExzCmhSCc=
github.com/zexi/influxql-to-metricsql v0.1.0 h1:gSXhnYEu16fHNs5e1NSdNYceKdlKjMijTHEpnnMVYN4=
github.com/zexi/influxql-to-metricsql v0.1.0/go.mod h1:JlC5FY+6De9ZPxG47G5DOgva8P9X1VaKS4ExzCmhSCc=
github.com/zexi/promql/v2 v2.12.1 h1:crHKpULdLLsBZ9b78Rg6qQkugzlk6BHeCj93tw/F5RU=
@@ -1281,10 +1279,10 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241125120206-f98ea0037b11 h1:RvExqhcidd6FD/J3/A91uWNUfbtmnl+BtI3UPWAeP1g=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241125120206-f98ea0037b11/go.mod h1:rj/pb3DitJlQaQD8UW1oxx/KD+PzDZqoywzqRJaFE9A=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241231161259-655e77daa48e h1:EzO1fhJG5Mp7mfoVpqzMwesTUNTMD8gHH3blbS6uGRo=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241231161259-655e77daa48e/go.mod h1:KQ/jWx7bZlmjCE711KEWuvHW/dzpdr/UTlBjjutkj0Y=
yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8=
yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f h1:N7V0uLqX9mED1HEPeIKcYpkWglnHL5hyn4ENX+VBeqM=
yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo=
@@ -1296,10 +1294,10 @@ yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVB
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318 h1:Fm7I8ypXHxeObY4u/VUGz78NsambemzTZ9fECyGKNi8=
yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/pkg v1.10.2 h1:oZhedvlvDsebIWcOvrOMJQ31SxzjxxqUXUBYseBjh7w=
yunion.io/x/pkg v1.10.2/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1 h1:HWPqY1I5JSmM6Sks6FyK9hnq/MjL7FDghM6M8DXHob0=
yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A=
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
pkg/apis/compute/backup.go
+2
View File
@@ -189,6 +189,8 @@ type SBackupStorageAccessInfo struct {
ObjectAccessKey string `json:"object_access_key"`
// description: secret of object storage
ObjectSecret string `json:"object_secret"`
// description: signing version, can be v2/v4, default is v4
ObjectSignVer string `json:"object_sign_ver"`
}
func (ba *SBackupStorageAccessInfo) String() string {
pkg/compute/models/backup_storage.go
+5
View File
@@ -169,6 +169,7 @@ func (bs *SBackupStorage) getMoreDetails(ctx context.Context, out api.BackupStor
out.NfsSharedDir = bs.AccessInfo.NfsSharedDir
out.ObjectBucketUrl = bs.AccessInfo.ObjectBucketUrl
out.ObjectAccessKey = bs.AccessInfo.ObjectAccessKey
out.ObjectSignVer = bs.AccessInfo.ObjectSignVer
// should not return secret
out.ObjectSecret = "" // bs.AccessInfo.ObjectSecret
return out
@@ -316,6 +317,10 @@ func (bs *SBackupStorage) PostUpdate(ctx context.Context, userCred mcclient.Toke
accessInfo.ObjectSecret = sec
accessInfoChanged = true
}
if input.ObjectSignVer != accessInfo.ObjectSignVer {
accessInfo.ObjectSignVer = input.ObjectSignVer
accessInfoChanged = true
}
}
if accessInfoChanged {
_, err = db.Update(bs, func() error {
pkg/compute/models/cloudaccounts.go
+7 -8
View File
@@ -252,7 +252,7 @@ func (acnt *SCloudaccount) enableAccountOnly(ctx context.Context, userCred mccli
}
func (acnt *SCloudaccount) PerformEnable(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input apis.PerformEnableInput) (jsonutils.JSONObject, error) {
if strings.Index(acnt.Status, "delet") >= 0 {
if strings.Contains(acnt.Status, "delet") {
return nil, httperrors.NewInvalidStatusError("Cannot enable deleting account")
}
_, err := acnt.enableAccountOnly(ctx, userCred, query, input)
@@ -747,7 +747,12 @@ func (acnt *SCloudaccount) PerformTestConnectivity(ctx context.Context, userCred
return nil, nil
}
func (acnt *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
func (acnt *SCloudaccount) PerformUpdateCredential(
ctx context.Context,
userCred mcclient.TokenCredential,
query jsonutils.JSONObject,
input cloudprovider.SCloudaccountCredential,
) (jsonutils.JSONObject, error) {
if !acnt.GetEnabled() {
return nil, httperrors.NewInvalidStatusError("Account disabled")
}
@@ -757,12 +762,6 @@ func (acnt *SCloudaccount) PerformUpdateCredential(ctx context.Context, userCred
return nil, httperrors.NewBadRequestError("failed to found provider factory error: %v", err)
}
input := cloudprovider.SCloudaccountCredential{}
err = data.Unmarshal(&input)
if err != nil {
return nil, httperrors.NewInputParameterError("failed to unmarshal input params: %v", err)
}
account, err := providerDriver.ValidateUpdateCloudaccountCredential(ctx, input, acnt.Account)
if err != nil {
return nil, err
pkg/hostman/storageman/backupstorage/object/factory.go
+2 -1
View File
@@ -15,6 +15,7 @@
package object
import (
"yunion.io/x/cloudmux/pkg/multicloud/objectstore"
"yunion.io/x/jsonutils"
"yunion.io/x/pkg/errors"
@@ -40,7 +41,7 @@ func (factory *sObjectBackupStorageFactory) NewBackupStore(backupStroageId strin
if len(accessInfo.ObjectSecret) == 0 {
return nil, errors.Wrap(httperrors.ErrInputParameter, "need object_secret in backup_storage_access_info")
}
return newObjectBackupStorage(backupStroageId, accessInfo.ObjectBucketUrl, accessInfo.ObjectAccessKey, accessInfo.ObjectSecret)
return newObjectBackupStorage(backupStroageId, accessInfo.ObjectBucketUrl, accessInfo.ObjectAccessKey, accessInfo.ObjectSecret, objectstore.S3SignVersion(accessInfo.ObjectSignVer))
}
func init() {
pkg/hostman/storageman/backupstorage/object/object.go
+7 -9
View File
@@ -30,31 +30,29 @@ import (
type SObjectBackupStorage struct {
BackupStorageId string
endpoint string
bucket string
accessKey string
secret string
bucket string
store *objectstore.SObjectStoreClient
}
func newObjectBackupStorage(backupStorageId, bucketUrl, accessKey, secret string) (*SObjectBackupStorage, error) {
func newObjectBackupStorage(backupStorageId, bucketUrl, accessKey, secret string, signVer objectstore.S3SignVersion) (*SObjectBackupStorage, error) {
bucket, endpoint, err := parseBucketUrl(bucketUrl)
if err != nil {
return nil, errors.Wrapf(err, "parseBucketUrl %s", bucketUrl)
}
cfg := objectstore.NewObjectStoreClientConfig(endpoint, accessKey, secret)
if len(signVer) > 0 {
cfg = cfg.SignVersion(signVer)
}
store, err := objectstore.NewObjectStoreClient(cfg)
if err != nil {
return nil, errors.Wrap(err, "NewObjectStoreClient")
}
return &SObjectBackupStorage{
BackupStorageId: backupStorageId,
endpoint: endpoint,
bucket: bucket,
accessKey: accessKey,
secret: secret,
bucket: bucket,
store: store,
}, nil
pkg/mcclient/options/compute/backup.go
+2
View File
@@ -120,6 +120,7 @@ type BackupStorageCreateOptions struct {
ObjectBucketUrl string `help:"object bucket url, required when storage_type is object"`
ObjectAccessKey string `help:"object storage access key, required when storage_type is object"`
ObjectSecret string `help:"object storage secret, required when storage_type is object"`
ObjectSignVer string `help:"object storage signing alogirithm version, optional" choices:"v2|v4"`
CapacityMb int `help:"capacity, unit mb"`
}
@@ -137,6 +138,7 @@ type BackupStorageUpdateOptions struct {
ObjectBucketUrl string `help:"object bucket url, required when storage_type is object"`
ObjectAccessKey string `help:"object storage access key, required when storage_type is object"`
ObjectSecret string `help:"object storage secret, required when storage_type is object"`
ObjectSignVer string `help:"object storage signing alogirithm version, optional" choices:"v2|v4"`
}
func (opts *BackupStorageUpdateOptions) Params() (jsonutils.JSONObject, error) {
pkg/mcclient/options/compute/cloudaccounts.go
+6
View File
@@ -405,11 +405,17 @@ type SS3CloudAccountCreateOptions struct {
SCloudAccountCreateBaseOptions
SAccessKeyCredential
Endpoint string `help:"S3 endpoint" required:"true" positional:"true" json:"endpoint"`
OptionSignVer string `help:"signing algorithm version" choices:"v2|v4"`
}
func (opts *SS3CloudAccountCreateOptions) Params() (jsonutils.JSONObject, error) {
params := jsonutils.Marshal(opts)
params.(*jsonutils.JSONDict).Add(jsonutils.NewString("S3"), "provider")
options := jsonutils.NewDict()
if len(opts.OptionSignVer) > 0 {
options.Add(jsonutils.NewString(opts.OptionSignVer), "sign_ver")
}
return params, nil
}
vendor/modules.txt
Vendored
+4 -4
View File
@@ -1579,7 +1579,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241125120206-f98ea0037b11
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20241231161259-655e77daa48e
## explicit; go 1.21
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
@@ -1659,7 +1659,7 @@ yunion.io/x/cloudmux/pkg/multicloud/volcengine
yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider
yunion.io/x/cloudmux/pkg/multicloud/zstack
yunion.io/x/cloudmux/pkg/multicloud/zstack/provider
# yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
# yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256
## explicit; go 1.12
yunion.io/x/executor/apis
yunion.io/x/executor/client
@@ -1676,7 +1676,7 @@ yunion.io/x/log/hooks
yunion.io/x/ovsdb/cli_util
yunion.io/x/ovsdb/schema/ovn_nb
yunion.io/x/ovsdb/types
# yunion.io/x/pkg v1.10.1-0.20240905110705-77c46e716318
# yunion.io/x/pkg v1.10.2
## explicit; go 1.18
yunion.io/x/pkg/appctx
yunion.io/x/pkg/errors
@@ -1728,7 +1728,7 @@ yunion.io/x/pkg/util/vmdkutils
yunion.io/x/pkg/util/wait
yunion.io/x/pkg/util/workqueue
yunion.io/x/pkg/utils
# yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
# yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
## explicit; go 1.12
yunion.io/x/s3cli
# yunion.io/x/sqlchemy v1.1.3-0.20240926163039-d41512b264e1
vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/buckets.go
Generated Vendored
+3 -1
View File
@@ -22,6 +22,7 @@ import (
"strings"
"time"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
"yunion.io/x/s3cli"
@@ -243,7 +244,8 @@ func (bucket *SBucket) PutObject(ctx context.Context, key string, input io.Reade
}
err = obj.SetAcl(cannedAcl)
if err != nil && errors.Cause(err) != cloudprovider.ErrNotImplemented {
return errors.Wrap(err, "obj.SetAcl")
log.Errorf("PubObject SetAcl fail %s", err)
// return errors.Wrap(err, "obj.SetAcl")
}
return nil
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/objectstore.go
Generated Vendored
+29 -1
View File
@@ -34,6 +34,14 @@ import (
"yunion.io/x/cloudmux/pkg/multicloud"
)
type S3SignVersion string
const (
S3SignAlgDefault = S3SignVersion("")
S3SignAlgV4 = S3SignVersion("v4")
S3SignAlgV2 = S3SignVersion("v2")
)
type ObjectStoreClientConfig struct {
cpcfg cloudprovider.ProviderConfig
@@ -41,6 +49,8 @@ type ObjectStoreClientConfig struct {
accessKey string
accessSecret string
signVer S3SignVersion
debug bool
}
@@ -58,6 +68,11 @@ func (cfg *ObjectStoreClientConfig) CloudproviderConfig(cpcfg cloudprovider.Prov
return cfg
}
func (cfg *ObjectStoreClientConfig) SignVersion(signVer S3SignVersion) *ObjectStoreClientConfig {
cfg.signVer = signVer
return cfg
}
func (cfg *ObjectStoreClientConfig) Debug(debug bool) *ObjectStoreClientConfig {
cfg.debug = debug
return cfg
@@ -115,7 +130,20 @@ func NewObjectStoreClientAndFetch(cfg *ObjectStoreClientConfig, doFetch bool) (*
if parts.Scheme == "https" {
useSsl = true
}
cli, err := s3cli.New(
s3cliNewFunc := s3cli.New
switch cfg.signVer {
case S3SignAlgV4:
log.Debugf("Use v4 signing algorithm")
s3cliNewFunc = s3cli.NewV4
case S3SignAlgV2:
log.Debugf("Use v2 signing algorithm")
s3cliNewFunc = s3cli.NewV2
default:
log.Debugf("s3 sign algirithm version not set, use default")
}
cli, err := s3cliNewFunc(
parts.Host,
client.accessKey,
client.accessSecret,
vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/provider/provider.go
Generated Vendored
+44 -33
View File
@@ -29,15 +29,15 @@ type SObjectStoreProviderFactory struct {
cloudprovider.SPremiseBaseProviderFactory
}
func (self *SObjectStoreProviderFactory) GetId() string {
func (factory *SObjectStoreProviderFactory) GetId() string {
return api.CLOUD_PROVIDER_GENERICS3
}
func (self *SObjectStoreProviderFactory) GetName() string {
func (factory *SObjectStoreProviderFactory) GetName() string {
return api.CLOUD_PROVIDER_GENERICS3
}
func (self *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
func (factory *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx context.Context, input cloudprovider.SCloudaccountCredential) (cloudprovider.SCloudaccount, error) {
output := cloudprovider.SCloudaccount{}
if len(input.AccessKeyId) == 0 {
return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_id")
@@ -54,7 +54,7 @@ func (self *SObjectStoreProviderFactory) ValidateCreateCloudaccountData(ctx cont
return output, nil
}
func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
func (factory *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ctx context.Context, input cloudprovider.SCloudaccountCredential, cloudaccount string) (cloudprovider.SCloudaccount, error) {
output := cloudprovider.SCloudaccount{}
if len(input.AccessKeyId) == 0 {
return output, errors.Wrap(cloudprovider.ErrMissingParameter, "access_key_id")
@@ -69,25 +69,36 @@ func (self *SObjectStoreProviderFactory) ValidateUpdateCloudaccountCredential(ct
return output, nil
}
func (self *SObjectStoreProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) {
client, err := objectstore.NewObjectStoreClient(
objectstore.NewObjectStoreClientConfig(
cfg.URL, cfg.Account, cfg.Secret,
).CloudproviderConfig(cfg),
)
func (factory *SObjectStoreProviderFactory) GetProvider(cfg cloudprovider.ProviderConfig) (cloudprovider.ICloudProvider, error) {
storeCfg := objectstore.NewObjectStoreClientConfig(
cfg.URL, cfg.Account, cfg.Secret,
).CloudproviderConfig(cfg)
var signVer string
if cfg.Options != nil {
signVer, _ = cfg.Options.GetString("sign_ver")
}
if len(signVer) > 0 {
storeCfg = storeCfg.SignVersion(objectstore.S3SignVersion(signVer))
}
client, err := objectstore.NewObjectStoreClient(storeCfg)
if err != nil {
return nil, err
}
return NewObjectStoreProvider(self, client, []string{
return NewObjectStoreProvider(factory, client, []string{
string(cloudprovider.ACLPrivate),
}), nil
}
func (self *SObjectStoreProviderFactory) GetClientRC(info cloudprovider.SProviderInfo) (map[string]string, error) {
func (factory *SObjectStoreProviderFactory) GetClientRC(info cloudprovider.SProviderInfo) (map[string]string, error) {
var signVer string
if info.Options != nil {
signVer, _ = info.Options.GetString("sign_ver")
}
return map[string]string{
"S3_ACCESS_KEY": info.Account,
"S3_SECRET": info.Secret,
"S3_ACCESS_URL": info.Url,
"S3_SIGN_VER": signVer,
"S3_BACKEND": api.CLOUD_PROVIDER_GENERICS3,
}, nil
}
@@ -111,15 +122,15 @@ func NewObjectStoreProvider(factory cloudprovider.ICloudProviderFactory, client
}
}
func (self *SObjectStoreProvider) GetIRegions() ([]cloudprovider.ICloudRegion, error) {
func (provider *SObjectStoreProvider) GetIRegions() ([]cloudprovider.ICloudRegion, error) {
return nil, cloudprovider.ErrNotSupported
}
func (self *SObjectStoreProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) {
func (provider *SObjectStoreProvider) GetIRegionById(id string) (cloudprovider.ICloudRegion, error) {
return nil, cloudprovider.ErrNotSupported
}
func (self *SObjectStoreProvider) GetBalance() (*cloudprovider.SBalanceInfo, error) {
func (provider *SObjectStoreProvider) GetBalance() (*cloudprovider.SBalanceInfo, error) {
return &cloudprovider.SBalanceInfo{
Amount: 0.0,
Currency: "CNY",
@@ -127,42 +138,42 @@ func (self *SObjectStoreProvider) GetBalance() (*cloudprovider.SBalanceInfo, err
}, cloudprovider.ErrNotSupported
}
func (self *SObjectStoreProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) {
return self.client, nil
func (provider *SObjectStoreProvider) GetOnPremiseIRegion() (cloudprovider.ICloudRegion, error) {
return provider.client, nil
}
func (self *SObjectStoreProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {
func (provider *SObjectStoreProvider) GetIProjects() ([]cloudprovider.ICloudProject, error) {
return nil, cloudprovider.ErrNotSupported
}
func (self *SObjectStoreProvider) GetSysInfo() (jsonutils.JSONObject, error) {
return self.client.About(), nil
func (provider *SObjectStoreProvider) GetSysInfo() (jsonutils.JSONObject, error) {
return provider.client.About(), nil
}
func (self *SObjectStoreProvider) GetVersion() string {
return self.client.GetVersion()
func (provider *SObjectStoreProvider) GetVersion() string {
return provider.client.GetVersion()
}
func (self *SObjectStoreProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
return self.client.GetSubAccounts()
func (provider *SObjectStoreProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) {
return provider.client.GetSubAccounts()
}
func (self *SObjectStoreProvider) GetAccountId() string {
return self.client.GetAccountId()
func (provider *SObjectStoreProvider) GetAccountId() string {
return provider.client.GetAccountId()
}
func (self *SObjectStoreProvider) GetStorageClasses(regionId string) []string {
func (provider *SObjectStoreProvider) GetStorageClasses(regionId string) []string {
return []string{}
}
func (self *SObjectStoreProvider) GetBucketCannedAcls(regionId string) []string {
return self.supportedAcls
func (provider *SObjectStoreProvider) GetBucketCannedAcls(regionId string) []string {
return provider.supportedAcls
}
func (self *SObjectStoreProvider) GetObjectCannedAcls(regionId string) []string {
return self.supportedAcls
func (provider *SObjectStoreProvider) GetObjectCannedAcls(regionId string) []string {
return provider.supportedAcls
}
func (self *SObjectStoreProvider) GetCapabilities() []string {
return self.client.GetCapabilities()
func (provider *SObjectStoreProvider) GetCapabilities() []string {
return provider.client.GetCapabilities()
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/objectstore/shell.go
Generated Vendored
+29
View File
@@ -838,6 +838,35 @@ func S3Shell() {
return nil
})
type BucketObjectBatchDownloadOptions struct {
BUCKET string `help:"name of bucket"`
PREFIX string `help:"Prefix of object"`
Output string `help:"target output directory, default to current directory"`
}
shellutils.R(&BucketObjectBatchDownloadOptions{}, "batch-download", "Download objects recursively", func(cli cloudprovider.ICloudRegion, args *BucketObjectBatchDownloadOptions) error {
bucket, err := cli.GetIBucketById(args.BUCKET)
if err != nil {
return err
}
marker := ""
const maxCount = 1000
for {
results, err := bucket.ListObjects(args.PREFIX, marker, "", maxCount)
if err != nil {
return errors.Wrapf(err, "ListObjects prefix %s marker %s", args.PREFIX, marker)
}
for _, obj := range results.Objects {
fmt.Println(obj.GetKey())
}
if !results.IsTruncated {
break
} else {
marker = results.NextMarker
}
}
return nil
})
type BucketObjectDownloadOptions struct {
BUCKET string `help:"name of bucket"`
KEY string `help:"Key of object"`
vendor/yunion.io/x/executor/server/server.go
Generated Vendored
+8 -8
View File
@@ -144,14 +144,6 @@ func (e *Executor) Wait(ctx context.Context, in *apis.Sn) (*apis.WaitResponse, e
err error
)
if m.stdout != nil {
<-m.stdoutCh
}
if m.stderr != nil {
<-m.stderrCh
}
m.wg.Wait()
err = m.c.Wait()
var (
exitStatus uint32
@@ -172,6 +164,14 @@ func (e *Executor) Wait(ctx context.Context, in *apis.Sn) (*apis.WaitResponse, e
} else {
exitStatus = 0
}
if m.stdout != nil {
<-m.stdoutCh
}
if m.stderr != nil {
<-m.stderrCh
}
m.wg.Wait()
cmds.Delete(in.Sn)
return &apis.WaitResponse{
ExitStatus: exitStatus,
vendor/yunion.io/x/s3cli/api-s3-datatypes.go
Generated Vendored
+6 -6
View File
@@ -187,7 +187,7 @@ type ListObjectPartsResult struct {
// initiateMultipartUploadResult container for InitiateMultiPartUpload
// response.
type InitiateMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"`
XMLName xml.Name `xml:"InitiateMultipartUploadResult" json:"-"`
Bucket string `xml:"Bucket"`
Key string `xml:"Key"`
@@ -197,7 +197,7 @@ type InitiateMultipartUploadResult struct {
// completeMultipartUploadResult container for completed multipart
// upload response.
type CompleteMultipartUploadResult struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"`
XMLName xml.Name `xml:"CompleteMultipartUploadResult" json:"-"`
Location string `xml:"Location"`
Bucket string `xml:"Bucket"`
@@ -208,7 +208,7 @@ type CompleteMultipartUploadResult struct {
// CompletePart sub container lists individual part numbers and their
// md5sum, part of completeMultipartUpload.
type CompletePart struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"`
XMLName xml.Name `xml:"Part" json:"-"`
// Part number identifies the part.
PartNumber int `xml:"PartNumber"`
@@ -217,13 +217,13 @@ type CompletePart struct {
// completeMultipartUpload container for completing multipart upload.
type CompleteMultipartUpload struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"`
XMLName xml.Name `xml:"CompleteMultipartUpload" json:"-"`
Parts []CompletePart `xml:"Part"`
}
// createBucketConfiguration container for bucket configuration.
type CreateBucketConfiguration struct {
XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CreateBucketConfiguration" json:"-"`
XMLName xml.Name `xml:"CreateBucketConfiguration" json:"-"`
Location string `xml:"LocationConstraint"`
}
@@ -267,7 +267,7 @@ type deleteMultiObjectsResult struct {
}
type VersioningConfiguration struct {
XMLName xml.Name `xmlns:"http://s3.amazonaws.com/doc/2006-03-01/" xml:"VersioningConfiguration"`
XMLName xml.Name `xml:"VersioningConfiguration"`
Status string `xml:"Status,omitempty"`
MfaDelete string `xml:"MfaDelete,omitempty"`
}