diff --git a/go.mod b/go.mod index f1a59da552..827c12e197 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101074831-349be92afd17 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101101511-84cd9015cb57 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19 yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361 diff --git a/go.sum b/go.sum index 4b24f08499..7cd3db7d35 100644 --- a/go.sum +++ b/go.sum @@ -1179,8 +1179,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101074831-349be92afd17 h1:qBjN6Q/QRhKLAkxyGrTLOAI6acfQpsxuQR1Ajk0VMp8= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101074831-349be92afd17/go.mod h1:McRjoG2gaOUisB+Qa41kLCNZhr0lsCu4apEjTiphXVY= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101101511-84cd9015cb57 h1:qA0jIVE7pdmPRIDWslK77ajeKOdWig3CR8GTeUai3yA= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101101511-84cd9015cb57/go.mod h1:McRjoG2gaOUisB+Qa41kLCNZhr0lsCu4apEjTiphXVY= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/vendor/modules.txt b/vendor/modules.txt index 001dfa4f75..700da6b211 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1438,7 +1438,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101074831-349be92afd17 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231101101511-84cd9015cb57 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go index a6de036d0a..1006c6536a 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/bucket.go @@ -91,28 +91,25 @@ func (b *SBucket) GetAccessUrls() []cloudprovider.SBucketAccessUrl { } func grantToCannedAcl(acls []tos.GrantV2) cloudprovider.TBucketACLType { - switch { - case len(acls) == 1: - if acls[0].Permission == enum.PermissionFullControl { - return cloudprovider.ACLPrivate + isWrite, isRead := false, false + for _, acl := range acls { + if acl.GranteeV2.Type != enum.GranteeGroup || acl.GranteeV2.Canned != enum.CannedAllUsers { + continue } - case len(acls) == 2: - for _, g := range acls { - if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAuthenticatedUsers && g.Permission == enum.PermissionRead { - return cloudprovider.ACLAuthRead - } - if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAllUsers && g.Permission == enum.PermissionRead { - return cloudprovider.ACLPublicRead - } - } - case len(acls) == 3: - for _, g := range acls { - if g.GranteeV2.Type == enum.GranteeGroup && g.GranteeV2.Canned == enum.CannedAllUsers && g.Permission == enum.PermissionRead { - return cloudprovider.ACLPublicReadWrite - } + switch acl.Permission { + case enum.PermissionWrite: + isWrite = true + case enum.PermissionRead: + isRead = true } } - return cloudprovider.ACLUnknown + if isWrite && isRead { + return cloudprovider.ACLPublicReadWrite + } + if isRead { + return cloudprovider.ACLPublicRead + } + return cloudprovider.ACLPrivate } func (b *SBucket) GetAcl() cloudprovider.TBucketACLType { @@ -148,7 +145,26 @@ func (b *SBucket) SetAcl(aclStr cloudprovider.TBucketACLType) error { } func (b *SBucket) NewMultipartUpload(ctx context.Context, key string, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) (string, error) { - return "", errors.ErrNotImplemented + toscli, err := b.region.GetTosClient() + if err != nil { + return "", errors.Wrapf(err, "GetTosClient") + } + input := &tos.CreateMultipartUploadV2Input{ + Bucket: b.Name, + Key: key, + ACL: enum.ACLType(cannedAcl), + StorageClass: enum.StorageClassType(storageClassStr), + Meta: map[string]string{}, + } + for k := range meta { + input.Meta[k] = meta.Get(k) + } + + output, err := toscli.CreateMultipartUploadV2(ctx, input) + if err != nil { + return "", err + } + return output.UploadID, nil } func (b *SBucket) AbortMultipartUpload(ctx context.Context, key string, uploadId string) error { @@ -215,7 +231,7 @@ func (b *SBucket) CopyObject(ctx context.Context, destKey string, srcBucket, src } else { metaDir = "COPY" } - input := tos.CopyObjectInput{Bucket: b.Name, Key: destKey, SrcKey: fmt.Sprintf("%s/%s", srcBucket, url.PathEscape(srcKey)), StorageClass: enum.StorageClassType(storageClassStr), ACL: enum.ACLType(cannedAcl), MetadataDirective: enum.MetadataDirectiveType(metaDir)} + input := tos.CopyObjectInput{SrcBucket: srcBucket, Bucket: b.Name, Key: destKey, SrcKey: url.PathEscape(srcKey), StorageClass: enum.StorageClassType(storageClassStr), ACL: enum.ACLType(cannedAcl), MetadataDirective: enum.MetadataDirectiveType(metaDir)} if len(cacheControl) > 0 { input.CacheControl = cacheControl } @@ -280,7 +296,10 @@ func (b *SBucket) DeleteObject(ctx context.Context, key string) error { if err != nil { return errors.Wrap(err, "GetTosClient") } - input := tos.DeleteObjectV2Input{} + input := tos.DeleteObjectV2Input{ + Bucket: b.Name, + Key: key, + } _, err = toscli.DeleteObjectV2(ctx, &input) if err != nil { return errors.Wrap(err, "DeleteObject") @@ -293,7 +312,12 @@ func (b *SBucket) GetObject(ctx context.Context, key string, rangeOpt *cloudprov if err != nil { return nil, errors.Wrap(err, "GetTosClient") } - input := tos.GetObjectV2Input{} + input := tos.GetObjectV2Input{ + Bucket: b.Name, + Key: key, + RangeStart: rangeOpt.Start, + RangeEnd: rangeOpt.End, + } output, err := toscli.GetObjectV2(ctx, &input) if err != nil { return nil, errors.Wrap(err, "DeleteObject") @@ -353,7 +377,21 @@ func (b *SBucket) ListObjects(prefix string, marker string, delimiter string, ma } func (b *SBucket) GetTempUrl(method string, key string, expire time.Duration) (string, error) { - return "", errors.ErrNotImplemented + toscli, err := b.region.GetTosClient() + if err != nil { + return "", errors.Wrapf(err, "GetTosClient") + } + input := &tos.PreSignedURLInput{ + HTTPMethod: enum.HttpMethodGet, + Bucket: b.Name, + Key: key, + Expires: int64(expire.Seconds()), + } + output, err := toscli.PreSignedURL(input) + if err != nil { + return "", err + } + return output.SignedUrl, nil } func (b *SBucket) PutObject(ctx context.Context, key string, body io.Reader, sizeBytes int64, cannedAcl cloudprovider.TBucketACLType, storageClassStr string, meta http.Header) error { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go index be526e7996..513dabe806 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/image.go @@ -23,6 +23,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/imagetools" + "yunion.io/x/pkg/util/osprofile" api "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/cloudmux/pkg/cloudprovider" @@ -67,6 +68,7 @@ type SImage struct { Size int Status ImageStatusType Usage string + BootMode string } func (img *SImage) GetMinRamSizeMb() int { @@ -159,11 +161,11 @@ func (img *SImage) GetImageStatus() string { } func (img *SImage) Refresh() error { - new, err := img.storageCache.region.GetImage(img.ImageId) + image, err := img.storageCache.region.GetImage(img.ImageId) if err != nil { return err } - return jsonutils.Update(img, new) + return jsonutils.Update(img, image) } func (img *SImage) GetImageType() cloudprovider.TImageType { @@ -211,11 +213,17 @@ func (img *SImage) GetOsLang() string { } func (img *SImage) GetOsArch() string { - return img.getNormalizedImageInfo().OsArch + if strings.Contains(img.Architecture, "arm") { + return osprofile.OS_ARCH_ARM + } + return osprofile.OS_ARCH_X86_64 } func (img *SImage) GetBios() cloudprovider.TBiosType { - return cloudprovider.ToBiosType(img.getNormalizedImageInfo().OsBios) + if img.BootMode == "UEFI" { + return cloudprovider.UEFI + } + return cloudprovider.BIOS } func (img *SImage) GetMinOsDiskSizeGb() int { @@ -231,25 +239,29 @@ func (img *SImage) GetCreatedAt() time.Time { } func (region *SRegion) GetImage(imageId string) (*SImage, error) { - images, _, err := region.GetImages("", "", []string{imageId}, "", 1, "") + images, err := region.GetImages("", []string{imageId}, "") if err != nil { return nil, err } - if len(images) == 0 { - return nil, cloudprovider.ErrNotFound + for i := range images { + if images[i].ImageId == imageId { + return &images[i], nil + } } - return &images[0], nil + return nil, errors.Wrapf(cloudprovider.ErrNotFound, imageId) } func (region *SRegion) GetImageByName(name string) (*SImage, error) { - images, _, err := region.GetImages("", "", nil, name, 1, "") + images, err := region.GetImages("", nil, name) if err != nil { return nil, err } - if len(images) == 0 { - return nil, cloudprovider.ErrNotFound + for i := range images { + if images[i].ImageName == name { + return &images[i], nil + } } - return &images[0], nil + return nil, errors.Wrapf(cloudprovider.ErrNotFound, name) } func (region *SRegion) GetImageStatus(imageId string) (ImageStatusType, error) { @@ -260,50 +272,41 @@ func (region *SRegion) GetImageStatus(imageId string) (ImageStatusType, error) { return image.Status, nil } -func (region *SRegion) GetImages(status ImageStatusType, owner ImageOwnerType, imageId []string, name string, limit int, token string) ([]SImage, string, error) { - if limit > 100 || limit <= 0 { - limit = 100 - } +func (region *SRegion) GetImages(visibility string, imageIds []string, name string) ([]SImage, error) { params := make(map[string]string) - params["MaxResults"] = fmt.Sprintf("%d", limit) - if len(token) > 0 { - params["NextToken"] = token + params["MaxResults"] = "100" + for i, id := range imageIds { + params[fmt.Sprintf("ImageIds.%d", i+1)] = id } - if len(status) > 0 { - params["Status"] = string(status) - } else { - allStatus := []string{"available", "creating", "error"} - for idx, status := range allStatus { - params[fmt.Sprintf("Status.%d", idx+1)] = status - } - } - if len(imageId) > 0 { - params["ImageId"] = strings.Join(imageId, ",") - } - if len(owner) > 0 { - params["ImageOwnerAlias"] = string(owner) + if len(visibility) > 0 { + params["Visibility"] = visibility } if len(name) > 0 { params["ImageName"] = name } - return region.getImages(params) -} - -func (region *SRegion) getImages(params map[string]string) ([]SImage, string, error) { - body, err := region.ecsRequest("DescribeImages", params) - if err != nil { - return nil, "", errors.Wrapf(err, "DescribeImages fail") + ret := []SImage{} + for { + resp, err := region.ecsRequest("DescribeImages", params) + if err != nil { + return nil, errors.Wrapf(err, "DescribeImages") + } + part := struct { + Images []SImage + NextToken string + }{} + err = resp.Unmarshal(&part) + if err != nil { + return nil, err + } + ret = append(ret, part.Images...) + if len(part.NextToken) == 0 || len(part.Images) == 0 { + break + } + params["NextToken"] = part.NextToken } - - images := make([]SImage, 0) - err = body.Unmarshal(&images, "Images") - if err != nil { - return nil, "", errors.Wrapf(err, "Unmarshal images fail") - } - nextToken, _ := body.GetString("NextToken") - return images, nextToken, nil + return ret, nil } func (region *SRegion) DeleteImage(imageId string) error { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go index 2343af7e4a..6f32dccbed 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/objects.go @@ -59,7 +59,8 @@ func (obj *SObject) SetAcl(aclStr cloudprovider.TBucketACLType) error { if err != nil { return errors.Wrap(err, "GetTosClient") } - _, err = toscli.PutObjectACL(context.Background(), &tos.PutObjectACLInput{Key: obj.Key, ACL: enum.ACLType(aclStr)}) + input := &tos.PutObjectACLInput{Bucket: obj.bucket.Name, Key: obj.Key, ACL: enum.ACLType(aclStr)} + _, err = toscli.PutObjectACL(context.Background(), input) if err != nil { return errors.Wrapf(err, "PutObjectACL") } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go index 37b0e071ce..5a734572e1 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/provider/provider.go @@ -153,7 +153,25 @@ func (self *SVolcEngineProvider) GetBalance() (*cloudprovider.SBalanceInfo, erro } func (self *SVolcEngineProvider) GetBucketCannedAcls(regionId string) []string { - return nil + return []string{ + string(cloudprovider.ACLPrivate), + string(cloudprovider.ACLPublicRead), + string(cloudprovider.ACLPublicReadWrite), + } +} + +func (self *SVolcEngineProvider) GetObjectCannedAcls(regionId string) []string { + return []string{ + string(cloudprovider.ACLPrivate), + string(cloudprovider.ACLPublicRead), + string(cloudprovider.ACLPublicReadWrite), + } +} + +func (self *SVolcEngineProvider) GetStorageClasses(regionId string) []string { + return []string{ + "STANDARD", "IA", "ARCHIVE_FR", "INTELLIGENT_TIERING", "COLD_ARCHIVE", + } } func (self *SVolcEngineProvider) GetCapabilities() []string { @@ -172,14 +190,6 @@ func (self *SVolcEngineProvider) GetIRegions() []cloudprovider.ICloudRegion { return self.client.GetIRegions() } -func (self *SVolcEngineProvider) GetObjectCannedAcls(regionId string) []string { - return nil -} - -func (self *SVolcEngineProvider) GetStorageClasses(regionId string) []string { - return nil -} - func (self *SVolcEngineProvider) GetSubAccounts() ([]cloudprovider.SSubAccount, error) { return self.client.GetSubAccounts() } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go index 1ce571142f..4d3b6d277f 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/volcengine/storagecache.go @@ -62,18 +62,9 @@ func (scache *SStoragecache) GetICloudImages() ([]cloudprovider.ICloudImage, err } func (scache *SStoragecache) GetICustomizedCloudImages() ([]cloudprovider.ICloudImage, error) { - images := make([]SImage, 0) - token := "" - for { - parts, nextToken, err := scache.region.GetImages(ImageStatusType(""), ImageOwnerPrivate, nil, "", 50, token) - if err != nil { - return nil, errors.Wrapf(err, "GetImages") - } - images = append(images, parts...) - if len(nextToken) == 0 { - break - } - token = nextToken + images, err := scache.region.GetImages("private", nil, "") + if err != nil { + return nil, errors.Wrapf(err, "GetImages") } ret := []cloudprovider.ICloudImage{} for i := range images {