opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-29 03:51:54 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(region): update gp3 disk size (#24811)

Browse Source
This commit is contained in:
屈轩
2026-05-12 17:27:29 +08:00
committed by GitHub
parent 8ef501a7da
commit 9beb4f54c0
15 changed files with 363 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2 -2
View File
@@ -94,12 +94,12 @@ require (
k8s.io/client-go v0.26.1
k8s.io/cluster-bootstrap v0.19.3
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260411042910-ada2ae80ee32
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260512084607-77c927d96c32
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005
yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429
yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1
yunion.io/x/sqlchemy v1.1.3-0.20251231025938-b0a38f6e9fab
yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
go.sum
+4 -4
View File
@@ -1090,8 +1090,8 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260411042910-ada2ae80ee32 h1:ZleBMSl5WDTRcE+N2IuMUFTU0r0l1cpNdBCoKKB4YY8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260411042910-ada2ae80ee32/go.mod h1:GdAwZ78fiqj13i2r1zNuE/D6YOlhh2q4jZjuyHiZTq8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260512084607-77c927d96c32 h1:PeQ1Q+AWVb1vGcRl7Fg9zc5nt8tgp6Oj1U1TUUaVSsk=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260512084607-77c927d96c32/go.mod h1:GdAwZ78fiqj13i2r1zNuE/D6YOlhh2q4jZjuyHiZTq8=
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 h1:3sWwcjGXGjG9mLBWa7AyLq+QSi0udTAx21pfVQRFMBE=
yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
@@ -1105,8 +1105,8 @@ yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVB
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc=
yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429 h1:yskc2HFnBZXEzfYeQBFdL2gXzVfdPQ8yT4qVDyuhajM=
yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978 h1:+BS9DValwi0xHa92xrB+RR6Cxp7bDbXVysa/KiC18rY=
yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM=
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4=
yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v1.1.3-0.20251231025938-b0a38f6e9fab h1:5m/bSzW3uTAk83rp9eethbYsxJFYInFVeU1RDkteW4E=
pkg/apis/compute/storage_const.go
+1 -1
View File
@@ -187,7 +187,7 @@ var (
STORAGE_PUBLIC_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD_PL0, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_ESSD_PL2, STORAGE_CLOUD_ESSD_PL3,
STORAGE_EPHEMERAL_SSD, STORAGE_CLOUD_EFFICIENCY,
STORAGE_STANDARD_LRS, STORAGE_STANDARDSSD_LRS, STORAGE_PREMIUM_LRS,
STORAGE_GP2_SSD, STORAGE_GP3_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD,
STORAGE_GP2_SSD, STORAGE_GP3_SSD, STORAGE_IO1_SSD, STORAGE_IO2_SSD, STORAGE_ST1_HDD, STORAGE_SC1_HDD, STORAGE_STANDARD_HDD,
STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PREMIUM,
STORAGE_HUAWEI_SSD, STORAGE_HUAWEI_SAS, STORAGE_HUAWEI_SATA,
STORAGE_OPENSTACK_ISCSI, STORAGE_UCLOUD_CLOUD_NORMAL, STORAGE_UCLOUD_CLOUD_SSD,
pkg/compute/guestdrivers/aws.go
+21 -9
View File
@@ -84,6 +84,15 @@ func (self *SAwsGuestDriver) GetWindowsUserDataType() string {
return cloudprovider.CLOUD_EC2
}
// AWS EBS 单卷容量：参见 https://docs.aws.amazon.com/ebs/latest/userguide/ebs-volume-types.html
const (
awsEbsGp2MaxSizeGiB = 16384 // gp2 最大 16 TiB
awsEbsGp3MaxSizeGiB = 65536 // gp3 最大 64 TiB
awsEbsIoMaxSizeGiB = 16384 // io1/io2 常规卷最大 16 TiB（io2 Block Express 另受实例等约束）
awsEbsHddMaxSizeGiB = 16384 // st1/sc1 最大 16 TiB
awsEbsSt1Sc1MinSizeGiB = 125 // st1/sc1 最小 125 GiB
)
func (self *SAwsGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapability {
return cloudprovider.SInstanceCapability{
Hypervisor: self.GetHypervisor(),
@@ -100,18 +109,21 @@ func (self *SAwsGuestDriver) GetInstanceCapability() cloudprovider.SInstanceCapa
},
Storages: cloudprovider.Storage{
DataDisk: []cloudprovider.StorageInfo{
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP3_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: 16384, MinSizeGb: 4, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: awsEbsGp2MaxSizeGiB, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP3_SSD, MaxSizeGb: awsEbsGp3MaxSizeGiB, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: awsEbsIoMaxSizeGiB, MinSizeGb: 4, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_IO2_SSD, MaxSizeGb: awsEbsIoMaxSizeGiB, MinSizeGb: 4, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: awsEbsHddMaxSizeGiB, MinSizeGb: awsEbsSt1Sc1MinSizeGiB, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: awsEbsHddMaxSizeGiB, MinSizeGb: awsEbsSt1Sc1MinSizeGiB, StepSizeGb: 1, Resizable: true},
cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_HDD, MaxSizeGb: 1024, MinSizeGb: 1, StepSizeGb: 1, Resizable: true},
},
SysDisk: []cloudprovider.StorageInfo{
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: 16384, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: 16384, MinSizeGb: 4, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: 16384, MinSizeGb: 500, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP2_SSD, MaxSizeGb: awsEbsGp2MaxSizeGiB, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_GP3_SSD, MaxSizeGb: awsEbsGp3MaxSizeGiB, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_IO1_SSD, MaxSizeGb: awsEbsIoMaxSizeGiB, MinSizeGb: 4, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_IO2_SSD, MaxSizeGb: awsEbsIoMaxSizeGiB, MinSizeGb: 4, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_ST1_HDD, MaxSizeGb: awsEbsHddMaxSizeGiB, MinSizeGb: awsEbsSt1Sc1MinSizeGiB, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_SC1_HDD, MaxSizeGb: awsEbsHddMaxSizeGiB, MinSizeGb: awsEbsSt1Sc1MinSizeGiB, StepSizeGb: 1, Resizable: false},
cloudprovider.StorageInfo{StorageType: api.STORAGE_STANDARD_HDD, MaxSizeGb: 1024, MinSizeGb: 1, StepSizeGb: 1, Resizable: false},
},
},
pkg/compute/hostdrivers/aws.go
+7 -3
View File
@@ -44,17 +44,21 @@ func (self *SAwsHostDriver) GetHypervisor() string {
}
func (self *SAwsHostDriver) ValidateDiskSize(storage *models.SStorage, sizeGb int) error {
if storage.StorageType == api.STORAGE_GP2_SSD || storage.StorageType == api.STORAGE_GP3_SSD {
if storage.StorageType == api.STORAGE_GP2_SSD {
if sizeGb < 1 || sizeGb > 16384 {
return fmt.Errorf("The %s disk size must be in the range of 1G ~ 16384GB", storage.StorageType)
}
} else if storage.StorageType == api.STORAGE_GP3_SSD {
if sizeGb < 1 || sizeGb > 65536 {
return fmt.Errorf("The %s disk size must be in the range of 1G ~ 65536GB", storage.StorageType)
}
} else if storage.StorageType == api.STORAGE_IO1_SSD || storage.StorageType == api.STORAGE_IO2_SSD {
if sizeGb < 4 || sizeGb > 16384 {
return fmt.Errorf("The %s disk size must be in the range of 4G ~ 16384GB", storage.StorageType)
}
} else if utils.IsInStringArray(storage.StorageType, []string{api.STORAGE_ST1_HDD, api.STORAGE_SC1_HDD}) {
if sizeGb < 500 || sizeGb > 16384 {
return fmt.Errorf("The %s disk size must be in the range of 500G ~ 16384GB", storage.StorageType)
if sizeGb < 125 || sizeGb > 16384 {
return fmt.Errorf("The %s disk size must be in the range of 125G ~ 16384GB", storage.StorageType)
}
} else if storage.StorageType == api.STORAGE_STANDARD_HDD {
if sizeGb < 1 || sizeGb > 1024 {
vendor/modules.txt
Vendored
+2 -2
View File
@@ -1830,7 +1830,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.3.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260411042910-ada2ae80ee32
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260512084607-77c927d96c32
## explicit; go 1.21
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
@@ -1938,7 +1938,7 @@ yunion.io/x/log/hooks
yunion.io/x/ovsdb/cli_util
yunion.io/x/ovsdb/schema/ovn_nb
yunion.io/x/ovsdb/types
# yunion.io/x/pkg v1.10.4-0.20260224071200-5c269fa62429
# yunion.io/x/pkg v1.10.4-0.20260422030155-01b100134978
## explicit; go 1.18
yunion.io/x/pkg/appctx
yunion.io/x/pkg/errors
vendor/yunion.io/x/cloudmux/pkg/multicloud/aliyun/aliyun.go
Generated Vendored
+1 -1
View File
@@ -200,7 +200,7 @@ func doRequest(client *sdk.Client, domain, apiVersion, apiName string, params ma
if e, ok := errors.Cause(err).(*alierr.ServerError); ok {
code := e.ErrorCode()
switch code {
case "InternalError":
case "InternalError", "AuthSiteFail":
if apiName != "QueryAccountBalance" {
return nil, err
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/bucket.go
Generated Vendored
+229 -1
View File
@@ -154,8 +154,236 @@ func (b *SBucket) GetWebsiteUrl() string {
return fmt.Sprintf("http://%s.%s", b.Name, b.region.getS3WebsiteEndpoint())
}
// S3 daily storage metrics (BucketSizeBytes / NumberOfObjects) are published to CloudWatch once per day
// with Period 86400. See https://docs.aws.amazon.com/AmazonS3/latest/userguide/metrics-dimensions.html
const (
s3BucketMetricsNamespace = "AWS/S3"
s3BucketMetricsPeriod = "86400"
)
type s3CwMetricDimension struct {
Name string `xml:"Name"`
Value string `xml:"Value"`
}
type s3CwListedMetric struct {
Namespace string `xml:"Namespace"`
MetricName string `xml:"MetricName"`
Dimensions []s3CwMetricDimension `xml:"Dimensions>member"`
}
// s3CwListMetricsOutput decodes the ListMetricsResult element only: cloudmux Unmarshal
// uses DecodeElement(..., StartElement{Name: Operation+"Result"}) so fields must match
// direct children of ListMetricsResult (Metrics, NextToken), not wrap ListMetricsResult again.
type s3CwListMetricsOutput struct {
Metrics []s3CwListedMetric `xml:"Metrics>member"`
NextToken string `xml:"NextToken"`
}
func s3CwMetricHasFilterId(dimensions []s3CwMetricDimension) bool {
for i := range dimensions {
if dimensions[i].Name == "FilterId" {
return true
}
}
return false
}
func s3CwFindDimensionValue(dimensions []s3CwMetricDimension, name string) string {
for i := range dimensions {
if dimensions[i].Name == name {
return dimensions[i].Value
}
}
return ""
}
// s3CwKnownStorageTypes probes GetMetricStatistics when ListMetrics omits metrics
// (e.g. no data reported in past two weeks per AWS ListMetrics behavior).
var s3CwKnownStorageTypes = []string{
"StandardStorage",
"StandardIAStorage",
"OneZoneIAStorage",
"ReducedRedundancyStorage",
"IntelligentTieringFAStorage",
"IntelligentTieringAIAStorage",
"IntelligentTieringArchiveStorage",
"IntelligentTieringDeepArchiveStorage",
"GlacierStorage",
"GlacierInstantRetrievalStorage",
"DeepArchiveStorage",
"OutpostsStandardStorage",
"OutpostsOneZoneIAStorage",
"ExpressOneZoneStorage",
"ExpressOneZoneIAStorage",
}
func (b *SBucket) listS3BucketStorageTypes() (map[string]struct{}, error) {
storageTypes := map[string]struct{}{}
var nextToken string
for {
params := map[string]string{
"Namespace": s3BucketMetricsNamespace,
"Dimensions.member.1.Name": "BucketName",
"Dimensions.member.1.Value": b.Name,
}
if len(nextToken) > 0 {
params["NextToken"] = nextToken
}
out := s3CwListMetricsOutput{}
err := b.region.client.monitorRequest(b.region.RegionId, "ListMetrics", params, &out)
if err != nil {
return nil, err
}
for _, m := range out.Metrics {
if m.MetricName != "BucketSizeBytes" && m.MetricName != "NumberOfObjects" {
continue
}
if s3CwMetricHasFilterId(m.Dimensions) {
continue
}
st := s3CwFindDimensionValue(m.Dimensions, "StorageType")
if len(st) == 0 {
continue
}
storageTypes[st] = struct{}{}
}
nextToken = out.NextToken
if len(nextToken) == 0 {
break
}
}
return storageTypes, nil
}
// s3CwLatestPointValue picks the latest datapoint; S3 daily storage metrics document
// Average as the valid statistic, while some samples use Maximum — take whichever is set.
func s3CwLatestPointValue(dps Datapoints) float64 {
var bestTs time.Time
var bestVal float64
found := false
for i := range dps.Datapoints {
d := dps.Datapoints[i]
if !found || d.Timestamp.After(bestTs) {
bestTs = d.Timestamp
// S3 daily storage metrics publish Average; Maximum may be unset in the XML response.
bestVal = d.Average
if bestVal == 0 {
bestVal = d.Maximum
}
if bestVal == 0 {
bestVal = d.Sum
}
found = true
}
}
if !found {
return 0
}
return bestVal
}
func (b *SBucket) getS3BucketMetricDatapoints(metricName, storageType string, start, end time.Time) (Datapoints, error) {
params := map[string]string{
"Namespace": s3BucketMetricsNamespace,
"MetricName": metricName,
"Dimensions.member.1.Name": "BucketName",
"Dimensions.member.1.Value": b.Name,
"Dimensions.member.2.Name": "StorageType",
"Dimensions.member.2.Value": storageType,
"StartTime": start.UTC().Format(time.RFC3339),
"EndTime": end.UTC().Format(time.RFC3339),
"Period": s3BucketMetricsPeriod,
"Statistics.member.1": "Average",
"Statistics.member.2": "Maximum",
}
ret := Datapoints{}
err := b.region.client.monitorRequest(b.region.RegionId, "GetMetricStatistics", params, &ret)
if err != nil {
return ret, err
}
return ret, nil
}
func s3CwHasDatapoints(dps Datapoints) bool {
return len(dps.Datapoints) > 0
}
func (b *SBucket) discoverS3BucketStorageTypesByProbe(start, end time.Time) (map[string]struct{}, error) {
storageTypes := map[string]struct{}{}
for _, st := range s3CwKnownStorageTypes {
dps, err := b.getS3BucketMetricDatapoints("BucketSizeBytes", st, start, end)
if err != nil {
return nil, err
}
if s3CwHasDatapoints(dps) {
storageTypes[st] = struct{}{}
continue
}
dps2, err := b.getS3BucketMetricDatapoints("NumberOfObjects", st, start, end)
if err != nil {
return nil, err
}
if s3CwHasDatapoints(dps2) {
storageTypes[st] = struct{}{}
}
}
return storageTypes, nil
}
func (b *SBucket) getS3BucketMetricValue(metricName, storageType string, start, end time.Time) (float64, error) {
ret, err := b.getS3BucketMetricDatapoints(metricName, storageType, start, end)
if err != nil {
return 0, err
}
return s3CwLatestPointValue(ret), nil
}
func (b *SBucket) getStatsFromCloudWatch() (cloudprovider.SBucketStats, error) {
stats := cloudprovider.SBucketStats{}
if b.region == nil || b.region.client == nil {
return stats, errors.Error("bucket region or client is nil")
}
end := time.Now().UTC()
start := end.Add(-8 * 24 * time.Hour)
storageTypes, err := b.listS3BucketStorageTypes()
if err != nil {
return stats, err
}
if len(storageTypes) == 0 {
storageTypes, err = b.discoverS3BucketStorageTypesByProbe(start, end)
if err != nil {
return stats, err
}
}
if len(storageTypes) == 0 {
return stats, nil
}
var sizeSum int64
var countSum int64
for st := range storageTypes {
sz, err := b.getS3BucketMetricValue("BucketSizeBytes", st, start, end)
if err != nil {
return stats, errors.Wrapf(err, "BucketSizeBytes %s", st)
}
sizeSum += int64(sz)
cnt, err := b.getS3BucketMetricValue("NumberOfObjects", st, start, end)
if err != nil {
return stats, errors.Wrapf(err, "NumberOfObjects %s", st)
}
countSum += int64(cnt)
}
stats.SizeBytes = sizeSum
stats.ObjectCount = int(countSum)
return stats, nil
}
func (b *SBucket) GetStats() cloudprovider.SBucketStats {
stats, _ := cloudprovider.GetIBucketStats(b)
stats, err := b.getStatsFromCloudWatch()
if err != nil {
log.Errorf("getStatsFromCloudWatch %s: %v", b.Name, err)
return cloudprovider.SBucketStats{SizeBytes: -1, ObjectCount: -1}
}
return stats
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/disk.go
Generated Vendored
+21 -8
View File
@@ -351,17 +351,29 @@ func (self *SRegion) ResizeDisk(diskId string, sizeGb int64) error {
return self.ec2Request("ModifyVolume", params, &ret)
}
// io1类型的卷需要指定IOPS参数,最大不超过32000。这里根据aws网站的建议值进行设置
// io2类型的卷需要指定IOPS参数,最大不超过64000。
// GenDiskIops Base 100, 卷每增加2G。IOPS增加1。最多到3000 iops
// io1 卷 IOPS 默认按容量估算，上限 32000（参见 AWS Provisioned IOPS 文档）
// io2 卷 IOPS 上限 64000
// GenDiskIops: Base 100, 卷每增加 2GiB IOPS 增加 1
func GenDiskIops(diskType string, sizeGB int) int64 {
switch diskType {
case api.STORAGE_IO1_SSD, api.STORAGE_IO2_SSD:
case api.STORAGE_IO1_SSD:
iops := int64(100 + sizeGB/2)
if iops < 32000 {
return iops
if iops < 100 {
return 100
}
return 100
if iops > 32000 {
return 32000
}
return iops
case api.STORAGE_IO2_SSD:
iops := int64(100 + sizeGB/2)
if iops < 100 {
return 100
}
if iops > 64000 {
return 64000
}
return iops
case api.STORAGE_GP3_SSD:
return 3000
}
@@ -395,7 +407,8 @@ func (self *SRegion) CreateDisk(zoneId string, volumeType string, opts *cloudpro
if len(opts.SnapshotId) > 0 {
params["SnapshotId"] = opts.SnapshotId
}
if opts.Throughput >= 125 && opts.Throughput <= 1000 && volumeType == api.STORAGE_GP3_SSD {
// gp3 吞吐：125–2000 MiB/s（参见 AWS gp3 文档）
if opts.Throughput >= 125 && opts.Throughput <= 2000 && volumeType == api.STORAGE_GP3_SSD {
params["Throughput"] = fmt.Sprintf("%d", opts.Throughput)
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/instance.go
Generated Vendored
+1 -1
View File
@@ -798,7 +798,7 @@ func (self *SRegion) CreateInstance(name string, image *SImage, instanceType str
}) {
params[fmt.Sprintf("BlockDeviceMapping.%d.Ebs.Iops", i+1)] = fmt.Sprintf("%d", iops)
}
if disk.Throughput >= 125 && disk.Throughput <= 1000 && disk.StorageType == api.STORAGE_GP3_SSD {
if disk.Throughput >= 125 && disk.Throughput <= 2000 && disk.StorageType == api.STORAGE_GP3_SSD {
params[fmt.Sprintf("BlockDeviceMapping.%d.Ebs.Throughput", i+1)] = fmt.Sprintf("%d", disk.Throughput)
}
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/storagecache.go
Generated Vendored
+1 -1
View File
@@ -62,7 +62,7 @@ func (self *SStoragecache) IsEmulated() bool {
}
func (self *SStoragecache) GetICustomizedCloudImages() ([]cloudprovider.ICloudImage, error) {
images, err := self.region.GetImages("", ImageOwnerSelf, nil, "", "hvm", nil, "", true)
images, err := self.region.GetImages("", ImageOwnerSelf, nil, "", "hvm", nil, "", false)
if err != nil {
return nil, errors.Wrapf(err, "GetImages")
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/azure/azure.go
Generated Vendored
+12 -8
View File
@@ -579,17 +579,21 @@ func (self *SAzureClient) getUniqName(resourceGroup, resourceType, name string)
newName := name
for i := 0; i < 20; i++ {
err := self._get(prefix+newName, nil, url.Values{}, false)
if err == nil {
info := strings.Split(newName, "-")
num, err := strconv.Atoi(info[len(info)-1])
if err != nil {
info = append(info, "1")
} else {
info[len(info)-1] = fmt.Sprintf("%d", num+1)
}
newName = strings.Join(info, "-")
continue
}
if errors.Cause(err) == cloudprovider.ErrNotFound {
return newName, nil
}
info := strings.Split(newName, "-")
num, err := strconv.Atoi(info[len(info)-1])
if err != nil {
info = append(info, "1")
} else {
info[len(info)-1] = fmt.Sprintf("%d", num+1)
}
newName = strings.Join(info, "-")
return "", err
}
return "", fmt.Errorf("not find uniq name for %s[%s]", resourceType, name)
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/azure/eip.go
Generated Vendored
+3
View File
@@ -69,6 +69,9 @@ func (self *SRegion) AllocateEIP(name, projectId string) (*SEipAddress, error) {
params := map[string]interface{}{
"Location": self.Name,
"Name": name,
"Sku": map[string]string{
"Name": "Standard",
},
"Properties": map[string]string{
"PublicIPAddressVersion": "IPv4",
"PublicIPAllocationMethod": "Static",
vendor/yunion.io/x/pkg/util/compare/syncresults.go
Generated Vendored
+41 -8
View File
@@ -17,11 +17,14 @@ package compare
import (
"fmt"
"strings"
"sync"
"yunion.io/x/pkg/errors"
)
type SyncResult struct {
mu sync.Mutex `json:"-"`
AddCnt int `json:"add_cnt,omitzero"`
AddErrCnt int `json:"add_err_cnt,omitzero"`
UpdateCnt int `json:"update_cnt,omitzero"`
@@ -31,41 +34,59 @@ type SyncResult struct {
errors []error
}
func (self *SyncResult) Error(msg error) {
func (self *SyncResult) appendError(msg error) {
if self.errors == nil {
self.errors = make([]error, 0)
}
self.errors = append(self.errors, msg)
}
func (self *SyncResult) Error(msg error) {
self.mu.Lock()
defer self.mu.Unlock()
self.appendError(msg)
}
func (self *SyncResult) Add() {
self.mu.Lock()
defer self.mu.Unlock()
self.AddCnt += 1
}
func (self *SyncResult) AddError(msg error) {
self.mu.Lock()
defer self.mu.Unlock()
self.AddErrCnt += 1
self.Error(msg)
self.appendError(msg)
}
func (self *SyncResult) Update() {
self.mu.Lock()
defer self.mu.Unlock()
self.UpdateCnt += 1
}
func (self *SyncResult) UpdateError(msg error) {
self.mu.Lock()
defer self.mu.Unlock()
self.UpdateErrCnt += 1
self.Error(msg)
self.appendError(msg)
}
func (self *SyncResult) Delete() {
self.mu.Lock()
defer self.mu.Unlock()
self.DelCnt += 1
}
func (self *SyncResult) DeleteError(msg error) {
self.mu.Lock()
defer self.mu.Unlock()
self.DelErrCnt += 1
self.Error(msg)
self.appendError(msg)
}
func (self *SyncResult) AllError() error {
func (self *SyncResult) allErrorUnlocked() error {
msgs := make(map[string]bool)
for _, e := range self.errors {
msg := e.Error()
@@ -80,11 +101,21 @@ func (self *SyncResult) AllError() error {
return fmt.Errorf(strings.Join(ret, ";"))
}
func (self *SyncResult) AllError() error {
self.mu.Lock()
defer self.mu.Unlock()
return self.allErrorUnlocked()
}
func (self *SyncResult) IsError() bool {
return self.errors != nil && len(self.errors) > 0
self.mu.Lock()
defer self.mu.Unlock()
return len(self.errors) > 0
}
func (self *SyncResult) IsGenerateError() bool {
self.mu.Lock()
defer self.mu.Unlock()
for _, err := range self.errors {
if e := errors.Cause(err); e != errors.ErrNotImplemented && e != errors.ErrNotSupported && e != errors.ErrNotEmpty {
return true
@@ -94,9 +125,11 @@ func (self *SyncResult) IsGenerateError() bool {
}
func (self *SyncResult) Result() string {
self.mu.Lock()
defer self.mu.Unlock()
msg := fmt.Sprintf("removed %d failed %d updated %d failed %d added %d failed %d", self.DelCnt, self.DelErrCnt, self.UpdateCnt, self.UpdateErrCnt, self.AddCnt, self.AddErrCnt)
if self.IsError() {
msg = fmt.Sprintf("%s;%s", msg, self.AllError())
if len(self.errors) > 0 {
msg = fmt.Sprintf("%s;%s", msg, self.allErrorUnlocked())
}
return msg
}
vendor/yunion.io/x/pkg/util/imagetools/imagetools.go
Generated Vendored
+17 -10
View File
@@ -156,7 +156,7 @@ func normalizeOsDistribution(osDist string, imageName string) string {
return OS_DIST_KYLIN
} else if strings.Contains(osDist, "uos") {
return OS_DIST_UOS
} else if strings.Contains(osDist, "windows") || regexp.MustCompile(".+win(xp|7|8|10|11|2003|2008|2012|2016|2019|2022)*").MatchString(osDist) {
} else if strings.Contains(osDist, "windows") || regexp.MustCompile("(^|[^a-z0-9])win(xp|7|8|10|11|2003|2008|2012|2016|2019|2022)([^a-z0-9]|$)").MatchString(osDist) {
for _, ver := range []string{"2003", "2008", "2012", "2016", "2019", "2022"} {
if strings.Contains(osDist, ver) {
return OS_DIST_WINDOWS_SERVER
@@ -197,18 +197,18 @@ var imageVersions = map[string][]string{
OS_DIST_OPEN_EULER: {"2.0 SP1", "2.0 SP2", "2.0 SP3", "2.0 SP8", "3.0", "22.03", "23.09"},
OS_DIST_EULER_OS: {"2"},
// 阿里云Linux：补充1代和2/3代版本
OS_DIST_ALIYUN: {"1", "2.1903", "3.2104", "3.2304"},
OS_DIST_ALIYUN: {"1", "2.1903", "2", "3.2104", "3.2304", "3"},
// 阿里云轻量版：补充完整版本
OS_DIST_ALIBABA_CLOUD_LINUX: {"2.1903", "3.2104", "3.2304", "3.2404"},
OS_DIST_ALIBABA_CLOUD_LINUX: {"2.1903", "2", "3.2104", "3.2304", "3.2404", "3"},
// 龙蜥OS：补充7/8系列完整小版本
OS_DIST_ANOLIS: {"7.6", "7.9", "8.2", "8.4", "8.6", "8.8", "9.0", "9.2"},
OS_DIST_ANOLIS: {"7.6", "7.9", "7", "8.2", "8.4", "8.6", "8.8", "8", "9.0", "9.2", "9"},
// Rocky Linux：补充8/9全系列小版本
OS_DIST_ROCKY_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5"},
OS_DIST_ROCKY_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "8", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9"},
// Fedora：补充近年主流版本（33到40）
OS_DIST_FEDORA: {"33", "34", "35", "36", "37", "38", "39", "40"},
// AlmaLinux：补充8/9全系列
OS_DIST_ALMA_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5"},
OS_DIST_ALMA_LINUX: {"8.5", "8.6", "8.7", "8.8", "8.9", "8.10", "8", "9.0", "9.1", "9.2", "9.3", "9.4", "9.5", "9"},
// Amazon Linux：补充1/2/2023版本
OS_DIST_AMAZON_LINUX: {"2022", "2023", "1", "2"},
@@ -242,7 +242,7 @@ func normalizeOsVersion(imageName string, osDist string, osVersion string) strin
parts = append(parts, fmt.Sprintf(`(?P<verstr>%s[.\d]*)`, version), "")
regexpStr := strings.Join(parts, `[\s-_]*`)
m := regexp.MustCompile(regexpStr).FindAllStringSubmatch(strings.ToLower(imageName), -1)
if m != nil && len(m) > 0 && len(m[0]) > 1 {
if len(m) > 0 && len(m[0]) > 1 {
verStr := m[0][1]
if strings.HasPrefix(verStr, version) && len(verStr) > len(version) && !strings.HasPrefix(verStr, version+".") {
verStr = version + "." + verStr[len(version):]
@@ -251,11 +251,18 @@ func normalizeOsVersion(imageName string, osDist string, osVersion string) strin
}
}
}
for i := len(versions) - 1; i > 0; i-- {
if strings.Contains(imageName, versions[i]) {
return versions[i]
imageNameLower := strings.ToLower(imageName)
bestMatch := ""
for i := 0; i < len(versions); i++ {
version := strings.ToLower(versions[i])
pattern := fmt.Sprintf(`(^|[^0-9])%s([^0-9]|$)`, regexp.QuoteMeta(version))
if regexp.MustCompile(pattern).MatchString(imageNameLower) && len(versions[i]) > len(bestMatch) {
bestMatch = versions[i]
}
}
if len(bestMatch) > 0 {
return bestMatch
}
}
return ""
}