opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(region): fix modelarts createFailed

Browse Source
This commit is contained in:
mhf
2023-04-04 17:31:14 +08:00
parent 605aaa5945
commit dcbdfe7b09
8 changed files with 134 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -83,7 +83,7 @@ require (
k8s.io/client-go v0.19.3
k8s.io/cluster-bootstrap v0.19.3
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd
yunion.io/x/log v1.0.0
go.sum
+2 -2
View File
@@ -1164,8 +1164,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 h1:w9vF4QzyJtkkDBPY7TOH1kUvUzPi6kGDmuFYHZrbmG8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5/go.mod h1:VBAtfyGMDW11x5JmF0vUEmn/PmNpZ8CKBAQCiyz+cDY=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1 h1:0/cYUmz6Y84CeIqbmVgE9ohxP00UA6S+b//38xYlvcI=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1/go.mod h1:VBAtfyGMDW11x5JmF0vUEmn/PmNpZ8CKBAQCiyz+cDY=
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
pkg/compute/tasks/modelarts_pool_create_task.go
+1 -1
View File
@@ -93,7 +93,7 @@ func (modelartsCreateTask *ModelartsPoolCreateTask) WaitStatus(ctx context.Conte
return nil, err
}
if ipool.GetStatus() == api.MODELARTS_POOL_STATUS_CREATE_FAILED {
return nil, errors.Wrap(err, "create_failed")
return nil, errors.Error("create_failed")
} else {
return nil, nil
}
vendor/modules.txt
Vendored
+1 -1
View File
@@ -1442,7 +1442,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1
## explicit; go 1.18
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/monitor.go
Generated Vendored
+74 -16
View File
@@ -24,6 +24,7 @@ import (
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/utils"
api "yunion.io/x/cloudmux/pkg/apis/compute"
"yunion.io/x/cloudmux/pkg/cloudprovider"
)
@@ -241,10 +242,49 @@ func (self *SApsaraClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions)
"diskusage_utilization": "",
}
case cloudprovider.VM_METRIC_TYPE_PROCESS_NUMBER:
metricTags = map[string]string{
"process.number": "",
}
metric := "process.number"
tagKey = cloudprovider.METRIC_TAG_PROCESS_NAME
ret := []cloudprovider.MetricValues{}
regions := self.GetRegions()
for r := range regions {
vms := make([]SInstance, 0)
for {
part, total, err := regions[r].GetInstances("", nil, len(vms), 50)
if err != nil {
return ret, errors.Wrapf(err, "GetInstances")
}
vms = append(vms, part...)
if len(vms) >= total || len(vms) == 0 {
break
}
}
for i := range vms {
if vms[i].GetStatus() != api.VM_RUNNING {
continue
}
dimensions := jsonutils.Marshal([]map[string]string{{"instanceId": vms[i].InstanceId}}).String()
result, err := self.listMetrics(vms[i].Department, "acs_ecs_dashboard", metric, dimensions, opts.StartTime, opts.EndTime)
if err != nil {
log.Errorf("ListMetric(%s) error: %v", metric, err)
continue
}
for j := range result {
dataTag := result[j].GetTags()
ret = append(ret, cloudprovider.MetricValues{
Id: result[j].InstanceId,
MetricType: opts.MetricType,
Values: []cloudprovider.MetricValue{
{
Timestamp: time.UnixMilli(result[j].Timestamp),
Value: result[j].GetValue(),
Tags: dataTag,
},
},
})
}
}
}
return ret, nil
case cloudprovider.VM_METRIC_TYPE_NET_TCP_CONNECTION:
metricTags = map[string]string{
"net_tcpconnection": "",
@@ -255,11 +295,7 @@ func (self *SApsaraClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions)
}
ret := []cloudprovider.MetricValues{}
for metric, tag := range metricTags {
dimensions := ""
if opts.MetricType == cloudprovider.VM_METRIC_TYPE_PROCESS_NUMBER {
dimensions = jsonutils.Marshal(map[string]string{"instanceId": opts.ResourceId}).String()
}
result, err := self.ListMetrics("acs_ecs_dashboard", metric, dimensions, opts.StartTime, opts.EndTime)
result, err := self.ListMetrics("acs_ecs_dashboard", metric, "", opts.StartTime, opts.EndTime)
if err != nil {
log.Errorf("ListMetric(%s) error: %v", metric, err)
continue
@@ -290,6 +326,7 @@ func (self *SApsaraClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions)
}
func (self *SApsaraClient) GetOssMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) {
ret := []cloudprovider.MetricValues{}
metricTags, tagKey := map[string]string{}, ""
switch opts.MetricType {
case cloudprovider.BUCKET_METRIC_TYPE_LATECY:
@@ -334,19 +371,40 @@ func (self *SApsaraClient) GetOssMetrics(opts *cloudprovider.MetricListOptions)
"SuccessCount": "",
}
case cloudprovider.BUCKET_METRIC_TYPE_STORAGE_SIZE:
metricTags = map[string]string{
"MeteringStorageUtilization": "",
metricName := "MeteringStorageUtilization"
regions := self.GetRegions()
for i := range regions {
region := regions[i]
buckets, err := region.GetBuckets()
if err != nil {
return ret, errors.Wrapf(err, "GetBuckets")
}
for _, bucket := range buckets {
dimensions := jsonutils.Marshal([]map[string]string{{"BucketName": bucket.Name}}).String()
result, err := self.listMetrics(bucket.Department, "acs_oss_dashboard", metricName, dimensions, opts.StartTime, opts.EndTime)
if err != nil {
log.Errorf("ListMetric(%s) error: %v", metricName, err)
}
for j := range result {
ret = append(ret, cloudprovider.MetricValues{
Id: result[j].BucketName,
MetricType: opts.MetricType,
Values: []cloudprovider.MetricValue{
{
Timestamp: time.UnixMilli(result[j].Timestamp),
Value: result[j].GetValue(),
},
},
})
}
}
return ret, nil
}
default:
return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "%s", opts.MetricType)
}
ret := []cloudprovider.MetricValues{}
for metric, tag := range metricTags {
dimensions := ""
if opts.MetricType == cloudprovider.BUCKET_METRIC_TYPE_STORAGE_SIZE {
dimensions = jsonutils.Marshal(map[string]string{"BucketName": opts.ResourceId}).String()
}
result, err := self.ListMetrics("acs_oss_dashboard", metric, dimensions, opts.StartTime, opts.EndTime)
result, err := self.ListMetrics("acs_oss_dashboard", metric, "", opts.StartTime, opts.EndTime)
if err != nil {
log.Errorf("ListMetric(%s) error: %v", metric, err)
continue
vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go
Generated Vendored
+5 -1
View File
@@ -40,6 +40,9 @@ type akClient struct {
}
func (self *akClient) Do(req *http.Request) (*http.Response, error) {
req.Header.Del("Host")
req.Header.Del("Authorization")
req.Header.Del("X-Sdk-Date")
req.Header.Del("Accept")
if req.Method == string(httputils.GET) || req.Method == string(httputils.DELETE) || req.Method == string(httputils.PATCH) {
req.Header.Del("Content-Length")
@@ -102,13 +105,14 @@ func (self *SHuaweiClient) request(method httputils.THttpMethod, url string, que
}
var resp jsonutils.JSONObject
var err error
for i := 0; i < 3; i++ {
_, resp, err = requestWithRetry(client, context.Background(), method, url, header, body, self.debug)
if method == httputils.GET && needRetry(err) {
time.Sleep(time.Second * 15)
continue
}
return resp, err
break
}
return resp, err
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go
Generated Vendored
+25 -5
View File
@@ -316,13 +316,26 @@ func (self *SModelartsPool) GetName() string {
func (self *SModelartsPool) GetStatus() string {
res := strings.ToLower(self.Status.Phase)
availableCount := 0
for _, node := range self.Status.Resource.Available {
availableCount += node.Count
}
switch {
case res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0:
case res == compute.MODELARTS_POOL_STATUS_RUNNING && availableCount == self.GetNodeCount():
res = compute.MODELARTS_POOL_STATUS_RUNNING
case res == compute.MODELARTS_POOL_STATUS_DELETING:
res = compute.MODELARTS_POOL_STATUS_DELETING
case res == compute.MODELARTS_POOL_STATUS_ERROR:
res = compute.MODELARTS_POOL_STATUS_ERROR
case (res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0) || res == compute.MODELARTS_POOL_STATUS_CREATING:
res = compute.MODELARTS_POOL_STATUS_CREATING
case self.Status.Phase == "CreationFailed":
res = compute.MODELARTS_POOL_STATUS_CREATE_FAILED
case self.Status.Phase == "SeclingFailed":
res = compute.MODELARTS_POOL_STATUS_CHANGE_CONFIG_FAILED
default:
res = compute.MODELARTS_POOL_STATUS_UNKNOWN
}
return res
}
@@ -379,25 +392,28 @@ func (self *SModelartsPool) Refresh() error {
pool.Unmarshal(self)
return nil
}
if err != errors.ErrNotFound {
if !strings.Contains(err.Error(), "not found") {
return errors.Wrap(err, "modelartsPoolById")
}
pools := make([]SModelartsPool, 0)
resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil)
if err != nil {
return errors.Wrap(err, "modelartsPoolListWithStatus")
}
err = resObj.Unmarshal(&pools, "items")
if err != nil {
return errors.Wrap(err, "resObj unmarshal")
}
for _, pool := range pools {
if pool.GetId() == self.GetId() {
self.Status.Phase = "CreationFailed"
break
return jsonutils.Update(self, pool)
}
}
return jsonutils.Update(self, pool)
return nil
}
func (self *SModelartsPool) SetTags(tags map[string]string, replace bool) error {
@@ -425,7 +441,11 @@ func (self *SModelartsPool) GetNodeCount() int {
if len(self.Spec.Resource) < 1 {
return 0
}
return self.Spec.Resource[0].Count
nodeCount := 0
for _, v := range self.Spec.Resource {
nodeCount += v.Count
}
return nodeCount
}
func (self *SModelartsPool) ChangeConfig(opts *cloudprovider.ModelartsPoolChangeConfigOptions) error {
vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go
Generated Vendored
+25 -5
View File
@@ -315,13 +315,26 @@ func (self *SModelartsPool) GetName() string {
func (self *SModelartsPool) GetStatus() string {
res := strings.ToLower(self.Status.Phase)
availableCount := 0
for _, node := range self.Status.Resource.Available {
availableCount += node.Count
}
switch {
case res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0:
case res == compute.MODELARTS_POOL_STATUS_RUNNING && availableCount == self.GetNodeCount():
res = compute.MODELARTS_POOL_STATUS_RUNNING
case res == compute.MODELARTS_POOL_STATUS_DELETING:
res = compute.MODELARTS_POOL_STATUS_DELETING
case res == compute.MODELARTS_POOL_STATUS_ERROR:
res = compute.MODELARTS_POOL_STATUS_ERROR
case (res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0) || res == compute.MODELARTS_POOL_STATUS_CREATING:
res = compute.MODELARTS_POOL_STATUS_CREATING
case self.Status.Phase == "CreationFailed":
res = compute.MODELARTS_POOL_STATUS_CREATE_FAILED
case self.Status.Phase == "SeclingFailed":
res = compute.MODELARTS_POOL_STATUS_CHANGE_CONFIG_FAILED
default:
res = compute.MODELARTS_POOL_STATUS_UNKNOWN
}
return res
}
@@ -378,25 +391,28 @@ func (self *SModelartsPool) Refresh() error {
pool.Unmarshal(self)
return nil
}
if err != errors.ErrNotFound {
if !strings.Contains(err.Error(), "not found") {
return errors.Wrap(err, "modelartsPoolById")
}
pools := make([]SModelartsPool, 0)
resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil)
if err != nil {
return errors.Wrap(err, "modelartsPoolListWithStatus")
}
err = resObj.Unmarshal(&pools, "items")
if err != nil {
return errors.Wrap(err, "resObj unmarshal")
}
for _, pool := range pools {
if pool.GetId() == self.GetId() {
self.Status.Phase = "CreationFailed"
break
return jsonutils.Update(self, pool)
}
}
return jsonutils.Update(self, pool)
return nil
}
func (self *SModelartsPool) SetTags(tags map[string]string, replace bool) error {
@@ -424,7 +440,11 @@ func (self *SModelartsPool) GetNodeCount() int {
if len(self.Spec.Resource) < 1 {
return 0
}
return self.Spec.Resource[0].Count
nodeCount := 0
for _, v := range self.Spec.Resource {
nodeCount += v.Count
}
return nodeCount
}
func (self *SModelartsPool) ChangeConfig(opts *cloudprovider.ModelartsPoolChangeConfigOptions) error {