diff --git a/go.mod b/go.mod index d798244058..4b9692bdc9 100644 --- a/go.mod +++ b/go.mod @@ -83,7 +83,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd yunion.io/x/log v1.0.0 diff --git a/go.sum b/go.sum index 4775cb1ef8..96c21f5e92 100644 --- a/go.sum +++ b/go.sum @@ -1164,8 +1164,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 h1:w9vF4QzyJtkkDBPY7TOH1kUvUzPi6kGDmuFYHZrbmG8= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5/go.mod h1:VBAtfyGMDW11x5JmF0vUEmn/PmNpZ8CKBAQCiyz+cDY= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1 h1:0/cYUmz6Y84CeIqbmVgE9ohxP00UA6S+b//38xYlvcI= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1/go.mod h1:VBAtfyGMDW11x5JmF0vUEmn/PmNpZ8CKBAQCiyz+cDY= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/compute/tasks/modelarts_pool_create_task.go b/pkg/compute/tasks/modelarts_pool_create_task.go index e126298101..07aa29975f 100644 --- a/pkg/compute/tasks/modelarts_pool_create_task.go +++ b/pkg/compute/tasks/modelarts_pool_create_task.go @@ -93,7 +93,7 @@ func (modelartsCreateTask *ModelartsPoolCreateTask) WaitStatus(ctx context.Conte return nil, err } if ipool.GetStatus() == api.MODELARTS_POOL_STATUS_CREATE_FAILED { - return nil, errors.Wrap(err, "create_failed") + return nil, errors.Error("create_failed") } else { return nil, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 258d29a013..47cc9e823f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1442,7 +1442,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230403081338-c406d6df0cf1 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/monitor.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/monitor.go index cb329c44d7..74b64c2722 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/monitor.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/monitor.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + api "yunion.io/x/cloudmux/pkg/apis/compute" "yunion.io/x/cloudmux/pkg/cloudprovider" ) @@ -241,10 +242,49 @@ func (self *SApsaraClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions) "diskusage_utilization": "", } case cloudprovider.VM_METRIC_TYPE_PROCESS_NUMBER: - metricTags = map[string]string{ - "process.number": "", - } + metric := "process.number" tagKey = cloudprovider.METRIC_TAG_PROCESS_NAME + ret := []cloudprovider.MetricValues{} + regions := self.GetRegions() + for r := range regions { + vms := make([]SInstance, 0) + for { + part, total, err := regions[r].GetInstances("", nil, len(vms), 50) + if err != nil { + return ret, errors.Wrapf(err, "GetInstances") + } + vms = append(vms, part...) + if len(vms) >= total || len(vms) == 0 { + break + } + } + for i := range vms { + if vms[i].GetStatus() != api.VM_RUNNING { + continue + } + dimensions := jsonutils.Marshal([]map[string]string{{"instanceId": vms[i].InstanceId}}).String() + result, err := self.listMetrics(vms[i].Department, "acs_ecs_dashboard", metric, dimensions, opts.StartTime, opts.EndTime) + if err != nil { + log.Errorf("ListMetric(%s) error: %v", metric, err) + continue + } + for j := range result { + dataTag := result[j].GetTags() + ret = append(ret, cloudprovider.MetricValues{ + Id: result[j].InstanceId, + MetricType: opts.MetricType, + Values: []cloudprovider.MetricValue{ + { + Timestamp: time.UnixMilli(result[j].Timestamp), + Value: result[j].GetValue(), + Tags: dataTag, + }, + }, + }) + } + } + } + return ret, nil case cloudprovider.VM_METRIC_TYPE_NET_TCP_CONNECTION: metricTags = map[string]string{ "net_tcpconnection": "", @@ -255,11 +295,7 @@ func (self *SApsaraClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions) } ret := []cloudprovider.MetricValues{} for metric, tag := range metricTags { - dimensions := "" - if opts.MetricType == cloudprovider.VM_METRIC_TYPE_PROCESS_NUMBER { - dimensions = jsonutils.Marshal(map[string]string{"instanceId": opts.ResourceId}).String() - } - result, err := self.ListMetrics("acs_ecs_dashboard", metric, dimensions, opts.StartTime, opts.EndTime) + result, err := self.ListMetrics("acs_ecs_dashboard", metric, "", opts.StartTime, opts.EndTime) if err != nil { log.Errorf("ListMetric(%s) error: %v", metric, err) continue @@ -290,6 +326,7 @@ func (self *SApsaraClient) GetEcsMetrics(opts *cloudprovider.MetricListOptions) } func (self *SApsaraClient) GetOssMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + ret := []cloudprovider.MetricValues{} metricTags, tagKey := map[string]string{}, "" switch opts.MetricType { case cloudprovider.BUCKET_METRIC_TYPE_LATECY: @@ -334,19 +371,40 @@ func (self *SApsaraClient) GetOssMetrics(opts *cloudprovider.MetricListOptions) "SuccessCount": "", } case cloudprovider.BUCKET_METRIC_TYPE_STORAGE_SIZE: - metricTags = map[string]string{ - "MeteringStorageUtilization": "", + metricName := "MeteringStorageUtilization" + regions := self.GetRegions() + for i := range regions { + region := regions[i] + buckets, err := region.GetBuckets() + if err != nil { + return ret, errors.Wrapf(err, "GetBuckets") + } + for _, bucket := range buckets { + dimensions := jsonutils.Marshal([]map[string]string{{"BucketName": bucket.Name}}).String() + result, err := self.listMetrics(bucket.Department, "acs_oss_dashboard", metricName, dimensions, opts.StartTime, opts.EndTime) + if err != nil { + log.Errorf("ListMetric(%s) error: %v", metricName, err) + } + for j := range result { + ret = append(ret, cloudprovider.MetricValues{ + Id: result[j].BucketName, + MetricType: opts.MetricType, + Values: []cloudprovider.MetricValue{ + { + Timestamp: time.UnixMilli(result[j].Timestamp), + Value: result[j].GetValue(), + }, + }, + }) + } + } + return ret, nil } default: return nil, errors.Wrapf(cloudprovider.ErrNotImplemented, "%s", opts.MetricType) } - ret := []cloudprovider.MetricValues{} for metric, tag := range metricTags { - dimensions := "" - if opts.MetricType == cloudprovider.BUCKET_METRIC_TYPE_STORAGE_SIZE { - dimensions = jsonutils.Marshal(map[string]string{"BucketName": opts.ResourceId}).String() - } - result, err := self.ListMetrics("acs_oss_dashboard", metric, dimensions, opts.StartTime, opts.EndTime) + result, err := self.ListMetrics("acs_oss_dashboard", metric, "", opts.StartTime, opts.EndTime) if err != nil { log.Errorf("ListMetric(%s) error: %v", metric, err) continue diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go index 7025ad321c..e83425dec4 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go @@ -40,6 +40,9 @@ type akClient struct { } func (self *akClient) Do(req *http.Request) (*http.Response, error) { + req.Header.Del("Host") + req.Header.Del("Authorization") + req.Header.Del("X-Sdk-Date") req.Header.Del("Accept") if req.Method == string(httputils.GET) || req.Method == string(httputils.DELETE) || req.Method == string(httputils.PATCH) { req.Header.Del("Content-Length") @@ -102,13 +105,14 @@ func (self *SHuaweiClient) request(method httputils.THttpMethod, url string, que } var resp jsonutils.JSONObject var err error + for i := 0; i < 3; i++ { _, resp, err = requestWithRetry(client, context.Background(), method, url, header, body, self.debug) if method == httputils.GET && needRetry(err) { time.Sleep(time.Second * 15) continue } - return resp, err + break } return resp, err } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go index fc67a1129b..06b6513fd4 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go @@ -316,13 +316,26 @@ func (self *SModelartsPool) GetName() string { func (self *SModelartsPool) GetStatus() string { res := strings.ToLower(self.Status.Phase) + availableCount := 0 + for _, node := range self.Status.Resource.Available { + availableCount += node.Count + } + switch { - case res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0: + case res == compute.MODELARTS_POOL_STATUS_RUNNING && availableCount == self.GetNodeCount(): + res = compute.MODELARTS_POOL_STATUS_RUNNING + case res == compute.MODELARTS_POOL_STATUS_DELETING: + res = compute.MODELARTS_POOL_STATUS_DELETING + case res == compute.MODELARTS_POOL_STATUS_ERROR: + res = compute.MODELARTS_POOL_STATUS_ERROR + case (res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0) || res == compute.MODELARTS_POOL_STATUS_CREATING: res = compute.MODELARTS_POOL_STATUS_CREATING case self.Status.Phase == "CreationFailed": res = compute.MODELARTS_POOL_STATUS_CREATE_FAILED case self.Status.Phase == "SeclingFailed": res = compute.MODELARTS_POOL_STATUS_CHANGE_CONFIG_FAILED + default: + res = compute.MODELARTS_POOL_STATUS_UNKNOWN } return res } @@ -379,25 +392,28 @@ func (self *SModelartsPool) Refresh() error { pool.Unmarshal(self) return nil } - if err != errors.ErrNotFound { + if !strings.Contains(err.Error(), "not found") { return errors.Wrap(err, "modelartsPoolById") } + pools := make([]SModelartsPool, 0) resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil) if err != nil { return errors.Wrap(err, "modelartsPoolListWithStatus") } + err = resObj.Unmarshal(&pools, "items") if err != nil { return errors.Wrap(err, "resObj unmarshal") } + for _, pool := range pools { if pool.GetId() == self.GetId() { self.Status.Phase = "CreationFailed" - break + return jsonutils.Update(self, pool) } } - return jsonutils.Update(self, pool) + return nil } func (self *SModelartsPool) SetTags(tags map[string]string, replace bool) error { @@ -425,7 +441,11 @@ func (self *SModelartsPool) GetNodeCount() int { if len(self.Spec.Resource) < 1 { return 0 } - return self.Spec.Resource[0].Count + nodeCount := 0 + for _, v := range self.Spec.Resource { + nodeCount += v.Count + } + return nodeCount } func (self *SModelartsPool) ChangeConfig(opts *cloudprovider.ModelartsPoolChangeConfigOptions) error { diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go index 2d9cbd0830..c3cb175e80 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go @@ -315,13 +315,26 @@ func (self *SModelartsPool) GetName() string { func (self *SModelartsPool) GetStatus() string { res := strings.ToLower(self.Status.Phase) + availableCount := 0 + for _, node := range self.Status.Resource.Available { + availableCount += node.Count + } + switch { - case res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0: + case res == compute.MODELARTS_POOL_STATUS_RUNNING && availableCount == self.GetNodeCount(): + res = compute.MODELARTS_POOL_STATUS_RUNNING + case res == compute.MODELARTS_POOL_STATUS_DELETING: + res = compute.MODELARTS_POOL_STATUS_DELETING + case res == compute.MODELARTS_POOL_STATUS_ERROR: + res = compute.MODELARTS_POOL_STATUS_ERROR + case (res == compute.MODELARTS_POOL_STATUS_RUNNING && len(self.Status.Resource.Creating) != 0) || res == compute.MODELARTS_POOL_STATUS_CREATING: res = compute.MODELARTS_POOL_STATUS_CREATING case self.Status.Phase == "CreationFailed": res = compute.MODELARTS_POOL_STATUS_CREATE_FAILED case self.Status.Phase == "SeclingFailed": res = compute.MODELARTS_POOL_STATUS_CHANGE_CONFIG_FAILED + default: + res = compute.MODELARTS_POOL_STATUS_UNKNOWN } return res } @@ -378,25 +391,28 @@ func (self *SModelartsPool) Refresh() error { pool.Unmarshal(self) return nil } - if err != errors.ErrNotFound { + if !strings.Contains(err.Error(), "not found") { return errors.Wrap(err, "modelartsPoolById") } + pools := make([]SModelartsPool, 0) resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil) if err != nil { return errors.Wrap(err, "modelartsPoolListWithStatus") } + err = resObj.Unmarshal(&pools, "items") if err != nil { return errors.Wrap(err, "resObj unmarshal") } + for _, pool := range pools { if pool.GetId() == self.GetId() { self.Status.Phase = "CreationFailed" - break + return jsonutils.Update(self, pool) } } - return jsonutils.Update(self, pool) + return nil } func (self *SModelartsPool) SetTags(tags map[string]string, replace bool) error { @@ -424,7 +440,11 @@ func (self *SModelartsPool) GetNodeCount() int { if len(self.Spec.Resource) < 1 { return 0 } - return self.Spec.Resource[0].Count + nodeCount := 0 + for _, v := range self.Spec.Resource { + nodeCount += v.Count + } + return nodeCount } func (self *SModelartsPool) ChangeConfig(opts *cloudprovider.ModelartsPoolChangeConfigOptions) error {