From 8617675df4ca4ee90b06e46cb34af2a63d2a35f5 Mon Sep 17 00:00:00 2001 From: mhf Date: Wed, 29 Mar 2023 16:53:10 +0800 Subject: [PATCH] fix(cloudmon): add Slb metrics --- go.mod | 6 +- go.sum | 12 +- pkg/apis/compute/loadbalancer.go | 17 ++ pkg/cloudmon/providerdriver/base.go | 65 ++++++ vendor/modules.txt | 7 +- .../x/cloudmux/pkg/cloudprovider/metrics.go | 7 +- .../x/cloudmux/pkg/multicloud/hcso/hcso.go | 47 +++- .../cloudmux/pkg/multicloud/hcso/modelarts.go | 15 +- .../x/cloudmux/pkg/multicloud/hcso/monitor.go | 83 ++++++++ .../pkg/multicloud/huawei/modelarts_pool.go | 15 +- .../x/pkg/sortedstring/sortedstrings.go | 201 ++++++++++++++++++ .../x/pkg/util/httputils/httputils.go | 2 + .../backends/clickhouse/clickhouse.go | 8 +- .../x/sqlchemy/backends/clickhouse/column.go | 29 ++- .../backends/clickhouse/columninfo.go | 27 ++- .../x/sqlchemy/backends/clickhouse/sync.go | 67 +++++- vendor/yunion.io/x/sqlchemy/column.go | 15 ++ vendor/yunion.io/x/sqlchemy/query.go | 2 + vendor/yunion.io/x/sqlchemy/subquery.go | 2 +- vendor/yunion.io/x/sqlchemy/sync.go | 6 + vendor/yunion.io/x/sqlchemy/table.go | 57 ++++- 21 files changed, 630 insertions(+), 60 deletions(-) create mode 100644 vendor/yunion.io/x/pkg/sortedstring/sortedstrings.go diff --git a/go.mod b/go.mod index d3cd1bc824..d798244058 100644 --- a/go.mod +++ b/go.mod @@ -83,14 +83,14 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd yunion.io/x/log v1.0.0 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 - yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c + yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5 + yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6 yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 ) diff --git a/go.sum b/go.sum index 07c6e3c210..4775cb1ef8 100644 --- a/go.sum +++ b/go.sum @@ -1164,8 +1164,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add h1:w0kPDxDar0F0p9fWSBoqTCsBojhFrRzxsiJYiGO3bLY= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add/go.mod h1:sWqblYRhQCO63xeKdxdAT3wczCCY8Mc+TDa3WD/a5Kg= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 h1:w9vF4QzyJtkkDBPY7TOH1kUvUzPi6kGDmuFYHZrbmG8= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5/go.mod h1:VBAtfyGMDW11x5JmF0vUEmn/PmNpZ8CKBAQCiyz+cDY= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= @@ -1182,12 +1182,12 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v1.0.1-0.20220819080140-55b265a8658e/go.mod h1:nYOMflWG1wQC6301rGVglQva9bYbJxHThEJUxRAgMFc= -yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c h1:qkkyHkL8bFWumxWvtdbYhY4aDfKi+sBS21Im/i6Jeds= -yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk= +yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373 h1:VdQgvUo6JoOZNsPzRdoFKbD+3tEt4saHRhI/kUf6DeM= +yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v1.0.1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= -yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5 h1:7+m2F8B8eKJ4ELtIWpEEnEhL3WsIjpLocAaMx+83U5o= -yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE= +yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6 h1:qs8BU3AH7MsKeWLzuaj2bCc2/VnwyYqzl55VJXS25X4= +yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE= yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 h1:WuWXhY3DvhdRTzWCJ/kwt3Ss6KIq7+KqJwb+esvNGwU= yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/compute/loadbalancer.go b/pkg/apis/compute/loadbalancer.go index eff833b470..37c11662e7 100644 --- a/pkg/apis/compute/loadbalancer.go +++ b/pkg/apis/compute/loadbalancer.go @@ -230,3 +230,20 @@ type LoadbalancerDissociateEipInput struct { // 是否自动释放 AudoDelete *bool `json:"auto_delete"` } + +func (self LoadbalancerDetails) GetMetricTags() map[string]string { + ret := map[string]string{ + "id": self.Id, + "brand": self.Brand, + "backend_group": self.BackendGroup, + "cloudregion": self.Cloudregion, + "domain_id": self.DomainId, + "project_domain": self.ProjectDomain, + "region_ext_id": self.RegionExtId, + "status": self.Status, + "tenant": self.Project, + "tenant_id": self.ProjectId, + "external_id": self.ExternalId, + } + return ret +} diff --git a/pkg/cloudmon/providerdriver/base.go b/pkg/cloudmon/providerdriver/base.go index d11f7593c1..942caa7e02 100644 --- a/pkg/cloudmon/providerdriver/base.go +++ b/pkg/cloudmon/providerdriver/base.go @@ -583,6 +583,71 @@ func (self *SCollectByResourceIdDriver) CollectK8sMetrics(ctx context.Context, m return self.sendMetrics(ctx, manager, "k8s", len(res), metrics) } +func (self *SCollectByResourceIdDriver) CollectLoadbalancerMetrics(ctx context.Context, manager api.CloudproviderDetails, provider cloudprovider.ICloudProvider, res map[string]api.LoadbalancerDetails, start, end time.Time) error { + metrics := []influxdb.SMetricData{} + var wg sync.WaitGroup + var mu sync.Mutex + for i := range res { + wg.Add(1) + go func(lb api.LoadbalancerDetails) { + defer func() { + wg.Done() + }() + opts := &cloudprovider.MetricListOptions{ + ResourceType: cloudprovider.METRIC_RESOURCE_TYPE_LB, + StartTime: start, + EndTime: end, + } + opts.ResourceId = lb.ExternalId + + tags := []influxdb.SKeyValue{} + for k, v := range lb.GetMetricTags() { + tags = append(tags, influxdb.SKeyValue{ + Key: k, + Value: v, + }) + } + + data, err := provider.GetMetrics(opts) + if err != nil { + if errors.Cause(err) != cloudprovider.ErrNotImplemented && errors.Cause(err) != cloudprovider.ErrNotSupported { + log.Errorf("get loadbalancers %s(%s) error: %v", lb.Name, lb.Id, err) + return + } + return + } + + for _, values := range data { + for _, value := range values.Values { + metric := influxdb.SMetricData{ + Name: values.MetricType.Name(), + Timestamp: value.Timestamp, + Tags: tags, + Metrics: []influxdb.SKeyValue{ + { + Key: values.MetricType.Key(), + Value: strconv.FormatFloat(value.Value, 'E', -1, 64), + }, + }, + } + for k, v := range value.Tags { + metric.Tags = append(metric.Tags, influxdb.SKeyValue{ + Key: k, + Value: v, + }) + } + mu.Lock() + metrics = append(metrics, metric) + mu.Unlock() + } + } + }(res[i]) + } + wg.Wait() + + return self.sendMetrics(ctx, manager, "slb", len(res), metrics) +} + type SCollectByMetricTypeDriver struct { SBaseCollectDriver } diff --git a/vendor/modules.txt b/vendor/modules.txt index dfcd21d6ab..258d29a013 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1442,7 +1442,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230323062012-21f8a2ca0add +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230329090243-cc4cdbd90fd5 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1537,7 +1537,7 @@ yunion.io/x/log/hooks yunion.io/x/ovsdb/cli_util yunion.io/x/ovsdb/schema/ovn_nb yunion.io/x/ovsdb/types -# yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c +# yunion.io/x/pkg v1.0.1-0.20230327034731-79f520e7d373 ## explicit; go 1.18 yunion.io/x/pkg/appctx yunion.io/x/pkg/errors @@ -1545,6 +1545,7 @@ yunion.io/x/pkg/gotypes yunion.io/x/pkg/object yunion.io/x/pkg/prettytable yunion.io/x/pkg/sortedmap +yunion.io/x/pkg/sortedstring yunion.io/x/pkg/trace yunion.io/x/pkg/tristate yunion.io/x/pkg/util/billing @@ -1591,7 +1592,7 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.2-0.20230223125758-76ea06389ac5 +# yunion.io/x/sqlchemy v1.1.2-0.20230326161357-912d44dd66d6 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go index 7b0e1bed92..f4c938a4ab 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/metrics.go @@ -222,10 +222,13 @@ const ( // 支持平台: aliyun, apsara LB_METRIC_TYPE_NET_PACKET_TX TMetricType = "haproxy.packet_tx" // 非活跃连接数 - // 支持平台: apsara, aliyun + // 支持平台: apsara, aliyun, huawei LB_METRIC_TYPE_NET_INACTIVE_CONNECTION = "haproxy.inactive_connection" + // 活跃连接数 + // 支持平台: apsara, aliyun, huawei + LB_METRIC_TYPE_NET_ACTIVE_CONNECTION = "haproxy.active_connection" // 最大并发数 - // 支持平台: apsara, aliyun + // 支持平台: apsara, aliyun, huawei LB_METRIC_TYPE_MAX_CONNECTION = "haproxy.max_connection" // 后端异常ECS实例个数 // 支持平台: apsara, aliyun diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go index 4e5a313a81..7025ad321c 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/hcso.go @@ -17,10 +17,12 @@ package hcso import ( "context" "fmt" + "net" "net/http" "net/url" "strconv" "strings" + "time" "github.com/huaweicloud/huaweicloud-sdk-go/auth/aksk" @@ -98,16 +100,34 @@ func (self *SHuaweiClient) request(method httputils.THttpMethod, url string, que if strings.Contains(url, "/OS-CREDENTIAL/credentials") && len(self.ownerId) > 0 { header.Set("X-Domain-Id", self.ownerId) } - _, resp, err := httputils.JSONRequest(client, context.Background(), method, url, header, body, self.debug) - if err != nil { - if e, ok := err.(*httputils.JSONClientError); ok && e.Code == 404 { - return nil, errors.Wrapf(cloudprovider.ErrNotFound, err.Error()) + var resp jsonutils.JSONObject + var err error + for i := 0; i < 3; i++ { + _, resp, err = requestWithRetry(client, context.Background(), method, url, header, body, self.debug) + if method == httputils.GET && needRetry(err) { + time.Sleep(time.Second * 15) + continue } - return nil, err + return resp, err } return resp, err } +func requestWithRetry(client *akClient, ctx context.Context, method httputils.THttpMethod, urlStr string, header http.Header, body jsonutils.JSONObject, debug bool) (http.Header, jsonutils.JSONObject, error) { + var bodystr string + if !gotypes.IsNil(body) { + bodystr = body.String() + } + jbody := strings.NewReader(bodystr) + if header == nil { + header = http.Header{} + } + header.Set("Content-Length", strconv.FormatInt(int64(len(bodystr)), 10)) + header.Set("Content-Type", "application/json") + resp, err := httputils.RequestWithRetry(client, ctx, method, urlStr, header, jbody, debug) + return httputils.ParseJSONResponse(bodystr, resp, err, debug) +} + func (self *SHuaweiClient) resetEndpoint(endpoint, serviceName string) string { if len(endpoint) == 0 { domain := self.HuaweiClientConfig.endpoints.EndpointDomain @@ -258,3 +278,20 @@ func (self *SHuaweiClient) patchRequest(method httputils.THttpMethod, url string } return respValue, err } + +func needRetry(err error) bool { + if err == nil { + return false + } + switch e := err.(type) { + case *url.Error: + switch e.Err.(type) { + case *net.DNSError, *net.OpError, net.UnknownNetworkError: + return true + } + if strings.Contains(err.Error(), "The throttling threshold has been reached: policy ip over ratelimit") { + return true + } + } + return false +} diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go index d86c35ad17..fc67a1129b 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/modelarts.go @@ -373,6 +373,15 @@ func (self *SModelartsPool) SetAutoRenew(bc billing.SBillingCycle) error { } func (self *SModelartsPool) Refresh() error { + self.Status.Resource = SNodeStatus{} + pool, err := self.region.client.modelartsPoolById(self.GetId()) + if err == nil { + pool.Unmarshal(self) + return nil + } + if err != errors.ErrNotFound { + return errors.Wrap(err, "modelartsPoolById") + } pools := make([]SModelartsPool, 0) resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil) if err != nil { @@ -385,13 +394,9 @@ func (self *SModelartsPool) Refresh() error { for _, pool := range pools { if pool.GetId() == self.GetId() { self.Status.Phase = "CreationFailed" + break } } - self.Status.Resource = SNodeStatus{} - pool, err := self.region.client.modelartsPoolById(self.GetId()) - if err != nil { - return errors.Wrapf(err, "GetModelartsPool(%s)", self.GetId()) - } return jsonutils.Update(self, pool) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go index e31b6e00f1..d80a1b5394 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/hcso/monitor.go @@ -353,6 +353,87 @@ func (self *SHuaweiClient) getBucketMetrics(opts *cloudprovider.MetricListOption return result, nil } +func (self *SHuaweiClient) getLoadbalancerMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { + namespace, dimesionName, metricNames := "SYS.ELB", "lbaas_instance_id", []string{ + "m1_cps", + "m2_act_conn", + "m7_in_Bps", + "m8_out_Bps", + "mc_l7_http_2xx", + "md_l7_http_3xx", + "me_l7_http_4xx", + "mf_l7_http_5xx", + } + result := []cloudprovider.MetricValues{} + + for _, metricName := range metricNames { + temp := make(map[string]string) + temp["namespace"] = namespace + temp["metric_name"] = metricName + temp["from"] = strconv.Itoa(int(opts.StartTime.UnixMilli())) + temp["to"] = strconv.Itoa(int(opts.EndTime.UnixMilli())) + temp["period"] = "1" + temp["filter"] = "average" + temp["dim.0"] = fmt.Sprintf("%s,%s", dimesionName, opts.ResourceId) + resp, err := self.commonMonitor(temp) + if err != nil { + return nil, err + } + metricData := MetricData{} + err = resp.Unmarshal(&metricData) + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") + } + ret := cloudprovider.MetricValues{ + Id: opts.ResourceId, + Unit: metricData.Unit, + Values: []cloudprovider.MetricValue{}, + } + tags := map[string]string{} + switch metricData.MetricName { + case "m1_cps": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_MAX_CONNECTION + case "m2_act_conn": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_ACTIVE_CONNECTION + case "m7_in_Bps": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_RX + case "m8_out_Bps": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_NET_BPS_TX + case "mc_l7_http_2xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "2xx"} + case "md_l7_http_3xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "3xx"} + case "md_l7_http_4xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "4xx"} + case "md_l7_http_5xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "5xx"} + case "me_l7_http_4xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "4xx"} + case "mf_l7_http_5xx": + ret.MetricType = cloudprovider.LB_METRIC_TYPE_HRSP_COUNT + tags = map[string]string{"request": "5xx"} + default: + log.Warningf("invalid metricName %s for %s %s", metricData.MetricName, opts.ResourceType, opts.ResourceId) + continue + } + for _, value := range metricData.Datapoints { + metricValue := cloudprovider.MetricValue{ + Value: value.Average, + Timestamp: time.UnixMilli(value.Timestamp), + Tags: tags, + } + ret.Values = append(ret.Values, metricValue) + } + result = append(result, ret) + } + return result, nil +} + func (self *SHuaweiClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([]cloudprovider.MetricValues, error) { switch opts.ResourceType { case cloudprovider.METRIC_RESOURCE_TYPE_SERVER: @@ -363,6 +444,8 @@ func (self *SHuaweiClient) GetMetrics(opts *cloudprovider.MetricListOptions) ([] return self.getBucketMetrics(opts) case cloudprovider.METRIC_RESOURCE_TYPE_MODELARTS_POOL: return self.getModelartsPoolMetrics(opts) + case cloudprovider.METRIC_RESOURCE_TYPE_LB: + return self.getLoadbalancerMetrics(opts) default: return nil, errors.Wrapf(cloudprovider.ErrNotSupported, "%s", opts.ResourceType) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go index d412fcebaa..2d9cbd0830 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/huawei/modelarts_pool.go @@ -372,6 +372,15 @@ func (self *SModelartsPool) SetAutoRenew(bc billing.SBillingCycle) error { } func (self *SModelartsPool) Refresh() error { + self.Status.Resource = SNodeStatus{} + pool, err := self.region.client.modelartsPoolById(self.GetId()) + if err == nil { + pool.Unmarshal(self) + return nil + } + if err != errors.ErrNotFound { + return errors.Wrap(err, "modelartsPoolById") + } pools := make([]SModelartsPool, 0) resObj, err := self.region.client.modelartsPoolListWithStatus("pools", "failed", nil) if err != nil { @@ -384,13 +393,9 @@ func (self *SModelartsPool) Refresh() error { for _, pool := range pools { if pool.GetId() == self.GetId() { self.Status.Phase = "CreationFailed" + break } } - self.Status.Resource = SNodeStatus{} - pool, err := self.region.client.modelartsPoolById(self.GetId()) - if err != nil { - return errors.Wrapf(err, "GetModelartsPool(%s)", self.GetId()) - } return jsonutils.Update(self, pool) } diff --git a/vendor/yunion.io/x/pkg/sortedstring/sortedstrings.go b/vendor/yunion.io/x/pkg/sortedstring/sortedstrings.go new file mode 100644 index 0000000000..56134f6812 --- /dev/null +++ b/vendor/yunion.io/x/pkg/sortedstring/sortedstrings.go @@ -0,0 +1,201 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sortedstring + +import ( + "sort" +) + +type SSortedStrings []string + +func NewSortedStrings(strs []string) SSortedStrings { + if strs == nil { + return SSortedStrings{} + } + sort.Strings(strs) + return SSortedStrings(strs) +} + +func Append(ss SSortedStrings, ele ...string) SSortedStrings { + ss = ss.Append(ele...) + return ss +} + +func (ss SSortedStrings) Append(ele ...string) SSortedStrings { + if ss == nil { + ss = NewSortedStrings([]string{}) + } + for _, e := range ele { + pos, find := ss.Index(e) + if find { + continue + } + ss = append(ss, e) + copy(ss[pos+1:], ss[pos:]) + ss[pos] = e + } + return ss +} + +func (ss SSortedStrings) Remove(ele ...string) SSortedStrings { + if ss == nil { + return ss + } + for _, e := range ele { + pos, find := ss.Index(e) + if !find { + continue + } + if pos < len(ss)-1 { + copy(ss[pos:], ss[pos+1:]) + } + ss = ss[:len(ss)-1] + } + return ss +} + +func (ss SSortedStrings) Index(needle string) (int, bool) { + i := 0 + j := len(ss) - 1 + for i <= j { + m := (i + j) / 2 + if ss[m] < needle { + i = m + 1 + } else if ss[m] > needle { + j = m - 1 + } else { + return m, true + } + } + return j + 1, false +} + +func (ss SSortedStrings) Contains(needle string) bool { + _, find := ss.Index(needle) + return find +} + +func (ss SSortedStrings) ContainsAny(needles ...string) bool { + for i := range needles { + _, find := ss.Index(needles[i]) + if find { + return true + } + } + return false +} + +func (ss SSortedStrings) ContainsAll(needles ...string) bool { + for i := range needles { + _, find := ss.Index(needles[i]) + if !find { + return false + } + } + return true +} + +func Contains(a, b SSortedStrings) bool { + _, _, bNoA := Split(a, b) + if len(bNoA) == 0 { + return true + } else { + return false + } +} + +func Equals(a, b SSortedStrings) bool { + aNoB, _, bNoA := Split(a, b) + if len(aNoB) == 0 && len(bNoA) == 0 { + return true + } else { + return false + } +} + +func Split(a, b SSortedStrings) (aNoB SSortedStrings, aAndB SSortedStrings, bNoA SSortedStrings) { + a_b := make([]string, 0) + b_a := make([]string, 0) + anb := make([]string, 0) + i := 0 + j := 0 + for i < len(a) && j < len(b) { + if a[i] == b[j] { + anb = append(anb, a[i]) + i += 1 + j += 1 + } else if a[i] < b[j] { + a_b = append(a_b, a[i]) + i += 1 + } else if a[i] > b[j] { + b_a = append(b_a, b[j]) + j += 1 + } + } + if i < len(a) { + a_b = append(a_b, a[i:]...) + } + if j < len(b) { + b_a = append(b_a, b[j:]...) + } + aNoB = SSortedStrings(a_b) + aAndB = SSortedStrings(anb) + bNoA = SSortedStrings(b_a) + return +} + +func Merge(a, b SSortedStrings) SSortedStrings { + ret := make([]string, 0) + i := 0 + j := 0 + for i < len(a) && j < len(b) { + if a[i] == b[j] { + ret = append(ret, a[i]) + i += 1 + j += 1 + } else if a[i] < b[j] { + ret = append(ret, a[i]) + i += 1 + } else if a[i] > b[j] { + ret = append(ret, b[j]) + j += 1 + } + } + if i < len(a) { + ret = append(ret, a[i:]...) + } + if j < len(b) { + ret = append(ret, b[j:]...) + } + return SSortedStrings(ret) +} + +func Intersect(a, b SSortedStrings) SSortedStrings { + ret := make([]string, 0) + i := 0 + j := 0 + for i < len(a) && j < len(b) { + if a[i] == b[j] { + ret = append(ret, a[i]) + i += 1 + j += 1 + } else if a[i] < b[j] { + i += 1 + } else if a[i] > b[j] { + j += 1 + } + } + return SSortedStrings(ret) +} diff --git a/vendor/yunion.io/x/pkg/util/httputils/httputils.go b/vendor/yunion.io/x/pkg/util/httputils/httputils.go index 592fc291c2..e77b9d6b27 100644 --- a/vendor/yunion.io/x/pkg/util/httputils/httputils.go +++ b/vendor/yunion.io/x/pkg/util/httputils/httputils.go @@ -487,6 +487,8 @@ func isHTTPReqErrorRetryable(err error) bool { } else if strings.Contains(err.Error(), "net/http: timeout awaiting response headers") { // Retry errors due to server not sending the response before timeout return true + } else if strings.Contains(err.Error(), "dial tcp: lookup") { + return true } } return false diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go index 59a717ea54..f6d29108a3 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go @@ -173,7 +173,7 @@ func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) ( if err != nil { return nil, errors.Wrap(err, "show create table") } - primaries, orderbys, partition, ttl := parseCreateTable(defStr) + primaries, orderbys, partitions, ttl := parseCreateTable(defStr) var ttlCfg sColumnTTL if len(ttl) > 0 { ttlCfg, err = parseTTLExpression(ttl) @@ -189,8 +189,10 @@ func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) ( if utils.IsInStringArray(clickSpec.Name(), orderbys) { clickSpec.SetOrderBy(true) } - if strings.Contains(partition, clickSpec.Name()) { - clickSpec.SetPartitionBy(partition) + for _, part := range partitions { + if strings.Contains(part, clickSpec.Name()) { + clickSpec.SetPartitionBy(part) + } } if ttlCfg.ColName == clickSpec.Name() { clickSpec.SetTTL(ttlCfg.Count, ttlCfg.Unit) diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go index 78e16c7c98..3b01fdfba0 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/column.go @@ -181,7 +181,7 @@ func (c *SBooleanColumn) IsZero(val interface{}) bool { return bVal == nil } bVal := val.(bool) - return bVal == false + return !bVal } // NewBooleanColumn return an instance of SBooleanColumn @@ -241,10 +241,10 @@ func (c *STristateColumn) IsZero(val interface{}) bool { // NewTristateColumn return an instance of STristateColumn func NewTristateColumn(name string, tagmap map[string]string, isPointer bool) STristateColumn { - if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok { - // tristate always nullable - delete(tagmap, sqlchemy.TAG_NULLABLE) - } + //if _, ok := tagmap[sqlchemy.TAG_NULLABLE]; ok { + // tristate always nullable + delete(tagmap, sqlchemy.TAG_NULLABLE) + //} bc := STristateColumn{SClickhouseBaseColumn: NewClickhouseBaseColumn(name, "UInt8", tagmap, isPointer)} return bc } @@ -298,7 +298,6 @@ func (c *SIntegerColumn) ConvertFromString(str string) interface{} { default: panic(fmt.Sprintf("unsupported type %s", ctype)) } - return val } else { val, _ := strconv.ParseInt(str, 10, 64) switch ctype { @@ -313,7 +312,6 @@ func (c *SIntegerColumn) ConvertFromString(str string) interface{} { default: panic(fmt.Sprintf("unsupported type %s", ctype)) } - return val } } @@ -359,16 +357,16 @@ func (c *SFloatColumn) IsZero(val interface{}) bool { if c.IsPointer() { switch val.(type) { case *float32: - return val.(*float32) == nil + return val == nil case *float64: - return val.(*float64) == nil + return val == nil } } else { switch val.(type) { case float32: - return val.(float32) == 0.0 + return val == 0.0 case float64: - return val.(float64) == 0.0 + return val == 0.0 } } return true @@ -386,7 +384,6 @@ func (c *SFloatColumn) ConvertFromString(str string) interface{} { default: panic(fmt.Sprintf("unsupported type %s", ctype)) } - return val } // NewFloatColumn returns an instance of SFloatColumn @@ -425,16 +422,16 @@ func (c *SDecimalColumn) IsZero(val interface{}) bool { if c.IsPointer() { switch val.(type) { case *float32: - return val.(*float32) == nil + return val == nil case *float64: - return val.(*float64) == nil + return val == nil } } else { switch val.(type) { case float32: - return val.(float32) == 0.0 + return val == 0.0 case float64: - return val.(float64) == 0.0 + return val == 0.0 } } return true diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go index d4c6b5f741..e8102ea433 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/columninfo.go @@ -16,6 +16,7 @@ package clickhouse import ( "regexp" + "sort" "strings" "yunion.io/x/log" @@ -140,6 +141,7 @@ func parseKeys(keyStr string) []string { key = strings.TrimSpace(key) ret = append(ret, key) } + sort.Strings(ret) return ret } @@ -163,7 +165,27 @@ func findSegment(sqlStr string, prefix string) string { return "" } -func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, partition string, ttl string) { +func trimPartition(partStr string) string { + for { + partStr = strings.TrimSpace(partStr) + if partStr[0] == '(' { + partStr = partStr[1 : len(partStr)-1] + } else { + break + } + } + partStr = strings.ReplaceAll(partStr, " ", "") + return partStr +} + +func parsePartitions(partStr string) []string { + partStr = trimPartition(partStr) + parts := strings.Split(partStr, ",") + sort.Strings(parts) + return parts +} + +func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, partitions []string, ttl string) { matches := primaryKeyRegexp.FindAllStringSubmatch(sqlStr, -1) if len(matches) > 0 { primaries = parseKeys(matches[0][1]) @@ -172,7 +194,8 @@ func parseCreateTable(sqlStr string) (primaries []string, orderbys []string, par if len(matches) > 0 { orderbys = parseKeys(matches[0][1]) } - partition = findSegment(sqlStr, partitionByPrefix) + partitionStr := findSegment(sqlStr, partitionByPrefix) + partitions = parsePartitions(partitionStr) ttl = findSegment(sqlStr, ttlPrefix) return } diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go index 21415c3ea6..d652ad6c57 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go @@ -16,10 +16,14 @@ package clickhouse import ( "fmt" + "sort" "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" + "yunion.io/x/pkg/sortedstring" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" ) @@ -42,18 +46,32 @@ func findTtlColumn(cols []sqlchemy.IColumnSpec) sColumnTTL { return ret } +func findPartitions(cols []sqlchemy.IColumnSpec) []string { + parts := make([]string, 0) + for i := range cols { + if c, ok := cols[i].(IClickhouseColumnSpec); ok { + part := strings.ReplaceAll(c.PartitionBy(), " ", "") + if len(part) > 0 && !utils.IsInStringArray(part, parts) { + parts = append(parts, part) + } + } + } + sort.Strings(parts) + return parts +} + func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes sqlchemy.STableChanges) []string { ret := make([]string, 0) + needCopyTable := false + alters := make([]string, 0) // first check if primary key is modifed changePrimary := false - oldHasPrimary := false for _, col := range changes.RemoveColumns { if col.IsPrimary() { changePrimary = true - oldHasPrimary = true } } @@ -61,19 +79,21 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe if cols.OldCol.IsPrimary() != cols.NewCol.IsPrimary() { changePrimary = true } - if cols.OldCol.IsPrimary() { - oldHasPrimary = true - } } for _, col := range changes.AddColumns { if col.IsPrimary() { changePrimary = true } } - if changePrimary && oldHasPrimary { - sql := fmt.Sprintf("DROP PRIMARY KEY") - alters = append(alters, sql) + + if changePrimary { + log.Infof("primary key changed") + needCopyTable = true } + // if changePrimary && oldHasPrimary { + // sql := fmt.Sprintf("DROP PRIMARY KEY") + // alters = append(alters, sql) + // } /* IGNORE DROP STATEMENT */ for _, col := range changes.RemoveColumns { sql := fmt.Sprintf("DROP COLUMN `%s`", col.Name()) @@ -120,6 +140,7 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe } }*/ + // check TTL oldTtlSpec := findTtlColumn(changes.OldColumns) newTtlSpec := findTtlColumn(ts.Columns()) log.Debugf("old: %s new: %s", jsonutils.Marshal(oldTtlSpec), jsonutils.Marshal(newTtlSpec)) @@ -140,5 +161,35 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe ret = append(ret, sql) } + // check partitions + oldPartitions := findPartitions(changes.OldColumns) + newPartitions := findPartitions(ts.Columns()) + if !sortedstring.Equals(oldPartitions, newPartitions) { + log.Infof("partition inconsistemt: old=%s new=%s", oldPartitions, newPartitions) + needCopyTable = true + } + + // needCopyTable + if needCopyTable { + // create new table + alterTableName := fmt.Sprintf("%s_tmp_%d", ts.Name(), time.Now().Unix()) + alterTable := ts.(*sqlchemy.STableSpec).Clone(alterTableName, 0) + createSqls := alterTable.CreateSQLs() + ret = append(ret, createSqls...) + colNames := make([]string, 0) + for _, c := range ts.Columns() { + colNames = append(colNames, fmt.Sprintf("`%s`", c.Name())) + } + colNamesStr := strings.Join(colNames, ",") + // copy data + sql := fmt.Sprintf("INSERT INTO `%s` (%s) SELECT %s FROM `%s`", alterTableName, colNamesStr, colNamesStr, ts.Name()) + ret = append(ret, sql) + // rename tables + sql = fmt.Sprintf("RENAME TABLE `%s` TO `%s_backup`", ts.Name(), alterTableName) + ret = append(ret, sql) + sql = fmt.Sprintf("RENAME TABLE `%s` TO `%s`", alterTableName, ts.Name()) + ret = append(ret, sql) + } + return ret } diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index 682d85404c..6a9db90459 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -121,6 +121,11 @@ type IColumnSpec interface { IsString() bool IsDateTime() bool + + // index of column, to preserve the column position + GetColIndex() int + // setter of column index + SetColIndex(idx int) } // SBaseColumn is the base structure represents a column @@ -136,6 +141,7 @@ type SBaseColumn struct { isIndex bool isAllowZero bool tags map[string]string + colIndex int } // IsPointer implementation of SBaseColumn for IColumnSpec @@ -279,6 +285,14 @@ func (c *SBaseColumn) IsDateTime() bool { return false } +func (c *SBaseColumn) GetColIndex() int { + return c.colIndex +} + +func (c *SBaseColumn) SetColIndex(idx int) { + c.colIndex = idx +} + // NewBaseColumn returns an instance of SBaseColumn func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPointer bool) SBaseColumn { var val string @@ -333,6 +347,7 @@ func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPoin tags: tagmap, isPointer: isPointer, isAllowZero: isAllowZero, + colIndex: -1, } } diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 5c273ee645..886171e84b 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -81,6 +81,7 @@ func (tq *SQuery) HasField(f IQueryField) bool { } for i := range tq.fields { fi := tq.fields[i] + // log.Debugf("field at %d: %s", i, fi.Name()) if fi.Name() == f.Name() { return true } @@ -90,6 +91,7 @@ func (tq *SQuery) HasField(f IQueryField) bool { // AppendField appends query field to a query func (tq *SQuery) AppendField(f ...IQueryField) *SQuery { + // log.Debugf("AppendField tq has fields %d", len(tq.fields)) for i := range f { if !tq.HasField(f[i]) { tq.fields = append(tq.fields, f[i]) diff --git a/vendor/yunion.io/x/sqlchemy/subquery.go b/vendor/yunion.io/x/sqlchemy/subquery.go index 831bcd68ec..d786fc475c 100644 --- a/vendor/yunion.io/x/sqlchemy/subquery.go +++ b/vendor/yunion.io/x/sqlchemy/subquery.go @@ -49,7 +49,7 @@ func (sqf *SSubQueryField) Reference() string { // Label implementation of SSubQueryField for IQueryField func (sqf *SSubQueryField) Label(label string) IQueryField { - if len(label) > 0 && label != sqf.field.Name() { + if len(label) > 0 { sqf.alias = label } return sqf diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 5858a14879..dfad2cf3f5 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -32,6 +32,12 @@ func compareColumnSpec(c1, c2 IColumnSpec) int { return strings.Compare(c1.Name(), c2.Name()) } +func compareColumnIndex(c1, c2 IColumnSpec) int { + i1 := c1.GetColIndex() + i2 := c2.GetColIndex() + return i1 - i2 +} + type SUpdateColumnSpec struct { OldCol IColumnSpec NewCol IColumnSpec diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 31fe97cd62..587c03d54a 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -17,8 +17,10 @@ package sqlchemy import ( "fmt" "reflect" + "sort" "yunion.io/x/log" + "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" ) @@ -141,8 +143,61 @@ func (ts *STableSpec) Expression() string { return fmt.Sprintf("`%s`", ts.name) } +func (ts *STableSpec) SyncColumnIndexes() error { + if !ts.Exists() { + return errors.Wrap(errors.ErrNotFound, "table not exists") + } + + cols, err := ts.Database().backend.FetchTableColumnSpecs(ts) + if err != nil { + log.Errorf("fetchColumnDefs fail: %s", err) + return errors.Wrap(err, "FetchTableColumnSpecs") + } + if len(cols) != len(ts._columns) { + return errors.Wrapf(errors.ErrInvalidStatus, "ts col %d != actual col %d", len(ts._columns), len(cols)) + } + for i := range cols { + cols[i].SetColIndex(i) + } + // sort colums + sort.Slice(cols, func(i, j int) bool { + return compareColumnSpec(cols[i], cols[j]) < 0 + }) + sort.Slice(ts._columns, func(i, j int) bool { + return compareColumnSpec(ts._columns[i], ts._columns[j]) < 0 + }) + // compare columns and assign colindex + for i := range ts._columns { + comp := compareColumnSpec(cols[i], ts._columns[i]) + if comp != 0 { + return errors.Wrapf(errors.ErrInvalidStatus, "colname %s != %s", cols[i].Name(), ts._columns[i].Name()) + } + ts._columns[i].SetColIndex(cols[i].GetColIndex()) + } + + // sort columns according to colindex + sort.Slice(ts._columns, func(i, j int) bool { + return compareColumnIndex(ts._columns[i], ts._columns[j]) < 0 + }) + + return nil +} + // Clone makes a clone of a table, so we may create a new table of the same schema func (ts *STableSpec) Clone(name string, autoIncOffset int64) *STableSpec { + nts, _ := ts.CloneWithSyncColumnOrder(name, autoIncOffset, false) + return nts +} + +// Clone makes a clone of a table, so we may create a new table of the same schema +func (ts *STableSpec) CloneWithSyncColumnOrder(name string, autoIncOffset int64, syncColOrder bool) (*STableSpec, error) { + if ts.Exists() && syncColOrder { + // if table exists, sync column index + err := ts.SyncColumnIndexes() + if err != nil { + return nil, errors.Wrap(err, "SyncColumnIndexes") + } + } columns := ts.Columns() newCols := make([]IColumnSpec, len(columns)) for i := range newCols { @@ -170,7 +225,7 @@ func (ts *STableSpec) Clone(name string, autoIncOffset int64) *STableSpec { newIndexes[i] = ts._indexes[i].clone(nts) } nts._indexes = newIndexes - return nts + return nts, nil } // Columns implementation of STableSpec for ITableSpec