From 0c7bf6989fd507d5b0a4cab4437457cbb5a9e52d Mon Sep 17 00:00:00 2001 From: ioito Date: Tue, 5 Dec 2023 10:03:10 +0800 Subject: [PATCH] fix(region): avoid cloudmon no cloudprovider sync --- go.mod | 4 +- go.sum | 8 +- pkg/cloudmon/resources/resources.go | 4 +- vendor/modules.txt | 4 +- .../cloudmux/pkg/multicloud/apsara/apsara.go | 5 +- .../backends/clickhouse/clickhouse.go | 96 +++++++++++++------ .../x/sqlchemy/backends/clickhouse/consts.go | 11 +++ vendor/yunion.io/x/sqlchemy/extraopts.go | 47 +++++++++ vendor/yunion.io/x/sqlchemy/table.go | 21 +++- 9 files changed, 162 insertions(+), 38 deletions(-) create mode 100644 vendor/yunion.io/x/sqlchemy/extraopts.go diff --git a/go.mod b/go.mod index 498f67190b..8c122f55c5 100644 --- a/go.mod +++ b/go.mod @@ -87,14 +87,14 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19 yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 yunion.io/x/pkg v1.0.1-0.20231101105448-abef64cdc142 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0 + yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c ) diff --git a/go.sum b/go.sum index 15acd11b7c..ca65fc65fd 100644 --- a/go.sum +++ b/go.sum @@ -1192,8 +1192,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e h1:QmYmnMJjVijGbLi5haZS49d2CaNgA/JLphLn7oL8q94= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7 h1:fZ3sE1acOojrgXGtqLxXcfFhg4bT3shKx2o73J+Yat8= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug= yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= @@ -1211,7 +1211,7 @@ yunion.io/x/pkg v1.0.1-0.20231101105448-abef64cdc142 h1:L6LqxfP08eWUx+A6yQdrL6VB yunion.io/x/pkg v1.0.1-0.20231101105448-abef64cdc142/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0 h1:+MaykFV6YCakTLnHR3v31tovXhkvWgkBPFa83MuCekA= -yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ= +yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51 h1:XgvhXxYul4W85vVJVKCKOWw6Ea/YroI3TBxyLcQ7vx4= +yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/cloudmon/resources/resources.go b/pkg/cloudmon/resources/resources.go index 3977627776..2c8f32f666 100644 --- a/pkg/cloudmon/resources/resources.go +++ b/pkg/cloudmon/resources/resources.go @@ -86,11 +86,13 @@ func (self *SBaseResources) init(ctx context.Context) error { "order_by.1": "imported_at", "order": "asc", "pending_delete": "all", - "filter.0": "external_id.isnotempty()", } if self.manager.GetKeyword() == compute.Hosts.GetKeyword() { // private and vmware query["cloud_env"] = "private_or_onpremise" } + if self.manager.GetKeyword() != compute.Cloudproviders.GetKeyword() { + query["filter.0"] = "external_id.isnotempty()" + } offset := 0 for { query["offset"] = offset diff --git a/vendor/modules.txt b/vendor/modules.txt index e626bef386..c6002ded82 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1458,7 +1458,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7 ## explicit; go 1.18 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -1616,7 +1616,7 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0 +# yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go index 3b16cc9105..fe682f0174 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go @@ -262,6 +262,9 @@ func (self *SApsaraClient) getDefaultClient(regionId string) (*sdk.Client, error return nil, errors.Wrapf(err, "ParseQuery(%s)", req.URL.RawQuery) } action := params.Get("OpenApiAction") + if len(action) == 0 { + action = params.Get("Action") + } service := strings.ToLower(params.Get("Product")) respCheck := func(resp *http.Response) error { if self.cpcfg.UpdatePermission != nil { @@ -288,7 +291,7 @@ func (self *SApsaraClient) getDefaultClient(regionId string) (*sdk.Client, error } return nil } - if self.cpcfg.ReadOnly { + if self.cpcfg.ReadOnly && len(action) > 0 { for _, prefix := range []string{"Get", "List", "Describe"} { if strings.HasPrefix(action, prefix) { return respCheck, nil diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go index 72e690fde7..4195d54e4e 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go @@ -104,6 +104,17 @@ func (click *SClickhouseBackend) UpdateSQLTemplate() string { return "ALTER TABLE `{{ .Table }}` UPDATE {{ .Columns }} WHERE {{ .Conditions }}" } +func MySQLExtraOptions(hostport, database, table, user, passwd string) sqlchemy.TableExtraOptions { + return sqlchemy.TableExtraOptions{ + EXTRA_OPTION_ENGINE_KEY: EXTRA_OPTION_ENGINE_VALUE_MYSQL, + EXTRA_OPTION_CLICKHOUSE_MYSQL_HOSTPORT_KEY: hostport, + EXTRA_OPTION_CLICKHOUSE_MYSQL_DATABASE_KEY: database, + EXTRA_OPTION_CLICKHOUSE_MYSQL_TABLE_KEY: table, + EXTRA_OPTION_CLICKHOUSE_MYSQL_USERNAME_KEY: user, + EXTRA_OPTION_CLICKHOUSE_MYSQL_PASSWORD_KEY: passwd, + } +} + func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string { cols := make([]string, 0) primaries := make([]string, 0) @@ -129,35 +140,51 @@ func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string } } } - createSql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE MergeTree", ts.Name(), strings.Join(cols, ",\n")) - if len(orderbys) == 0 { - orderbys = primaries - } - if len(partitions) > 0 { - createSql += fmt.Sprintf("\nPARTITION BY (%s)", strings.Join(partitions, ", ")) - } - if len(primaries) > 0 { - createSql += fmt.Sprintf("\nPRIMARY KEY (%s)", strings.Join(primaries, ", ")) - newOrderBys := make([]string, len(primaries)) - copy(newOrderBys, primaries) - for _, f := range orderbys { - if !utils.IsInStringArray(f, newOrderBys) { - newOrderBys = append(newOrderBys, f) - } + createSql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE = ", ts.Name(), strings.Join(cols, ",\n")) + extraOpts := ts.GetExtraOptions() + engine := extraOpts.Get(EXTRA_OPTION_ENGINE_KEY) + switch engine { + case EXTRA_OPTION_ENGINE_VALUE_MYSQL: + // mysql + createSql += fmt.Sprintf("MySQL('%s', '%s', '%s', '%s', '%s')", + extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_HOSTPORT_KEY), + extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_DATABASE_KEY), + extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_TABLE_KEY), + extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_USERNAME_KEY), + extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_PASSWORD_KEY), + ) + default: + // mergetree + createSql += "MergeTree()" + if len(orderbys) == 0 { + orderbys = primaries } - orderbys = newOrderBys + if len(partitions) > 0 { + createSql += fmt.Sprintf("\nPARTITION BY (%s)", strings.Join(partitions, ", ")) + } + if len(primaries) > 0 { + createSql += fmt.Sprintf("\nPRIMARY KEY (%s)", strings.Join(primaries, ", ")) + newOrderBys := make([]string, len(primaries)) + copy(newOrderBys, primaries) + for _, f := range orderbys { + if !utils.IsInStringArray(f, newOrderBys) { + newOrderBys = append(newOrderBys, f) + } + } + orderbys = newOrderBys + } + if len(orderbys) > 0 { + createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", ")) + } else { + createSql += "\nORDER BY tuple()" + } + if ttlCol != nil { + ttlCount, ttlUnit := ttlCol.GetTTL() + createSql += fmt.Sprintf("\nTTL `%s` + INTERVAL %d %s", ttlCol.Name(), ttlCount, ttlUnit) + } + // set default time zone of table to UTC + createSql += "\nSETTINGS index_granularity=8192" } - if len(orderbys) > 0 { - createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", ")) - } else { - createSql += fmt.Sprintf("\nORDER BY tuple()") - } - if ttlCol != nil { - ttlCount, ttlUnit := ttlCol.GetTTL() - createSql += fmt.Sprintf("\nTTL `%s` + INTERVAL %d %s", ttlCol.Name(), ttlCount, ttlUnit) - } - // set default time zone of table to UTC - createSql += "\nSETTINGS index_granularity=8192" return []string{ createSql, } @@ -216,6 +243,21 @@ func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) ( } func (click *SClickhouseBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec { + extraOpts := table.GetExtraOptions() + engine := extraOpts.Get(EXTRA_OPTION_ENGINE_KEY) + isMySQLEngine := false + switch engine { + case EXTRA_OPTION_ENGINE_VALUE_MYSQL: + isMySQLEngine = true + } + colSpec := click.getColumnSpecByFieldTypeInternal(table, fieldType, fieldname, tagmap, isPointer) + if isMySQLEngine && colSpec.IsPrimary() { + colSpec.SetPrimary(false) + } + return colSpec +} + +func (click *SClickhouseBackend) getColumnSpecByFieldTypeInternal(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec { switch fieldType { case tristate.TriStateType: col := NewTristateColumn(table.Name(), fieldname, tagmap, isPointer) diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go index 112ec6e8f0..7b95c71ae0 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go @@ -23,4 +23,15 @@ const ( // TAG_TTL defines table TTL TAG_TTL = "clickhouse_ttl" + + EXTRA_OPTION_ENGINE_KEY = "clickhouse_engine" + EXTRA_OPTION_ENGINE_VALUE_MERGETRUE = "MergeTree" + EXTRA_OPTION_ENGINE_VALUE_MYSQL = "MySQL" + + // 'host:port', 'database', 'table', 'user', 'password' + EXTRA_OPTION_CLICKHOUSE_MYSQL_HOSTPORT_KEY = "clickhouse_mysql_hostport" + EXTRA_OPTION_CLICKHOUSE_MYSQL_DATABASE_KEY = "clickhouse_mysql_database" + EXTRA_OPTION_CLICKHOUSE_MYSQL_TABLE_KEY = "clickhouse_mysql_table" + EXTRA_OPTION_CLICKHOUSE_MYSQL_USERNAME_KEY = "clickhouse_mysql_username" + EXTRA_OPTION_CLICKHOUSE_MYSQL_PASSWORD_KEY = "clickhouse_mysql_password" ) diff --git a/vendor/yunion.io/x/sqlchemy/extraopts.go b/vendor/yunion.io/x/sqlchemy/extraopts.go new file mode 100644 index 0000000000..fb6318c1f7 --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/extraopts.go @@ -0,0 +1,47 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlchemy + +type TableExtraOptions map[string]string + +func (opts TableExtraOptions) Get(key string) string { + return opts[key] +} + +func (opts TableExtraOptions) Set(key string, val string) TableExtraOptions { + opts[key] = val + return opts +} + +func (opts TableExtraOptions) Contains(key string) bool { + if _, ok := opts[key]; ok { + return true + } + return false +} + +func (ts *STableSpec) GetExtraOptions() TableExtraOptions { + return ts.extraOptions +} + +func (ts *STableSpec) SetExtraOptions(opts TableExtraOptions) { + if ts.extraOptions == nil { + ts.extraOptions = opts + return + } + for k := range opts { + ts.extraOptions[k] = opts[k] + } +} diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 67421c46ea..aada193e1a 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -85,6 +85,12 @@ type ITableSpec interface { // Drop drops table Drop() error + + // getter of Extra Options + GetExtraOptions() TableExtraOptions + + // setter of Extra Options + SetExtraOptions(opts TableExtraOptions) } // STableSpec defines the table specification, which implements ITableSpec @@ -95,6 +101,8 @@ type STableSpec struct { _indexes []STableIndex _contraints []STableConstraint + extraOptions TableExtraOptions + sDBReferer } @@ -129,7 +137,18 @@ func NewTableSpecFromStructWithDBName(s interface{}, name string, dbName DBName) dbName: dbName, }, } - // table.struct2TableSpec(val) + return table +} + +func NewTableSpecFromISpecWithDBName(spec ITableSpec, name string, dbName DBName, extraOpts TableExtraOptions) *STableSpec { + table := &STableSpec{ + name: name, + structType: spec.DataType(), + sDBReferer: sDBReferer{ + dbName: dbName, + }, + extraOptions: extraOpts, + } return table }