fix(region): avoid cloudmon no cloudprovider sync

go.mod

@@ -87,14 +87,14 @@ require (
 	k8s.io/client-go v0.19.3
 	k8s.io/cluster-bootstrap v0.19.3
 	moul.io/http2curl/v2 v2.3.0
-	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e
+	yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7
 	yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32
 	yunion.io/x/jsonutils v1.0.1-0.20230613121553-0f3b41e2ef19
 	yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
 	yunion.io/x/pkg v1.0.1-0.20231101105448-abef64cdc142
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0
+	yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
 )
 

go.sum

@@ -1192,8 +1192,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e h1:QmYmnMJjVijGbLi5haZS49d2CaNgA/JLphLn7oL8q94=
-yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7 h1:fZ3sE1acOojrgXGtqLxXcfFhg4bT3shKx2o73J+Yat8=
+yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7/go.mod h1:aj1gR9PPb6eqqKOwvANe26CoZFY8ydmXy0fuvgKYXH0=
 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32 h1:v7POYkQwo1XzOxBoIoRVr/k0V9Y5JyjpshlIFa9raug=
 yunion.io/x/executor v0.0.0-20230705125604-c5ac3141db32/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
@@ -1211,7 +1211,7 @@ yunion.io/x/pkg v1.0.1-0.20231101105448-abef64cdc142 h1:L6LqxfP08eWUx+A6yQdrL6VB
 yunion.io/x/pkg v1.0.1-0.20231101105448-abef64cdc142/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0 h1:+MaykFV6YCakTLnHR3v31tovXhkvWgkBPFa83MuCekA=
-yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ=
+yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51 h1:XgvhXxYul4W85vVJVKCKOWw6Ea/YroI3TBxyLcQ7vx4=
+yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/cloudmon/resources/resources.go

@@ -86,11 +86,13 @@ func (self *SBaseResources) init(ctx context.Context) error {
 		"order_by.1":     "imported_at",
 		"order":          "asc",
 		"pending_delete": "all",
-		"filter.0":       "external_id.isnotempty()",
 	}
 	if self.manager.GetKeyword() == compute.Hosts.GetKeyword() { // private and vmware
 		query["cloud_env"] = "private_or_onpremise"
 	}
+	if self.manager.GetKeyword() != compute.Cloudproviders.GetKeyword() {
+		query["filter.0"] = "external_id.isnotempty()"
+	}
 	offset := 0
 	for {
 		query["offset"] = offset

vendor/modules.txt

@@ -1458,7 +1458,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.2.0
 ## explicit; go 1.12
 sigs.k8s.io/yaml
-# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204065558-c4edc0938f2e
+# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20231204130443-d54a8b16c7b7
 ## explicit; go 1.18
 yunion.io/x/cloudmux/pkg/apis
 yunion.io/x/cloudmux/pkg/apis/billing
@@ -1616,7 +1616,7 @@ yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 ## explicit; go 1.12
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v1.1.2-0.20231201052514-97026b18ccf0
+# yunion.io/x/sqlchemy v1.1.2-0.20231204175132-1eb294922a51
 ## explicit; go 1.17
 yunion.io/x/sqlchemy
 yunion.io/x/sqlchemy/backends

vendor/yunion.io/x/cloudmux/pkg/multicloud/apsara/apsara.go

@@ -262,6 +262,9 @@ func (self *SApsaraClient) getDefaultClient(regionId string) (*sdk.Client, error
 					return nil, errors.Wrapf(err, "ParseQuery(%s)", req.URL.RawQuery)
 				}
 				action := params.Get("OpenApiAction")
+				if len(action) == 0 {
+					action = params.Get("Action")
+				}
 				service := strings.ToLower(params.Get("Product"))
 				respCheck := func(resp *http.Response) error {
 					if self.cpcfg.UpdatePermission != nil {
@@ -288,7 +291,7 @@ func (self *SApsaraClient) getDefaultClient(regionId string) (*sdk.Client, error
 					}
 					return nil
 				}
-				if self.cpcfg.ReadOnly {
+				if self.cpcfg.ReadOnly && len(action) > 0 {
 					for _, prefix := range []string{"Get", "List", "Describe"} {
 						if strings.HasPrefix(action, prefix) {
 							return respCheck, nil

vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go

@@ -104,6 +104,17 @@ func (click *SClickhouseBackend) UpdateSQLTemplate() string {
 	return "ALTER TABLE `{{ .Table }}` UPDATE {{ .Columns }} WHERE {{ .Conditions }}"
 }
 
+func MySQLExtraOptions(hostport, database, table, user, passwd string) sqlchemy.TableExtraOptions {
+	return sqlchemy.TableExtraOptions{
+		EXTRA_OPTION_ENGINE_KEY:                    EXTRA_OPTION_ENGINE_VALUE_MYSQL,
+		EXTRA_OPTION_CLICKHOUSE_MYSQL_HOSTPORT_KEY: hostport,
+		EXTRA_OPTION_CLICKHOUSE_MYSQL_DATABASE_KEY: database,
+		EXTRA_OPTION_CLICKHOUSE_MYSQL_TABLE_KEY:    table,
+		EXTRA_OPTION_CLICKHOUSE_MYSQL_USERNAME_KEY: user,
+		EXTRA_OPTION_CLICKHOUSE_MYSQL_PASSWORD_KEY: passwd,
+	}
+}
+
 func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string {
 	cols := make([]string, 0)
 	primaries := make([]string, 0)
@@ -129,35 +140,51 @@ func (click *SClickhouseBackend) GetCreateSQLs(ts sqlchemy.ITableSpec) []string
 			}
 		}
 	}
-	createSql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE MergeTree", ts.Name(), strings.Join(cols, ",\n"))
-	if len(orderbys) == 0 {
-		orderbys = primaries
-	}
-	if len(partitions) > 0 {
-		createSql += fmt.Sprintf("\nPARTITION BY (%s)", strings.Join(partitions, ", "))
-	}
-	if len(primaries) > 0 {
-		createSql += fmt.Sprintf("\nPRIMARY KEY (%s)", strings.Join(primaries, ", "))
-		newOrderBys := make([]string, len(primaries))
-		copy(newOrderBys, primaries)
-		for _, f := range orderbys {
-			if !utils.IsInStringArray(f, newOrderBys) {
-				newOrderBys = append(newOrderBys, f)
-			}
+	createSql := fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` (\n%s\n) ENGINE = ", ts.Name(), strings.Join(cols, ",\n"))
+	extraOpts := ts.GetExtraOptions()
+	engine := extraOpts.Get(EXTRA_OPTION_ENGINE_KEY)
+	switch engine {
+	case EXTRA_OPTION_ENGINE_VALUE_MYSQL:
+		// mysql
+		createSql += fmt.Sprintf("MySQL('%s', '%s', '%s', '%s', '%s')",
+			extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_HOSTPORT_KEY),
+			extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_DATABASE_KEY),
+			extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_TABLE_KEY),
+			extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_USERNAME_KEY),
+			extraOpts.Get(EXTRA_OPTION_CLICKHOUSE_MYSQL_PASSWORD_KEY),
+		)
+	default:
+		// mergetree
+		createSql += "MergeTree()"
+		if len(orderbys) == 0 {
+			orderbys = primaries
 		}
-		orderbys = newOrderBys
+		if len(partitions) > 0 {
+			createSql += fmt.Sprintf("\nPARTITION BY (%s)", strings.Join(partitions, ", "))
+		}
+		if len(primaries) > 0 {
+			createSql += fmt.Sprintf("\nPRIMARY KEY (%s)", strings.Join(primaries, ", "))
+			newOrderBys := make([]string, len(primaries))
+			copy(newOrderBys, primaries)
+			for _, f := range orderbys {
+				if !utils.IsInStringArray(f, newOrderBys) {
+					newOrderBys = append(newOrderBys, f)
+				}
+			}
+			orderbys = newOrderBys
+		}
+		if len(orderbys) > 0 {
+			createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", "))
+		} else {
+			createSql += "\nORDER BY tuple()"
+		}
+		if ttlCol != nil {
+			ttlCount, ttlUnit := ttlCol.GetTTL()
+			createSql += fmt.Sprintf("\nTTL `%s` + INTERVAL %d %s", ttlCol.Name(), ttlCount, ttlUnit)
+		}
+		// set default time zone of table to UTC
+		createSql += "\nSETTINGS index_granularity=8192"
 	}
-	if len(orderbys) > 0 {
-		createSql += fmt.Sprintf("\nORDER BY (%s)", strings.Join(orderbys, ", "))
-	} else {
-		createSql += fmt.Sprintf("\nORDER BY tuple()")
-	}
-	if ttlCol != nil {
-		ttlCount, ttlUnit := ttlCol.GetTTL()
-		createSql += fmt.Sprintf("\nTTL `%s` + INTERVAL %d %s", ttlCol.Name(), ttlCount, ttlUnit)
-	}
-	// set default time zone of table to UTC
-	createSql += "\nSETTINGS index_granularity=8192"
 	return []string{
 		createSql,
 	}
@@ -216,6 +243,21 @@ func (click *SClickhouseBackend) FetchTableColumnSpecs(ts sqlchemy.ITableSpec) (
 }
 
 func (click *SClickhouseBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec {
+	extraOpts := table.GetExtraOptions()
+	engine := extraOpts.Get(EXTRA_OPTION_ENGINE_KEY)
+	isMySQLEngine := false
+	switch engine {
+	case EXTRA_OPTION_ENGINE_VALUE_MYSQL:
+		isMySQLEngine = true
+	}
+	colSpec := click.getColumnSpecByFieldTypeInternal(table, fieldType, fieldname, tagmap, isPointer)
+	if isMySQLEngine && colSpec.IsPrimary() {
+		colSpec.SetPrimary(false)
+	}
+	return colSpec
+}
+
+func (click *SClickhouseBackend) getColumnSpecByFieldTypeInternal(table *sqlchemy.STableSpec, fieldType reflect.Type, fieldname string, tagmap map[string]string, isPointer bool) sqlchemy.IColumnSpec {
 	switch fieldType {
 	case tristate.TriStateType:
 		col := NewTristateColumn(table.Name(), fieldname, tagmap, isPointer)

vendor/yunion.io/x/sqlchemy/backends/clickhouse/consts.go

@@ -23,4 +23,15 @@ const (
 
 	// TAG_TTL defines table TTL
 	TAG_TTL = "clickhouse_ttl"
+
+	EXTRA_OPTION_ENGINE_KEY             = "clickhouse_engine"
+	EXTRA_OPTION_ENGINE_VALUE_MERGETRUE = "MergeTree"
+	EXTRA_OPTION_ENGINE_VALUE_MYSQL     = "MySQL"
+
+	// 'host:port', 'database', 'table', 'user', 'password'
+	EXTRA_OPTION_CLICKHOUSE_MYSQL_HOSTPORT_KEY = "clickhouse_mysql_hostport"
+	EXTRA_OPTION_CLICKHOUSE_MYSQL_DATABASE_KEY = "clickhouse_mysql_database"
+	EXTRA_OPTION_CLICKHOUSE_MYSQL_TABLE_KEY    = "clickhouse_mysql_table"
+	EXTRA_OPTION_CLICKHOUSE_MYSQL_USERNAME_KEY = "clickhouse_mysql_username"
+	EXTRA_OPTION_CLICKHOUSE_MYSQL_PASSWORD_KEY = "clickhouse_mysql_password"
 )

vendor/yunion.io/x/sqlchemy/extraopts.go

@@ -0,0 +1,47 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlchemy
+
+type TableExtraOptions map[string]string
+
+func (opts TableExtraOptions) Get(key string) string {
+	return opts[key]
+}
+
+func (opts TableExtraOptions) Set(key string, val string) TableExtraOptions {
+	opts[key] = val
+	return opts
+}
+
+func (opts TableExtraOptions) Contains(key string) bool {
+	if _, ok := opts[key]; ok {
+		return true
+	}
+	return false
+}
+
+func (ts *STableSpec) GetExtraOptions() TableExtraOptions {
+	return ts.extraOptions
+}
+
+func (ts *STableSpec) SetExtraOptions(opts TableExtraOptions) {
+	if ts.extraOptions == nil {
+		ts.extraOptions = opts
+		return
+	}
+	for k := range opts {
+		ts.extraOptions[k] = opts[k]
+	}
+}

vendor/yunion.io/x/sqlchemy/table.go

@@ -85,6 +85,12 @@ type ITableSpec interface {
 
 	// Drop drops table
 	Drop() error
+
+	// getter of Extra Options
+	GetExtraOptions() TableExtraOptions
+
+	// setter of Extra Options
+	SetExtraOptions(opts TableExtraOptions)
 }
 
 // STableSpec defines the table specification, which implements ITableSpec
@@ -95,6 +101,8 @@ type STableSpec struct {
 	_indexes    []STableIndex
 	_contraints []STableConstraint
 
+	extraOptions TableExtraOptions
+
 	sDBReferer
 }
 
@@ -129,7 +137,18 @@ func NewTableSpecFromStructWithDBName(s interface{}, name string, dbName DBName)
 			dbName: dbName,
 		},
 	}
-	// table.struct2TableSpec(val)
+	return table
+}
+
+func NewTableSpecFromISpecWithDBName(spec ITableSpec, name string, dbName DBName, extraOpts TableExtraOptions) *STableSpec {
+	table := &STableSpec{
+		name:       name,
+		structType: spec.DataType(),
+		sDBReferer: sDBReferer{
+			dbName: dbName,
+		},
+		extraOptions: extraOpts,
+	}
 	return table
 }