fix: mapping mysql tables into clickhouse with Engine MySQL (#18877)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

pkg/cloudcommon/db/modelbase.go

@@ -28,12 +28,15 @@ import (
 	"yunion.io/x/pkg/util/rbacscope"
 	"yunion.io/x/pkg/util/version"
 	"yunion.io/x/sqlchemy"
+	"yunion.io/x/sqlchemy/backends/clickhouse"
 
 	"yunion.io/x/onecloud/pkg/apis"
 	"yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
 	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/dbutils"
 	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/onecloud/pkg/util/splitable"
 	"yunion.io/x/onecloud/pkg/util/stringutils2"
@@ -83,6 +86,29 @@ func NewModelBaseManagerWithSplitableDBName(model interface{}, tableName string,
 	return modelMan
 }
 
+func NewModelBaseManagerWithClickhouseMapping(manager IModelManager, tblname, keyword, keywordPlural string) SModelBaseManager {
+	ots := manager.TableSpec()
+	var extraOpts sqlchemy.TableExtraOptions
+	switch consts.DefaultDBDialect() {
+	case "mysql":
+		cfg := dbutils.ParseMySQLConnStr(consts.DefaultDBConnStr())
+		err := cfg.Validate()
+		if err != nil {
+			panic(fmt.Sprintf("invalid mysql connection string %s", consts.DefaultDBConnStr()))
+		}
+		extraOpts = clickhouse.MySQLExtraOptions(cfg.Hostport, cfg.Database, ots.Name(), cfg.Username, cfg.Password)
+	default:
+		panic(fmt.Sprintf("unsupport dialect %s to be backend of clickhouse", consts.DefaultDBDialect()))
+	}
+	nts := newClickhouseTableSpecFromMySQL(ots, tblname, ClickhouseDB, extraOpts)
+	modelMan := SModelBaseManager{
+		tableSpec:     nts,
+		keyword:       keyword,
+		keywordPlural: keywordPlural,
+	}
+	return modelMan
+}
+
 func (manager *SModelBaseManager) CreateByInsertOrUpdate() bool {
 	return true
 }

pkg/cloudcommon/db/models.go

@@ -103,6 +103,7 @@ func mustCheckModelManager(modelMan IModelManager) {
 
 func tableSpecId(tableSpec ITableSpec) string {
 	keys := []string{
+		string(tableSpec.GetDBName()),
 		tableSpec.Name(),
 	}
 	for _, c := range tableSpec.Columns() {
@@ -151,6 +152,7 @@ func CheckSync(autoSync bool, enableChecksumTables bool, skipInitChecksum bool)
 		tableSpec := modelMan.TableSpec()
 		tableKey := tableSpecId(tableSpec)
 		if _, ok := processedTbl[tableKey]; ok {
+			log.Warningf("table %s has been synced!", tableKey)
 			continue
 		}
 		processedTbl[tableKey] = true

pkg/cloudcommon/db/tablespec.go

@@ -82,6 +82,13 @@ func newTableSpec(model interface{}, tableName string, indexField string, dateFi
 	}
 }
 
+func newClickhouseTableSpecFromMySQL(spec ITableSpec, name string, dbName sqlchemy.DBName, extraOpts sqlchemy.TableExtraOptions) ITableSpec {
+	itbl := sqlchemy.NewTableSpecFromISpecWithDBName(spec.(*sTableSpec).ITableSpec, name, dbName, extraOpts)
+	return &sTableSpec{
+		ITableSpec: itbl,
+	}
+}
+
 func (ts *sTableSpec) GetSplitTable() *splitable.SSplitTableSpec {
 	sts, ok := ts.ITableSpec.(*splitable.SSplitTableSpec)
 	if ok {