From e73ed9d0815dfda29aad39177a432e3d9e441a0a Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Thu, 29 Feb 2024 16:37:21 +0800 Subject: [PATCH] feature: dameng database support (#19570) Co-authored-by: Qiu Jian --- pkg/cloudcommon/database.go | 8 ++++++++ pkg/cloudcommon/db/modelbase.go | 2 +- pkg/cloudcommon/db/tablespec.go | 3 +++ pkg/compute/models/networkaddresses.go | 27 ++++++++++++++++---------- pkg/compute/models/server_skus.go | 8 ++++---- pkg/keystone/models/tokencache.go | 2 +- pkg/util/splitable/sync.go | 9 +++++++-- 7 files changed, 41 insertions(+), 18 deletions(-) diff --git a/pkg/cloudcommon/database.go b/pkg/cloudcommon/database.go index 791a4cc8b0..80b5f1e709 100644 --- a/pkg/cloudcommon/database.go +++ b/pkg/cloudcommon/database.go @@ -17,6 +17,7 @@ package cloudcommon import ( "context" "database/sql" + "strings" "time" "github.com/mattn/go-sqlite3" @@ -25,6 +26,7 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/sqlchemy" + _ "yunion.io/x/sqlchemy/backends" noapi "yunion.io/x/onecloud/pkg/apis/notify" "yunion.io/x/onecloud/pkg/cloudcommon/consts" @@ -43,6 +45,8 @@ func InitDB(options *common_options.DBOptions) { sqlchemy.DEBUG_SQLCHEMY = true } + log.Infof("Registered SQL drivers: %s", strings.Join(sql.Drivers(), ", ")) + consts.QueryOffsetOptimization = options.QueryOffsetOptimization if options.HistoricalUniqueName { @@ -64,6 +68,10 @@ func InitDB(options *common_options.DBOptions) { } backend := sqlchemy.MySQLBackend switch dialect { + case "dm": + backend = sqlchemy.DamengBackend + dialect = "dm" + sqlStr = "dm://" + sqlStr case "sqlite3": backend = sqlchemy.SQLiteBackend dialect = "sqlite3_with_extensions" diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index a2aa2c5d0f..54b1e886e5 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -580,7 +580,7 @@ func (manager *SModelBaseManager) CustomizedTotalCount(ctx context.Context, user ret := apis.TotalCountBase{} err := totalQ.First(&ret) if err != nil { - return -1, nil, errors.Wrap(err, "SModelBaseManager Query total") + return -1, nil, errors.Wrapf(err, "SModelBaseManager Query total %s", totalQ.DebugString()) } return ret.Count, nil, nil } diff --git a/pkg/cloudcommon/db/tablespec.go b/pkg/cloudcommon/db/tablespec.go index a3e7e062e5..d6587be9fb 100644 --- a/pkg/cloudcommon/db/tablespec.go +++ b/pkg/cloudcommon/db/tablespec.go @@ -17,6 +17,7 @@ package db import ( "context" "reflect" + "runtime/debug" "time" "yunion.io/x/jsonutils" @@ -277,6 +278,7 @@ func (ts *sTableSpec) inform(ctx context.Context, dt interface{}, f func(ctx con obj, err := ts.newInformerModel(dt) if err != nil { log.Warningf("newInformerModel error: %v", err) + debug.PrintStack() return } if err := f(ctx, obj); err != nil { @@ -298,6 +300,7 @@ func (ts *sTableSpec) informUpdate(ctx context.Context, dt interface{}, oldObj * obj, err := ts.newInformerModel(dt) if err != nil { log.Warningf("newInformerModel error: %v", err) + debug.PrintStack() return } if err := informer.Update(ctx, obj, oldObj); err != nil { diff --git a/pkg/compute/models/networkaddresses.go b/pkg/compute/models/networkaddresses.go index 8109dc99fa..b417b82034 100644 --- a/pkg/compute/models/networkaddresses.go +++ b/pkg/compute/models/networkaddresses.go @@ -517,12 +517,15 @@ func (man *SNetworkAddressManager) ListItemFilter(ctx context.Context, q *sqlche q, err = managedResourceFilterByAccount( ctx, q, input.ManagedResourceListInput, "network_id", func() *sqlchemy.SQuery { - networks := NetworkManager.Query().SubQuery() - wires := WireManager.Query().SubQuery() - vpcs := VpcManager.Query().SubQuery() - subq := networks.Query(networks.Field("id")) - subq = subq.Join(wires, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) - subq = subq.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) + networks := NetworkManager.Query() + wires := WireManager.Query("id", "vpc_id", "manager_id").SubQuery() + vpcs := VpcManager.Query("id", "manager_id").SubQuery() + networks = networks.Join(wires, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id"))) + networks = networks.Join(vpcs, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id"))) + networks = networks.AppendField(networks.Field("id")) + networks = networks.AppendField(sqlchemy.NewFunction(sqlchemy.NewCase().When(sqlchemy.IsNullOrEmpty(wires.Field("manager_id")), vpcs.Field("manager_id")).Else(wires.Field("manager_id")), "manager_id")) + subq := networks.SubQuery().Query() + subq = subq.AppendField(subq.Field("id")) return subq }) if err != nil { @@ -537,13 +540,17 @@ func (man *SNetworkAddressManager) OrderByExtraFields( q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query api.NetworkAddressListInput, -) (retq *sqlchemy.SQuery, err error) { - retq, err = man.SNetworkResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.NetworkFilterListInput) +) (*sqlchemy.SQuery, error) { + var err error + q, err = man.SNetworkResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.NetworkFilterListInput) if err != nil { return nil, err } - retq, err = man.SStandaloneAnonResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneAnonResourceListInput) - return retq, nil + q, err = man.SStandaloneAnonResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.StandaloneAnonResourceListInput) + if err != nil { + return nil, err + } + return q, nil } func (man *SNetworkAddressManager) FetchCustomizeColumns( diff --git a/pkg/compute/models/server_skus.go b/pkg/compute/models/server_skus.go index 59aed57fae..8e5a2ec25d 100644 --- a/pkg/compute/models/server_skus.go +++ b/pkg/compute/models/server_skus.go @@ -98,19 +98,19 @@ type SServerSku struct { SysDiskMinSizeGB int `nullable:"true" list:"user" create:"admin_optional" update:"admin"` // not required。 windows比较新的版本都是50G左右。 SysDiskMaxSizeGB int `nullable:"true" list:"user" create:"admin_optional" update:"admin"` // not required - AttachedDiskType string `nullable:"true" list:"user" create:"admin_optional" update:"admin"` + AttachedDiskType string `width:"32" nullable:"true" list:"user" create:"admin_optional" update:"admin"` AttachedDiskSizeGB int `nullable:"true" list:"user" create:"admin_optional" update:"admin"` AttachedDiskCount int `nullable:"true" list:"user" create:"admin_optional" update:"admin"` DataDiskTypes string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"` DataDiskMaxCount int `nullable:"true" list:"user" create:"admin_optional" update:"admin"` - NicType string `nullable:"true" list:"user" create:"admin_optional" update:"admin"` + NicType string `width:"32" nullable:"true" list:"user" create:"admin_optional" update:"admin"` NicMaxCount int `default:"1" nullable:"true" list:"user" create:"admin_optional" update:"admin"` GpuAttachable tristate.TriState `default:"true" list:"user" create:"admin_optional" update:"admin"` GpuSpec string `width:"128" charset:"ascii" nullable:"true" list:"user" create:"admin_optional" update:"admin"` - GpuCount string `nullable:"true" list:"user" create:"admin_optional" update:"admin"` + GpuCount string `width:"16" nullable:"true" list:"user" create:"admin_optional" update:"admin"` GpuMaxCount int `nullable:"true" list:"user" create:"admin_optional" update:"admin"` Provider string `width:"64" charset:"ascii" nullable:"true" list:"user" default:"OneCloud" create:"admin_optional"` @@ -556,7 +556,7 @@ func (manager *SServerSkuManager) GetPropertyInstanceSpecs(ctx context.Context, q = q.Asc(q.Field("cpu_core_count"), q.Field("memory_size_mb")) err = db.FetchModelObjects(manager, q, &skus) if err != nil { - log.Infof("FetchModelObjects %s", err) + log.Errorf("FetchModelObjects %s: %s", q.DebugString(), err) return nil, httperrors.NewBadRequestError("instance specs list query error") } diff --git a/pkg/keystone/models/tokencache.go b/pkg/keystone/models/tokencache.go index 9e5705550a..9cbb976766 100644 --- a/pkg/keystone/models/tokencache.go +++ b/pkg/keystone/models/tokencache.go @@ -158,7 +158,7 @@ func (manager *STokenCacheManager) FetchToken(tokenStr string) (*STokenCache, er } func (manager *STokenCacheManager) removeObsolete() error { - sql := fmt.Sprintf("DELETE FROM `%s` WHERE `expired_at` < ?", manager.TableSpec().Name()) + sql := fmt.Sprintf("DELETE FROM %s WHERE expired_at < ?", manager.TableSpec().Name()) db := sqlchemy.GetDBWithName(manager.TableSpec().GetDBName()) now := timeutils.UtcNow() _, err := db.Exec(sql, now.Add(-24*time.Hour)) diff --git a/pkg/util/splitable/sync.go b/pkg/util/splitable/sync.go index ee64feda2f..b3609390d9 100644 --- a/pkg/util/splitable/sync.go +++ b/pkg/util/splitable/sync.go @@ -133,8 +133,13 @@ func (spec *SSplitTableSpec) SyncSQL() []string { Start: indexCol.AutoIncrementOffset(), } // insert the first meta - sql := fmt.Sprintf("INSERT INTO `%s`(`table`, `deleted`, `created_at`) VALUES('%s', 0, '%s')", spec.metaSpec.Name(), meta.Table, timeutils.MysqlTime(now)) - sqls = append(sqls, sql) + insertResult, err := spec.metaSpec.InsertSqlPrep(&meta, false) + if err != nil { + log.Errorf("spec.metaSpec.InsertSqlPrep fail %s", err) + return nil + } + // sql := fmt.Sprintf("INSERT INTO `%s`(`table`, `deleted`, `created_at`) VALUES('%s', 0, '%s')", spec.metaSpec.Name(), meta.Table, timeutils.MysqlTime(now)) + sqls = append(sqls, sqlchemy.SQLPrintf(insertResult.Sql, insertResult.Values)) // create the first table newtable := spec.GetTableSpec(meta) nsql := newtable.SyncSQL()