From d0304058b22ae86be66b33c4347570c3caa3d7df Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Sun, 11 Feb 2024 00:26:16 +0800 Subject: [PATCH] fix: IModelManager add NewQuery to initialize query with params (#19482) Co-authored-by: Qiu Jian --- go.mod | 2 +- go.sum | 4 +- pkg/cloudcommon/db/db_dispatcher.go | 6 +- pkg/cloudcommon/db/interface.go | 3 +- pkg/cloudcommon/db/modelbase.go | 8 + pkg/cloudcommon/db/standalone_anon.go | 21 ++- pkg/compute/models/storages.go | 2 + vendor/modules.txt | 2 +- .../x/sqlchemy/backends/clickhouse/sync.go | 25 +++- .../x/sqlchemy/backends/mysql/sync.go | 9 +- .../x/sqlchemy/backends/sqlite/sync.go | 27 +++- vendor/yunion.io/x/sqlchemy/case.go | 2 +- vendor/yunion.io/x/sqlchemy/column.go | 18 +++ vendor/yunion.io/x/sqlchemy/const.go | 3 + vendor/yunion.io/x/sqlchemy/debug.go | 29 +++- vendor/yunion.io/x/sqlchemy/query.go | 138 ++++++++++-------- vendor/yunion.io/x/sqlchemy/querydefs.go | 33 ++++- vendor/yunion.io/x/sqlchemy/queryfield.go | 74 ++++++++++ vendor/yunion.io/x/sqlchemy/subquery.go | 30 ++-- vendor/yunion.io/x/sqlchemy/sync.go | 36 +++++ vendor/yunion.io/x/sqlchemy/table.go | 8 +- vendor/yunion.io/x/sqlchemy/union.go | 5 +- vendor/yunion.io/x/sqlchemy/update.go | 5 +- 23 files changed, 362 insertions(+), 128 deletions(-) create mode 100644 vendor/yunion.io/x/sqlchemy/queryfield.go diff --git a/go.mod b/go.mod index c108ee2653..b6ec1b5734 100644 --- a/go.mod +++ b/go.mod @@ -95,7 +95,7 @@ require ( yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b + yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c ) diff --git a/go.sum b/go.sum index 2557743efb..0296f8ad4d 100644 --- a/go.sum +++ b/go.sum @@ -1220,7 +1220,7 @@ yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f h1:nLMBj8S3/ev2pB1wzGSi6hO yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b h1:W87Fe3H4wVm2CIrsVXNuypnO3yV7nhkiWozd2lDruKM= -yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ= +yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04 h1:30AHnjPBnE/bvyQu8px71/IP6mFsfNBnCYOsBVldIJc= +yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ= yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 3de8831eb7..b783d363dd 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -682,11 +682,7 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok } q = union.Query() } else { - if useRawQuery { - q = manager.RawQuery() - } else { - q = manager.Query() - } + q = manager.NewQuery(ctx, userCred, queryDict, useRawQuery) } q, err = listItemQueryFiltersRaw(manager, ctx, q, userCred, queryDict, policy.PolicyActionList, true, useRawQuery) diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 62b9b52191..95dea9511e 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -69,9 +69,10 @@ type IModelManager interface { // OrderByExtraFields dynmically called by dispatcher // OrderByExtraFields(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) + NewQuery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, useRawQuery bool) *sqlchemy.SQuery // fetch hook Query(val ...string) *sqlchemy.SQuery - RawQuery(val ...string) *sqlchemy.SQuery + // RawQuery(val ...string) *sqlchemy.SQuery FilterById(q *sqlchemy.SQuery, idStr string) *sqlchemy.SQuery FilterByNotId(q *sqlchemy.SQuery, idStr string) *sqlchemy.SQuery diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 8e80ea40f7..a2aa2c5d0f 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -234,6 +234,14 @@ func (manager *SModelBaseManager) ExtraSearchConditions(ctx context.Context, q * return nil } +func (manager *SModelBaseManager) NewQuery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, useRawQuery bool) *sqlchemy.SQuery { + if useRawQuery { + return manager.Query() + } else { + return manager.GetIModelManager().Query() + } +} + // fetch hook func (manager *SModelBaseManager) getTable() *sqlchemy.STable { return manager.TableSpec().Instance() diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index 5139d3353b..d3f74bcd68 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -990,20 +990,24 @@ func GetTagValueCountMap( query jsonutils.JSONObject, ) ([]map[string]string, error) { var err error - objSubQ := manager.Query().SubQuery() + objQ := manager.Query() + objQ, err = ListItemQueryFilters(manager, ctx, objQ, userCred, query, policy.PolicyActionList) + if err != nil { + return nil, errors.Wrap(err, "ListItemQueryFilters") + } + objSubQ := objQ.SubQuery().Query() + objSubQ = objSubQ.AppendField(objSubQ.Field(tagIdField)) + objSubQ = objSubQ.GroupBy(objSubQ.Field(tagIdField)) var sumFieldQ sqlchemy.IQueryField if len(sumField) > 0 { sumFieldQ = sqlchemy.SUM("_sub_count_", objSubQ.Field(sumField)) } else { sumFieldQ = sqlchemy.COUNT("_sub_count_") } - objQ := objSubQ.Query(objSubQ.Field(tagIdField), sumFieldQ) - objQ, err = ListItemQueryFilters(manager, ctx, objQ, userCred, query, policy.PolicyActionList) - if err != nil { - return nil, errors.Wrap(err, "ListItemQueryFilters") - } - objQ = objQ.GroupBy(objSubQ.Field(tagIdField)) - q := objQ.SubQuery().Query(sqlchemy.SUM(tagValueCountKey, objQ.Field("_sub_count_"))) + objSubQ = objSubQ.AppendField(sumFieldQ) + + q := objSubQ.SubQuery().Query() + q = q.AppendField(sqlchemy.SUM(tagValueCountKey, objSubQ.Field("_sub_count_"))) metadataMan := GetMetadaManagerInContext(ctx) metadataSQ := metadataMan.Query().Equals("obj_type", tagObjType).In("key", keys).SubQuery() @@ -1021,6 +1025,7 @@ func GetTagValueCountMap( groupBy = append(groupBy, q.Field(valueFieldName)) } q = q.GroupBy(groupBy...) + valueMap, err := q.AllStringMap() if err != nil { return nil, errors.Wrap(err, "AllStringAmp") diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 32a45c5add..dd47b67fad 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -816,6 +816,7 @@ func (self *SStorage) GetAttachedHosts() ([]SHost, error) { func (self *SStorage) SyncStatusWithHosts(ctx context.Context) { hosts, err := self.GetAttachedHosts() if err != nil { + log.Errorf("storage.SyncStatusWithHosts: GetAttachedHosts fail %s", err) return } total := 0 @@ -848,6 +849,7 @@ func (self *SStorage) SyncStatusWithHosts(ctx context.Context) { status = api.STORAGE_UNMOUNT } if status != self.Status { + log.Infof("Storage %s(%s) status %s expect %s online %d", self.Name, self.Id, self.Status, status, online) self.SetStatus(ctx, nil, status, "SyncStatusWithHosts") } } diff --git a/vendor/modules.txt b/vendor/modules.txt index b577a6ff01..eece485c0d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1616,7 +1616,7 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b +# yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go index f265707f4b..3fe1f91328 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go @@ -71,8 +71,6 @@ func arrayContainsWord(strs []string, word string) bool { } func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes sqlchemy.STableChanges) []string { - ret := make([]string, 0) - needCopyTable := false alters := make([]string, 0) @@ -132,7 +130,10 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe oldPartitions := findPartitions(changes.OldColumns) for _, cols := range changes.UpdatedColumns { - if cols.OldCol.IsNullable() && !cols.NewCol.IsNullable() && arrayContainsWord(oldPartitions, cols.NewCol.Name()) { + if cols.OldCol.Name() != cols.NewCol.Name() { + sql := fmt.Sprintf("RENAME COLUMN %s TO %s", cols.OldCol.Name(), cols.NewCol.Name()) + alters = append(alters, sql) + } else if cols.OldCol.IsNullable() && !cols.NewCol.IsNullable() && arrayContainsWord(oldPartitions, cols.NewCol.Name()) { needCopyTable = true } else { sql := fmt.Sprintf("MODIFY COLUMN %s", cols.NewCol.DefinitionString()) @@ -172,11 +173,6 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe } } - if len(alters) > 0 { - sql := fmt.Sprintf("ALTER TABLE `%s` %s;", ts.Name(), strings.Join(alters, ", ")) - ret = append(ret, sql) - } - // check partitions newPartitions := findPartitions(ts.Columns()) if !sortedstring.Equals(oldPartitions, newPartitions) { @@ -184,6 +180,8 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe needCopyTable = true } + ret := make([]string, 0) + // needCopyTable if needCopyTable { // create new table @@ -204,6 +202,17 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe ret = append(ret, sql) sql = fmt.Sprintf("RENAME TABLE `%s` TO `%s`", alterTableName, ts.Name()) ret = append(ret, sql) + } else if len(alters) > 0 { + tableSpec := ts.(*sqlchemy.STableSpec) + if tableSpec.IsLinked { + // if the table is a linked table, simply re-create the table + ret = append(ret, fmt.Sprintf("DROP TABLE IF EXISTS `%s`", tableSpec.Name())) + createSqls := tableSpec.CreateSQLs() + ret = append(ret, createSqls...) + } else { + sql := fmt.Sprintf("ALTER TABLE `%s` %s;", ts.Name(), strings.Join(alters, ", ")) + ret = append(ret, sql) + } } return ret diff --git a/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go b/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go index 753c410ff4..495d7344a2 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go @@ -99,8 +99,13 @@ func (mysql *SMySQLBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes } } for _, cols := range changes.UpdatedColumns { - sql := fmt.Sprintf("MODIFY COLUMN %s", cols.NewCol.DefinitionString()) - alters = append(alters, sql) + if cols.OldCol.Name() != cols.NewCol.Name() { + sql := fmt.Sprintf("CHANGE COLUMN `%s` %s", cols.OldCol.Name(), cols.NewCol.DefinitionString()) + alters = append(alters, sql) + } else { + sql := fmt.Sprintf("MODIFY COLUMN %s", cols.NewCol.DefinitionString()) + alters = append(alters, sql) + } } for _, col := range changes.AddColumns { sql := fmt.Sprintf("ADD COLUMN %s", col.DefinitionString()) diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go index bf7be49d4b..52e946a390 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go @@ -90,16 +90,31 @@ func (sqlite *SSqliteBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, chang newTableName := fmt.Sprintf("%s_tmp", ts.Name()) oldTableName := fmt.Sprintf("%s_old", ts.Name()) // create a table with alter name - var newTable *sqlchemy.STableSpec - newTable = ts.(*sqlchemy.STableSpec).Clone(newTableName, 0) + // var newTable *sqlchemy.STableSpec + newTable := ts.(*sqlchemy.STableSpec).Clone(newTableName, 0) createSqls := newTable.CreateSQLs() ret = append(ret, createSqls...) // insert - colNames := make([]string, 0) - for _, col := range ts.Columns() { - colNames = append(colNames, fmt.Sprintf("`%s`", col.Name())) + colNameMap := make(map[string]string) + + for _, cols := range changes.UpdatedColumns { + if cols.OldCol.Name() != cols.NewCol.Name() { + colNameMap[cols.NewCol.Name()] = cols.OldCol.Name() + } + } - sql := fmt.Sprintf("INSERT INTO `%s` SELECT %s FROM `%s`", newTableName, strings.Join(colNames, ", "), ts.Name()) + colNames := make([]string, 0) + srcCols := make([]string, 0) + for _, col := range ts.Columns() { + colName := col.Name() + srcName := colName + if n, ok := colNameMap[colName]; ok { + srcName = n + } + colNames = append(colNames, fmt.Sprintf("`%s`", colName)) + srcCols = append(srcCols, fmt.Sprintf("`%s`", srcName)) + } + sql := fmt.Sprintf("INSERT INTO `%s`(%s) SELECT %s FROM `%s`", newTableName, strings.Join(colNames, ", "), strings.Join(srcCols, ", "), ts.Name()) ret = append(ret, sql) // change name sql = fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", ts.Name(), oldTableName) diff --git a/vendor/yunion.io/x/sqlchemy/case.go b/vendor/yunion.io/x/sqlchemy/case.go index d63b3d87b4..7d062c473f 100644 --- a/vendor/yunion.io/x/sqlchemy/case.go +++ b/vendor/yunion.io/x/sqlchemy/case.go @@ -59,7 +59,7 @@ func (cf *SCaseFunction) expression() string { buf.WriteString(cf.branches[i].thenField.Reference()) } buf.WriteString(" ELSE ") - buf.WriteString(cf.elseField.Expression()) + buf.WriteString(cf.elseField.Reference()) buf.WriteString(" END") return buf.String() } diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index bae5393653..6c787bf85d 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -127,10 +127,17 @@ type IColumnSpec interface { SetColIndex(idx int) } +type iColumnInternal interface { + IColumnSpec + + Oldname() string +} + // SBaseColumn is the base structure represents a column type SBaseColumn struct { name string dbName string + oldName string sqlType string defaultString string isPointer bool @@ -156,6 +163,11 @@ func (c *SBaseColumn) Name() string { return c.name } +// Name implementation of SBaseColumn for IColumnSpec +func (c *SBaseColumn) Oldname() string { + return c.oldName +} + // ColType implementation of SBaseColumn for IColumnSpec func (c *SBaseColumn) ColType() string { return c.sqlType @@ -301,6 +313,11 @@ func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPoin if ok { dbName = val } + oldName := "" + tagmap, val, ok = utils.TagPop(tagmap, TAG_OLD_NAME) + if ok { + oldName = val + } defStr := "" tagmap, val, ok = utils.TagPop(tagmap, TAG_DEFAULT) if ok { @@ -337,6 +354,7 @@ func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPoin return SBaseColumn{ name: name, dbName: dbName, + oldName: oldName, sqlType: sqltype, defaultString: defStr, isNullable: isNullable, diff --git a/vendor/yunion.io/x/sqlchemy/const.go b/vendor/yunion.io/x/sqlchemy/const.go index 09b01149bd..35d59eda2b 100644 --- a/vendor/yunion.io/x/sqlchemy/const.go +++ b/vendor/yunion.io/x/sqlchemy/const.go @@ -81,4 +81,7 @@ const ( TAG_CREATE_TIMESTAMP = "created_at" // TAG_ALLOW_ZERO is a field tag that indicates whether the column allow zero value TAG_ALLOW_ZERO = "allow_zero" + // TAG_OLD_NAME is a field indicate the colume was renamed from an old name, + // sync table will do renaming of coolumn instead of creating a new column + TAG_OLD_NAME = "old_name" ) diff --git a/vendor/yunion.io/x/sqlchemy/debug.go b/vendor/yunion.io/x/sqlchemy/debug.go index c9d159c839..d84bce4220 100644 --- a/vendor/yunion.io/x/sqlchemy/debug.go +++ b/vendor/yunion.io/x/sqlchemy/debug.go @@ -27,9 +27,12 @@ var ( DEBUG_SQLCHEMY = false ) -func sqlDebug(sqlstr string, variables []interface{}) { +func sqlDebug(key, sqlstr string, variables []interface{}) { sqlstr = _sqlDebug(sqlstr, variables) - log.Debugln("SQuery ", sqlstr) + if key == "" { + key = "SQUery" + } + log.Debugln(key, sqlstr) } func _sqlDebug(sqlstr string, variables []interface{}) string { @@ -48,16 +51,30 @@ func _sqlDebug(sqlstr string, variables []interface{}) string { // DebugQuery show the full query string for debug func (tq *SQuery) DebugQuery() { + tq.DebugQuery2("") +} + +// DebugQuery show the full query string for debug +func (tq *SQuery) DebugQuery2(key string) { sqlstr := tq.String() vars := tq.Variables() - sqlDebug(sqlstr, vars) + sqlDebug(key, sqlstr, vars) +} + +func (tq *SQuery) DebugString() string { + return _sqlDebug(tq.String(), tq.Variables()) +} + +// DebugQuery show the full query string for a subquery for debug +func (sqf *SSubQuery) DebugQuery2(key string) { + sqlstr := sqf.Expression() + vars := sqf.query.Variables() + sqlDebug(key, sqlstr, vars) } // DebugQuery show the full query string for a subquery for debug func (sqf *SSubQuery) DebugQuery() { - sqlstr := sqf.Expression() - vars := sqf.query.Variables() - sqlDebug(sqlstr, vars) + sqf.DebugQuery2("") } // DebugInsert does insert with debug mode on diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 9d189d190e..9ea7083985 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -63,43 +63,43 @@ type SQuery struct { limit int offset int - fieldCache map[string]IQueryField + refFieldMap map[string]IQueryField snapshot string db *SDatabase } -func (self *SQuery) Copy() *SQuery { +func (tq *SQuery) Copy() *SQuery { q := &SQuery{ - rawSql: self.rawSql, - fields: []IQueryField{}, - distinct: self.distinct, - from: self.from, - joins: []sQueryJoin{}, - where: self.where, - groupBy: []IQueryField{}, - orderBy: []sQueryOrder{}, - limit: self.limit, - offset: self.offset, - fieldCache: map[string]IQueryField{}, - snapshot: self.snapshot, - db: self.db, + rawSql: tq.rawSql, + fields: []IQueryField{}, + refFieldMap: map[string]IQueryField{}, + distinct: tq.distinct, + from: tq.from, + joins: []sQueryJoin{}, + where: tq.where, + groupBy: []IQueryField{}, + orderBy: []sQueryOrder{}, + limit: tq.limit, + offset: tq.offset, + snapshot: tq.snapshot, + db: tq.db, } - for i := range self.fields { - q.fields = append(q.fields, self.fields[i]) + for i := range tq.fields { + q.fields = append(q.fields, tq.fields[i]) } - for i := range self.joins { - q.joins = append(q.joins, self.joins[i]) + for k := range tq.refFieldMap { + q.refFieldMap[k] = tq.refFieldMap[k] } - for i := range self.groupBy { - q.groupBy = append(q.groupBy, self.groupBy[i]) + for i := range tq.joins { + q.joins = append(q.joins, tq.joins[i]) } - for i := range self.orderBy { - q.orderBy = append(q.orderBy, self.orderBy[i]) + for i := range tq.groupBy { + q.groupBy = append(q.groupBy, tq.groupBy[i]) } - for k, field := range self.fieldCache { - q.fieldCache[k] = field + for i := range tq.orderBy { + q.orderBy = append(q.orderBy, tq.orderBy[i]) } return q } @@ -109,14 +109,12 @@ func (tq *SQuery) IsGroupBy() bool { return len(tq.groupBy) > 0 } -func (tq *SQuery) HasField(f IQueryField) bool { +func (tq *SQuery) hasField(f IQueryField) bool { if len(tq.fields) == 0 { return false } for i := range tq.fields { - fi := tq.fields[i] - // log.Debugf("field at %d: %s", i, fi.Name()) - if fi.Name() == f.Name() { + if tq.fields[i].Name() == f.Name() { return true } } @@ -127,15 +125,33 @@ func (tq *SQuery) HasField(f IQueryField) bool { func (tq *SQuery) AppendField(f ...IQueryField) *SQuery { // log.Debugf("AppendField tq has fields %d", len(tq.fields)) for i := range f { - if !tq.HasField(f[i]) { - tq.fields = append(tq.fields, f[i]) + if !tq.hasField(f[i]) { + if refField, ok := tq.refFieldMap[f[i].Name()]; ok { + tq.fields = append(tq.fields, refField) + delete(tq.refFieldMap, f[i].Name()) + } else { + tq.fields = append(tq.fields, f[i]) + } + } + } + return tq +} + +func (tq *SQuery) addRefField(f IQueryField) *SQuery { + if tq.refFieldMap == nil { + tq.refFieldMap = make(map[string]IQueryField) + } + if !tq.hasField(f) { + if _, ok := tq.refFieldMap[f.Name()]; !ok { + tq.refFieldMap[f.Name()] = f } } return tq } func (tq *SQuery) ResetFields() *SQuery { - tq.fields = nil + tq.fields = make([]IQueryField, 0) + tq.refFieldMap = make(map[string]IQueryField) return tq } @@ -236,6 +252,10 @@ func (tq *SQuery) Offset(offset int) *SQuery { return tq } +func (tq *SQuery) FieldCount() int { + return len(tq.fields) +} + // QueryFields of SQuery returns fields in SELECT clause of a query func (tq *SQuery) QueryFields() []IQueryField { if len(tq.fields) > 0 { @@ -320,7 +340,11 @@ func (tq *SQuery) Distinct() *SQuery { // SubQuery of SQuery generates a SSubQuery from a Query func (tq *SQuery) SubQuery() *SSubQuery { - sq := SSubQuery{query: tq, alias: getTableAliasName()} + sq := SSubQuery{ + query: tq, + alias: getTableAliasName(), + referedFields: make(map[string]IQueryField), + } return &sq } @@ -333,7 +357,7 @@ func (tq *SQuery) Row() *sql.Row { sqlstr := tq.String() vars := tq.Variables() if DEBUG_SQLCHEMY { - sqlDebug(sqlstr, vars) + sqlDebug("SQuery.Row", sqlstr, vars) } if tq.db == nil { panic("tq.db") @@ -349,7 +373,7 @@ func (tq *SQuery) Rows() (*sql.Rows, error) { sqlstr := tq.String() vars := tq.Variables() if DEBUG_SQLCHEMY { - sqlDebug(sqlstr, vars) + sqlDebug("SQuery.Rows", sqlstr, vars) } return tq.db.db.Query(sqlstr, vars...) } @@ -391,27 +415,13 @@ func (tq *SQuery) CountWithError() (int, error) { // Field implementation of SQuery for IQuery func (tq *SQuery) Field(name string) IQueryField { f := tq.findField(name) - if DEBUG_SQLCHEMY && f == nil { - log.Debugf("cannot find field %s for query", name) + if f == nil { + log.Errorf("SQuery %s cannot find Field %s", tq.String(), name) } return f } func (tq *SQuery) findField(name string) IQueryField { - if tq.fieldCache == nil { - tq.fieldCache = make(map[string]IQueryField) - } - if _, ok := tq.fieldCache[name]; ok { - return tq.fieldCache[name] - } - f := tq.internalFindField(name) - if f != nil { - tq.fieldCache[name] = f - } - return f -} - -func (tq *SQuery) internalFindField(name string) IQueryField { for _, f := range tq.fields { if f.Name() == name { // switch f.(type) { @@ -421,25 +431,25 @@ func (tq *SQuery) internalFindField(name string) IQueryField { return f } } - f := tq.from.Field(name) - if f != nil { + if f, ok := tq.refFieldMap[name]; ok { return f } - /* for _, f := range tq.from.Fields() { - if f.Name() == name { - return f - } - }*/ + f := tq.from.Field(name) + if f != nil { + return newQueryField(tq.from, name) + } + finds := make([]IQueryField, 0) for _, join := range tq.joins { f = join.from.Field(name) if f != nil { - return f + finds = append(finds, newQueryField(join.from, name)) } - /* for _, f := range join.from.Fields() { - if f.Name() == name { - return f - } - }*/ + } + if len(finds) == 1 { + return finds[0] + } else if len(finds) > 1 { + log.Errorf("Field %s found duplicated %d, please specifify the field", name, len(finds)) + return finds[0] } return nil } diff --git a/vendor/yunion.io/x/sqlchemy/querydefs.go b/vendor/yunion.io/x/sqlchemy/querydefs.go index 3308f45f4a..530ddc160e 100644 --- a/vendor/yunion.io/x/sqlchemy/querydefs.go +++ b/vendor/yunion.io/x/sqlchemy/querydefs.go @@ -17,6 +17,7 @@ package sqlchemy import ( "bytes" "fmt" + "sort" ) // IQuery is an interface that reprsents a SQL query, e.g. @@ -106,16 +107,34 @@ func queryString(tq *SQuery, tmpFields ...IQueryField) string { if tq.distinct { buf.WriteString("DISTINCT ") } - fields := tq.fields + + fields := tmpFields if len(fields) == 0 { - fields = tmpFields - } - if len(fields) == 0 { - fields = tq.QueryFields() - for i := range fields { - tq.from.Field(fields[i].Name()) + fields = tq.fields + if len(fields) == 0 { + fields = tq.from.Fields() + for i := range fields { + tq.from.Field(fields[i].Name()) + } } } + { + // add reference query fields + queryFields := make(map[string]IQueryField) + for i, f := range fields { + queryFields[f.Name()] = fields[i] + } + refFields := make([]IQueryField, 0) + for _, f := range tq.refFieldMap { + if _, ok := queryFields[f.Name()]; !ok { + queryFields[f.Name()] = f + refFields = append(refFields, f) + } + } + sort.Sort(queryFieldList(refFields)) + fields = append(fields, refFields...) + } + for i := range fields { if i > 0 { buf.WriteString(", ") diff --git a/vendor/yunion.io/x/sqlchemy/queryfield.go b/vendor/yunion.io/x/sqlchemy/queryfield.go new file mode 100644 index 0000000000..d62c29ec1b --- /dev/null +++ b/vendor/yunion.io/x/sqlchemy/queryfield.go @@ -0,0 +1,74 @@ +// Copyright 2019 Yunion +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package sqlchemy + +import "fmt" + +type sQueryField struct { + from IQuerySource + name string + alias string +} + +// the string after select +func (sqf *sQueryField) Expression() string { + alias := sqf.name + if len(sqf.alias) > 0 { + alias = sqf.alias + } + return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.from.Alias(), sqf.name, alias) +} + +// the name of thie field +func (sqf *sQueryField) Name() string { + if len(sqf.alias) > 0 { + return sqf.alias + } + return sqf.name +} + +// the reference string in where clause +func (sqf *sQueryField) Reference() string { + return fmt.Sprintf("`%s`.`%s`", sqf.from.Alias(), sqf.Name()) +} + +// give this field an alias name +func (sqf *sQueryField) Label(label string) IQueryField { + sqf.alias = label + return sqf +} + +// return variables +func (sqf *sQueryField) Variables() []interface{} { + return nil +} + +// Database returns the database of this IQuerySource +func (sqf *sQueryField) database() *SDatabase { + return sqf.from.database() +} + +func newQueryField(from IQuerySource, name string) *sQueryField { + return &sQueryField{ + from: from, + name: name, + } +} + +type queryFieldList []IQueryField + +func (a queryFieldList) Len() int { return len(a) } +func (a queryFieldList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a queryFieldList) Less(i, j int) bool { return a[i].Name() < a[j].Name() } diff --git a/vendor/yunion.io/x/sqlchemy/subquery.go b/vendor/yunion.io/x/sqlchemy/subquery.go index d786fc475c..69051f5c4c 100644 --- a/vendor/yunion.io/x/sqlchemy/subquery.go +++ b/vendor/yunion.io/x/sqlchemy/subquery.go @@ -17,6 +17,8 @@ package sqlchemy import ( "fmt" "sort" + + "yunion.io/x/log" ) // SSubQueryField represents a field of subquery, which implements IQueryField @@ -28,10 +30,11 @@ type SSubQueryField struct { // Expression implementation of SSubQueryField for IQueryField func (sqf *SSubQueryField) Expression() string { + alias := sqf.field.Name() if len(sqf.alias) > 0 { - return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.query.alias, sqf.field.Name(), sqf.alias) + alias = sqf.alias } - return fmt.Sprintf("`%s`.`%s`", sqf.query.alias, sqf.field.Name()) + return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.query.alias, sqf.field.Name(), alias) } // Name implementation of SSubQueryField for IQueryField @@ -99,24 +102,31 @@ func (sq *SSubQuery) Variables() []interface{} { } func (sq *SSubQuery) findField(id string) IQueryField { - if sq.referedFields == nil { - sq.referedFields = make(map[string]IQueryField) - } if _, ok := sq.referedFields[id]; ok { return sq.referedFields[id] } - queryFields := sq.query.QueryFields() - for i := range queryFields { - if queryFields[i].Name() == id { - sq.referedFields[id] = sq.query.Field(queryFields[i].Name()) - return sq.referedFields[id] + f := sq.query.Field(id) + if f != nil { + sq.referedFields[id] = f + switch tq := sq.query.(type) { + case *SQuery: + tq.addRefField(f) } + return f } return nil } // Field implementation of SSubQuery for IQuerySource func (sq *SSubQuery) Field(id string, alias ...string) IQueryField { + f := sq.field(id, alias...) + if f == nil { + log.Errorf("subquery %s AS %s cannot find field %s", sq.query.String(), sq.alias, id) + } + return f +} + +func (sq *SSubQuery) field(id string, alias ...string) IQueryField { f := sq.findField(id) if f == nil { return nil diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index dfad2cf3f5..a18b81f550 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -89,6 +89,42 @@ func DiffCols(tableName string, cols1 []IColumnSpec, cols2 []IColumnSpec) ([]ICo j++ } } + for i := 0; i < len(add); { + intCol := add[i].(iColumnInternal) + if len(intCol.Oldname()) > 0 { + // find delete column + rmIdx := -1 + for j := range remove { + if remove[j].Name() == intCol.Oldname() { + // remove from + rmIdx = j + break + } + } + if rmIdx >= 0 { + oldCol := remove[rmIdx] + { + // remove from remove + copy(remove[rmIdx:], remove[rmIdx+1:]) + remove = remove[:len(remove)-1] + } + { + // remove from add + copy(add[i:], add[i+1:]) + add = add[:len(add)-1] + } + { + update = append(update, SUpdateColumnSpec{ + OldCol: oldCol, + NewCol: intCol, + }) + } + // do not increase i + continue + } + } + i++ + } return remove, update, add } diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index aada193e1a..dbfaded3b2 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -104,6 +104,8 @@ type STableSpec struct { extraOptions TableExtraOptions sDBReferer + + IsLinked bool } // STable is an instance of table for query, system will automatically give a alias to this table @@ -148,6 +150,7 @@ func NewTableSpecFromISpecWithDBName(spec ITableSpec, name string, dbName DBName dbName: dbName, }, extraOptions: extraOpts, + IsLinked: true, } return table } @@ -351,10 +354,11 @@ func (tbl *STable) Variables() []interface{} { // Expression implementation of STableField for IQueryField func (c *STableField) Expression() string { + alias := c.spec.Name() if len(c.alias) > 0 { - return fmt.Sprintf("`%s`.`%s` as `%s`", c.table.Alias(), c.spec.Name(), c.alias) + alias = c.alias } - return fmt.Sprintf("`%s`.`%s`", c.table.Alias(), c.spec.Name()) + return fmt.Sprintf("`%s`.`%s` AS `%s`", c.table.Alias(), c.spec.Name(), alias) } // Name implementation of STableField for IQueryField diff --git a/vendor/yunion.io/x/sqlchemy/union.go b/vendor/yunion.io/x/sqlchemy/union.go index 0c31e4f8ca..905ba57997 100644 --- a/vendor/yunion.io/x/sqlchemy/union.go +++ b/vendor/yunion.io/x/sqlchemy/union.go @@ -32,10 +32,11 @@ type SUnionQueryField struct { // Expression implementation of SUnionQueryField for IQueryField func (sqf *SUnionQueryField) Expression() string { + alias := sqf.name if len(sqf.alias) > 0 { - return fmt.Sprintf("`%s`.`%s` as `%s`", sqf.union.Alias(), sqf.name, sqf.alias) + alias = sqf.alias } - return fmt.Sprintf("`%s`.`%s`", sqf.union.Alias(), sqf.name) + return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.union.Alias(), sqf.name, alias) } // Name implementation of SUnionQueryField for IQueryField diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index 9e2b0e9803..6dac487556 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -222,8 +222,9 @@ func (us *SUpdateSession) SaveUpdateSql(dt interface{}) (*SUpdateSQLResult, erro }) if DEBUG_SQLCHEMY { - log.Infof("Update: %s %s", updateSql, vars) + log.Infof("Update: %s", _sqlDebug(updateSql, vars)) } + log.Infof("Update: %s", _sqlDebug(updateSql, vars)) return &SUpdateSQLResult{ Sql: updateSql, @@ -268,7 +269,7 @@ func (ts *STableSpec) execUpdateSql(dt interface{}, result *SUpdateSQLResult) er } err = q.First(dt) if err != nil { - return errors.Wrap(err, "query after update failed") + return errors.Wrapf(err, "query after update failed %s", q.DebugString()) } return nil }