fix: IModelManager add NewQuery to initialize query with params (#19482)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

go.mod

@@ -95,7 +95,7 @@ require (
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
 	yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b
+	yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
 )
 

go.sum

@@ -1220,7 +1220,7 @@ yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f h1:nLMBj8S3/ev2pB1wzGSi6hO
 yunion.io/x/pkg v1.10.1-0.20240127153242-cdf9dc071f4f/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b h1:W87Fe3H4wVm2CIrsVXNuypnO3yV7nhkiWozd2lDruKM=
-yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ=
+yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04 h1:30AHnjPBnE/bvyQu8px71/IP6mFsfNBnCYOsBVldIJc=
+yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/cloudcommon/db/db_dispatcher.go

@@ -682,11 +682,7 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		}
 		q = union.Query()
 	} else {
-		if useRawQuery {
-			q = manager.RawQuery()
-		} else {
-			q = manager.Query()
-		}
+		q = manager.NewQuery(ctx, userCred, queryDict, useRawQuery)
 	}
 
 	q, err = listItemQueryFiltersRaw(manager, ctx, q, userCred, queryDict, policy.PolicyActionList, true, useRawQuery)

pkg/cloudcommon/db/interface.go

@@ -69,9 +69,10 @@ type IModelManager interface {
 	// OrderByExtraFields dynmically called by dispatcher
 	// OrderByExtraFields(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error)
 
+	NewQuery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, useRawQuery bool) *sqlchemy.SQuery
 	// fetch hook
 	Query(val ...string) *sqlchemy.SQuery
-	RawQuery(val ...string) *sqlchemy.SQuery
+	// RawQuery(val ...string) *sqlchemy.SQuery
 
 	FilterById(q *sqlchemy.SQuery, idStr string) *sqlchemy.SQuery
 	FilterByNotId(q *sqlchemy.SQuery, idStr string) *sqlchemy.SQuery

pkg/cloudcommon/db/modelbase.go

@@ -234,6 +234,14 @@ func (manager *SModelBaseManager) ExtraSearchConditions(ctx context.Context, q *
 	return nil
 }
 
+func (manager *SModelBaseManager) NewQuery(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, useRawQuery bool) *sqlchemy.SQuery {
+	if useRawQuery {
+		return manager.Query()
+	} else {
+		return manager.GetIModelManager().Query()
+	}
+}
+
 // fetch hook
 func (manager *SModelBaseManager) getTable() *sqlchemy.STable {
 	return manager.TableSpec().Instance()

pkg/cloudcommon/db/standalone_anon.go

@@ -990,20 +990,24 @@ func GetTagValueCountMap(
 	query jsonutils.JSONObject,
 ) ([]map[string]string, error) {
 	var err error
-	objSubQ := manager.Query().SubQuery()
+	objQ := manager.Query()
+	objQ, err = ListItemQueryFilters(manager, ctx, objQ, userCred, query, policy.PolicyActionList)
+	if err != nil {
+		return nil, errors.Wrap(err, "ListItemQueryFilters")
+	}
+	objSubQ := objQ.SubQuery().Query()
+	objSubQ = objSubQ.AppendField(objSubQ.Field(tagIdField))
+	objSubQ = objSubQ.GroupBy(objSubQ.Field(tagIdField))
 	var sumFieldQ sqlchemy.IQueryField
 	if len(sumField) > 0 {
 		sumFieldQ = sqlchemy.SUM("_sub_count_", objSubQ.Field(sumField))
 	} else {
 		sumFieldQ = sqlchemy.COUNT("_sub_count_")
 	}
-	objQ := objSubQ.Query(objSubQ.Field(tagIdField), sumFieldQ)
-	objQ, err = ListItemQueryFilters(manager, ctx, objQ, userCred, query, policy.PolicyActionList)
-	if err != nil {
-		return nil, errors.Wrap(err, "ListItemQueryFilters")
-	}
-	objQ = objQ.GroupBy(objSubQ.Field(tagIdField))
-	q := objQ.SubQuery().Query(sqlchemy.SUM(tagValueCountKey, objQ.Field("_sub_count_")))
+	objSubQ = objSubQ.AppendField(sumFieldQ)
+
+	q := objSubQ.SubQuery().Query()
+	q = q.AppendField(sqlchemy.SUM(tagValueCountKey, objSubQ.Field("_sub_count_")))
 
 	metadataMan := GetMetadaManagerInContext(ctx)
 	metadataSQ := metadataMan.Query().Equals("obj_type", tagObjType).In("key", keys).SubQuery()
@@ -1021,6 +1025,7 @@ func GetTagValueCountMap(
 		groupBy = append(groupBy, q.Field(valueFieldName))
 	}
 	q = q.GroupBy(groupBy...)
+
 	valueMap, err := q.AllStringMap()
 	if err != nil {
 		return nil, errors.Wrap(err, "AllStringAmp")

pkg/compute/models/storages.go

@@ -816,6 +816,7 @@ func (self *SStorage) GetAttachedHosts() ([]SHost, error) {
 func (self *SStorage) SyncStatusWithHosts(ctx context.Context) {
 	hosts, err := self.GetAttachedHosts()
 	if err != nil {
+		log.Errorf("storage.SyncStatusWithHosts: GetAttachedHosts fail %s", err)
 		return
 	}
 	total := 0
@@ -848,6 +849,7 @@ func (self *SStorage) SyncStatusWithHosts(ctx context.Context) {
 		status = api.STORAGE_UNMOUNT
 	}
 	if status != self.Status {
+		log.Infof("Storage %s(%s) status %s expect %s online %d", self.Name, self.Id, self.Status, status, online)
 		self.SetStatus(ctx, nil, status, "SyncStatusWithHosts")
 	}
 }

vendor/modules.txt

@@ -1616,7 +1616,7 @@ yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 ## explicit; go 1.12
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v1.1.3-0.20240103021517-2be4fb49bd3b
+# yunion.io/x/sqlchemy v1.1.3-0.20240210082039-494595ecac04
 ## explicit; go 1.17
 yunion.io/x/sqlchemy
 yunion.io/x/sqlchemy/backends

vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go

@@ -71,8 +71,6 @@ func arrayContainsWord(strs []string, word string) bool {
 }
 
 func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes sqlchemy.STableChanges) []string {
-	ret := make([]string, 0)
-
 	needCopyTable := false
 
 	alters := make([]string, 0)
@@ -132,7 +130,10 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 
 	oldPartitions := findPartitions(changes.OldColumns)
 	for _, cols := range changes.UpdatedColumns {
-		if cols.OldCol.IsNullable() && !cols.NewCol.IsNullable() && arrayContainsWord(oldPartitions, cols.NewCol.Name()) {
+		if cols.OldCol.Name() != cols.NewCol.Name() {
+			sql := fmt.Sprintf("RENAME COLUMN %s TO %s", cols.OldCol.Name(), cols.NewCol.Name())
+			alters = append(alters, sql)
+		} else if cols.OldCol.IsNullable() && !cols.NewCol.IsNullable() && arrayContainsWord(oldPartitions, cols.NewCol.Name()) {
 			needCopyTable = true
 		} else {
 			sql := fmt.Sprintf("MODIFY COLUMN %s", cols.NewCol.DefinitionString())
@@ -172,11 +173,6 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 		}
 	}
 
-	if len(alters) > 0 {
-		sql := fmt.Sprintf("ALTER TABLE `%s` %s;", ts.Name(), strings.Join(alters, ", "))
-		ret = append(ret, sql)
-	}
-
 	// check partitions
 	newPartitions := findPartitions(ts.Columns())
 	if !sortedstring.Equals(oldPartitions, newPartitions) {
@@ -184,6 +180,8 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 		needCopyTable = true
 	}
 
+	ret := make([]string, 0)
+
 	// needCopyTable
 	if needCopyTable {
 		// create new table
@@ -204,6 +202,17 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe
 		ret = append(ret, sql)
 		sql = fmt.Sprintf("RENAME TABLE `%s` TO `%s`", alterTableName, ts.Name())
 		ret = append(ret, sql)
+	} else if len(alters) > 0 {
+		tableSpec := ts.(*sqlchemy.STableSpec)
+		if tableSpec.IsLinked {
+			// if the table is a linked table, simply re-create the table
+			ret = append(ret, fmt.Sprintf("DROP TABLE IF EXISTS `%s`", tableSpec.Name()))
+			createSqls := tableSpec.CreateSQLs()
+			ret = append(ret, createSqls...)
+		} else {
+			sql := fmt.Sprintf("ALTER TABLE `%s` %s;", ts.Name(), strings.Join(alters, ", "))
+			ret = append(ret, sql)
+		}
 	}
 
 	return ret

vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go

@@ -99,8 +99,13 @@ func (mysql *SMySQLBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes
 		}
 	}
 	for _, cols := range changes.UpdatedColumns {
-		sql := fmt.Sprintf("MODIFY COLUMN %s", cols.NewCol.DefinitionString())
-		alters = append(alters, sql)
+		if cols.OldCol.Name() != cols.NewCol.Name() {
+			sql := fmt.Sprintf("CHANGE COLUMN `%s` %s", cols.OldCol.Name(), cols.NewCol.DefinitionString())
+			alters = append(alters, sql)
+		} else {
+			sql := fmt.Sprintf("MODIFY COLUMN %s", cols.NewCol.DefinitionString())
+			alters = append(alters, sql)
+		}
 	}
 	for _, col := range changes.AddColumns {
 		sql := fmt.Sprintf("ADD COLUMN %s", col.DefinitionString())

vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go

@@ -90,16 +90,31 @@ func (sqlite *SSqliteBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, chang
 		newTableName := fmt.Sprintf("%s_tmp", ts.Name())
 		oldTableName := fmt.Sprintf("%s_old", ts.Name())
 		// create a table with alter name
-		var newTable *sqlchemy.STableSpec
-		newTable = ts.(*sqlchemy.STableSpec).Clone(newTableName, 0)
+		// var newTable *sqlchemy.STableSpec
+		newTable := ts.(*sqlchemy.STableSpec).Clone(newTableName, 0)
 		createSqls := newTable.CreateSQLs()
 		ret = append(ret, createSqls...)
 		// insert
-		colNames := make([]string, 0)
-		for _, col := range ts.Columns() {
-			colNames = append(colNames, fmt.Sprintf("`%s`", col.Name()))
+		colNameMap := make(map[string]string)
+
+		for _, cols := range changes.UpdatedColumns {
+			if cols.OldCol.Name() != cols.NewCol.Name() {
+				colNameMap[cols.NewCol.Name()] = cols.OldCol.Name()
+			}
+
 		}
-		sql := fmt.Sprintf("INSERT INTO `%s` SELECT %s FROM `%s`", newTableName, strings.Join(colNames, ", "), ts.Name())
+		colNames := make([]string, 0)
+		srcCols := make([]string, 0)
+		for _, col := range ts.Columns() {
+			colName := col.Name()
+			srcName := colName
+			if n, ok := colNameMap[colName]; ok {
+				srcName = n
+			}
+			colNames = append(colNames, fmt.Sprintf("`%s`", colName))
+			srcCols = append(srcCols, fmt.Sprintf("`%s`", srcName))
+		}
+		sql := fmt.Sprintf("INSERT INTO `%s`(%s) SELECT %s FROM `%s`", newTableName, strings.Join(colNames, ", "), strings.Join(srcCols, ", "), ts.Name())
 		ret = append(ret, sql)
 		// change name
 		sql = fmt.Sprintf("ALTER TABLE `%s` RENAME TO `%s`", ts.Name(), oldTableName)

vendor/yunion.io/x/sqlchemy/case.go

@@ -59,7 +59,7 @@ func (cf *SCaseFunction) expression() string {
 		buf.WriteString(cf.branches[i].thenField.Reference())
 	}
 	buf.WriteString(" ELSE ")
-	buf.WriteString(cf.elseField.Expression())
+	buf.WriteString(cf.elseField.Reference())
 	buf.WriteString(" END")
 	return buf.String()
 }

vendor/yunion.io/x/sqlchemy/column.go

@@ -127,10 +127,17 @@ type IColumnSpec interface {
 	SetColIndex(idx int)
 }
 
+type iColumnInternal interface {
+	IColumnSpec
+
+	Oldname() string
+}
+
 // SBaseColumn is the base structure represents a column
 type SBaseColumn struct {
 	name          string
 	dbName        string
+	oldName       string
 	sqlType       string
 	defaultString string
 	isPointer     bool
@@ -156,6 +163,11 @@ func (c *SBaseColumn) Name() string {
 	return c.name
 }
 
+// Name implementation of SBaseColumn for IColumnSpec
+func (c *SBaseColumn) Oldname() string {
+	return c.oldName
+}
+
 // ColType implementation of SBaseColumn for IColumnSpec
 func (c *SBaseColumn) ColType() string {
 	return c.sqlType
@@ -301,6 +313,11 @@ func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPoin
 	if ok {
 		dbName = val
 	}
+	oldName := ""
+	tagmap, val, ok = utils.TagPop(tagmap, TAG_OLD_NAME)
+	if ok {
+		oldName = val
+	}
 	defStr := ""
 	tagmap, val, ok = utils.TagPop(tagmap, TAG_DEFAULT)
 	if ok {
@@ -337,6 +354,7 @@ func NewBaseColumn(name string, sqltype string, tagmap map[string]string, isPoin
 	return SBaseColumn{
 		name:          name,
 		dbName:        dbName,
+		oldName:       oldName,
 		sqlType:       sqltype,
 		defaultString: defStr,
 		isNullable:    isNullable,

vendor/yunion.io/x/sqlchemy/const.go

@@ -81,4 +81,7 @@ const (
 	TAG_CREATE_TIMESTAMP = "created_at"
 	// TAG_ALLOW_ZERO is a field tag that indicates whether the column allow zero value
 	TAG_ALLOW_ZERO = "allow_zero"
+	// TAG_OLD_NAME is a field indicate the colume was renamed from an old name,
+	// sync table will do renaming of coolumn instead of creating a new column
+	TAG_OLD_NAME = "old_name"
 )

vendor/yunion.io/x/sqlchemy/debug.go

@@ -27,9 +27,12 @@ var (
 	DEBUG_SQLCHEMY = false
 )
 
-func sqlDebug(sqlstr string, variables []interface{}) {
+func sqlDebug(key, sqlstr string, variables []interface{}) {
 	sqlstr = _sqlDebug(sqlstr, variables)
-	log.Debugln("SQuery ", sqlstr)
+	if key == "" {
+		key = "SQUery"
+	}
+	log.Debugln(key, sqlstr)
 }
 
 func _sqlDebug(sqlstr string, variables []interface{}) string {
@@ -48,16 +51,30 @@ func _sqlDebug(sqlstr string, variables []interface{}) string {
 
 // DebugQuery show the full query string for debug
 func (tq *SQuery) DebugQuery() {
+	tq.DebugQuery2("")
+}
+
+// DebugQuery show the full query string for debug
+func (tq *SQuery) DebugQuery2(key string) {
 	sqlstr := tq.String()
 	vars := tq.Variables()
-	sqlDebug(sqlstr, vars)
+	sqlDebug(key, sqlstr, vars)
+}
+
+func (tq *SQuery) DebugString() string {
+	return _sqlDebug(tq.String(), tq.Variables())
+}
+
+// DebugQuery show the full query string for a subquery for debug
+func (sqf *SSubQuery) DebugQuery2(key string) {
+	sqlstr := sqf.Expression()
+	vars := sqf.query.Variables()
+	sqlDebug(key, sqlstr, vars)
 }
 
 // DebugQuery show the full query string for a subquery for debug
 func (sqf *SSubQuery) DebugQuery() {
-	sqlstr := sqf.Expression()
-	vars := sqf.query.Variables()
-	sqlDebug(sqlstr, vars)
+	sqf.DebugQuery2("")
 }
 
 // DebugInsert does insert with debug mode on

vendor/yunion.io/x/sqlchemy/query.go

@@ -63,43 +63,43 @@ type SQuery struct {
 	limit  int
 	offset int
 
-	fieldCache map[string]IQueryField
+	refFieldMap map[string]IQueryField
 
 	snapshot string
 
 	db *SDatabase
 }
 
-func (self *SQuery) Copy() *SQuery {
+func (tq *SQuery) Copy() *SQuery {
 	q := &SQuery{
-		rawSql:     self.rawSql,
-		fields:     []IQueryField{},
-		distinct:   self.distinct,
-		from:       self.from,
-		joins:      []sQueryJoin{},
-		where:      self.where,
-		groupBy:    []IQueryField{},
-		orderBy:    []sQueryOrder{},
-		limit:      self.limit,
-		offset:     self.offset,
-		fieldCache: map[string]IQueryField{},
-		snapshot:   self.snapshot,
-		db:         self.db,
+		rawSql:      tq.rawSql,
+		fields:      []IQueryField{},
+		refFieldMap: map[string]IQueryField{},
+		distinct:    tq.distinct,
+		from:        tq.from,
+		joins:       []sQueryJoin{},
+		where:       tq.where,
+		groupBy:     []IQueryField{},
+		orderBy:     []sQueryOrder{},
+		limit:       tq.limit,
+		offset:      tq.offset,
+		snapshot:    tq.snapshot,
+		db:          tq.db,
 	}
-	for i := range self.fields {
-		q.fields = append(q.fields, self.fields[i])
+	for i := range tq.fields {
+		q.fields = append(q.fields, tq.fields[i])
 	}
-	for i := range self.joins {
-		q.joins = append(q.joins, self.joins[i])
+	for k := range tq.refFieldMap {
+		q.refFieldMap[k] = tq.refFieldMap[k]
 	}
-	for i := range self.groupBy {
-		q.groupBy = append(q.groupBy, self.groupBy[i])
+	for i := range tq.joins {
+		q.joins = append(q.joins, tq.joins[i])
 	}
-	for i := range self.orderBy {
-		q.orderBy = append(q.orderBy, self.orderBy[i])
+	for i := range tq.groupBy {
+		q.groupBy = append(q.groupBy, tq.groupBy[i])
 	}
-	for k, field := range self.fieldCache {
-		q.fieldCache[k] = field
+	for i := range tq.orderBy {
+		q.orderBy = append(q.orderBy, tq.orderBy[i])
 	}
 	return q
 }
@@ -109,14 +109,12 @@ func (tq *SQuery) IsGroupBy() bool {
 	return len(tq.groupBy) > 0
 }
 
-func (tq *SQuery) HasField(f IQueryField) bool {
+func (tq *SQuery) hasField(f IQueryField) bool {
 	if len(tq.fields) == 0 {
 		return false
 	}
 	for i := range tq.fields {
-		fi := tq.fields[i]
-		// log.Debugf("field at %d: %s", i, fi.Name())
-		if fi.Name() == f.Name() {
+		if tq.fields[i].Name() == f.Name() {
 			return true
 		}
 	}
@@ -127,15 +125,33 @@ func (tq *SQuery) HasField(f IQueryField) bool {
 func (tq *SQuery) AppendField(f ...IQueryField) *SQuery {
 	// log.Debugf("AppendField tq has fields %d", len(tq.fields))
 	for i := range f {
-		if !tq.HasField(f[i]) {
-			tq.fields = append(tq.fields, f[i])
+		if !tq.hasField(f[i]) {
+			if refField, ok := tq.refFieldMap[f[i].Name()]; ok {
+				tq.fields = append(tq.fields, refField)
+				delete(tq.refFieldMap, f[i].Name())
+			} else {
+				tq.fields = append(tq.fields, f[i])
+			}
+		}
+	}
+	return tq
+}
+
+func (tq *SQuery) addRefField(f IQueryField) *SQuery {
+	if tq.refFieldMap == nil {
+		tq.refFieldMap = make(map[string]IQueryField)
+	}
+	if !tq.hasField(f) {
+		if _, ok := tq.refFieldMap[f.Name()]; !ok {
+			tq.refFieldMap[f.Name()] = f
 		}
 	}
 	return tq
 }
 
 func (tq *SQuery) ResetFields() *SQuery {
-	tq.fields = nil
+	tq.fields = make([]IQueryField, 0)
+	tq.refFieldMap = make(map[string]IQueryField)
 	return tq
 }
 
@@ -236,6 +252,10 @@ func (tq *SQuery) Offset(offset int) *SQuery {
 	return tq
 }
 
+func (tq *SQuery) FieldCount() int {
+	return len(tq.fields)
+}
+
 // QueryFields of SQuery returns fields in SELECT clause of a query
 func (tq *SQuery) QueryFields() []IQueryField {
 	if len(tq.fields) > 0 {
@@ -320,7 +340,11 @@ func (tq *SQuery) Distinct() *SQuery {
 
 // SubQuery of SQuery generates a SSubQuery from a Query
 func (tq *SQuery) SubQuery() *SSubQuery {
-	sq := SSubQuery{query: tq, alias: getTableAliasName()}
+	sq := SSubQuery{
+		query:         tq,
+		alias:         getTableAliasName(),
+		referedFields: make(map[string]IQueryField),
+	}
 	return &sq
 }
 
@@ -333,7 +357,7 @@ func (tq *SQuery) Row() *sql.Row {
 	sqlstr := tq.String()
 	vars := tq.Variables()
 	if DEBUG_SQLCHEMY {
-		sqlDebug(sqlstr, vars)
+		sqlDebug("SQuery.Row", sqlstr, vars)
 	}
 	if tq.db == nil {
 		panic("tq.db")
@@ -349,7 +373,7 @@ func (tq *SQuery) Rows() (*sql.Rows, error) {
 	sqlstr := tq.String()
 	vars := tq.Variables()
 	if DEBUG_SQLCHEMY {
-		sqlDebug(sqlstr, vars)
+		sqlDebug("SQuery.Rows", sqlstr, vars)
 	}
 	return tq.db.db.Query(sqlstr, vars...)
 }
@@ -391,27 +415,13 @@ func (tq *SQuery) CountWithError() (int, error) {
 // Field implementation of SQuery for IQuery
 func (tq *SQuery) Field(name string) IQueryField {
 	f := tq.findField(name)
-	if DEBUG_SQLCHEMY && f == nil {
-		log.Debugf("cannot find field %s for query", name)
+	if f == nil {
+		log.Errorf("SQuery %s cannot find Field %s", tq.String(), name)
 	}
 	return f
 }
 
 func (tq *SQuery) findField(name string) IQueryField {
-	if tq.fieldCache == nil {
-		tq.fieldCache = make(map[string]IQueryField)
-	}
-	if _, ok := tq.fieldCache[name]; ok {
-		return tq.fieldCache[name]
-	}
-	f := tq.internalFindField(name)
-	if f != nil {
-		tq.fieldCache[name] = f
-	}
-	return f
-}
-
-func (tq *SQuery) internalFindField(name string) IQueryField {
 	for _, f := range tq.fields {
 		if f.Name() == name {
 			// switch f.(type) {
@@ -421,25 +431,25 @@ func (tq *SQuery) internalFindField(name string) IQueryField {
 			return f
 		}
 	}
-	f := tq.from.Field(name)
-	if f != nil {
+	if f, ok := tq.refFieldMap[name]; ok {
 		return f
 	}
-	/* for _, f := range tq.from.Fields() {
-		if f.Name() == name {
-			return f
-		}
-	}*/
+	f := tq.from.Field(name)
+	if f != nil {
+		return newQueryField(tq.from, name)
+	}
+	finds := make([]IQueryField, 0)
 	for _, join := range tq.joins {
 		f = join.from.Field(name)
 		if f != nil {
-			return f
+			finds = append(finds, newQueryField(join.from, name))
 		}
-		/* for _, f := range join.from.Fields() {
-			if f.Name() == name {
-				return f
-			}
-		}*/
+	}
+	if len(finds) == 1 {
+		return finds[0]
+	} else if len(finds) > 1 {
+		log.Errorf("Field %s found duplicated %d, please specifify the field", name, len(finds))
+		return finds[0]
 	}
 	return nil
 }

vendor/yunion.io/x/sqlchemy/querydefs.go

@@ -17,6 +17,7 @@ package sqlchemy
 import (
 	"bytes"
 	"fmt"
+	"sort"
 )
 
 // IQuery is an interface that reprsents a SQL query, e.g.
@@ -106,16 +107,34 @@ func queryString(tq *SQuery, tmpFields ...IQueryField) string {
 	if tq.distinct {
 		buf.WriteString("DISTINCT ")
 	}
-	fields := tq.fields
+
+	fields := tmpFields
 	if len(fields) == 0 {
-		fields = tmpFields
-	}
-	if len(fields) == 0 {
-		fields = tq.QueryFields()
-		for i := range fields {
-			tq.from.Field(fields[i].Name())
+		fields = tq.fields
+		if len(fields) == 0 {
+			fields = tq.from.Fields()
+			for i := range fields {
+				tq.from.Field(fields[i].Name())
+			}
 		}
 	}
+	{
+		// add reference query fields
+		queryFields := make(map[string]IQueryField)
+		for i, f := range fields {
+			queryFields[f.Name()] = fields[i]
+		}
+		refFields := make([]IQueryField, 0)
+		for _, f := range tq.refFieldMap {
+			if _, ok := queryFields[f.Name()]; !ok {
+				queryFields[f.Name()] = f
+				refFields = append(refFields, f)
+			}
+		}
+		sort.Sort(queryFieldList(refFields))
+		fields = append(fields, refFields...)
+	}
+
 	for i := range fields {
 		if i > 0 {
 			buf.WriteString(", ")

vendor/yunion.io/x/sqlchemy/queryfield.go

@@ -0,0 +1,74 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package sqlchemy
+
+import "fmt"
+
+type sQueryField struct {
+	from  IQuerySource
+	name  string
+	alias string
+}
+
+// the string after select
+func (sqf *sQueryField) Expression() string {
+	alias := sqf.name
+	if len(sqf.alias) > 0 {
+		alias = sqf.alias
+	}
+	return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.from.Alias(), sqf.name, alias)
+}
+
+// the name of thie field
+func (sqf *sQueryField) Name() string {
+	if len(sqf.alias) > 0 {
+		return sqf.alias
+	}
+	return sqf.name
+}
+
+// the reference string in where clause
+func (sqf *sQueryField) Reference() string {
+	return fmt.Sprintf("`%s`.`%s`", sqf.from.Alias(), sqf.Name())
+}
+
+// give this field an alias name
+func (sqf *sQueryField) Label(label string) IQueryField {
+	sqf.alias = label
+	return sqf
+}
+
+// return variables
+func (sqf *sQueryField) Variables() []interface{} {
+	return nil
+}
+
+// Database returns the database of this IQuerySource
+func (sqf *sQueryField) database() *SDatabase {
+	return sqf.from.database()
+}
+
+func newQueryField(from IQuerySource, name string) *sQueryField {
+	return &sQueryField{
+		from: from,
+		name: name,
+	}
+}
+
+type queryFieldList []IQueryField
+
+func (a queryFieldList) Len() int           { return len(a) }
+func (a queryFieldList) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a queryFieldList) Less(i, j int) bool { return a[i].Name() < a[j].Name() }

vendor/yunion.io/x/sqlchemy/subquery.go

@@ -17,6 +17,8 @@ package sqlchemy
 import (
 	"fmt"
 	"sort"
+
+	"yunion.io/x/log"
 )
 
 // SSubQueryField represents a field of subquery, which implements IQueryField
@@ -28,10 +30,11 @@ type SSubQueryField struct {
 
 // Expression implementation of SSubQueryField for IQueryField
 func (sqf *SSubQueryField) Expression() string {
+	alias := sqf.field.Name()
 	if len(sqf.alias) > 0 {
-		return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.query.alias, sqf.field.Name(), sqf.alias)
+		alias = sqf.alias
 	}
-	return fmt.Sprintf("`%s`.`%s`", sqf.query.alias, sqf.field.Name())
+	return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.query.alias, sqf.field.Name(), alias)
 }
 
 // Name implementation of SSubQueryField for IQueryField
@@ -99,24 +102,31 @@ func (sq *SSubQuery) Variables() []interface{} {
 }
 
 func (sq *SSubQuery) findField(id string) IQueryField {
-	if sq.referedFields == nil {
-		sq.referedFields = make(map[string]IQueryField)
-	}
 	if _, ok := sq.referedFields[id]; ok {
 		return sq.referedFields[id]
 	}
-	queryFields := sq.query.QueryFields()
-	for i := range queryFields {
-		if queryFields[i].Name() == id {
-			sq.referedFields[id] = sq.query.Field(queryFields[i].Name())
-			return sq.referedFields[id]
+	f := sq.query.Field(id)
+	if f != nil {
+		sq.referedFields[id] = f
+		switch tq := sq.query.(type) {
+		case *SQuery:
+			tq.addRefField(f)
 		}
+		return f
 	}
 	return nil
 }
 
 // Field implementation of SSubQuery for IQuerySource
 func (sq *SSubQuery) Field(id string, alias ...string) IQueryField {
+	f := sq.field(id, alias...)
+	if f == nil {
+		log.Errorf("subquery %s AS %s cannot find field %s", sq.query.String(), sq.alias, id)
+	}
+	return f
+}
+
+func (sq *SSubQuery) field(id string, alias ...string) IQueryField {
 	f := sq.findField(id)
 	if f == nil {
 		return nil

vendor/yunion.io/x/sqlchemy/sync.go

@@ -89,6 +89,42 @@ func DiffCols(tableName string, cols1 []IColumnSpec, cols2 []IColumnSpec) ([]ICo
 			j++
 		}
 	}
+	for i := 0; i < len(add); {
+		intCol := add[i].(iColumnInternal)
+		if len(intCol.Oldname()) > 0 {
+			// find delete column
+			rmIdx := -1
+			for j := range remove {
+				if remove[j].Name() == intCol.Oldname() {
+					// remove from
+					rmIdx = j
+					break
+				}
+			}
+			if rmIdx >= 0 {
+				oldCol := remove[rmIdx]
+				{
+					// remove from remove
+					copy(remove[rmIdx:], remove[rmIdx+1:])
+					remove = remove[:len(remove)-1]
+				}
+				{
+					// remove from add
+					copy(add[i:], add[i+1:])
+					add = add[:len(add)-1]
+				}
+				{
+					update = append(update, SUpdateColumnSpec{
+						OldCol: oldCol,
+						NewCol: intCol,
+					})
+				}
+				// do not increase i
+				continue
+			}
+		}
+		i++
+	}
 	return remove, update, add
 }
 

vendor/yunion.io/x/sqlchemy/table.go

@@ -104,6 +104,8 @@ type STableSpec struct {
 	extraOptions TableExtraOptions
 
 	sDBReferer
+
+	IsLinked bool
 }
 
 // STable is an instance of table for query, system will automatically give a alias to this table
@@ -148,6 +150,7 @@ func NewTableSpecFromISpecWithDBName(spec ITableSpec, name string, dbName DBName
 			dbName: dbName,
 		},
 		extraOptions: extraOpts,
+		IsLinked:     true,
 	}
 	return table
 }
@@ -351,10 +354,11 @@ func (tbl *STable) Variables() []interface{} {
 
 // Expression implementation of STableField for IQueryField
 func (c *STableField) Expression() string {
+	alias := c.spec.Name()
 	if len(c.alias) > 0 {
-		return fmt.Sprintf("`%s`.`%s` as `%s`", c.table.Alias(), c.spec.Name(), c.alias)
+		alias = c.alias
 	}
-	return fmt.Sprintf("`%s`.`%s`", c.table.Alias(), c.spec.Name())
+	return fmt.Sprintf("`%s`.`%s` AS `%s`", c.table.Alias(), c.spec.Name(), alias)
 }
 
 // Name implementation of STableField for IQueryField

vendor/yunion.io/x/sqlchemy/union.go

@@ -32,10 +32,11 @@ type SUnionQueryField struct {
 
 // Expression implementation of SUnionQueryField for IQueryField
 func (sqf *SUnionQueryField) Expression() string {
+	alias := sqf.name
 	if len(sqf.alias) > 0 {
-		return fmt.Sprintf("`%s`.`%s` as `%s`", sqf.union.Alias(), sqf.name, sqf.alias)
+		alias = sqf.alias
 	}
-	return fmt.Sprintf("`%s`.`%s`", sqf.union.Alias(), sqf.name)
+	return fmt.Sprintf("`%s`.`%s` AS `%s`", sqf.union.Alias(), sqf.name, alias)
 }
 
 // Name implementation of SUnionQueryField for IQueryField

vendor/yunion.io/x/sqlchemy/update.go

@@ -222,8 +222,9 @@ func (us *SUpdateSession) SaveUpdateSql(dt interface{}) (*SUpdateSQLResult, erro
 	})
 
 	if DEBUG_SQLCHEMY {
-		log.Infof("Update: %s %s", updateSql, vars)
+		log.Infof("Update: %s", _sqlDebug(updateSql, vars))
 	}
+	log.Infof("Update: %s", _sqlDebug(updateSql, vars))
 
 	return &SUpdateSQLResult{
 		Sql:       updateSql,
@@ -268,7 +269,7 @@ func (ts *STableSpec) execUpdateSql(dt interface{}, result *SUpdateSQLResult) er
 	}
 	err = q.First(dt)
 	if err != nil {
-		return errors.Wrap(err, "query after update failed")
+		return errors.Wrapf(err, "query after update failed %s", q.DebugString())
 	}
 	return nil
 }