From 721ac4b9121752ee1f47401f10e2af8ade8af382 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Fri, 2 Nov 2018 17:40:00 +0800 Subject: [PATCH] optimize export list make dep --- Gopkg.lock | 4 +- pkg/cloudcommon/db/db_dispatcher.go | 38 +++++++--- pkg/cloudcommon/db/interface.go | 3 +- pkg/cloudcommon/db/modelbase.go | 12 ++-- pkg/compute/models/guests.go | 91 ++++++++++++++++++++++-- vendor/yunion.io/x/sqlchemy/functions.go | 5 ++ vendor/yunion.io/x/sqlchemy/query.go | 23 ++++-- 7 files changed, 147 insertions(+), 29 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 989987b1bc..df6d4af012 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1262,11 +1262,11 @@ [[projects]] branch = "master" - digest = "1:141fd68bff3a443a9e5ba7aa2bcde1da32d10e8cd9d84c34a58cbd3a09d13364" + digest = "1:66b13d3803b8ff8f26c3ca0422fb18ca0362dff62240e436b0aa0a921f833bb9" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "a3f19609eaa576cde3dff0e9099891f95f9aeec8" + revision = "7de55aae3ef9b019c47d21ff8fa4add1a9c80eb7" [[projects]] branch = "master" diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index b1fc9958ac..898f60673f 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -269,6 +269,12 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem if err != nil { return nil, err } + if query.Contains("export_keys") { + q, err = manager.ListItemExportKeys(ctx, q, userCred, query) + if err != nil { + return nil, err + } + } q, err = listItemsQueryByColumn(manager, q, userCred, query) if err != nil { return nil, err @@ -323,29 +329,41 @@ func query2List(manager IModelManager, ctx context.Context, userCred mcclient.To for rows.Next() { itemValue := reflect.Indirect(reflect.ValueOf(item)) itemValue.Set(itemInitValue) - err = q.Row2Struct(rows, item) - if err != nil { - return nil, err + extraData := jsonutils.NewDict() + if query.Contains("export_keys") { + RowMap, err := q.Row2Map(rows) + if err != nil { + return nil, err + } + extraKeys := manager.GetExportExtraKeys(ctx, query, RowMap) + if extraKeys != nil { + extraData.Update(extraKeys) + } + err = q.RowMap2Struct(RowMap, item) + if err != nil { + return nil, err + } + } else { + err = q.Row2Struct(rows, item) + if err != nil { + return nil, err + } } + jsonData := jsonutils.Marshal(item) jsonDict, ok := jsonData.(*jsonutils.JSONDict) if !ok { return nil, fmt.Errorf("invalid model data structure, not a dict") } jsonDict = jsonDict.CopyIncludes(listF...) - if showDetails { + jsonDict.Update(extraData) + if showDetails && !query.Contains("export_keys") { extraDict := item.GetCustomizeColumns(ctx, userCred, query) if extraDict != nil { jsonDict.Update(extraDict) } jsonDict = getModelExtraDetails(item, ctx, jsonDict) } - if query.Contains("export_keys") { - exportDict := item.GetExportItems(ctx, userCred, query) - if exportDict != nil { - jsonDict.Update(exportDict) - } - } results = append(results, jsonDict) } return results, nil diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index a4be1bca6c..51756e3fa2 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -30,6 +30,8 @@ type IModelManager interface { ValidateListConditions(ctx context.Context, userCred mcclient.TokenCredential, query *jsonutils.JSONDict) (*jsonutils.JSONDict, error) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition + GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict + ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) // fetch hook Query(val ...string) *sqlchemy.SQuery @@ -74,7 +76,6 @@ type IModel interface { // get hooks AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict - GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict // create hooks CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 3941a80abd..33f247546c 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -149,6 +149,14 @@ func (manager *SModelBaseManager) InitializeData() error { return nil } +func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + return q, nil +} + +func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { + return jsonutils.NewDict() +} + func (model *SModelBase) GetId() string { return "" } @@ -180,10 +188,6 @@ func (model *SModelBase) GetCustomizeColumns(ctx context.Context, userCred mccli return jsonutils.NewDict() } -func (model *SModelBase) GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { - return jsonutils.NewDict() -} - // get hooks func (model *SModelBase) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return false diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 81138d00c6..563da4848b 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1103,16 +1103,95 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token return self.moreExtraInfo(extra) } -func (self *SGuest) GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { +func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { exportKeys, _ := query.GetString("export_keys") keys := strings.Split(exportKeys, ",") - res := jsonutils.NewDict() + + // guest_id as filter key + if utils.IsInStringArray("ips", keys) { + guestIpsQuery := GuestnetworkManager.Query("guest_id").GroupBy("guest_id") + guestIpsQuery.AppendField(sqlchemy.GROUP_CONCAT("concat_ip_addr", guestIpsQuery.Field("ip_addr"))) + ipsSubQuery := guestIpsQuery.SubQuery() + guestIpsQuery.DebugQuery() + q.LeftJoin(ipsSubQuery, sqlchemy.Equals(q.Field("id"), ipsSubQuery.Field("guest_id"))) + q.AppendField(ipsSubQuery.Field("concat_ip_addr")) + } + if utils.IsInStringArray("disk", keys) { + guestDisksQuery := GuestdiskManager.Query("guest_id", "disk_id").GroupBy("guest_id") + diskQuery := DiskManager.Query("id", "disk_size").SubQuery() + guestDisksQuery.Join(diskQuery, sqlchemy.Equals(diskQuery.Field("id"), guestDisksQuery.Field("disk_id"))) + guestDisksQuery.AppendField(sqlchemy.SUM("disk_size", diskQuery.Field("disk_size"))) + guestDisksSubQuery := guestDisksQuery.SubQuery() + guestDisksSubQuery.DebugQuery() + q.LeftJoin(guestDisksSubQuery, sqlchemy.Equals(q.Field("id"), guestDisksSubQuery. + Field("guest_id"))) + q.AppendField(guestDisksSubQuery.Field("disk_size")) + } + if utils.IsInStringArray("eip", keys) { + eipsQuery := ElasticipManager.Query("associate_id", "ip_addr").Equals("associate_type", "server").GroupBy("associate_id") + eipsSubQuery := eipsQuery.SubQuery() + eipsSubQuery.DebugQuery() + q.LeftJoin(eipsSubQuery, sqlchemy.Equals(q.Field("id"), eipsSubQuery.Field("associate_id"))) + q.AppendField(eipsSubQuery.Field("ip_addr", "eip")) + } + + // host_id as filter key + if utils.IsInStringArray("region", keys) { + zoneQuery := ZoneManager.Query("id", "cloudregion_id").SubQuery() + hostQuery := HostManager.Query("id", "zone_id").GroupBy("id") + cloudregionQuery := CloudregionManager.Query("id", "name").SubQuery() + hostQuery.LeftJoin(zoneQuery, sqlchemy.Equals(hostQuery.Field("zone_id"), zoneQuery.Field("id"))). + LeftJoin(cloudregionQuery, sqlchemy.OR(sqlchemy.Equals(cloudregionQuery.Field("id"), + zoneQuery.Field("cloudregion_id")), sqlchemy.Equals(cloudregionQuery.Field("id"), "default"))) + hostQuery.AppendField(cloudregionQuery.Field("name", "region")) + hostSubQuery := hostQuery.SubQuery() + q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id"))) + q.AppendField(hostSubQuery.Field("region")) + } + if utils.IsInStringArray("manager", keys) { + hostQuery := HostManager.Query("id", "manager_id").GroupBy("id") + cloudProviderQuery := CloudproviderManager.Query("id", "name").SubQuery() + hostQuery.LeftJoin(cloudProviderQuery, sqlchemy.Equals(hostQuery.Field("manager_id"), + cloudProviderQuery.Field("id"))) + hostQuery.AppendField(cloudProviderQuery.Field("name", "manager")) + hostSubQuery := hostQuery.SubQuery() + q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id"))) + q.AppendField(hostSubQuery.Field("manager")) + } + return q, nil +} + +func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { + res := manager.SStatusStandaloneResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap) + exportKeys, _ := query.GetString("export_keys") + keys := strings.Split(exportKeys, ",") + if ips, ok := rowMap["concat_ip_addr"]; ok && len(ips) > 0 { + res.Set("ips", jsonutils.NewString(ips)) + } + if eip, ok := rowMap["eip"]; ok && len(eip) > 0 { + res.Set("eip", jsonutils.NewString(eip)) + } + if disk, ok := rowMap["disk_size"]; ok { + res.Set("disk", jsonutils.NewString(disk)) + } + if region, ok := rowMap["region"]; ok && len(region) > 0 { + res.Set("region", jsonutils.NewString(region)) + } + if manager, ok := rowMap["manager"]; ok && len(manager) > 0 { + res.Set("manager", jsonutils.NewString(manager)) + } + if utils.IsInStringArray("tenant", keys) { + if projectId, ok := rowMap["tenant_id"]; ok { + tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId) + if err == nil { + res.Set("tenant", jsonutils.NewString(tenant.GetName())) + } + } + } if utils.IsInStringArray("os_distribution", keys) { - osType := self.GetMetadata("os_distribution", userCred) - if len(osType) == 0 { - osType = self.OsType + if osType, ok := rowMap["os_type"]; ok { + res.Set("os_distribution", jsonutils.NewString(osType)) } - res.Set("os_distribution", jsonutils.NewString(osType)) } return res } diff --git a/vendor/yunion.io/x/sqlchemy/functions.go b/vendor/yunion.io/x/sqlchemy/functions.go index 2e97454bb7..c55920f836 100644 --- a/vendor/yunion.io/x/sqlchemy/functions.go +++ b/vendor/yunion.io/x/sqlchemy/functions.go @@ -57,3 +57,8 @@ func DISTINCT(name string, field IQueryField) IQueryField { ff := NewFunctionField(name, "DISTINCT(%s)", field) return &ff } + +func GROUP_CONCAT(name string, field IQueryField) IQueryField { + ff := NewFunctionField(name, "GROUP_CONCAT(%s)", field) + return &ff +} diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index d3ae2dfeca..b72b412856 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -146,6 +146,10 @@ func DoQuery(from IQuerySource, f ...IQueryField) *SQuery { return &tq } +func (q *SQuery) AppendField(f ...IQueryField) { + q.fields = append(q.fields, f...) +} + func (table *SSubQuery) Query(f ...IQueryField) *SQuery { return DoQuery(table, f...) } @@ -527,22 +531,29 @@ func (q *SQuery) All(dest interface{}) error { return err } -func (q *SQuery) Row2Struct(row IRowScanner, dest interface{}) error { - result, err := q.rowScan2StringMap(row) - if err != nil { - return err - } +func (q *SQuery) Row2Map(row IRowScanner) (map[string]string, error) { + return q.rowScan2StringMap(row) +} +func (q *SQuery) RowMap2Struct(result map[string]string, dest interface{}) error { destPtrValue := reflect.ValueOf(dest) if destPtrValue.Kind() != reflect.Ptr { return fmt.Errorf("input must be a pointer") } destValue := destPtrValue.Elem() - err = mapString2Struct(result, destValue) + err := mapString2Struct(result, destValue) if err != nil { return err } callAfterQuery(destPtrValue) return nil } + +func (q *SQuery) Row2Struct(row IRowScanner, dest interface{}) error { + result, err := q.rowScan2StringMap(row) + if err != nil { + return err + } + return q.RowMap2Struct(result, dest) +}