diff --git a/Gopkg.lock b/Gopkg.lock index cca49bc99d..a27afdf85c 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -1236,11 +1236,7 @@ [[projects]] branch = "master" -<<<<<<< HEAD - digest = "1:2dd5cfa0e15bc1351229c0be3b09b0b7fe47d23e99d19edfba5b2402ddb1cf7b" -======= - digest = "1:71658437b1f5aca7d253eca0d622301d06ed943f32ca2f5da7b5dd1dac8bfcb4" ->>>>>>> c845dda83df2ddff6d7d384354ed3d214b64b27a + digest = "1:bf987f904fd821c6395fefdab956d070596eb401657fff40420014a3a8e3f45c" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -1274,27 +1270,15 @@ "utils", ] pruneopts = "UT" -<<<<<<< HEAD - revision = "194153d51e1c1bd8a3b9545ee0bc1504ddcdcb4d" + revision = "7614d751299a6703a05f757a13e6dc900332e31b" [[projects]] branch = "master" - digest = "1:d8daebdff1c94f0acac6952014812ac02fc10e1894004dd2efd8377a2517ba06" + digest = "1:66b13d3803b8ff8f26c3ca0422fb18ca0362dff62240e436b0aa0a921f833bb9" name = "yunion.io/x/sqlchemy" packages = ["."] pruneopts = "UT" - revision = "95a4726da196a27611ca7df4e08f16ece628f708" -======= - revision = "526a04d2eb834022e20100d991d1c84b78d87c07" - -[[projects]] - branch = "master" - digest = "1:141fd68bff3a443a9e5ba7aa2bcde1da32d10e8cd9d84c34a58cbd3a09d13364" - name = "yunion.io/x/sqlchemy" - packages = ["."] - pruneopts = "UT" - revision = "a3f19609eaa576cde3dff0e9099891f95f9aeec8" ->>>>>>> c845dda83df2ddff6d7d384354ed3d214b64b27a + revision = "7de55aae3ef9b019c47d21ff8fa4add1a9c80eb7" [[projects]] branch = "master" @@ -1302,7 +1286,7 @@ name = "yunion.io/x/structarg" packages = ["."] pruneopts = "UT" - revision = "ba8620a6258308b3faa4174d4c2d81a6f70b57ea" + revision = "e0cc2c73375327d401b100bd8988fa340c2687ee" [solve-meta] analyzer-name = "dep" @@ -1342,6 +1326,7 @@ "github.com/coredns/coredns/plugin/nsid", "github.com/coredns/coredns/plugin/pkg/dnsutil", "github.com/coredns/coredns/plugin/pkg/fall", + "github.com/coredns/coredns/plugin/pkg/log", "github.com/coredns/coredns/plugin/pkg/upstream", "github.com/coredns/coredns/plugin/proxy", "github.com/coredns/coredns/plugin/reload", diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 69a9813c35..6ca19c3b74 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -275,6 +275,12 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem if err != nil { return nil, err } + if query.Contains("export_keys") { + q, err = manager.ListItemExportKeys(ctx, q, userCred, query) + if err != nil { + return nil, err + } + } q, err = listItemsQueryByColumn(manager, q, userCred, query) if err != nil { return nil, err @@ -329,29 +335,41 @@ func query2List(manager IModelManager, ctx context.Context, userCred mcclient.To for rows.Next() { itemValue := reflect.Indirect(reflect.ValueOf(item)) itemValue.Set(itemInitValue) - err = q.Row2Struct(rows, item) - if err != nil { - return nil, err + extraData := jsonutils.NewDict() + if query.Contains("export_keys") { + RowMap, err := q.Row2Map(rows) + if err != nil { + return nil, err + } + extraKeys := manager.GetExportExtraKeys(ctx, query, RowMap) + if extraKeys != nil { + extraData.Update(extraKeys) + } + err = q.RowMap2Struct(RowMap, item) + if err != nil { + return nil, err + } + } else { + err = q.Row2Struct(rows, item) + if err != nil { + return nil, err + } } + jsonData := jsonutils.Marshal(item) jsonDict, ok := jsonData.(*jsonutils.JSONDict) if !ok { return nil, fmt.Errorf("invalid model data structure, not a dict") } jsonDict = jsonDict.CopyIncludes(listF...) - if showDetails { + jsonDict.Update(extraData) + if showDetails && !query.Contains("export_keys") { extraDict := item.GetCustomizeColumns(ctx, userCred, query) if extraDict != nil { jsonDict.Update(extraDict) } jsonDict = getModelExtraDetails(item, ctx, jsonDict) } - if query.Contains("export_keys") { - exportDict := item.GetExportItems(ctx, userCred, query) - if exportDict != nil { - jsonDict.Update(exportDict) - } - } results = append(results, jsonDict) } return results, nil diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 0f21d9592f..e63c2ac02e 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -31,6 +31,8 @@ type IModelManager interface { ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*CustomizeListFilters, error) ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition + GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict + ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) // fetch hook Query(val ...string) *sqlchemy.SQuery @@ -78,7 +80,6 @@ type IModel interface { // get hooks AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict - GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict // create hooks CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 5182e42470..d4c9a704a9 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -162,6 +162,14 @@ func (manager *SModelBaseManager) DoCreate(ctx context.Context, userCred mcclien return nil, fmt.Errorf("Do create not implement?") } +func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { + return q, nil +} + +func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { + return jsonutils.NewDict() +} + func (model *SModelBase) GetId() string { return "" } @@ -193,10 +201,6 @@ func (model *SModelBase) GetCustomizeColumns(ctx context.Context, userCred mccli return jsonutils.NewDict() } -func (model *SModelBase) GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { - return jsonutils.NewDict() -} - // get hooks func (model *SModelBase) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { return false diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 226703f551..937000249f 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -1137,16 +1137,95 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token return self.moreExtraInfo(extra) } -func (self *SGuest) GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict { +func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) { exportKeys, _ := query.GetString("export_keys") keys := strings.Split(exportKeys, ",") - res := jsonutils.NewDict() + + // guest_id as filter key + if utils.IsInStringArray("ips", keys) { + guestIpsQuery := GuestnetworkManager.Query("guest_id").GroupBy("guest_id") + guestIpsQuery.AppendField(sqlchemy.GROUP_CONCAT("concat_ip_addr", guestIpsQuery.Field("ip_addr"))) + ipsSubQuery := guestIpsQuery.SubQuery() + guestIpsQuery.DebugQuery() + q.LeftJoin(ipsSubQuery, sqlchemy.Equals(q.Field("id"), ipsSubQuery.Field("guest_id"))) + q.AppendField(ipsSubQuery.Field("concat_ip_addr")) + } + if utils.IsInStringArray("disk", keys) { + guestDisksQuery := GuestdiskManager.Query("guest_id", "disk_id").GroupBy("guest_id") + diskQuery := DiskManager.Query("id", "disk_size").SubQuery() + guestDisksQuery.Join(diskQuery, sqlchemy.Equals(diskQuery.Field("id"), guestDisksQuery.Field("disk_id"))) + guestDisksQuery.AppendField(sqlchemy.SUM("disk_size", diskQuery.Field("disk_size"))) + guestDisksSubQuery := guestDisksQuery.SubQuery() + guestDisksSubQuery.DebugQuery() + q.LeftJoin(guestDisksSubQuery, sqlchemy.Equals(q.Field("id"), guestDisksSubQuery. + Field("guest_id"))) + q.AppendField(guestDisksSubQuery.Field("disk_size")) + } + if utils.IsInStringArray("eip", keys) { + eipsQuery := ElasticipManager.Query("associate_id", "ip_addr").Equals("associate_type", "server").GroupBy("associate_id") + eipsSubQuery := eipsQuery.SubQuery() + eipsSubQuery.DebugQuery() + q.LeftJoin(eipsSubQuery, sqlchemy.Equals(q.Field("id"), eipsSubQuery.Field("associate_id"))) + q.AppendField(eipsSubQuery.Field("ip_addr", "eip")) + } + + // host_id as filter key + if utils.IsInStringArray("region", keys) { + zoneQuery := ZoneManager.Query("id", "cloudregion_id").SubQuery() + hostQuery := HostManager.Query("id", "zone_id").GroupBy("id") + cloudregionQuery := CloudregionManager.Query("id", "name").SubQuery() + hostQuery.LeftJoin(zoneQuery, sqlchemy.Equals(hostQuery.Field("zone_id"), zoneQuery.Field("id"))). + LeftJoin(cloudregionQuery, sqlchemy.OR(sqlchemy.Equals(cloudregionQuery.Field("id"), + zoneQuery.Field("cloudregion_id")), sqlchemy.Equals(cloudregionQuery.Field("id"), "default"))) + hostQuery.AppendField(cloudregionQuery.Field("name", "region")) + hostSubQuery := hostQuery.SubQuery() + q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id"))) + q.AppendField(hostSubQuery.Field("region")) + } + if utils.IsInStringArray("manager", keys) { + hostQuery := HostManager.Query("id", "manager_id").GroupBy("id") + cloudProviderQuery := CloudproviderManager.Query("id", "name").SubQuery() + hostQuery.LeftJoin(cloudProviderQuery, sqlchemy.Equals(hostQuery.Field("manager_id"), + cloudProviderQuery.Field("id"))) + hostQuery.AppendField(cloudProviderQuery.Field("name", "manager")) + hostSubQuery := hostQuery.SubQuery() + q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id"))) + q.AppendField(hostSubQuery.Field("manager")) + } + return q, nil +} + +func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict { + res := manager.SStatusStandaloneResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap) + exportKeys, _ := query.GetString("export_keys") + keys := strings.Split(exportKeys, ",") + if ips, ok := rowMap["concat_ip_addr"]; ok && len(ips) > 0 { + res.Set("ips", jsonutils.NewString(ips)) + } + if eip, ok := rowMap["eip"]; ok && len(eip) > 0 { + res.Set("eip", jsonutils.NewString(eip)) + } + if disk, ok := rowMap["disk_size"]; ok { + res.Set("disk", jsonutils.NewString(disk)) + } + if region, ok := rowMap["region"]; ok && len(region) > 0 { + res.Set("region", jsonutils.NewString(region)) + } + if manager, ok := rowMap["manager"]; ok && len(manager) > 0 { + res.Set("manager", jsonutils.NewString(manager)) + } + if utils.IsInStringArray("tenant", keys) { + if projectId, ok := rowMap["tenant_id"]; ok { + tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId) + if err == nil { + res.Set("tenant", jsonutils.NewString(tenant.GetName())) + } + } + } if utils.IsInStringArray("os_distribution", keys) { - osType := self.GetMetadata("os_distribution", userCred) - if len(osType) == 0 { - osType = self.OsType + if osType, ok := rowMap["os_type"]; ok { + res.Set("os_distribution", jsonutils.NewString(osType)) } - res.Set("os_distribution", jsonutils.NewString(osType)) } return res } diff --git a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go index c88c6caa91..3f9de2fe1b 100644 --- a/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go +++ b/vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go @@ -20,12 +20,29 @@ const ( STORAGE_STANDARD_RAGRS = "standard_ragrs" STORAGE_STANDARD_ZRS = "standard_zrs" STORAGE_PREMIUM_LRS = "premium_lrs" + + // aws storage type + STORAGE_GP2_SSD = "gp2" // aws general purpose ssd + STORAGE_IO1_SSD = "io1" // aws Provisioned IOPS SSD + STORAGE_ST1_HDD = "st1" // aws Throughput Optimized HDD + STORAGE_SC1_SSD = "sc1" // aws Cold HDD + STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes + + // qcloud storage type + // STORAGE_CLOUD_SSD ="cloud_ssd" + STORAGE_LOCAL_BASIC = "local_basic" + STORAGE_LOCAL_SSD = "local_ssd" + STORAGE_CLOUD_BASIC = "cloud_basic" + STORAGE_CLOUD_PERMIUM = "cloud_permium" ) var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG, STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN, STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY, - STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS} + STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS, + STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD, + STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM, +} var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL} diff --git a/vendor/yunion.io/x/sqlchemy/functions.go b/vendor/yunion.io/x/sqlchemy/functions.go index 2e97454bb7..c55920f836 100644 --- a/vendor/yunion.io/x/sqlchemy/functions.go +++ b/vendor/yunion.io/x/sqlchemy/functions.go @@ -57,3 +57,8 @@ func DISTINCT(name string, field IQueryField) IQueryField { ff := NewFunctionField(name, "DISTINCT(%s)", field) return &ff } + +func GROUP_CONCAT(name string, field IQueryField) IQueryField { + ff := NewFunctionField(name, "GROUP_CONCAT(%s)", field) + return &ff +} diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index d3ae2dfeca..b72b412856 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -146,6 +146,10 @@ func DoQuery(from IQuerySource, f ...IQueryField) *SQuery { return &tq } +func (q *SQuery) AppendField(f ...IQueryField) { + q.fields = append(q.fields, f...) +} + func (table *SSubQuery) Query(f ...IQueryField) *SQuery { return DoQuery(table, f...) } @@ -527,22 +531,29 @@ func (q *SQuery) All(dest interface{}) error { return err } -func (q *SQuery) Row2Struct(row IRowScanner, dest interface{}) error { - result, err := q.rowScan2StringMap(row) - if err != nil { - return err - } +func (q *SQuery) Row2Map(row IRowScanner) (map[string]string, error) { + return q.rowScan2StringMap(row) +} +func (q *SQuery) RowMap2Struct(result map[string]string, dest interface{}) error { destPtrValue := reflect.ValueOf(dest) if destPtrValue.Kind() != reflect.Ptr { return fmt.Errorf("input must be a pointer") } destValue := destPtrValue.Elem() - err = mapString2Struct(result, destValue) + err := mapString2Struct(result, destValue) if err != nil { return err } callAfterQuery(destPtrValue) return nil } + +func (q *SQuery) Row2Struct(row IRowScanner, dest interface{}) error { + result, err := q.rowScan2StringMap(row) + if err != nil { + return err + } + return q.RowMap2Struct(result, dest) +}