Automatic merge from release/2.3.0 -> release/2.4.0

* commit 'f98c3275baf2daf3393a8443f4c6e972213f8d56':
  optimize export list make dep

Gopkg.lock

@@ -1236,11 +1236,7 @@
 
 [[projects]]
   branch = "master"
-<<<<<<< HEAD
-  digest = "1:2dd5cfa0e15bc1351229c0be3b09b0b7fe47d23e99d19edfba5b2402ddb1cf7b"
-=======
-  digest = "1:71658437b1f5aca7d253eca0d622301d06ed943f32ca2f5da7b5dd1dac8bfcb4"
->>>>>>> c845dda83df2ddff6d7d384354ed3d214b64b27a
+  digest = "1:bf987f904fd821c6395fefdab956d070596eb401657fff40420014a3a8e3f45c"
   name = "yunion.io/x/pkg"
   packages = [
     "gotypes",
@@ -1274,27 +1270,15 @@
     "utils",
   ]
   pruneopts = "UT"
-<<<<<<< HEAD
-  revision = "194153d51e1c1bd8a3b9545ee0bc1504ddcdcb4d"
+  revision = "7614d751299a6703a05f757a13e6dc900332e31b"
 
 [[projects]]
   branch = "master"
-  digest = "1:d8daebdff1c94f0acac6952014812ac02fc10e1894004dd2efd8377a2517ba06"
+  digest = "1:66b13d3803b8ff8f26c3ca0422fb18ca0362dff62240e436b0aa0a921f833bb9"
   name = "yunion.io/x/sqlchemy"
   packages = ["."]
   pruneopts = "UT"
-  revision = "95a4726da196a27611ca7df4e08f16ece628f708"
-=======
-  revision = "526a04d2eb834022e20100d991d1c84b78d87c07"
-
-[[projects]]
-  branch = "master"
-  digest = "1:141fd68bff3a443a9e5ba7aa2bcde1da32d10e8cd9d84c34a58cbd3a09d13364"
-  name = "yunion.io/x/sqlchemy"
-  packages = ["."]
-  pruneopts = "UT"
-  revision = "a3f19609eaa576cde3dff0e9099891f95f9aeec8"
->>>>>>> c845dda83df2ddff6d7d384354ed3d214b64b27a
+  revision = "7de55aae3ef9b019c47d21ff8fa4add1a9c80eb7"
 
 [[projects]]
   branch = "master"
@@ -1302,7 +1286,7 @@
   name = "yunion.io/x/structarg"
   packages = ["."]
   pruneopts = "UT"
-  revision = "ba8620a6258308b3faa4174d4c2d81a6f70b57ea"
+  revision = "e0cc2c73375327d401b100bd8988fa340c2687ee"
 
 [solve-meta]
   analyzer-name = "dep"
@@ -1342,6 +1326,7 @@
     "github.com/coredns/coredns/plugin/nsid",
     "github.com/coredns/coredns/plugin/pkg/dnsutil",
     "github.com/coredns/coredns/plugin/pkg/fall",
+    "github.com/coredns/coredns/plugin/pkg/log",
     "github.com/coredns/coredns/plugin/pkg/upstream",
     "github.com/coredns/coredns/plugin/proxy",
     "github.com/coredns/coredns/plugin/reload",

pkg/cloudcommon/db/db_dispatcher.go

@@ -275,6 +275,12 @@ func listItemQueryFilters(manager IModelManager, ctx context.Context, q *sqlchem
 	if err != nil {
 		return nil, err
 	}
+	if query.Contains("export_keys") {
+		q, err = manager.ListItemExportKeys(ctx, q, userCred, query)
+		if err != nil {
+			return nil, err
+		}
+	}
 	q, err = listItemsQueryByColumn(manager, q, userCred, query)
 	if err != nil {
 		return nil, err
@@ -329,29 +335,41 @@ func query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 	for rows.Next() {
 		itemValue := reflect.Indirect(reflect.ValueOf(item))
 		itemValue.Set(itemInitValue)
-		err = q.Row2Struct(rows, item)
-		if err != nil {
-			return nil, err
+		extraData := jsonutils.NewDict()
+		if query.Contains("export_keys") {
+			RowMap, err := q.Row2Map(rows)
+			if err != nil {
+				return nil, err
+			}
+			extraKeys := manager.GetExportExtraKeys(ctx, query, RowMap)
+			if extraKeys != nil {
+				extraData.Update(extraKeys)
+			}
+			err = q.RowMap2Struct(RowMap, item)
+			if err != nil {
+				return nil, err
+			}
+		} else {
+			err = q.Row2Struct(rows, item)
+			if err != nil {
+				return nil, err
+			}
 		}
+
 		jsonData := jsonutils.Marshal(item)
 		jsonDict, ok := jsonData.(*jsonutils.JSONDict)
 		if !ok {
 			return nil, fmt.Errorf("invalid model data structure, not a dict")
 		}
 		jsonDict = jsonDict.CopyIncludes(listF...)
-		if showDetails {
+		jsonDict.Update(extraData)
+		if showDetails && !query.Contains("export_keys") {
 			extraDict := item.GetCustomizeColumns(ctx, userCred, query)
 			if extraDict != nil {
 				jsonDict.Update(extraDict)
 			}
 			jsonDict = getModelExtraDetails(item, ctx, jsonDict)
 		}
-		if query.Contains("export_keys") {
-			exportDict := item.GetExportItems(ctx, userCred, query)
-			if exportDict != nil {
-				jsonDict.Update(exportDict)
-			}
-		}
 		results = append(results, jsonDict)
 	}
 	return results, nil

pkg/cloudcommon/db/interface.go

@@ -31,6 +31,8 @@ type IModelManager interface {
 	ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error)
 	CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*CustomizeListFilters, error)
 	ExtraSearchConditions(ctx context.Context, q *sqlchemy.SQuery, like string) []sqlchemy.ICondition
+	GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict
+	ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error)
 
 	// fetch hook
 	Query(val ...string) *sqlchemy.SQuery
@@ -78,7 +80,6 @@ type IModel interface {
 	// get hooks
 	AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool
 	GetExtraDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict
-	GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict
 
 	// create hooks
 	CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) error

pkg/cloudcommon/db/modelbase.go

@@ -162,6 +162,14 @@ func (manager *SModelBaseManager) DoCreate(ctx context.Context, userCred mcclien
 	return nil, fmt.Errorf("Do create not implement?")
 }
 
+func (manager *SModelBaseManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	return q, nil
+}
+
+func (manager *SModelBaseManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict {
+	return jsonutils.NewDict()
+}
+
 func (model *SModelBase) GetId() string {
 	return ""
 }
@@ -193,10 +201,6 @@ func (model *SModelBase) GetCustomizeColumns(ctx context.Context, userCred mccli
 	return jsonutils.NewDict()
 }
 
-func (model *SModelBase) GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
-	return jsonutils.NewDict()
-}
-
 // get hooks
 func (model *SModelBase) AllowGetDetails(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {
 	return false

pkg/compute/models/guests.go

@@ -1137,16 +1137,95 @@ func (self *SGuest) GetExtraDetails(ctx context.Context, userCred mcclient.Token
 	return self.moreExtraInfo(extra)
 }
 
-func (self *SGuest) GetExportItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) *jsonutils.JSONDict {
+func (manager *SGuestManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	exportKeys, _ := query.GetString("export_keys")
 	keys := strings.Split(exportKeys, ",")
-	res := jsonutils.NewDict()
+
+	// guest_id as filter key
+	if utils.IsInStringArray("ips", keys) {
+		guestIpsQuery := GuestnetworkManager.Query("guest_id").GroupBy("guest_id")
+		guestIpsQuery.AppendField(sqlchemy.GROUP_CONCAT("concat_ip_addr", guestIpsQuery.Field("ip_addr")))
+		ipsSubQuery := guestIpsQuery.SubQuery()
+		guestIpsQuery.DebugQuery()
+		q.LeftJoin(ipsSubQuery, sqlchemy.Equals(q.Field("id"), ipsSubQuery.Field("guest_id")))
+		q.AppendField(ipsSubQuery.Field("concat_ip_addr"))
+	}
+	if utils.IsInStringArray("disk", keys) {
+		guestDisksQuery := GuestdiskManager.Query("guest_id", "disk_id").GroupBy("guest_id")
+		diskQuery := DiskManager.Query("id", "disk_size").SubQuery()
+		guestDisksQuery.Join(diskQuery, sqlchemy.Equals(diskQuery.Field("id"), guestDisksQuery.Field("disk_id")))
+		guestDisksQuery.AppendField(sqlchemy.SUM("disk_size", diskQuery.Field("disk_size")))
+		guestDisksSubQuery := guestDisksQuery.SubQuery()
+		guestDisksSubQuery.DebugQuery()
+		q.LeftJoin(guestDisksSubQuery, sqlchemy.Equals(q.Field("id"), guestDisksSubQuery.
+			Field("guest_id")))
+		q.AppendField(guestDisksSubQuery.Field("disk_size"))
+	}
+	if utils.IsInStringArray("eip", keys) {
+		eipsQuery := ElasticipManager.Query("associate_id", "ip_addr").Equals("associate_type", "server").GroupBy("associate_id")
+		eipsSubQuery := eipsQuery.SubQuery()
+		eipsSubQuery.DebugQuery()
+		q.LeftJoin(eipsSubQuery, sqlchemy.Equals(q.Field("id"), eipsSubQuery.Field("associate_id")))
+		q.AppendField(eipsSubQuery.Field("ip_addr", "eip"))
+	}
+
+	// host_id as filter key
+	if utils.IsInStringArray("region", keys) {
+		zoneQuery := ZoneManager.Query("id", "cloudregion_id").SubQuery()
+		hostQuery := HostManager.Query("id", "zone_id").GroupBy("id")
+		cloudregionQuery := CloudregionManager.Query("id", "name").SubQuery()
+		hostQuery.LeftJoin(zoneQuery, sqlchemy.Equals(hostQuery.Field("zone_id"), zoneQuery.Field("id"))).
+			LeftJoin(cloudregionQuery, sqlchemy.OR(sqlchemy.Equals(cloudregionQuery.Field("id"),
+				zoneQuery.Field("cloudregion_id")), sqlchemy.Equals(cloudregionQuery.Field("id"), "default")))
+		hostQuery.AppendField(cloudregionQuery.Field("name", "region"))
+		hostSubQuery := hostQuery.SubQuery()
+		q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id")))
+		q.AppendField(hostSubQuery.Field("region"))
+	}
+	if utils.IsInStringArray("manager", keys) {
+		hostQuery := HostManager.Query("id", "manager_id").GroupBy("id")
+		cloudProviderQuery := CloudproviderManager.Query("id", "name").SubQuery()
+		hostQuery.LeftJoin(cloudProviderQuery, sqlchemy.Equals(hostQuery.Field("manager_id"),
+			cloudProviderQuery.Field("id")))
+		hostQuery.AppendField(cloudProviderQuery.Field("name", "manager"))
+		hostSubQuery := hostQuery.SubQuery()
+		q.LeftJoin(hostSubQuery, sqlchemy.Equals(q.Field("host_id"), hostSubQuery.Field("id")))
+		q.AppendField(hostSubQuery.Field("manager"))
+	}
+	return q, nil
+}
+
+func (manager *SGuestManager) GetExportExtraKeys(ctx context.Context, query jsonutils.JSONObject, rowMap map[string]string) *jsonutils.JSONDict {
+	res := manager.SStatusStandaloneResourceBaseManager.GetExportExtraKeys(ctx, query, rowMap)
+	exportKeys, _ := query.GetString("export_keys")
+	keys := strings.Split(exportKeys, ",")
+	if ips, ok := rowMap["concat_ip_addr"]; ok && len(ips) > 0 {
+		res.Set("ips", jsonutils.NewString(ips))
+	}
+	if eip, ok := rowMap["eip"]; ok && len(eip) > 0 {
+		res.Set("eip", jsonutils.NewString(eip))
+	}
+	if disk, ok := rowMap["disk_size"]; ok {
+		res.Set("disk", jsonutils.NewString(disk))
+	}
+	if region, ok := rowMap["region"]; ok && len(region) > 0 {
+		res.Set("region", jsonutils.NewString(region))
+	}
+	if manager, ok := rowMap["manager"]; ok && len(manager) > 0 {
+		res.Set("manager", jsonutils.NewString(manager))
+	}
+	if utils.IsInStringArray("tenant", keys) {
+		if projectId, ok := rowMap["tenant_id"]; ok {
+			tenant, err := db.TenantCacheManager.FetchTenantById(ctx, projectId)
+			if err == nil {
+				res.Set("tenant", jsonutils.NewString(tenant.GetName()))
+			}
+		}
+	}
 	if utils.IsInStringArray("os_distribution", keys) {
-		osType := self.GetMetadata("os_distribution", userCred)
-		if len(osType) == 0 {
-			osType = self.OsType
+		if osType, ok := rowMap["os_type"]; ok {
+			res.Set("os_distribution", jsonutils.NewString(osType))
 		}
-		res.Set("os_distribution", jsonutils.NewString(osType))
 	}
 	return res
 }

vendor/yunion.io/x/pkg/util/sysutils/storagetypes.go

@@ -20,12 +20,29 @@ const (
 	STORAGE_STANDARD_RAGRS = "standard_ragrs"
 	STORAGE_STANDARD_ZRS   = "standard_zrs"
 	STORAGE_PREMIUM_LRS    = "premium_lrs"
+
+	// aws storage type
+	STORAGE_GP2_SSD      = "gp2"      // aws general purpose ssd
+	STORAGE_IO1_SSD      = "io1"      // aws Provisioned IOPS SSD
+	STORAGE_ST1_HDD      = "st1"      // aws Throughput Optimized HDD
+	STORAGE_SC1_SSD      = "sc1"      // aws Cold HDD
+	STORAGE_STANDARD_SSD = "standard" // aws Magnetic volumes
+
+	// qcloud storage type
+	// STORAGE_CLOUD_SSD ="cloud_ssd"
+	STORAGE_LOCAL_BASIC   = "local_basic"
+	STORAGE_LOCAL_SSD     = "local_ssd"
+	STORAGE_CLOUD_BASIC   = "cloud_basic"
+	STORAGE_CLOUD_PERMIUM = "cloud_permium"
 )
 
 var STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL, STORAGE_SHEEPDOG,
 	STORAGE_RBD, STORAGE_DOCKER, STORAGE_NAS, STORAGE_VSAN,
 	STORAGE_CLOUD, STORAGE_CLOUD_SSD, STORAGE_CLOUD_ESSD, STORAGE_CLOUD_EFFICIENCY,
-	STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS}
+	STORAGE_STANDARD_GRS, STORAGE_STANDARD_LRS, STORAGE_STANDARD_RAGRS, STORAGE_STANDARD_ZRS, STORAGE_PREMIUM_LRS,
+	STORAGE_GP2_SSD, STORAGE_IO1_SSD, STORAGE_ST1_HDD, STORAGE_SC1_SSD, STORAGE_STANDARD_SSD,
+	STORAGE_LOCAL_BASIC, STORAGE_LOCAL_SSD, STORAGE_CLOUD_BASIC, STORAGE_CLOUD_PERMIUM,
+}
 
 var LOCAL_STORAGE_TYPES = []string{STORAGE_LOCAL, STORAGE_BAREMETAL}
 

vendor/yunion.io/x/sqlchemy/functions.go

@@ -57,3 +57,8 @@ func DISTINCT(name string, field IQueryField) IQueryField {
 	ff := NewFunctionField(name, "DISTINCT(%s)", field)
 	return &ff
 }
+
+func GROUP_CONCAT(name string, field IQueryField) IQueryField {
+	ff := NewFunctionField(name, "GROUP_CONCAT(%s)", field)
+	return &ff
+}

vendor/yunion.io/x/sqlchemy/query.go

@@ -146,6 +146,10 @@ func DoQuery(from IQuerySource, f ...IQueryField) *SQuery {
 	return &tq
 }
 
+func (q *SQuery) AppendField(f ...IQueryField) {
+	q.fields = append(q.fields, f...)
+}
+
 func (table *SSubQuery) Query(f ...IQueryField) *SQuery {
 	return DoQuery(table, f...)
 }
@@ -527,22 +531,29 @@ func (q *SQuery) All(dest interface{}) error {
 	return err
 }
 
-func (q *SQuery) Row2Struct(row IRowScanner, dest interface{}) error {
-	result, err := q.rowScan2StringMap(row)
-	if err != nil {
-		return err
-	}
+func (q *SQuery) Row2Map(row IRowScanner) (map[string]string, error) {
+	return q.rowScan2StringMap(row)
+}
 
+func (q *SQuery) RowMap2Struct(result map[string]string, dest interface{}) error {
 	destPtrValue := reflect.ValueOf(dest)
 	if destPtrValue.Kind() != reflect.Ptr {
 		return fmt.Errorf("input must be a pointer")
 	}
 
 	destValue := destPtrValue.Elem()
-	err = mapString2Struct(result, destValue)
+	err := mapString2Struct(result, destValue)
 	if err != nil {
 		return err
 	}
 	callAfterQuery(destPtrValue)
 	return nil
 }
+
+func (q *SQuery) Row2Struct(row IRowScanner, dest interface{}) error {
+	result, err := q.rowScan2StringMap(row)
+	if err != nil {
+		return err
+	}
+	return q.RowMap2Struct(result, dest)
+}