Add unified cloud provider and env filter for any resource list

pkg/apis/compute/cloudaccount_const.go

@@ -13,7 +13,7 @@ const (
 	CLOUD_PROVIDER_SYNC_STATUS_SYNCING = "syncing"
 	CLOUD_PROVIDER_SYNC_STATUS_IDLE    = "idle"
 
-	CLOUD_PROVIDER_KVM       = "KVM"
+	CLOUD_PROVIDER_ONECLOUD  = "OneCloud"
 	CLOUD_PROVIDER_VMWARE    = "VMware"
 	CLOUD_PROVIDER_ALIYUN    = "Aliyun"
 	CLOUD_PROVIDER_QCLOUD    = "Qcloud"
@@ -34,7 +34,7 @@ var (
 	CLOUD_PROVIDER_VALID_STATUS = []string{CLOUD_PROVIDER_CONNECTED}
 
 	CLOUD_PROVIDERS = []string{
-		CLOUD_PROVIDER_KVM,
+		CLOUD_PROVIDER_ONECLOUD,
 		CLOUD_PROVIDER_VMWARE,
 		CLOUD_PROVIDER_ALIYUN,
 		CLOUD_PROVIDER_QCLOUD,
@@ -45,3 +45,9 @@ var (
 		CLOUD_PROVIDER_UCLOUD,
 	}
 )
+
+const (
+	CLOUD_ENV_PUBLIC_CLOUD  = "public"
+	CLOUD_ENV_PRIVATE_CLOUD = "private"
+	CLOUD_ENV_ON_PREMISE    = "onpremise"
+)

pkg/compute/models/cloudaccounts.go

@@ -911,8 +911,7 @@ func (self *SCloudaccount) PerformChangeProject(ctx context.Context, userCred mc
 func (manager *SCloudaccountManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
 	accountStr, _ := query.GetString("account")
 	if len(accountStr) > 0 {
-		queryDict := query.(*jsonutils.JSONDict)
-		queryDict.Remove("account")
+		query.(*jsonutils.JSONDict).Remove("account")
 		accountObj, err := manager.FetchByIdOrName(userCred, accountStr)
 		if err != nil {
 			if err == sql.ErrNoRows {
@@ -942,15 +941,16 @@ func (manager *SCloudaccountManager) ListItemFilter(ctx context.Context, q *sqlc
 		q = q.Equals("id", provider.CloudaccountId)
 	}
 
-	if jsonutils.QueryBoolean(query, "public_cloud", false) {
-		q = q.IsTrue("is_public_cloud")
+	cloudEnvStr, _ := query.GetString("cloud_env")
+	if cloudEnvStr == api.CLOUD_ENV_PUBLIC_CLOUD || jsonutils.QueryBoolean(query, "public_cloud", false) {
+		q = q.IsTrue("is_public_cloud").IsFalse("is_on_premise")
 	}
 
-	if jsonutils.QueryBoolean(query, "private_cloud", false) {
+	if cloudEnvStr == api.CLOUD_ENV_PRIVATE_CLOUD || jsonutils.QueryBoolean(query, "private_cloud", false) {
 		q = q.IsFalse("is_public_cloud").IsFalse("is_on_premise")
 	}
 
-	if jsonutils.QueryBoolean(query, "is_on_premise", false) {
+	if cloudEnvStr == api.CLOUD_ENV_ON_PREMISE || jsonutils.QueryBoolean(query, "is_on_premise", false) {
 		q = q.IsTrue("is_on_premise").IsFalse("is_public_cloud")
 	}
 

pkg/compute/models/cloudproviders.go

@@ -829,20 +829,22 @@ func (manager *SCloudproviderManager) ListItemFilter(ctx context.Context, q *sql
 		q = q.Equals("id", providerObj.GetId())
 	}
 
-	if jsonutils.QueryBoolean(query, "public_cloud", false) {
+	cloudEnvStr, _ := query.GetString("cloud_env")
+	if cloudEnvStr == api.CLOUD_ENV_PUBLIC_CLOUD || jsonutils.QueryBoolean(query, "public_cloud", false) {
 		cloudaccounts := CloudaccountManager.Query().SubQuery()
 		q = q.Join(cloudaccounts, sqlchemy.Equals(cloudaccounts.Field("id"), q.Field("cloudaccount_id")))
 		q = q.Filter(sqlchemy.IsTrue(cloudaccounts.Field("is_public_cloud")))
+		q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("is_on_premise")))
 	}
 
-	if jsonutils.QueryBoolean(query, "private_cloud", false) {
+	if cloudEnvStr == api.CLOUD_ENV_PRIVATE_CLOUD || jsonutils.QueryBoolean(query, "private_cloud", false) {
 		cloudaccounts := CloudaccountManager.Query().SubQuery()
 		q = q.Join(cloudaccounts, sqlchemy.Equals(cloudaccounts.Field("id"), q.Field("cloudaccount_id")))
 		q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("is_public_cloud")))
 		q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("is_on_premise")))
 	}
 
-	if jsonutils.QueryBoolean(query, "is_on_premise", false) {
+	if cloudEnvStr == api.CLOUD_ENV_ON_PREMISE || jsonutils.QueryBoolean(query, "is_on_premise", false) {
 		cloudaccounts := CloudaccountManager.Query().SubQuery()
 		q = q.Join(cloudaccounts, sqlchemy.Equals(cloudaccounts.Field("id"), q.Field("cloudaccount_id")))
 		q = q.Filter(sqlchemy.IsFalse(cloudaccounts.Field("is_public_cloud")))

pkg/compute/models/cloudregions.go

@@ -142,7 +142,7 @@ func (self *SCloudregion) GetVpcCount() int {
 func (self *SCloudregion) GetDriver() IRegionDriver {
 	provider := self.Provider
 	if len(provider) == 0 {
-		provider = api.CLOUD_PROVIDER_KVM
+		provider = api.CLOUD_PROVIDER_ONECLOUD
 	}
 	if !utils.IsInStringArray(provider, api.CLOUD_PROVIDERS) {
 		log.Fatalf("Unsupported region provider %s", provider)
@@ -400,23 +400,31 @@ func (manager *SCloudregionManager) InitializeData() error {
 }
 
 func (manager *SCloudregionManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
+	if len(providerStr) > 0 {
+		query.(*jsonutils.JSONDict).Remove("provider")
+		if providerStr == api.CLOUD_PROVIDER_ONECLOUD {
+			q = q.IsNullOrEmpty("provider")
+		} else {
+			q = q.Equals("provider", providerStr)
+		}
+	}
+
 	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
 	if err != nil {
 		return nil, err
 	}
 
-	if jsonutils.QueryBoolean(query, "is_public", false) || jsonutils.QueryBoolean(query, "public_cloud", false) {
+	cloudEnvStr, _ := query.GetString("cloud_env")
+	if cloudEnvStr == api.CLOUD_ENV_PUBLIC_CLOUD || jsonutils.QueryBoolean(query, "is_public", false) || jsonutils.QueryBoolean(query, "public_cloud", false) {
 		q = q.In("provider", cloudprovider.GetPublicProviders())
 	}
 
-	if jsonutils.QueryBoolean(query, "is_private", false) || jsonutils.QueryBoolean(query, "private_cloud", false) {
-		q = q.Filter(sqlchemy.OR(
-			sqlchemy.In(q.Field("provider"), cloudprovider.GetPrivateProviders()),
-			sqlchemy.IsNullOrEmpty(q.Field("provider")),
-		))
+	if cloudEnvStr == api.CLOUD_ENV_PRIVATE_CLOUD || jsonutils.QueryBoolean(query, "is_private", false) || jsonutils.QueryBoolean(query, "private_cloud", false) {
+		q = q.In("provider", cloudprovider.GetPrivateProviders())
 	}
 
-	if jsonutils.QueryBoolean(query, "is_on_premise", false) {
+	if cloudEnvStr == api.CLOUD_ENV_ON_PREMISE || jsonutils.QueryBoolean(query, "is_on_premise", false) {
 		q = q.Filter(sqlchemy.OR(
 			sqlchemy.In(q.Field("provider"), cloudprovider.GetOnPremiseProviders()),
 			sqlchemy.IsNullOrEmpty(q.Field("provider")),

pkg/compute/models/managedresource.go

@@ -23,6 +23,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/sqlchemy"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/appctx"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
@@ -157,15 +158,26 @@ func managedResourceFilterByAccount(q *sqlchemy.SQuery, query jsonutils.JSONObje
 	providerStr := jsonutils.GetAnyString(query, []string{"provider"})
 	if len(providerStr) > 0 {
 		queryDict.Remove("provider")
-		subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery()
-		if len(filterField) == 0 {
-			q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
-			queryDict.Remove("manager_id")
+		if providerStr == api.CLOUD_PROVIDER_ONECLOUD {
+			if len(filterField) == 0 {
+				q = q.Filter(sqlchemy.IsNullOrEmpty(q.Field("manager_id")))
+			} else {
+				sq := subqFunc()
+				sq = sq.Filter(sqlchemy.IsNullOrEmpty(sq.Field("manager_id")))
+				q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
+				queryDict.Remove(filterField)
+			}
 		} else {
-			sq := subqFunc()
-			sq = sq.Filter(sqlchemy.In(sq.Field("manager_id"), subq))
-			q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
-			queryDict.Remove(filterField)
+			subq := CloudproviderManager.Query("id").Equals("provider", providerStr).SubQuery()
+			if len(filterField) == 0 {
+				q = q.Filter(sqlchemy.In(q.Field("manager_id"), subq))
+				queryDict.Remove("manager_id")
+			} else {
+				sq := subqFunc()
+				sq = sq.Filter(sqlchemy.In(sq.Field("manager_id"), subq))
+				q = q.Filter(sqlchemy.In(q.Field(filterField), sq.SubQuery()))
+				queryDict.Remove(filterField)
+			}
 		}
 	}
 
@@ -173,7 +185,9 @@ func managedResourceFilterByAccount(q *sqlchemy.SQuery, query jsonutils.JSONObje
 }
 
 func managedResourceFilterByCloudType(q *sqlchemy.SQuery, query jsonutils.JSONObject, filterField string, subqFunc func() *sqlchemy.SQuery) *sqlchemy.SQuery {
-	if jsonutils.QueryBoolean(query, "public_cloud", false) || jsonutils.QueryBoolean(query, "is_public", false) {
+	cloudEnvStr, _ := query.GetString("cloud_env")
+
+	if cloudEnvStr == api.CLOUD_ENV_PUBLIC_CLOUD || jsonutils.QueryBoolean(query, "public_cloud", false) || jsonutils.QueryBoolean(query, "is_public", false) {
 		if len(filterField) == 0 {
 			q = q.Filter(sqlchemy.In(q.Field("manager_id"), CloudproviderManager.GetPublicProviderIdsQuery()))
 		} else {
@@ -183,7 +197,7 @@ func managedResourceFilterByCloudType(q *sqlchemy.SQuery, query jsonutils.JSONOb
 		}
 	}
 
-	if jsonutils.QueryBoolean(query, "private_cloud", false) || jsonutils.QueryBoolean(query, "is_private", false) {
+	if cloudEnvStr == api.CLOUD_ENV_PRIVATE_CLOUD || jsonutils.QueryBoolean(query, "private_cloud", false) || jsonutils.QueryBoolean(query, "is_private", false) {
 		if len(filterField) == 0 {
 			q = q.Filter(sqlchemy.In(q.Field("manager_id"), CloudproviderManager.GetPrivateProviderIdsQuery()))
 		} else {
@@ -193,7 +207,7 @@ func managedResourceFilterByCloudType(q *sqlchemy.SQuery, query jsonutils.JSONOb
 		}
 	}
 
-	if jsonutils.QueryBoolean(query, "is_on_premise", false) {
+	if cloudEnvStr == api.CLOUD_ENV_ON_PREMISE || jsonutils.QueryBoolean(query, "is_on_premise", false) {
 		if len(filterField) == 0 {
 			q = q.Filter(
 				sqlchemy.OR(

pkg/compute/models/zones.go

@@ -525,22 +525,20 @@ func (manager *SZoneManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 		return nil, err
 	}
 
-	if jsonutils.QueryBoolean(query, "is_private", false) || jsonutils.QueryBoolean(query, "private", false) || jsonutils.QueryBoolean(query, "private_cloud", false) {
+	cloudEnvStr, _ := query.GetString("cloud_env")
+	if cloudEnvStr == api.CLOUD_ENV_PRIVATE_CLOUD || jsonutils.QueryBoolean(query, "is_private", false) || jsonutils.QueryBoolean(query, "private", false) || jsonutils.QueryBoolean(query, "private_cloud", false) {
 		regions := CloudregionManager.Query().SubQuery()
 		subq := regions.Query(regions.Field("id"))
-		subq = subq.Filter(sqlchemy.OR(
-			sqlchemy.In(regions.Field("provider"), cloudprovider.GetPrivateProviders()),
-			sqlchemy.IsNullOrEmpty(regions.Field("provider")),
-		))
+		subq = subq.Filter(sqlchemy.In(regions.Field("provider"), cloudprovider.GetPrivateProviders()))
 		q = q.In("cloudregion_id", subq.SubQuery())
 	}
-	if jsonutils.QueryBoolean(query, "is_public", false) || jsonutils.QueryBoolean(query, "public", false) || jsonutils.QueryBoolean(query, "public_cloud", false) {
+	if cloudEnvStr == api.CLOUD_ENV_PUBLIC_CLOUD || jsonutils.QueryBoolean(query, "is_public", false) || jsonutils.QueryBoolean(query, "public", false) || jsonutils.QueryBoolean(query, "public_cloud", false) {
 		regions := CloudregionManager.Query().SubQuery()
 		subq := regions.Query(regions.Field("id"))
 		subq = subq.Filter(sqlchemy.In(regions.Field("provider"), cloudprovider.GetPublicProviders()))
 		q = q.In("cloudregion_id", subq.SubQuery())
 	}
-	if jsonutils.QueryBoolean(query, "is_on_premise", false) {
+	if cloudEnvStr == api.CLOUD_ENV_ON_PREMISE || jsonutils.QueryBoolean(query, "is_on_premise", false) {
 		regions := CloudregionManager.Query().SubQuery()
 		subq := regions.Query(regions.Field("id"))
 		subq = subq.Filter(sqlchemy.OR(
@@ -624,8 +622,14 @@ func (manager *SZoneManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQu
 	}
 	providerStr, _ := query.GetString("provider")
 	if len(providerStr) > 0 {
-		subq := CloudregionManager.Query("id").Equals("provider", providerStr).SubQuery()
-		q = q.In("cloudregion_id", subq)
+		subq := CloudregionManager.Query("id")
+		if providerStr == api.CLOUD_PROVIDER_ONECLOUD {
+			subq = subq.IsNullOrEmpty("provider")
+		} else {
+			subq = subq.Equals("provider", providerStr)
+		}
+
+		q = q.In("cloudregion_id", subq.SubQuery())
 	}
 
 	city, _ := query.GetString("city")

pkg/compute/regiondrivers/kvm.go

@@ -40,7 +40,7 @@ func init() {
 }
 
 func (self *SKVMRegionDriver) GetProvider() string {
-	return api.CLOUD_PROVIDER_KVM
+	return api.CLOUD_PROVIDER_ONECLOUD
 }
 
 func (self *SKVMRegionDriver) ValidateCreateLoadbalancerData(ctx context.Context, userCred mcclient.TokenCredential, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {

pkg/mcclient/options/base.go

@@ -206,7 +206,8 @@ type BaseListOptions struct {
 
 	Manager      string `help:"List objects belonging to the cloud provider" json:"manager,omitempty"`
 	Account      string `help:"List objects belonging to the cloud account" json:"account,omitempty"`
-	Provider     string `help:"List objects from the provider" choices:"VMware|Aliyun|Qcloud|Azure|Aws|Huawei|Openstack" json:"provider,omitempty"`
+	Provider     string `help:"List objects from the provider" choices:"OneCloud|VMware|Aliyun|Qcloud|Azure|Aws|Huawei|Openstack|Ucloud" json:"provider,omitempty"`
+	CloudEnv     string `help:"Cloud environment" choices:"public|private|onpremise" json:"cloud_env,omitempty"`
 	PublicCloud  *bool  `help:"List objects belonging to public cloud" json:"public_cloud"`
 	PrivateCloud *bool  `help:"List objects belonging to private cloud" json:"private_cloud"`
 	IsOnPremise  *bool  `help:"List objects belonging to on premise infrastructures" token:"on-premise" json:"is_on_premise"`