fix: optimize filterByOwner query speed (#22446)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

pkg/cloudcommon/db/db_dispatcher.go

@@ -272,13 +272,6 @@ func listItemQueryFiltersRaw(
 		log.Debugf("policyTagFilers: %s", query)
 	}
 
-	if !useRawQuery {
-		// Specifically for joint resource, these filters will exclude
-		// deleted resources by joining with master/slave tables
-		q = manager.FilterByOwner(ctx, q, manager, userCred, ownerId, queryScope)
-		q = manager.FilterBySystemAttributes(q, userCred, query, queryScope)
-		q = manager.FilterByHiddenSystemAttributes(q, userCred, query, queryScope)
-	}
 	if query.Contains("export_keys") {
 		exportKeys, _ := query.GetString("export_keys")
 		keys := stringutils2.NewSortedStrings(strings.Split(exportKeys, ","))
@@ -324,6 +317,14 @@ func listItemQueryFiltersRaw(
 		return nil, errors.Wrap(err, "ListItemFilter")
 	}
 
+	if !useRawQuery {
+		// Specifically for joint resource, these filters will exclude
+		// deleted resources by joining with master/slave tables
+		q = manager.FilterByOwner(ctx, q, manager, userCred, ownerId, queryScope)
+		q = manager.FilterBySystemAttributes(q, userCred, query, queryScope)
+		q = manager.FilterByHiddenSystemAttributes(q, userCred, query, queryScope)
+	}
+
 	if isShowDetails(query) {
 		managerVal := reflect.ValueOf(manager)
 		fName := "ExtendListQuery"

pkg/cloudcommon/db/metadataresource.go

@@ -96,7 +96,6 @@ func ExtendQueryWithTag(ctx context.Context, q *sqlchemy.SQuery, idField string,
 
 func objIdQueryWithTags(ctx context.Context, objIdSubQ *sqlchemy.SSubQuery, idField string, modelName string, tagsList []map[string][]string) *sqlchemy.SQuery {
 	manager := GetMetadaManagerInContext(ctx)
-	metadataResQ := manager.Query().Equals("obj_type", modelName).SubQuery()
 
 	queries := make([]sqlchemy.IQuery, 0)
 	for _, tags := range tagsList {
@@ -106,19 +105,17 @@ func objIdQueryWithTags(ctx context.Context, objIdSubQ *sqlchemy.SSubQuery, idFi
 		objIdQ := objIdSubQ.Query()
 		objIdQ = objIdQ.AppendField(objIdQ.Field(idField))
 		for key, val := range tags {
-			sq := metadataResQ.Query().Equals("key", key).SubQuery()
-			objIdQ = objIdQ.LeftJoin(sq, sqlchemy.Equals(objIdQ.Field(idField), sq.Field("obj_id")))
+			sq := manager.Query("obj_id").Equals("obj_type", modelName).Equals("key", key)
 			if len(val) > 0 {
+				ssq := sq.In("value", val).SubQuery()
 				if utils.IsInArray(tagutils.NoValue, val) {
-					objIdQ = objIdQ.Filter(sqlchemy.OR(
-						sqlchemy.In(sq.Field("value"), val),
-						sqlchemy.IsNull(sq.Field("value")),
-					))
+					objIdQ = objIdQ.LeftJoin(ssq, sqlchemy.Equals(objIdQ.Field(idField), ssq.Field("obj_id")))
 				} else {
-					objIdQ = objIdQ.Filter(sqlchemy.In(sq.Field("value"), val))
+					objIdQ = objIdQ.Join(ssq, sqlchemy.Equals(objIdQ.Field(idField), ssq.Field("obj_id")))
 				}
 			} else {
-				objIdQ = objIdQ.Filter(sqlchemy.IsNotNull(sq.Field("obj_id")))
+				ssq := sq.SubQuery()
+				objIdQ = objIdQ.Join(ssq, sqlchemy.Equals(objIdQ.Field(idField), ssq.Field("obj_id")))
 			}
 		}
 		queries = append(queries, objIdQ.Distinct())

pkg/cloudcommon/db/project.go

@@ -149,13 +149,13 @@ func (manager *SProjectizedResourceBaseManager) ListItemFilter(
 		q = q.In("tenant_id", subq)
 	}
 	tagFilters := tagutils.STagFilters{}
-	if len(query.ProjectOrganizations) > 0 {
+	/*if len(query.ProjectOrganizations) > 0 {
 		orgFilters, err := FetchOrganizationTags(ctx, query.ProjectOrganizations, identityapi.OrgTypeProject)
 		if err != nil {
 			return nil, errors.Wrap(err, "FetchOrganizationTags")
 		}
 		tagFilters.AddFilters(orgFilters)
-	}
+	}*/
 	if !query.ProjectTags.IsEmpty() {
 		tagFilters.AddFilters(query.ProjectTags)
 	}