diff --git a/pkg/cloudcommon/db/fetch.go b/pkg/cloudcommon/db/fetch.go index bdf44ca30e..a381f59a59 100644 --- a/pkg/cloudcommon/db/fetch.go +++ b/pkg/cloudcommon/db/fetch.go @@ -597,6 +597,10 @@ func FetchField(modelMan IModelManager, field string, qCallback func(q *sqlchemy if qCallback != nil { q = qCallback(q) } + return FetchIds(q) +} + +func FetchIds(q *sqlchemy.SQuery) ([]string, error) { rows, err := q.Rows() if err != nil { if errors.Cause(err) == sql.ErrNoRows { diff --git a/pkg/cloudcommon/db/metadataresource.go b/pkg/cloudcommon/db/metadataresource.go index 8494f9b2f0..06b70c0970 100644 --- a/pkg/cloudcommon/db/metadataresource.go +++ b/pkg/cloudcommon/db/metadataresource.go @@ -16,7 +16,10 @@ package db import ( "context" + "crypto/md5" + "fmt" "strings" + "time" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -27,6 +30,7 @@ import ( "yunion.io/x/onecloud/pkg/apis" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" + "yunion.io/x/onecloud/pkg/util/hashcache" "yunion.io/x/onecloud/pkg/util/rbacutils" "yunion.io/x/onecloud/pkg/util/stringutils2" "yunion.io/x/onecloud/pkg/util/tagutils" @@ -58,6 +62,37 @@ func ObjectIdQueryWithPolicyResult(ctx context.Context, q *sqlchemy.SQuery, mana return q } +func ObjectIdQueryWithTagFiltersOptimized(ctx context.Context, q *sqlchemy.SQuery, idField string, modelName string, filters tagutils.STagFilters) *sqlchemy.SQuery { + if len(filters.Filters) > 0 || len(filters.NoFilters) > 0 { + idSubQ := q.Copy().SubQuery().Query() + idSubQ.AppendField(sqlchemy.DISTINCT(idField, idSubQ.Field(idField))) + subQ := idSubQ.SubQuery() + if len(filters.Filters) > 0 { + if GetMetadaManagerInContext(ctx) == Metadata { + sq := tenantIdQueryWithTags(ctx, modelName, filters.Filters) + q = q.In(idField, sq.SubQuery()) + } else { // clickhouse + ids := objIdQueryWithTagsOptimized(ctx, subQ, idField, modelName, filters.Filters) + if len(ids) > 0 { + q = q.In(idField, ids) + } + } + } + if len(filters.NoFilters) > 0 { + if GetMetadaManagerInContext(ctx) == Metadata { + sq := tenantIdQueryWithTags(ctx, modelName, filters.Filters) + q = q.NotIn(idField, sq.SubQuery()) + } else { // clickhouse + ids := objIdQueryWithTagsOptimized(ctx, subQ, idField, modelName, filters.NoFilters) + if len(ids) > 0 { + q = q.NotIn(idField, ids) + } + } + } + } + return q +} + func ObjectIdQueryWithTagFilters(ctx context.Context, q *sqlchemy.SQuery, idField string, modelName string, filters tagutils.STagFilters) *sqlchemy.SQuery { if len(filters.Filters) > 0 || len(filters.NoFilters) > 0 { idSubQ := q.Copy().SubQuery().Query() @@ -94,6 +129,30 @@ func ExtendQueryWithTag(ctx context.Context, q *sqlchemy.SQuery, idField string, return q } +func tenantIdQueryWithTags(ctx context.Context, modelName string, tagsList []map[string][]string) *sqlchemy.SQuery { + manager := GetMetadaManagerInContext(ctx) + + conditions := []sqlchemy.ICondition{} + sq := manager.Query("obj_id") + for _, tags := range tagsList { + if len(tags) == 0 { + continue + } + subconds := []sqlchemy.ICondition{} + for key, val := range tags { + if len(val) > 0 { + sqq := sq.Copy().Equals("obj_type", modelName).Equals("key", key).In("value", val) + subconds = append(subconds, sqlchemy.In(sq.Field("obj_id"), sqq.SubQuery())) + } else { + sqq := sq.Copy().Equals("obj_type", modelName).Equals("key", key) + subconds = append(subconds, sqlchemy.In(sq.Field("obj_id"), sqq.SubQuery())) + } + } + conditions = append(conditions, sqlchemy.AND(subconds...)) + } + return sq.Filter(sqlchemy.OR(conditions...)).Distinct() +} + func objIdQueryWithTags(ctx context.Context, objIdSubQ *sqlchemy.SSubQuery, idField string, modelName string, tagsList []map[string][]string) *sqlchemy.SQuery { manager := GetMetadaManagerInContext(ctx) @@ -133,6 +192,53 @@ func objIdQueryWithTags(ctx context.Context, objIdSubQ *sqlchemy.SSubQuery, idFi return query } +var ( + tagsCache = hashcache.NewCache(1024, time.Minute*15) +) + +func objIdQueryWithTagsOptimized(ctx context.Context, objIdSubQ *sqlchemy.SSubQuery, idField string, modelName string, tagsList []map[string][]string) []string { + manager := GetMetadaManagerInContext(ctx) + + ret := []string{} + for _, tags := range tagsList { + if len(tags) == 0 { + continue + } + objIdQ := objIdSubQ.Query() + objIdQ = objIdQ.AppendField(objIdQ.Field(idField)) + for key, val := range tags { + hashKeys := []string{idField, key, modelName} + hashKeys = append(hashKeys, val...) + hash := fmt.Sprintf("%x", md5.Sum([]byte(jsonutils.Marshal(hashKeys).String()))) + cache := tagsCache.Get(hash) + if cache != nil { + ids := cache.([]string) + ret = append(ret, ids...) + continue + } + sq := manager.Query("obj_id").Equals("obj_type", modelName).Equals("key", key) + if len(val) > 0 { + ssq := sq.In("value", val).SubQuery() + if utils.IsInArray(tagutils.NoValue, val) { + objIdQ = objIdQ.LeftJoin(ssq, sqlchemy.Equals(objIdQ.Field(idField), ssq.Field("obj_id"))) + } else { + objIdQ = objIdQ.Join(ssq, sqlchemy.Equals(objIdQ.Field(idField), ssq.Field("obj_id"))) + } + } else { + ssq := sq.SubQuery() + objIdQ = objIdQ.Join(ssq, sqlchemy.Equals(objIdQ.Field(idField), ssq.Field("obj_id"))) + } + ids, err := FetchIds(objIdQ) + if err != nil { + continue + } + ret = append(ret, ids...) + tagsCache.AtomicSet(hash, ids) + } + } + return ret +} + func (meta *SMetadataResourceBaseModelManager) ListItemFilter( ctx context.Context, manager IModelManager, diff --git a/pkg/cloudcommon/db/project.go b/pkg/cloudcommon/db/project.go index 23ec7000a2..9237568ac8 100644 --- a/pkg/cloudcommon/db/project.go +++ b/pkg/cloudcommon/db/project.go @@ -62,7 +62,7 @@ func (manager *SProjectizedResourceBaseManager) FilterByOwner(ctx context.Contex if !result.ObjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ObjectTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "id", man.Keyword(), policyTagFilters) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "id", man.Keyword(), policyTagFilters) } } case rbacscope.ScopeDomain: @@ -72,12 +72,12 @@ func (manager *SProjectizedResourceBaseManager) FilterByOwner(ctx context.Contex if !result.ProjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ProjectTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "tenant_id", "project", policyTagFilters) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "tenant_id", "project", policyTagFilters) } if !result.ObjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ObjectTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "id", man.Keyword(), policyTagFilters) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "id", man.Keyword(), policyTagFilters) } } case rbacscope.ScopeSystem: @@ -86,17 +86,17 @@ func (manager *SProjectizedResourceBaseManager) FilterByOwner(ctx context.Contex if !result.DomainTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.DomainTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "domain_id", "domain", policyTagFilters) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "domain_id", "domain", policyTagFilters) } if !result.ProjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ProjectTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "tenant_id", "project", policyTagFilters) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "tenant_id", "project", policyTagFilters) } if !result.ObjectTags.IsEmpty() { policyTagFilters := tagutils.STagFilters{} policyTagFilters.AddFilters(result.ObjectTags) - q = ObjectIdQueryWithTagFilters(ctx, q, "id", man.Keyword(), policyTagFilters) + q = ObjectIdQueryWithTagFiltersOptimized(ctx, q, "id", man.Keyword(), policyTagFilters) } } }