From b8b48b4d6e90ecfe4a76c7c4613d9ac13bfee6c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Thu, 16 May 2024 15:24:33 +0800 Subject: [PATCH] fix(region): query distinct fields (#20300) --- cmd/climc/shell/misc/distinctfield.go | 24 +++++++ pkg/apis/input.go | 6 ++ pkg/cloudcommon/db/interface.go | 1 + pkg/cloudcommon/db/modelbase.go | 97 +++++++++++++++++++++++++-- pkg/compute/models/guests.go | 11 +++ pkg/compute/models/networkresource.go | 13 ++++ 6 files changed, 148 insertions(+), 4 deletions(-) diff --git a/cmd/climc/shell/misc/distinctfield.go b/cmd/climc/shell/misc/distinctfield.go index d7da0776e8..1aeef45830 100644 --- a/cmd/climc/shell/misc/distinctfield.go +++ b/cmd/climc/shell/misc/distinctfield.go @@ -50,4 +50,28 @@ func init() { fmt.Println(result) return nil }) + + type DistinctFieldsOption struct { + MODULE string `help:"module name"` + Field []string `help:"distinct field name to query"` + ExtraField []string `help:"distinct field name to query"` + ExtraResource string + } + R(&DistinctFieldsOption{}, "distinct-fields", "Query values of a distinct fields for a module", func(s *mcclient.ClientSession, args *DistinctFieldsOption) error { + mod, err := modulebase.GetModule(s, args.MODULE) + if err != nil || mod == nil { + if err != nil { + return fmt.Errorf("module %s not found %s", args.MODULE, err) + } + return fmt.Errorf("No module %s found", args.MODULE) + } + params := jsonutils.Marshal(args) + result, err := mod.Get(s, "distinct-fields", params) + if err != nil { + return err + } + fmt.Println(result) + return nil + }) + } diff --git a/pkg/apis/input.go b/pkg/apis/input.go index 8ff8953d8a..1901a486f6 100644 --- a/pkg/apis/input.go +++ b/pkg/apis/input.go @@ -383,6 +383,12 @@ type DistinctFieldInput struct { ExtraField []string } +type DistinctFieldsInput struct { + Field []string + ExtraField []string + ExtraResource string +} + type PostpaidExpireInput struct { Duration string `json:"duration"` ExpireTime time.Time `json:"expire_time"` diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 13738c2ee9..9791e8839b 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -132,6 +132,7 @@ type IModelManager interface { // 如果error为非空，说明没有匹配的field，如果为空，说明匹配上了 QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) + QueryDistinctExtraFields(q *sqlchemy.SQuery, resource string, fields []string) (*sqlchemy.SQuery, error) GetPagingConfig() *SPagingConfig diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 09a2a2dde4..5895230ff0 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -24,6 +24,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/object" "yunion.io/x/pkg/util/rbacscope" "yunion.io/x/pkg/util/version" @@ -228,6 +229,10 @@ func (manager *SModelBaseManager) QueryDistinctExtraField(q *sqlchemy.SQuery, fi return q, httperrors.ErrNotFound } +func (manager *SModelBaseManager) QueryDistinctExtraFields(q *sqlchemy.SQuery, resource string, fields []string) (*sqlchemy.SQuery, error) { + return q, httperrors.ErrNotImplemented +} + func (manager *SModelBaseManager) CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*CustomizeListFilters, error) { return NewCustomizeListFilters(), nil } @@ -394,10 +399,6 @@ func (manager *SModelBaseManager) ResourceScope() rbacscope.TRbacScope { return rbacscope.ScopeSystem } -func (manager *SModelBaseManager) AllowGetPropertyDistinctField(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool { - return true -} - func (manager *SModelBaseManager) GetPagingConfig() *SPagingConfig { return nil } @@ -480,6 +481,94 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context, return res, nil } +func (manager *SModelBaseManager) GetPropertyDistinctFields(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) { + im, ok := manager.GetVirtualObject().(IModelManager) + if !ok { + im = manager + } + input := &apis.DistinctFieldsInput{} + query.Unmarshal(input) + if len(input.Field) == 0 && (len(input.ExtraResource) == 0 || len(input.ExtraField) == 0) { + return nil, httperrors.NewMissingParameterError("field") + } + // validate field + for _, fd := range input.Field { + var hasField = false + for _, field := range manager.getTable().Fields() { + if field.Name() == fd { + hasField = true + break + } + } + if !hasField { + return nil, httperrors.NewBadRequestError("model has no field %s", fd) + } + } + var err error + q := im.Query() + q, err = ListItemQueryFilters(im, ctx, q, userCred, query, policy.PolicyActionList) + if err != nil { + return nil, err + } + result := jsonutils.NewDict() + fields := jsonutils.NewArray() + if len(input.Field) > 0 { + sq := q.Copy() + // query field + for i := 0; i < len(input.Field); i++ { + sq = sq.AppendField(sq.Field(input.Field[i])) + } + rows, err := sq.Distinct().Rows() + if err != nil { + return nil, err + } + defer rows.Close() + for rows.Next() { + mMap, err := sq.Row2Map(rows) + if err != nil { + return nil, errors.Wrapf(err, "Row2Map") + } + fields.Add(jsonutils.Marshal(mMap)) + } + } + result.Set("fields", fields) + + extraFields := jsonutils.NewArray() + if len(input.ExtraResource) > 0 && len(input.ExtraField) > 0 { + // query extra field + sq := q.Copy() + em := GetModelManager(input.ExtraResource) + if gotypes.IsNil(em) { + return nil, httperrors.NewInputParameterError("invalid extra_resource %s", input.ExtraResource) + } + for _, field := range input.ExtraField { + if gotypes.IsNil(em.TableSpec().ColumnSpec(field)) { + return nil, httperrors.NewInputParameterError("resource %s does not have field %s", input.ExtraResource, field) + } + } + sq, err := im.QueryDistinctExtraFields(sq, input.ExtraResource, input.ExtraField) + if err != nil { + return nil, err + } + + rows, err := sq.Distinct().Rows() + if err != nil { + return nil, err + } + + defer rows.Close() + for rows.Next() { + mMap, err := sq.Row2Map(rows) + if err != nil { + return nil, errors.Wrapf(err, "Row2Map") + } + extraFields.Add(jsonutils.Marshal(mMap)) + } + } + result.Set("extra_fields", extraFields) + return result, nil +} + func (manager *SModelBaseManager) BatchPreValidate( ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict, count int, diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 87a3cbf06e..5ad9480844 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -808,6 +808,17 @@ func (manager *SGuestManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field return q, httperrors.ErrNotFound } +func (manager *SGuestManager) QueryDistinctExtraFields(q *sqlchemy.SQuery, resource string, fields []string) (*sqlchemy.SQuery, error) { + switch resource { + case NetworkManager.Keyword(): + guestnets := GuestnetworkManager.Query("guest_id", "network_id").SubQuery() + q = q.LeftJoin(guestnets, sqlchemy.Equals(q.Field("id"), guestnets.Field("guest_id"))) + + return manager.SNetworkResourceBaseManager.QueryDistinctExtraFields(q, resource, fields) + } + return q, httperrors.ErrNotFound +} + func (manager *SGuestManager) initHostname() error { guests := []SGuest{} q := manager.Query().IsNullOrEmpty("hostname") diff --git a/pkg/compute/models/networkresource.go b/pkg/compute/models/networkresource.go index 640da6bfee..d4a951e26b 100644 --- a/pkg/compute/models/networkresource.go +++ b/pkg/compute/models/networkresource.go @@ -178,6 +178,19 @@ func (manager *SNetworkResourceBaseManager) QueryDistinctExtraField(q *sqlchemy. return q, httperrors.ErrNotFound } +func (manager *SNetworkResourceBaseManager) QueryDistinctExtraFields(q *sqlchemy.SQuery, resource string, fields []string) (*sqlchemy.SQuery, error) { + switch resource { + case NetworkManager.Keyword(): + netQuery := NetworkManager.Query().SubQuery() + for _, field := range fields { + q = q.AppendField(netQuery.Field(field)) + } + q = q.Join(netQuery, sqlchemy.Equals(q.Field("network_id"), netQuery.Field("id"))) + return q, nil + } + return q, httperrors.ErrNotFound +} + func (manager *SNetworkResourceBaseManager) OrderByExtraFields( ctx context.Context, q *sqlchemy.SQuery,