Merge pull request #2733 from swordqiu/hotfix/qj-logger-optimization

fix: optimize logger query time

cmd/climc/shell/actions.go

@@ -36,6 +36,11 @@ type BaseActionListOptions struct {
 	Admin      bool     `help:"admin mode"`
 	Succ       bool     `help:"Show success action log only"`
 	Fail       bool     `help:"Show failed action log only"`
+
+	User    []string `help:"filter by operator user"`
+	Project []string `help:"filter by owner project"`
+
+	PagingMarker string `help:"marker for pagination"`
 }
 
 type ActionListOptions struct {
@@ -60,9 +65,18 @@ func doActionList(s *mcclient.ClientSession, args *ActionListOptions) error {
 	if len(args.Search) > 0 {
 		params.Add(jsonutils.NewString(args.Search), "search")
 	}
+	if len(args.User) > 0 {
+		params.Add(jsonutils.NewStringArray(args.User), "user")
+	}
+	if len(args.Project) > 0 {
+		params.Add(jsonutils.NewStringArray(args.Project), "project")
+	}
 	if len(args.Scope) > 0 {
 		params.Add(jsonutils.NewString(args.Scope), "scope")
 	}
+	if len(args.PagingMarker) > 0 {
+		params.Add(jsonutils.NewString(args.PagingMarker), "paging_marker")
+	}
 	if len(args.Since) > 0 {
 		params.Add(jsonutils.NewString(args.Since), "since")
 	}

cmd/climc/shell/events.go

@@ -32,6 +32,11 @@ type BaseEventListOptions struct {
 	Descending bool     `help:"Descending order"`
 	OrderBy    string   `help:"order by specific field"`
 	Action     []string `help:"Log action"`
+
+	User    []string `help:"filter by operator user"`
+	Project []string `help:"filter by owner project"`
+
+	PagingMarker string `help:"marker for pagination"`
 }
 
 type EventListOptions struct {
@@ -92,9 +97,18 @@ func doEventList(man modules.ResourceManager, s *mcclient.ClientSession, args *E
 	if len(args.Action) > 0 {
 		params.Add(jsonutils.NewStringArray(args.Action), "action")
 	}
+	if len(args.User) > 0 {
+		params.Add(jsonutils.NewStringArray(args.User), "user")
+	}
+	if len(args.Project) > 0 {
+		params.Add(jsonutils.NewStringArray(args.Project), "project")
+	}
 	if len(args.Scope) > 0 {
 		params.Add(jsonutils.NewString(args.Scope), "scope")
 	}
+	if len(args.PagingMarker) > 0 {
+		params.Add(jsonutils.NewString(args.PagingMarker), "paging_marker")
+	}
 	logs, err := man.List(s, params)
 	if err != nil {
 		return err

pkg/cloudcommon/db/db_dispatcher.go

@@ -320,7 +320,7 @@ func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.To
 	items := make([]IModel, 0)
 	results := make([]jsonutils.JSONObject, 0)
 	rows, err := q.Rows()
-	if err != nil {
+	if err != nil && err != sql.ErrNoRows {
 		return nil, err
 	}
 	defer rows.Close()
@@ -445,6 +445,7 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	var maxLimit int64 = 2048
 	limit, _ := query.Int("limit")
 	offset, _ := query.Int("offset")
+	pagingMarker, _ := query.GetString("paging_marker")
 	q := manager.Query()
 
 	queryDict, ok := query.(*jsonutils.JSONDict)
@@ -469,14 +470,22 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		return nil, err
 	}
 
-	totalCnt, err := q.CountWithError()
-	if err != nil {
-		return nil, err
-	}
-	// log.Debugf("total count %d", totalCnt)
-	if totalCnt == 0 {
-		emptyList := modules.ListResult{Data: []jsonutils.JSONObject{}}
-		return &emptyList, nil
+	var totalCnt int
+	pagingConf := manager.GetPagingConfig()
+	if pagingConf == nil {
+		totalCnt, err = q.CountWithError()
+		if err != nil {
+			return nil, err
+		}
+		// log.Debugf("total count %d", totalCnt)
+		if totalCnt == 0 {
+			emptyList := modules.ListResult{Data: []jsonutils.JSONObject{}}
+			return &emptyList, nil
+		}
+	} else {
+		if limit <= 0 {
+			limit = int64(pagingConf.DefaultLimit)
+		}
 	}
 	if int64(totalCnt) > maxLimit && (limit <= 0 || limit > maxLimit) {
 		limit = maxLimit
@@ -502,6 +511,10 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	}
 	orderQuery := query.(*jsonutils.JSONDict).Copy()
 	for _, orderByField := range orderBy {
+		if pagingConf != nil && orderByField == pagingConf.MarkerField {
+			// skip markerField
+			continue
+		}
 		colSpec := manager.TableSpec().ColumnSpec(orderByField)
 		if colSpec == nil {
 			orderQuery.Set(fmt.Sprintf("order_by_%s", orderByField), jsonutils.NewString(string(order)))
@@ -523,6 +536,10 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		}
 	}
 	for _, orderByField := range orderBy {
+		if pagingConf != nil && pagingConf.MarkerField == orderByField {
+			// skip markerField
+			continue
+		}
 		colSpec := manager.TableSpec().ColumnSpec(orderByField)
 		if colSpec != nil {
 			if order == sqlchemy.SQL_ORDER_ASC {
@@ -532,6 +549,44 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 			}
 		}
 	}
+	if pagingConf != nil {
+		if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
+			q = q.Asc(pagingConf.MarkerField)
+		} else {
+			q = q.Desc(pagingConf.MarkerField)
+		}
+	}
+
+	if pagingConf != nil {
+		q = q.Limit(int(limit) + 1)
+		if len(pagingMarker) > 0 {
+			if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
+				q = q.GE(pagingConf.MarkerField, pagingMarker)
+			} else {
+				q = q.LE(pagingConf.MarkerField, pagingMarker)
+			}
+		}
+		retList, err := Query2List(manager, ctx, userCred, q, queryDict, false)
+		if err != nil {
+			return nil, httperrors.NewGeneralError(err)
+		}
+		nextMarker := ""
+		if int64(len(retList)) > limit {
+			markerObj, _ := retList[limit].Get(pagingConf.MarkerField)
+			if markerObj != nil {
+				nextMarker = fmt.Sprintf("%s", markerObj)
+			}
+			retList = retList[:limit]
+		}
+		retResult := modules.ListResult{
+			Data: retList, Limit: int(limit),
+			NextMarker:  nextMarker,
+			MarkerField: pagingConf.MarkerField,
+			MarkerOrder: string(pagingConf.Order),
+		}
+		return &retResult, nil
+	}
+
 	customizeFilters, err := manager.CustomizeFilterList(ctx, q, userCred, queryDict)
 	if err != nil {
 		return nil, err

pkg/cloudcommon/db/interface.go

@@ -110,6 +110,8 @@ type IModelManager interface {
 	ResourceScope() rbacutils.TRbacScope
 
 	QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error)
+
+	GetPagingConfig() *SPagingConfig
 }
 
 type IModel interface {

pkg/cloudcommon/db/modelbase.go

@@ -272,6 +272,10 @@ func (manager *SModelBaseManager) AllowGetPropertyDistinctField(ctx context.Cont
 	return true
 }
 
+func (manager *SModelBaseManager) GetPagingConfig() *SPagingConfig {
+	return nil
+}
+
 func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
 	im, ok := manager.GetVirtualObject().(IModelManager)
 	if !ok {

pkg/cloudcommon/db/opslog.go

@@ -16,6 +16,7 @@ package db
 
 import (
 	"context"
+	"database/sql"
 	"fmt"
 	"strconv"
 	"strings"
@@ -227,12 +228,12 @@ type SOpsLogManager struct {
 type SOpsLog struct {
 	SModelBase
 
-	Id      int64  `primary:"true" auto_increment:"true" list:"user"`                           // = Column(BigInteger, primary_key=True)
-	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`  // = Column(VARCHAR(40, charset='ascii'), nullable=False)
-	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required"` //  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
-	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`  //= Column(VARCHAR(128, charset='utf8'), nullable=False)
-	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`   //= Column(VARCHAR(32, charset='ascii'), nullable=False)
-	Notes   string `width:"2048" charset:"utf8" list:"user" create:"required"`                  // = Column(VARCHAR(2048, charset='utf8'))
+	Id      int64  `primary:"true" auto_increment:"true" list:"user"`                                        // = Column(BigInteger, primary_key=True)
+	ObjType string `width:"40" charset:"ascii" nullable:"false" list:"user" create:"required"`               // = Column(VARCHAR(40, charset='ascii'), nullable=False)
+	ObjId   string `width:"128" charset:"ascii" nullable:"false" list:"user" create:"required" index:"true"` //  = Column(VARCHAR(ID_LENGTH, charset='ascii'), nullable=False)
+	ObjName string `width:"128" charset:"utf8" nullable:"false" list:"user" create:"required"`               //= Column(VARCHAR(128, charset='utf8'), nullable=False)
+	Action  string `width:"32" charset:"utf8" nullable:"false" list:"user" create:"required"`                //= Column(VARCHAR(32, charset='ascii'), nullable=False)
+	Notes   string `width:"2048" charset:"utf8" list:"user" create:"required"`                               // = Column(VARCHAR(2048, charset='utf8'))
 
 	ProjectId string `name:"tenant_id" width:"128" charset:"ascii" list:"user" create:"required" index:"true"` // = Column(VARCHAR(ID_LENGTH, charset='ascii'))
 	Project   string `name:"tenant" width:"128" charset:"utf8" list:"user" create:"required"`                  // tenant    = Column(VARCHAR(128, charset='utf8'))
@@ -386,19 +387,87 @@ func (manager *SOpsLogManager) LogDetachEvent(ctx context.Context, m1, m2 IModel
 }
 
 func (manager *SOpsLogManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
+	userStrs := jsonutils.GetQueryStringArray(query, "user")
+	if len(userStrs) > 0 {
+		for i := range userStrs {
+			usrObj, err := UserCacheManager.FetchUserByIdOrName(userStrs[i])
+			if err != nil {
+				if err == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2("user", userStrs[i])
+				} else if err == sqlchemy.ErrDuplicateEntry {
+					return nil, httperrors.NewDuplicateNameError("user", userStrs[i])
+				} else {
+					return nil, httperrors.NewGeneralError(err)
+				}
+			}
+			userStrs[i] = usrObj.GetId()
+		}
+		if len(userStrs) == 1 {
+			q = q.Filter(sqlchemy.Equals(q.Field("user_id"), userStrs[0]))
+		} else {
+			q = q.Filter(sqlchemy.In(q.Field("user_id"), userStrs))
+		}
+	}
+	projStrs := jsonutils.GetQueryStringArray(query, "project")
+	if len(projStrs) > 0 {
+		for i := range projStrs {
+			projObj, err := TenantCacheManager.FetchTenantByIdOrName(ctx, projStrs[i])
+			if err != nil {
+				if err == sql.ErrNoRows {
+					return nil, httperrors.NewResourceNotFoundError2("project", projStrs[i])
+				} else {
+					return nil, httperrors.NewGeneralError(err)
+				}
+			}
+			projStrs[i] = projObj.GetId()
+		}
+		if len(projStrs) == 1 {
+			q = q.Filter(sqlchemy.Equals(q.Field("owner_tenant_id"), projStrs[0]))
+		} else {
+			q = q.Filter(sqlchemy.In(q.Field("owner_tenant_id"), projStrs))
+		}
+	}
 	objTypes := jsonutils.GetQueryStringArray(query, "obj_type")
-	if objTypes != nil && len(objTypes) > 0 {
-		q = q.Filter(sqlchemy.In(q.Field("obj_type"), objTypes))
+	if len(objTypes) > 0 {
+		if len(objTypes) == 1 {
+			q = q.Filter(sqlchemy.Equals(q.Field("obj_type"), objTypes[0]))
+		} else {
+			q = q.Filter(sqlchemy.In(q.Field("obj_type"), objTypes))
+		}
+	}
+	objs := jsonutils.GetQueryStringArray(query, "obj")
+	if len(objs) > 0 {
+		if len(objs) == 1 {
+			q = q.Filter(sqlchemy.OR(sqlchemy.Equals(q.Field("obj_id"), objs[0]), sqlchemy.Equals(q.Field("obj_name"), objs[0])))
+		} else {
+			q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("obj_id"), objs), sqlchemy.In(q.Field("obj_name"), objs)))
+		}
 	}
 	objIds := jsonutils.GetQueryStringArray(query, "obj_id")
-	if objIds != nil && len(objIds) > 0 {
-		q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("obj_id"), objIds), sqlchemy.In(q.Field("obj_name"), objIds)))
+	if len(objIds) > 0 {
+		if len(objIds) == 1 {
+			q = q.Filter(sqlchemy.Equals(q.Field("obj_id"), objIds[0]))
+		} else {
+			q = q.Filter(sqlchemy.In(q.Field("obj_id"), objIds))
+		}
+	}
+	objNames := jsonutils.GetQueryStringArray(query, "obj_name")
+	if len(objNames) > 0 {
+		if len(objNames) == 1 {
+			q = q.Filter(sqlchemy.Equals(q.Field("obj_id"), objNames[0]))
+		} else {
+			q = q.Filter(sqlchemy.In(q.Field("obj_id"), objNames))
+		}
 	}
 	queryDict := query.(*jsonutils.JSONDict)
 	queryDict.Remove("obj_id")
 	action := jsonutils.GetQueryStringArray(query, "action")
 	if action != nil && len(action) > 0 {
-		q = q.Filter(sqlchemy.In(q.Field("action"), action))
+		if len(action) == 1 {
+			q = q.Filter(sqlchemy.Equals(q.Field("action"), action[0]))
+		} else {
+			q = q.Filter(sqlchemy.In(q.Field("action"), action))
+		}
 	}
 	//if !IsAdminAllowList(userCred, manager) {
 	// 	q = q.Filter(sqlchemy.OR(
@@ -515,3 +584,11 @@ func (self *SOpsLog) IsSharable(reqCred mcclient.IIdentityProvider) bool {
 func (manager *SOpsLogManager) ResourceScope() rbacutils.TRbacScope {
 	return rbacutils.ScopeProject
 }
+
+func (manager *SOpsLogManager) GetPagingConfig() *SPagingConfig {
+	return &SPagingConfig{
+		Order:        sqlchemy.SQL_ORDER_DESC,
+		MarkerField:  "id",
+		DefaultLimit: 20,
+	}
+}

pkg/cloudcommon/db/paging.go

@@ -0,0 +1,11 @@
+package db
+
+import (
+	"yunion.io/x/sqlchemy"
+)
+
+type SPagingConfig struct {
+	Order        sqlchemy.QueryOrderType
+	MarkerField  string
+	DefaultLimit int
+}

pkg/mcclient/modules/base.go

@@ -116,6 +116,10 @@ type ListResult struct {
 	Total  int
 	Limit  int
 	Offset int
+
+	NextMarker  string
+	MarkerField string
+	MarkerOrder string
 }
 
 func ListResult2JSONWithKey(result *ListResult, key string) jsonutils.JSONObject {
@@ -129,6 +133,15 @@ func ListResult2JSONWithKey(result *ListResult, key string) jsonutils.JSONObject
 	if result.Offset > 0 {
 		obj.Add(jsonutils.NewInt(int64(result.Offset)), "offset")
 	}
+	if len(result.NextMarker) > 0 {
+		obj.Add(jsonutils.NewString(result.NextMarker), "next_marker")
+	}
+	if len(result.MarkerField) > 0 {
+		obj.Add(jsonutils.NewString(result.MarkerField), "marker_field")
+	}
+	if len(result.MarkerOrder) > 0 {
+		obj.Add(jsonutils.NewString(result.MarkerOrder), "marker_order")
+	}
 	arr := jsonutils.NewArray(result.Data...)
 	obj.Add(arr, key)
 	return obj
@@ -142,8 +155,20 @@ func JSON2ListResult(result jsonutils.JSONObject) *ListResult {
 	total, _ := result.Int("total")
 	limit, _ := result.Int("limit")
 	offset, _ := result.Int("offset")
+	nextMarker, _ := result.GetString("next_marker")
+	markerField, _ := result.GetString("marker_field")
+	markerOrder, _ := result.GetString("marker_order")
 	data, _ := result.GetArray("data")
-	return &ListResult{Data: data, Total: int(total), Limit: int(limit), Offset: int(offset)}
+	if len(markerField) == 0 && total == 0 {
+		total = int64(len(data))
+	}
+	return &ListResult{
+		Data:  data,
+		Total: int(total), Limit: int(limit), Offset: int(offset),
+		NextMarker:  nextMarker,
+		MarkerField: markerField,
+		MarkerOrder: markerOrder,
+	}
 }
 
 func (this *BaseManager) _list(session *mcclient.ClientSession, path, responseKey string) (*ListResult, error) {
@@ -159,22 +184,22 @@ func (this *BaseManager) _list(session *mcclient.ClientSession, path, responseKe
 	if err != nil {
 		return nil, err
 	}
-	total, err := body.Int("total")
-	if err != nil {
+	nextMarker, _ := body.GetString("next_marker")
+	markerField, _ := body.GetString("marker_field")
+	markerOrder, _ := body.GetString("marker_order")
+	total, _ := body.Int("total")
+	limit, _ := body.Int("limit")
+	offset, _ := body.Int("offset")
+	if len(nextMarker) == 0 && total == 0 {
 		total = int64(len(rets))
 	}
-	if total == 0 {
-		total = int64(len(rets))
-	}
-	limit, err := body.Int("limit")
-	if err != nil {
-		limit = 0
-	}
-	offset, err := body.Int("offset")
-	if err != nil {
-		offset = 0
-	}
-	return &ListResult{rets, int(total), int(limit), int(offset)}, nil
+	return &ListResult{
+		Data:  rets,
+		Total: int(total), Limit: int(limit), Offset: int(offset),
+		NextMarker:  nextMarker,
+		MarkerField: markerField,
+		MarkerOrder: markerOrder,
+	}, nil
 }
 
 func (this *BaseManager) _submit(session *mcclient.ClientSession, method httputils.THttpMethod, path string, body jsonutils.JSONObject, respKey string) (jsonutils.JSONObject, error) {

pkg/mcclient/options/base.go

@@ -220,6 +220,8 @@ type BaseListOptions struct {
 	PrivateCloud *bool    `help:"List objects belonging to private cloud" json:"private_cloud"`
 	IsOnPremise  *bool    `help:"List objects belonging to on premise infrastructures" token:"on-premise" json:"is_on_premise"`
 	IsManaged    *bool    `help:"List objects managed by external providers" token:"managed" json:"is_managed"`
+
+	PagingMarker string `help:"Marker for pagination" json:"paging_marker"`
 }
 
 func (opts *BaseListOptions) addTag(prefix, tag string, idx int, params *jsonutils.JSONDict) error {

pkg/util/printutils/printjson.go

@@ -73,21 +73,29 @@ func PrintJSONList(list *modules.ListResult, columns []string) {
 		rows = append(rows, row)
 	}
 	fmt.Print(pt.GetString(rows))
+	var total int64
 	if list.Total == 0 {
-		list.Total = len(list.Data)
+		total = int64(len(list.Data))
 	}
-	title := fmt.Sprintf("Total: %d", list.Total)
-	if list.Limit == 0 && list.Total > len(list.Data) {
-		list.Limit = len(list.Data)
-	}
-	if list.Limit > 0 {
-		pages := int(list.Total / list.Limit)
-		if pages*list.Limit < list.Total {
-			pages += 1
+	title := fmt.Sprintf("Total: %d", total)
+	if len(list.MarkerField) > 0 {
+		title += fmt.Sprintf(" Field: %s Order: %s", list.MarkerField, list.MarkerOrder)
+		if len(list.NextMarker) > 0 {
+			title += fmt.Sprintf(" NextMarker: %s", list.NextMarker)
+		}
+	} else {
+		if list.Limit == 0 && total > int64(len(list.Data)) {
+			list.Limit = len(list.Data)
+		}
+		if list.Limit > 0 {
+			pages := int(total / int64(list.Limit))
+			if int64(pages*list.Limit) < total {
+				pages += 1
+			}
+			page := int(list.Offset/list.Limit) + 1
+			title = fmt.Sprintf("%s Pages: %d Limit: %d Offset: %d Page: %d",
+				title, pages, list.Limit, list.Offset, page)
 		}
-		page := int(list.Offset/list.Limit) + 1
-		title = fmt.Sprintf("%s Pages: %d Limit: %d Offset: %d Page: %d",
-			title, pages, list.Limit, list.Offset, page)
 	}
 	fmt.Println("*** ", title, " ***")
 }