fix: increment paging list order by multiple columns

pkg/cloudcommon/db/db_dispatcher.go

@@ -530,6 +530,17 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		limit = exportLimit
 	}
 
+	// orders defined in pagingConf should have the highest priority
+	if pagingConf != nil {
+		for _, f := range pagingConf.MarkerFields {
+			if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
+				q = q.Asc(f)
+			} else {
+				q = q.Desc(f)
+			}
+		}
+	}
+
 	var primaryCol sqlchemy.IColumnSpec
 	primaryCols := manager.TableSpec().PrimaryColumns()
 	if len(primaryCols) == 1 {
@@ -544,8 +555,8 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 	}
 	orderQuery := query.(*jsonutils.JSONDict).Copy()
 	for _, orderByField := range orderBy {
-		if pagingConf != nil && orderByField == pagingConf.MarkerField {
-			// skip markerField
+		if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
+			// skip markerField in pagingConf
 			continue
 		}
 		colSpec := manager.TableSpec().ColumnSpec(orderByField)
@@ -569,8 +580,8 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 		}
 	}
 	for _, orderByField := range orderBy {
-		if pagingConf != nil && pagingConf.MarkerField == orderByField {
-			// skip markerField
+		if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
+			// skip markerField in pagingConf
 			continue
 		}
 		colSpec := manager.TableSpec().ColumnSpec(orderByField)
@@ -582,36 +593,38 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
 			}
 		}
 	}
-	if pagingConf != nil {
-		if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
-			q = q.Asc(pagingConf.MarkerField)
-		} else {
-			q = q.Desc(pagingConf.MarkerField)
-		}
-	}
 
 	if pagingConf != nil {
 		q = q.Limit(int(limit) + 1)
 		if len(pagingMarker) > 0 {
-			if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
-				q = q.GE(pagingConf.MarkerField, pagingMarker)
-			} else {
-				q = q.LE(pagingConf.MarkerField, pagingMarker)
+			markers := decodePagingMarker(pagingMarker)
+			for markerIdx, marker := range markers {
+				if markerIdx < len(pagingConf.MarkerFields) {
+					if pagingConf.Order == sqlchemy.SQL_ORDER_ASC {
+						q = q.GE(pagingConf.MarkerFields[markerIdx], marker)
+					} else {
+						q = q.LE(pagingConf.MarkerFields[markerIdx], marker)
+					}
+				}
 			}
 		}
 		retList, err := Query2List(manager, ctx, userCred, q, queryDict, false)
 		if err != nil {
 			return nil, httperrors.NewGeneralError(err)
 		}
-		nextMarker := ""
+		nextMarkers := make([]string, 0)
 		if int64(len(retList)) > limit {
-			nextMarker, _ = retList[limit].GetString(pagingConf.MarkerField)
+			for _, markerField := range pagingConf.MarkerFields {
+				nextMarker, _ := retList[limit].GetString(markerField)
+				nextMarkers = append(nextMarkers, nextMarker)
+			}
 			retList = retList[:limit]
 		}
+		nextMarker := encodePagingMarker(nextMarkers)
 		retResult := modulebase.ListResult{
 			Data: retList, Limit: int(limit),
 			NextMarker:  nextMarker,
-			MarkerField: pagingConf.MarkerField,
+			MarkerField: strings.Join(pagingConf.MarkerFields, ","),
 			MarkerOrder: string(pagingConf.Order),
 		}
 		return &retResult, nil

pkg/cloudcommon/db/opslog.go

@@ -640,7 +640,7 @@ func (manager *SOpsLogManager) ResourceScope() rbacutils.TRbacScope {
 func (manager *SOpsLogManager) GetPagingConfig() *SPagingConfig {
 	return &SPagingConfig{
 		Order:        sqlchemy.SQL_ORDER_DESC,
-		MarkerField:  "id",
+		MarkerFields: []string{"id"},
 		DefaultLimit: 20,
 	}
 }

pkg/cloudcommon/db/paging.go

@@ -15,11 +15,38 @@
 package db
 
 import (
+	"yunion.io/x/jsonutils"
 	"yunion.io/x/sqlchemy"
 )
 
 type SPagingConfig struct {
 	Order        sqlchemy.QueryOrderType
-	MarkerField  string
+	MarkerFields []string
 	DefaultLimit int
 }
+
+func decodePagingMarker(markString string) []string {
+	if len(markString) > 2 && markString[0] == '[' && markString[len(markString)-1] == ']' {
+		markJson, _ := jsonutils.ParseString(markString)
+		if markJson != nil {
+			if markArray, ok := markJson.(*jsonutils.JSONArray); ok {
+				return markArray.GetStringArray()
+			}
+		}
+	}
+	if len(markString) > 0 {
+		return []string{markString}
+	}
+	return []string{}
+}
+
+func encodePagingMarker(markers []string) string {
+	switch len(markers) {
+	case 0:
+		return ""
+	case 1:
+		return markers[0]
+	default:
+		return jsonutils.Marshal(markers).String()
+	}
+}

pkg/cloudevent/models/cloudevents.go

@@ -154,7 +154,7 @@ func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred
 func (manager *SCloudeventManager) GetPagingConfig() *db.SPagingConfig {
 	return &db.SPagingConfig{
 		Order:        sqlchemy.SQL_ORDER_DESC,
-		MarkerField:  "created_at",
+		MarkerFields: []string{"created_at", "event_id"},
 		DefaultLimit: 20,
 	}
 }

pkg/logger/models/baremetalevents.go

@@ -80,7 +80,7 @@ func (manager *SBaremetalEventManager) ResourceScope() rbacutils.TRbacScope {
 func (manager *SBaremetalEventManager) GetPagingConfig() *db.SPagingConfig {
 	return &db.SPagingConfig{
 		Order:        sqlchemy.SQL_ORDER_DESC,
-		MarkerField:  "id",
+		MarkerFields: []string{"id"},
 		DefaultLimit: 20,
 	}
 }