opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-08-30 17:13:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

feature: list results carry extra total stats (#15682)

Browse Source 
Co-authored-by: Qiu Jian <qiujian@yunionyun.com>
This commit is contained in:
Jian Qiu
2023-01-02 14:32:36 +08:00
committed by GitHub
parent 6cdda3ead6
commit e656426e3f
13 changed files with 112 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+2 -2
View File
@@ -286,8 +286,8 @@ RELEASE_BRANCH:=master
GOPROXY ?= direct
mod:
GOPROXY=$(GOPROXY) go get -d yunion.io/x/cloudmux@$(RELEASE_BRANCH)
GOPROXY=$(GOPROXY) go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v '/cloudmux$$'))
GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go get -d yunion.io/x/cloudmux@$(RELEASE_BRANCH)
GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v '/cloudmux$$'))
go mod tidy
go mod vendor -v
go.mod
+2 -2
View File
@@ -86,9 +86,9 @@ require (
yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd
yunion.io/x/log v1.0.0
yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7
yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76
yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af
yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8
yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6
)
go.sum
+4 -4
View File
@@ -1176,12 +1176,12 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.0.1-0.20220819080140-55b265a8658e/go.mod h1:nYOMflWG1wQC6301rGVglQva9bYbJxHThEJUxRAgMFc=
yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76 h1:Kg/4ZcA4Cnac3KclHmD2OiaHaQdyRJZ3KWJKUfqFQfY=
yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c h1:qkkyHkL8bFWumxWvtdbYhY4aDfKi+sBS21Im/i6Jeds=
yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v1.0.1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af h1:WjaARkzdxBQBD6j5I1MDMlJ0Z3WOHzEp5bTQl+TScaU=
yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE=
yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8 h1:YJwxIFME9UoVX5NDGK2NnMHuIcRyEy0ubjBvzbIooNk=
yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE=
yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 h1:WuWXhY3DvhdRTzWCJ/kwt3Ss6KIq7+KqJwb+esvNGwU=
yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
pkg/apis/list.go
+4
View File
@@ -382,3 +382,7 @@ type IdNameDetails struct {
Id string `json:"id"`
Name string `json:"name"`
}
type TotalCountBase struct {
Count int `json:"count"`
}
pkg/cloudcommon/db/db_dispatcher.go
+15 -6
View File
@@ -648,10 +648,13 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
}
var totalCnt int
var totalJson jsonutils.JSONObject
if pagingConf == nil {
totalCnt, err = q.CountWithError()
// calculate total
totalQ := q.CountQuery()
totalCnt, totalJson, err = manager.CustomizedTotalCount(totalQ)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "CustomizedTotalCount")
}
//log.Debugf("total count %d", totalCnt)
if totalCnt == 0 {
@@ -780,7 +783,7 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
customizeFilters, err := manager.CustomizeFilterList(ctx, q, userCred, queryDict)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "CustomizeFilterList")
}
delayFetch := false
if customizeFilters.IsEmpty() {
@@ -818,10 +821,10 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok
// query not use Limit and Offset, do manual pagination
paginate = true
}
return calculateListResult(retList, int64(totalCnt), limit, offset, paginate), nil
return calculateListResult(retList, totalCnt, totalJson, int(limit), int(offset), paginate), nil
}
func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64, paginate bool) *printutils.ListResult {
func calculateListResult(data []jsonutils.JSONObject, total int, totalJson jsonutils.JSONObject, limit, offset int, paginate bool) *printutils.ListResult {
if paginate {
// do offset first
if offset > 0 {
@@ -839,7 +842,13 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64
}
}
retResult := printutils.ListResult{Data: data, Total: int(total), Limit: int(limit), Offset: int(offset)}
retResult := printutils.ListResult{
Data: data,
Total: total,
Limit: limit,
Offset: offset,
Totals: totalJson,
}
return &retResult
}
pkg/cloudcommon/db/interface.go
+2
View File
@@ -138,6 +138,8 @@ type IModelManager interface {
GetSplitTable() *splitable.SSplitTableSpec
CreateByInsertOrUpdate() bool
CustomizedTotalCount(totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error)
}
type IModel interface {
pkg/cloudcommon/db/modelbase.go
+9
View File
@@ -539,6 +539,15 @@ func (manager *SModelBaseManager) PerformPurgeSplitable(ctx context.Context, use
return jsonutils.Marshal(map[string][]string{"tables": ret}), nil
}
func (manager *SModelBaseManager) CustomizedTotalCount(totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error) {
ret := apis.TotalCountBase{}
err := totalQ.First(&ret)
if err != nil {
return -1, nil, errors.Wrap(err, "SModelBaseManager Query total")
}
return ret.Count, nil, nil
}
func (model *SModelBase) GetId() string {
return ""
}
pkg/compute/models/guests.go
+39
View File
@@ -6352,3 +6352,42 @@ func (guest *SGuest) GetGuestBackupMirrorJobStatus(ctx context.Context, userCred
func (guest *SGuest) ResetGuestQuorumChildIndex(ctx context.Context, userCred mcclient.TokenCredential) error {
return guest.SetMetadata(ctx, api.QUORUM_CHILD_INDEX, "", userCred)
}
type SGuestTotalCount struct {
apis.TotalCountBase
CpuCount int
MemMb int
DiskMb int64
DiskCount int
}
func (manager *SGuestManager) CustomizedTotalCount(totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error) {
results := SGuestTotalCount{}
totalQ = totalQ.AppendField(sqlchemy.SUM("cpu_count", totalQ.Field("vcpu_count")))
totalQ = totalQ.AppendField(sqlchemy.SUM("mem_mb", totalQ.Field("vmem_size")))
err := totalQ.First(&results)
if err != nil {
return -1, nil, errors.Wrap(err, "SGuestManager query total")
}
log.Debugf("CustomizedTotalCount %s", jsonutils.Marshal(results))
diskQ := DiskManager.Query()
diskGuestQ := GuestdiskManager.Query().SubQuery()
diskQ = diskQ.Join(diskGuestQ, sqlchemy.Equals(diskQ.Field("id"), diskGuestQ.Field("disk_id")))
totalSQ := totalQ.ResetFields().SubQuery()
diskQ = diskQ.Join(totalSQ, sqlchemy.Equals(diskGuestQ.Field("guest_id"), totalSQ.Field("id")))
diskQ = diskQ.AppendField(sqlchemy.COUNT("disk_count"))
diskQ = diskQ.AppendField(sqlchemy.SUM("disk_mb", diskQ.Field("disk_size")))
err = diskQ.First(&results)
if err != nil {
return -1, nil, errors.Wrap(err, "SGuestManager query total_disk")
}
log.Debugf("CustomizedTotalCount %s", jsonutils.Marshal(results))
return results.Count, jsonutils.Marshal(results), nil
}
pkg/mcclient/modulebase/base.go
+7
View File
@@ -154,6 +154,9 @@ func ListResult2JSONWithKey(result *printutils.ListResult, key string) jsonutils
if result.Total > 0 {
obj.Add(jsonutils.NewInt(int64(result.Total)), "total")
}
if result.Totals != nil {
obj.Add(result.Totals, "totals")
}
if result.Limit > 0 {
obj.Add(jsonutils.NewInt(int64(result.Limit)), "limit")
}
@@ -189,9 +192,11 @@ func JSON2ListResult(result jsonutils.JSONObject) *printutils.ListResult {
if len(markerField) == 0 && total == 0 {
total = int64(len(data))
}
totalJson, _ := result.Get("totals")
return &printutils.ListResult{
Data: data,
Total: int(total), Limit: int(limit), Offset: int(offset),
Totals: totalJson,
NextMarker: nextMarker,
MarkerField: markerField,
MarkerOrder: markerOrder,
@@ -219,9 +224,11 @@ func (this *BaseManager) _list(session *mcclient.ClientSession, path, responseKe
if len(nextMarker) == 0 && total == 0 {
total = int64(len(rets))
}
totalJson, _ := body.Get("totals")
return &printutils.ListResult{
Data: rets,
Total: int(total), Limit: int(limit), Offset: int(offset),
Totals: totalJson,
NextMarker: nextMarker,
MarkerField: markerField,
MarkerOrder: markerOrder,
vendor/modules.txt
Vendored
+2 -2
View File
@@ -1534,7 +1534,7 @@ yunion.io/x/log/hooks
yunion.io/x/ovsdb/cli_util
yunion.io/x/ovsdb/schema/ovn_nb
yunion.io/x/ovsdb/types
# yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76
# yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c
## explicit; go 1.18
yunion.io/x/pkg/appctx
yunion.io/x/pkg/errors
@@ -1588,7 +1588,7 @@ yunion.io/x/pkg/utils
# yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
## explicit; go 1.12
yunion.io/x/s3cli
# yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af
# yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8
## explicit; go 1.17
yunion.io/x/sqlchemy
yunion.io/x/sqlchemy/backends
vendor/yunion.io/x/pkg/util/printutils/printjson.go
Generated Vendored
+14
View File
@@ -132,6 +132,20 @@ func PrintJSONList(list *ListResult, columns []string) {
}
}
fmt.Println("*** ", title, " ***")
if list.Totals != nil {
if totalDict, ok := list.Totals.(*jsonutils.JSONDict); ok {
totalMap, err := totalDict.GetMap()
if err != nil {
fmt.Println("error to convert totals to JSONDict")
} else {
segs := make([]string, 0)
for k, v := range totalMap {
segs = append(segs, fmt.Sprintf("%s: %s", k, v.String()))
}
fmt.Printf("***%s***\n", strings.Join(segs, ", "))
}
}
}
if colTruncated {
fmt.Println(fmt.Sprintf("!!!Some text truncated, set env %s=-1 to show full text!!!", OS_MAX_COLUMN_TEXT_LENGTH))
}
vendor/yunion.io/x/pkg/util/printutils/struct.go
Generated Vendored
+7 -4
View File
@@ -19,10 +19,13 @@ import (
)
type ListResult struct {
Data []jsonutils.JSONObject `json:"data,allowempty"`
Total int
Limit int
Offset int
Data []jsonutils.JSONObject `json:"data,allowempty"`
Total int `json:"total"`
Limit int `json:"limit"`
Offset int `json:"offset"`
Totals jsonutils.JSONObject `json:"totals"`
NextMarker string
MarkerField string
vendor/yunion.io/x/sqlchemy/query.go
Generated Vendored
+5
View File
@@ -81,6 +81,11 @@ func (tq *SQuery) AppendField(f ...IQueryField) *SQuery {
return tq
}
func (tq *SQuery) ResetFields() *SQuery {
tq.fields = nil
return tq
}
// Query of SSubQuery generates a new query from a subquery
func (sq *SSubQuery) Query(f ...IQueryField) *SQuery {
return DoQuery(sq, f...)