diff --git a/Makefile b/Makefile index a232893b67..c78ad8ed2b 100644 --- a/Makefile +++ b/Makefile @@ -286,8 +286,8 @@ RELEASE_BRANCH:=release/3.10 GOPROXY ?= direct mod: - GOPROXY=$(GOPROXY) go get -d yunion.io/x/cloudmux@$(RELEASE_BRANCH) - GOPROXY=$(GOPROXY) go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v '/cloudmux$$')) + GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go get -d yunion.io/x/cloudmux@$(RELEASE_BRANCH) + GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p' | grep -v '/cloudmux$$')) go mod tidy go mod vendor -v diff --git a/go.mod b/go.mod index aa6e4e2020..9992f6362b 100644 --- a/go.mod +++ b/go.mod @@ -86,9 +86,9 @@ require ( yunion.io/x/jsonutils v1.0.1-0.20220819091305-3bab322ab4fd yunion.io/x/log v1.0.0 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 - yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76 + yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af + yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8 yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 ) diff --git a/go.sum b/go.sum index ce47c4785d..af0096e56f 100644 --- a/go.sum +++ b/go.sum @@ -1176,12 +1176,12 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v1.0.1-0.20220819080140-55b265a8658e/go.mod h1:nYOMflWG1wQC6301rGVglQva9bYbJxHThEJUxRAgMFc= -yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76 h1:Kg/4ZcA4Cnac3KclHmD2OiaHaQdyRJZ3KWJKUfqFQfY= -yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk= +yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c h1:qkkyHkL8bFWumxWvtdbYhY4aDfKi+sBS21Im/i6Jeds= +yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= yunion.io/x/sqlchemy v1.0.1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= -yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af h1:WjaARkzdxBQBD6j5I1MDMlJ0Z3WOHzEp5bTQl+TScaU= -yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE= +yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8 h1:YJwxIFME9UoVX5NDGK2NnMHuIcRyEy0ubjBvzbIooNk= +yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE= yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 h1:WuWXhY3DvhdRTzWCJ/kwt3Ss6KIq7+KqJwb+esvNGwU= yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/apis/list.go b/pkg/apis/list.go index f389cbe4ac..0825de85f6 100644 --- a/pkg/apis/list.go +++ b/pkg/apis/list.go @@ -382,3 +382,7 @@ type IdNameDetails struct { Id string `json:"id"` Name string `json:"name"` } + +type TotalCountBase struct { + Count int `json:"count"` +} diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index d18c4070bd..759a50295f 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -648,10 +648,13 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok } var totalCnt int + var totalJson jsonutils.JSONObject if pagingConf == nil { - totalCnt, err = q.CountWithError() + // calculate total + totalQ := q.CountQuery() + totalCnt, totalJson, err = manager.CustomizedTotalCount(totalQ) if err != nil { - return nil, err + return nil, errors.Wrap(err, "CustomizedTotalCount") } //log.Debugf("total count %d", totalCnt) if totalCnt == 0 { @@ -780,7 +783,7 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok customizeFilters, err := manager.CustomizeFilterList(ctx, q, userCred, queryDict) if err != nil { - return nil, err + return nil, errors.Wrap(err, "CustomizeFilterList") } delayFetch := false if customizeFilters.IsEmpty() { @@ -818,10 +821,10 @@ func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.Tok // query not use Limit and Offset, do manual pagination paginate = true } - return calculateListResult(retList, int64(totalCnt), limit, offset, paginate), nil + return calculateListResult(retList, totalCnt, totalJson, int(limit), int(offset), paginate), nil } -func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64, paginate bool) *printutils.ListResult { +func calculateListResult(data []jsonutils.JSONObject, total int, totalJson jsonutils.JSONObject, limit, offset int, paginate bool) *printutils.ListResult { if paginate { // do offset first if offset > 0 { @@ -839,7 +842,13 @@ func calculateListResult(data []jsonutils.JSONObject, total, limit, offset int64 } } - retResult := printutils.ListResult{Data: data, Total: int(total), Limit: int(limit), Offset: int(offset)} + retResult := printutils.ListResult{ + Data: data, + Total: total, + Limit: limit, + Offset: offset, + Totals: totalJson, + } return &retResult } diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index 9462eb6979..97cc1b8b0e 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -138,6 +138,8 @@ type IModelManager interface { GetSplitTable() *splitable.SSplitTableSpec CreateByInsertOrUpdate() bool + + CustomizedTotalCount(totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error) } type IModel interface { diff --git a/pkg/cloudcommon/db/modelbase.go b/pkg/cloudcommon/db/modelbase.go index 91b2a08634..e47befd324 100644 --- a/pkg/cloudcommon/db/modelbase.go +++ b/pkg/cloudcommon/db/modelbase.go @@ -539,6 +539,15 @@ func (manager *SModelBaseManager) PerformPurgeSplitable(ctx context.Context, use return jsonutils.Marshal(map[string][]string{"tables": ret}), nil } +func (manager *SModelBaseManager) CustomizedTotalCount(totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error) { + ret := apis.TotalCountBase{} + err := totalQ.First(&ret) + if err != nil { + return -1, nil, errors.Wrap(err, "SModelBaseManager Query total") + } + return ret.Count, nil, nil +} + func (model *SModelBase) GetId() string { return "" } diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 6db22330aa..4bef58a056 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -6352,3 +6352,42 @@ func (guest *SGuest) GetGuestBackupMirrorJobStatus(ctx context.Context, userCred func (guest *SGuest) ResetGuestQuorumChildIndex(ctx context.Context, userCred mcclient.TokenCredential) error { return guest.SetMetadata(ctx, api.QUORUM_CHILD_INDEX, "", userCred) } + +type SGuestTotalCount struct { + apis.TotalCountBase + CpuCount int + MemMb int + DiskMb int64 + DiskCount int +} + +func (manager *SGuestManager) CustomizedTotalCount(totalQ *sqlchemy.SQuery) (int, jsonutils.JSONObject, error) { + results := SGuestTotalCount{} + + totalQ = totalQ.AppendField(sqlchemy.SUM("cpu_count", totalQ.Field("vcpu_count"))) + totalQ = totalQ.AppendField(sqlchemy.SUM("mem_mb", totalQ.Field("vmem_size"))) + + err := totalQ.First(&results) + if err != nil { + return -1, nil, errors.Wrap(err, "SGuestManager query total") + } + + log.Debugf("CustomizedTotalCount %s", jsonutils.Marshal(results)) + + diskQ := DiskManager.Query() + diskGuestQ := GuestdiskManager.Query().SubQuery() + diskQ = diskQ.Join(diskGuestQ, sqlchemy.Equals(diskQ.Field("id"), diskGuestQ.Field("disk_id"))) + totalSQ := totalQ.ResetFields().SubQuery() + diskQ = diskQ.Join(totalSQ, sqlchemy.Equals(diskGuestQ.Field("guest_id"), totalSQ.Field("id"))) + diskQ = diskQ.AppendField(sqlchemy.COUNT("disk_count")) + diskQ = diskQ.AppendField(sqlchemy.SUM("disk_mb", diskQ.Field("disk_size"))) + + err = diskQ.First(&results) + if err != nil { + return -1, nil, errors.Wrap(err, "SGuestManager query total_disk") + } + + log.Debugf("CustomizedTotalCount %s", jsonutils.Marshal(results)) + + return results.Count, jsonutils.Marshal(results), nil +} diff --git a/pkg/mcclient/modulebase/base.go b/pkg/mcclient/modulebase/base.go index 6452cba7da..347dc01232 100644 --- a/pkg/mcclient/modulebase/base.go +++ b/pkg/mcclient/modulebase/base.go @@ -154,6 +154,9 @@ func ListResult2JSONWithKey(result *printutils.ListResult, key string) jsonutils if result.Total > 0 { obj.Add(jsonutils.NewInt(int64(result.Total)), "total") } + if result.Totals != nil { + obj.Add(result.Totals, "totals") + } if result.Limit > 0 { obj.Add(jsonutils.NewInt(int64(result.Limit)), "limit") } @@ -189,9 +192,11 @@ func JSON2ListResult(result jsonutils.JSONObject) *printutils.ListResult { if len(markerField) == 0 && total == 0 { total = int64(len(data)) } + totalJson, _ := result.Get("totals") return &printutils.ListResult{ Data: data, Total: int(total), Limit: int(limit), Offset: int(offset), + Totals: totalJson, NextMarker: nextMarker, MarkerField: markerField, MarkerOrder: markerOrder, @@ -219,9 +224,11 @@ func (this *BaseManager) _list(session *mcclient.ClientSession, path, responseKe if len(nextMarker) == 0 && total == 0 { total = int64(len(rets)) } + totalJson, _ := body.Get("totals") return &printutils.ListResult{ Data: rets, Total: int(total), Limit: int(limit), Offset: int(offset), + Totals: totalJson, NextMarker: nextMarker, MarkerField: markerField, MarkerOrder: markerOrder, diff --git a/vendor/modules.txt b/vendor/modules.txt index e0013be4f9..aae6070b6d 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1534,7 +1534,7 @@ yunion.io/x/log/hooks yunion.io/x/ovsdb/cli_util yunion.io/x/ovsdb/schema/ovn_nb yunion.io/x/ovsdb/types -# yunion.io/x/pkg v1.0.1-0.20221226030310-a8727011fc76 +# yunion.io/x/pkg v1.0.1-0.20230102060551-df05ccecb71c ## explicit; go 1.18 yunion.io/x/pkg/appctx yunion.io/x/pkg/errors @@ -1588,7 +1588,7 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.2-0.20220923040147-ddd3e96868af +# yunion.io/x/sqlchemy v1.1.2-0.20230102055317-b2aa2b5c26f8 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends diff --git a/vendor/yunion.io/x/pkg/util/printutils/printjson.go b/vendor/yunion.io/x/pkg/util/printutils/printjson.go index c373a7e238..85870d0f9e 100644 --- a/vendor/yunion.io/x/pkg/util/printutils/printjson.go +++ b/vendor/yunion.io/x/pkg/util/printutils/printjson.go @@ -132,6 +132,20 @@ func PrintJSONList(list *ListResult, columns []string) { } } fmt.Println("*** ", title, " ***") + if list.Totals != nil { + if totalDict, ok := list.Totals.(*jsonutils.JSONDict); ok { + totalMap, err := totalDict.GetMap() + if err != nil { + fmt.Println("error to convert totals to JSONDict") + } else { + segs := make([]string, 0) + for k, v := range totalMap { + segs = append(segs, fmt.Sprintf("%s: %s", k, v.String())) + } + fmt.Printf("***%s***\n", strings.Join(segs, ", ")) + } + } + } if colTruncated { fmt.Println(fmt.Sprintf("!!!Some text truncated, set env %s=-1 to show full text!!!", OS_MAX_COLUMN_TEXT_LENGTH)) } diff --git a/vendor/yunion.io/x/pkg/util/printutils/struct.go b/vendor/yunion.io/x/pkg/util/printutils/struct.go index 9c9ab0c71c..c296f356b6 100644 --- a/vendor/yunion.io/x/pkg/util/printutils/struct.go +++ b/vendor/yunion.io/x/pkg/util/printutils/struct.go @@ -19,10 +19,13 @@ import ( ) type ListResult struct { - Data []jsonutils.JSONObject `json:"data,allowempty"` - Total int - Limit int - Offset int + Data []jsonutils.JSONObject `json:"data,allowempty"` + + Total int `json:"total"` + Limit int `json:"limit"` + Offset int `json:"offset"` + + Totals jsonutils.JSONObject `json:"totals"` NextMarker string MarkerField string diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 7c13d02165..3f7c92e827 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -81,6 +81,11 @@ func (tq *SQuery) AppendField(f ...IQueryField) *SQuery { return tq } +func (tq *SQuery) ResetFields() *SQuery { + tq.fields = nil + return tq +} + // Query of SSubQuery generates a new query from a subquery func (sq *SSubQuery) Query(f ...IQueryField) *SQuery { return DoQuery(sq, f...)