opensource/teleport
3
0
Fork 0
mirror of https://github.com/gravitational/teleport.git synced 2026-10-28 03:19:47 +08:00
Code Issues Packages Projects Releases Wiki Activity

Return filtered total count with ListResources (#10573)

Browse Source 
* The web UI needed to show the total count of resources available after filter
* ListResources response returns as a struct to make adding extra fields easier
This commit is contained in:
Lisa Kim
2022-02-28 21:51:19 +00:00
committed by GitHub
parent 06d488ff31
commit e7eb6c4af8
18 changed files with 1266 additions and 814 deletions
Download Patch File Download Diff File Expand all files Collapse all files
api/client/client.go
+29 -24
View File
@@ -275,14 +275,16 @@ func connect(ctx context.Context, cfg Config) (*Client, error) {
}
}
type connectFunc func(ctx context.Context, params connectParams) (*Client, error)
type connectParams struct {
cfg Config
addr string
tlsConfig *tls.Config
dialer ContextDialer
sshConfig *ssh.ClientConfig
}
type (
connectFunc func(ctx context.Context, params connectParams) (*Client, error)
connectParams struct {
cfg Config
addr string
tlsConfig *tls.Config
dialer ContextDialer
sshConfig *ssh.ClientConfig
}
)
// authConnect connects to the Teleport Auth Server directly.
func authConnect(ctx context.Context, params connectParams) (*Client, error) {
@@ -1677,8 +1679,8 @@ func GetNodesWithLabels(ctx context.Context, clt NodeClient, namespace string, l
// ListNodes returns a paginated list of nodes that the user has access to in the given namespace.
// nextKey can be used as startKey in another call to ListNodes to retrieve the next page of nodes.
// ListNodes will return a trace.LimitExceeded error if the page of nodes retrieved exceeds 4MiB.
func (c *Client) ListNodes(ctx context.Context, req proto.ListNodesRequest) (nodes []types.Server, nextKey string, err error) {
resources, nextKey, err := c.ListResources(ctx, proto.ListResourcesRequest{
func (c *Client) ListNodes(ctx context.Context, req proto.ListNodesRequest) ([]types.Server, string, error) {
resp, err := c.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Namespace: req.Namespace,
StartKey: req.StartKey,
@@ -1693,8 +1695,8 @@ func (c *Client) ListNodes(ctx context.Context, req proto.ListNodesRequest) (nod
return nil, "", trace.Wrap(err)
}
servers := make([]types.Server, len(resources))
for i, resource := range resources {
servers := make([]types.Server, len(resp.Resources))
for i, resource := range resp.Resources {
server, ok := resource.(*types.ServerV2)
if !ok {
return nil, "", trace.BadParameter("invalid node type %T", resource)
@@ -1703,7 +1705,7 @@ func (c *Client) ListNodes(ctx context.Context, req proto.ListNodesRequest) (nod
servers[i] = server
}
return servers, nextKey, nil
return servers, resp.NextKey, nil
}
// listNodesFallback previous implementation of `ListNodes` function using
@@ -2386,17 +2388,17 @@ func (c *Client) GenerateCertAuthorityCRL(ctx context.Context, req *proto.CertAu
// `GetResources` function.
// It will return a `trace.LimitExceeded` error if the page exceeds gRPC max
// message size.
func (c *Client) ListResources(ctx context.Context, req proto.ListResourcesRequest) (resources []types.ResourceWithLabels, nextKey string, err error) {
func (c *Client) ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
if err := req.CheckAndSetDefaults(); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
resp, err := c.grpc.ListResources(ctx, &req, c.callOpts...)
if err != nil {
return nil, "", trail.FromGRPC(err)
return nil, trail.FromGRPC(err)
}
resources = make([]types.ResourceWithLabels, len(resp.GetResources()))
resources := make([]types.ResourceWithLabels, len(resp.GetResources()))
for i, respResource := range resp.GetResources() {
switch req.ResourceType {
case types.KindDatabaseServer:
@@ -2408,11 +2410,15 @@ func (c *Client) ListResources(ctx context.Context, req proto.ListResourcesReque
case types.KindKubeService:
resources[i] = respResource.GetKubeService()
default:
return nil, "", trace.NotImplemented("resource type %s does not support pagination", req.ResourceType)
return nil, trace.NotImplemented("resource type %s does not support pagination", req.ResourceType)
}
}
return resources, resp.NextKey, nil
return &types.ListResourcesResponse{
Resources: resources,
NextKey: resp.NextKey,
TotalCount: int(resp.TotalCount),
}, nil
}
// GetResources retrieves all pages from ListResourcesPage and return all
@@ -2425,7 +2431,7 @@ func (c *Client) GetResources(ctx context.Context, namespace, resourceType strin
)
for {
listResources, nextKey, err := c.ListResources(ctx, proto.ListResourcesRequest{
resp, err := c.ListResources(ctx, proto.ListResourcesRequest{
Namespace: namespace,
ResourceType: resourceType,
StartKey: startKey,
@@ -2444,9 +2450,9 @@ func (c *Client) GetResources(ctx context.Context, namespace, resourceType strin
return nil, trail.FromGRPC(err)
}
startKey = nextKey
resources = append(resources, listResources...)
if startKey == "" || len(listResources) == 0 {
startKey = resp.NextKey
resources = append(resources, resp.Resources...)
if startKey == "" || len(resp.Resources) == 0 {
break
}
@@ -2473,7 +2479,6 @@ func (c *Client) GetActiveSessionTrackers(ctx context.Context) ([]types.SessionT
stream, err := c.grpc.GetActiveSessionTrackers(ctx, &empty.Empty{})
if err != nil {
return nil, trail.FromGRPC(err)
}
var sessions []types.SessionTracker
api/client/client_test.go
+17 -7
View File
@@ -73,7 +73,8 @@ func (m *mockServer) ListResources(ctx context.Context, req *proto.ListResources
}
resp := &proto.ListResourcesResponse{
Resources: make([]*proto.PaginatedResource, 0),
Resources: make([]*proto.PaginatedResource, 0),
TotalCount: int32(len(resources)),
}
var (
@@ -494,18 +495,18 @@ func TestListResources(t *testing.T) {
for name, test := range testCases {
t.Run(name, func(t *testing.T) {
resources, nextKey, err := clt.ListResources(ctx, proto.ListResourcesRequest{
resp, err := clt.ListResources(ctx, proto.ListResourcesRequest{
Namespace: defaults.Namespace,
Limit: 10,
ResourceType: test.resourceType,
})
require.NoError(t, err)
require.NotEmpty(t, nextKey)
require.Len(t, resources, 10)
require.IsType(t, test.resourceStruct, resources[0])
require.NotEmpty(t, resp.NextKey)
require.Len(t, resp.Resources, 10)
require.IsType(t, test.resourceStruct, resp.Resources[0])
// exceed the limit
_, _, err = clt.ListResources(ctx, proto.ListResourcesRequest{
_, err = clt.ListResources(ctx, proto.ListResourcesRequest{
Namespace: defaults.Namespace,
Limit: 50,
ResourceType: test.resourceType,
@@ -514,6 +515,15 @@ func TestListResources(t *testing.T) {
require.IsType(t, &trace.LimitExceededError{}, err.(*trace.TraceErr).OrigError())
})
}
// Test a list with total count returned.
resp, err := clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Limit: 10,
NeedTotalCount: true,
})
require.NoError(t, err)
require.Equal(t, 50, resp.TotalCount)
}
func TestGetResources(t *testing.T) {
@@ -556,7 +566,7 @@ func TestGetResources(t *testing.T) {
require.NoError(t, err)
// listing everything at once breaks the ListResource.
_, _, err = clt.ListResources(ctx, proto.ListResourcesRequest{
_, err = clt.ListResources(ctx, proto.ListResourcesRequest{
Namespace: defaults.Namespace,
Limit: int32(len(expectedResources)),
ResourceType: test.resourceType,
api/client/proto/authservice.pb.go
+662 -580
View File
File diff suppressed because it is too large Load Diff
api/client/proto/authservice.proto
+4
View File
@@ -1446,6 +1446,8 @@ message ListResourcesRequest {
// SortBy describes which resource field and which direction to sort by.
types.SortBy SortBy = 8
[ (gogoproto.nullable) = false, (gogoproto.jsontag) = "sort_by,omitempty" ];
// NeedTotalCount indicates whether or not the caller also wants the total number of resources after filtering.
bool NeedTotalCount = 9 [ (gogoproto.jsontag) = "need_total_count,omitempty" ];
}
// ListResourceResponse response of ListResources.
@@ -1456,6 +1458,8 @@ message ListResourcesResponse {
// continue retrieving pages of resource. If NextKey is empty, there are no
// more pages.
string NextKey = 2 [ (gogoproto.jsontag) = "next_key,omitempty" ];
// TotalCount is the total number of resources available after filter, if any.
int32 TotalCount = 3 [ (gogoproto.jsontag) = "total_count,omitempty" ];
}
// CreateSessionTrackerRequest is a request to create a new session.
api/client/proto/types.go
+6
View File
@@ -84,3 +84,9 @@ func (req *ListResourcesRequest) CheckAndSetDefaults() error {
return nil
}
// RequiresFakePagination checks if we need to fallback to GetXXX calls
// that retrieves entire resources upfront rather than working with subsets.
func (req *ListResourcesRequest) RequiresFakePagination() bool {
return req.SortBy.Field != "" || req.NeedTotalCount
}
api/types/resource.go
+10
View File
@@ -360,3 +360,13 @@ func stringCompare(a string, b string, isDesc bool) bool {
}
return a < b
}
// ListResourcesResponse describes a non proto response to ListResources.
type ListResourcesResponse struct {
// Resources is a list of resource.
Resources []ResourceWithLabels
// NextKey is the next key to use as a starting point.
NextKey string
// TotalCount is the total number of resources available as a whole.
TotalCount int
}
lib/auth/api.go
+1 -1
View File
@@ -808,7 +808,7 @@ type Cache interface {
GetLocks(ctx context.Context, inForceOnly bool, targets ...types.LockTarget) ([]types.Lock, error)
// ListResources returns a paginated list of resources.
ListResources(ctx context.Context, req proto.ListResourcesRequest) (resources []types.ResourceWithLabels, nextKey string, err error)
ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error)
}
type NodeWrapper struct {
lib/auth/auth.go
+9 -9
View File
@@ -2706,25 +2706,25 @@ func (a *Server) IterateNodePages(ctx context.Context, req proto.ListNodesReques
type ResourcePageFunc func(next []types.ResourceWithLabels) (stop bool, err error)
// IterateResourcePages can be used to iterate over pages of resources.
func (a *Server) IterateResourcePages(ctx context.Context, req proto.ListResourcesRequest, f ResourcePageFunc) (string, error) {
func (a *Server) IterateResourcePages(ctx context.Context, req proto.ListResourcesRequest, f ResourcePageFunc) (*types.ListResourcesResponse, error) {
for {
nextPage, nextKey, err := a.ListResources(ctx, req)
resp, err := a.ListResources(ctx, req)
if err != nil {
return "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
stop, err := f(nextPage)
stop, err := f(resp.Resources)
if err != nil {
return "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
// Iterator stopped before end of pages or
// there are no more pages, return nextKey
if stop || nextKey == "" {
return nextKey, nil
if stop || resp.NextKey == "" {
return resp, nil
}
req.StartKey = nextKey
req.StartKey = resp.NextKey
}
}
@@ -3004,7 +3004,7 @@ func (a *Server) GetDatabase(ctx context.Context, name string) (types.Database,
}
// GetDatabases returns all database resources.
func (a *Server) ListResources(ctx context.Context, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func (a *Server) ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
return a.GetCache().ListResources(ctx, req)
}
lib/auth/auth_with_roles.go
+89 -8
View File
@@ -35,6 +35,7 @@ import (
"github.com/gravitational/teleport/lib/events"
"github.com/gravitational/teleport/lib/modules"
"github.com/gravitational/teleport/lib/services"
"github.com/gravitational/teleport/lib/services/local"
"github.com/gravitational/teleport/lib/session"
"github.com/gravitational/teleport/lib/utils"
@@ -877,14 +878,33 @@ func (a *ServerWithRoles) GetNodes(ctx context.Context, namespace string, opts .
}
// ListResources returns a paginated list of resources filtered by user access.
func (a *ServerWithRoles) ListResources(ctx context.Context, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func (a *ServerWithRoles) ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
// ListResources request coming through this auth layer gets request filters
// stripped off and saved to be applied later after items go through rbac checks.
// The list that gets returned from the backend comes back unfiltered and as
// we apply request filters, we might make multiple trips to get more subsets to
// reach our limit, which is fine b/c we can start query with our next key.
//
// But since sorting and counting totals requires us to work with entire list upfront,
// special handling is needed in this layer b/c if we try to mimic the "subset" here,
// we will be making unnecessary trips and doing needless work of deserializing every
// item for every subset.
if req.RequiresFakePagination() {
resp, err := a.listResourcesWithSort(ctx, req)
if err != nil {
return nil, trace.Wrap(err)
}
return resp, nil
}
// Start real pagination.
if err := req.CheckAndSetDefaults(); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
limit := int(req.Limit)
actionVerbs := []string{types.VerbList, types.VerbRead}
switch req.ResourceType {
case types.KindNode:
// We are checking list only for Nodes to keep backwards compatibility.
@@ -897,11 +917,11 @@ func (a *ServerWithRoles) ListResources(ctx context.Context, req proto.ListResou
case types.KindDatabaseServer, types.KindAppServer, types.KindKubeService:
default:
return nil, "", trace.NotImplemented("resource type %s does not support pagination", req.ResourceType)
return nil, trace.NotImplemented("resource type %s does not support pagination", req.ResourceType)
}
if err := a.action(req.Namespace, req.ResourceType, actionVerbs...); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
// Perform the label/search/expr filtering here (instead of at the backend
@@ -918,7 +938,7 @@ func (a *ServerWithRoles) ListResources(ctx context.Context, req proto.ListResou
req.PredicateExpression = ""
var resources []types.ResourceWithLabels
nextKey, err := a.authServer.IterateResourcePages(ctx, req, func(nextPage []types.ResourceWithLabels) (bool, error) {
resp, err := a.authServer.IterateResourcePages(ctx, req, func(nextPage []types.ResourceWithLabels) (bool, error) {
for _, resource := range nextPage {
if len(resources) == limit {
break
@@ -943,10 +963,13 @@ func (a *ServerWithRoles) ListResources(ctx context.Context, req proto.ListResou
return len(resources) == limit, nil
})
if err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
return resources, nextKey, nil
return &types.ListResourcesResponse{
Resources: resources,
NextKey: resp.NextKey,
}, nil
}
func (a *ServerWithRoles) checkAccessToResource(resource types.Resource) error {
@@ -973,6 +996,64 @@ func (a *ServerWithRoles) checkAccessToResource(resource types.Resource) error {
}
}
// listResourcesWithSort retrieves all resources of a certain resource type with rbac applied
// then afterwards applies request sorting and filtering.
func (a *ServerWithRoles) listResourcesWithSort(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
if err := req.CheckAndSetDefaults(); err != nil {
return nil, trace.Wrap(err)
}
var resources []types.ResourceWithLabels
switch req.ResourceType {
case types.KindNode:
nodes, err := a.GetNodes(ctx, req.Namespace)
if err != nil {
return nil, trace.Wrap(err)
}
servers := types.Servers(nodes)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
case types.KindAppServer:
appservers, err := a.GetApplicationServers(ctx, req.Namespace)
if err != nil {
return nil, trace.Wrap(err)
}
servers := types.AppServers(appservers)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
case types.KindDatabaseServer:
dbservers, err := a.GetDatabaseServers(ctx, req.Namespace)
if err != nil {
return nil, trace.Wrap(err)
}
servers := types.DatabaseServers(dbservers)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
default:
return nil, trace.NotImplemented("resource type %q is not supported for listResourcesWithSort", req.ResourceType)
}
// Apply request filters and get pagination info.
resp, err := local.FakePaginate(resources, req)
if err != nil {
return nil, trace.Wrap(err)
}
return resp, nil
}
// ListNodes returns a paginated list of nodes filtered by user access.
//
// DELETE IN 10.0.0 in favor of ListResources.
lib/auth/auth_with_roles_test.go
+129 -76
View File
@@ -67,7 +67,8 @@ func TestLocalUserCanReissueCerts(t *testing.T) {
// expiration limited to duration of the user's session (default 1 hour)
reqTTL: 4 * time.Hour,
expiresIn: 1 * time.Hour,
}, {
},
{
desc: "renewable",
renewable: true,
// expiration is allowed to be pushed out into the future
@@ -902,30 +903,30 @@ func TestGetAndList_Nodes(t *testing.T) {
}
// listing nodes 0-4 should list first 5 nodes
nodes, _, err := clt.ListResources(ctx, proto.ListResourcesRequest{
resp, err := clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Namespace: defaults.Namespace,
Limit: 5,
})
require.NoError(t, err)
require.Len(t, nodes, 5)
require.Len(t, resp.Resources, 5)
expectedNodes := testResources[:5]
require.Empty(t, cmp.Diff(expectedNodes, nodes))
require.Empty(t, cmp.Diff(expectedNodes, resp.Resources))
// remove permission for third node
role.SetNodeLabels(types.Deny, types.Labels{"name": {testResources[3].GetName()}})
require.NoError(t, srv.Auth().UpsertRole(ctx, role))
// listing nodes 0-4 should skip the third node and add the fifth to the end.
nodes, _, err = clt.ListResources(ctx, proto.ListResourcesRequest{
resp, err = clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Namespace: defaults.Namespace,
Limit: 5,
})
require.NoError(t, err)
require.Len(t, nodes, 5)
require.Len(t, resp.Resources, 5)
expectedNodes = append(testResources[:3], testResources[4:6]...)
require.Empty(t, cmp.Diff(expectedNodes, nodes))
require.Empty(t, cmp.Diff(expectedNodes, resp.Resources))
// Test various filtering.
baseRequest := proto.ListResourcesRequest{
@@ -937,26 +938,26 @@ func TestGetAndList_Nodes(t *testing.T) {
// Test label match.
withLabels := baseRequest
withLabels.Labels = map[string]string{"name": testResources[0].GetName()}
nodes, _, err = clt.ListResources(ctx, withLabels)
resp, err = clt.ListResources(ctx, withLabels)
require.NoError(t, err)
require.Len(t, nodes, 1)
require.Empty(t, cmp.Diff(testResources[0:1], nodes))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test search keywords match.
withSearchKeywords := baseRequest
withSearchKeywords.SearchKeywords = []string{"name", testResources[0].GetName()}
nodes, _, err = clt.ListResources(ctx, withSearchKeywords)
resp, err = clt.ListResources(ctx, withSearchKeywords)
require.NoError(t, err)
require.Len(t, nodes, 1)
require.Empty(t, cmp.Diff(testResources[0:1], nodes))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test expression match.
withExpression := baseRequest
withExpression.PredicateExpression = fmt.Sprintf(`labels.name == "%s"`, testResources[0].GetName())
nodes, _, err = clt.ListResources(ctx, withExpression)
resp, err = clt.ListResources(ctx, withExpression)
require.NoError(t, err)
require.Len(t, nodes, 1)
require.Empty(t, cmp.Diff(testResources[0:1], nodes))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
}
// TestAPILockedOut tests Auth API when there are locks involved.
@@ -1211,10 +1212,10 @@ func TestGetAndList_DatabaseServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, 1, len(servers))
require.Empty(t, cmp.Diff(testServers[0:1], servers))
resources, _, err := clt.ListResources(ctx, listRequest)
resp, err := clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// permit user to get all databases
role.SetDatabaseLabels(types.Allow, types.Labels{types.Wildcard: {types.Wildcard}})
@@ -1223,10 +1224,10 @@ func TestGetAndList_DatabaseServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, len(testServers), len(servers))
require.Empty(t, cmp.Diff(testServers, servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resources))
require.Len(t, resp.Resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resp.Resources))
// Test various filtering.
baseRequest := proto.ListResourcesRequest{
@@ -1238,26 +1239,26 @@ func TestGetAndList_DatabaseServers(t *testing.T) {
// list only database with label
withLabels := baseRequest
withLabels.Labels = map[string]string{"name": testServers[0].GetName()}
resources, _, err = clt.ListResources(ctx, withLabels)
resp, err = clt.ListResources(ctx, withLabels)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test search keywords match.
withSearchKeywords := baseRequest
withSearchKeywords.SearchKeywords = []string{"name", testServers[0].GetName()}
resources, _, err = clt.ListResources(ctx, withSearchKeywords)
resp, err = clt.ListResources(ctx, withSearchKeywords)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test expression match.
withExpression := baseRequest
withExpression.PredicateExpression = fmt.Sprintf(`labels.name == "%s"`, testServers[0].GetName())
resources, _, err = clt.ListResources(ctx, withExpression)
resp, err = clt.ListResources(ctx, withExpression)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// deny user to get the first database
role.SetDatabaseLabels(types.Deny, types.Labels{"name": {testServers[0].GetName()}})
@@ -1266,10 +1267,10 @@ func TestGetAndList_DatabaseServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, len(testServers[1:]), len(servers))
require.Empty(t, cmp.Diff(testServers[1:], servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources[1:]))
require.Empty(t, cmp.Diff(testResources[1:], resources))
require.Len(t, resp.Resources, len(testResources[1:]))
require.Empty(t, cmp.Diff(testResources[1:], resp.Resources))
// deny user to get all databases
role.SetDatabaseLabels(types.Deny, types.Labels{types.Wildcard: {types.Wildcard}})
@@ -1278,10 +1279,10 @@ func TestGetAndList_DatabaseServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, 0, len(servers))
require.Empty(t, cmp.Diff([]types.DatabaseServer{}, servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, 0)
require.Empty(t, cmp.Diff([]types.ResourceWithLabels{}, resources))
require.Len(t, resp.Resources, 0)
require.Empty(t, cmp.Diff([]types.ResourceWithLabels{}, resp.Resources))
}
// TestGetAndList_ApplicationServers verifies RBAC and filtering is applied when fetching app servers.
@@ -1336,10 +1337,10 @@ func TestGetAndList_ApplicationServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, 1, len(servers))
require.Empty(t, cmp.Diff(testServers[0:1], servers))
resources, _, err := clt.ListResources(ctx, listRequest)
resp, err := clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// permit user to get all apps
role.SetAppLabels(types.Allow, types.Labels{types.Wildcard: {types.Wildcard}})
@@ -1348,10 +1349,10 @@ func TestGetAndList_ApplicationServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, len(testServers), len(servers))
require.Empty(t, cmp.Diff(testServers, servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resources))
require.Len(t, resp.Resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resp.Resources))
// Test various filtering.
baseRequest := proto.ListResourcesRequest{
@@ -1363,26 +1364,26 @@ func TestGetAndList_ApplicationServers(t *testing.T) {
// list only application with label
withLabels := baseRequest
withLabels.Labels = map[string]string{"name": testServers[0].GetName()}
resources, _, err = clt.ListResources(ctx, withLabels)
resp, err = clt.ListResources(ctx, withLabels)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test search keywords match.
withSearchKeywords := baseRequest
withSearchKeywords.SearchKeywords = []string{"name", testServers[0].GetName()}
resources, _, err = clt.ListResources(ctx, withSearchKeywords)
resp, err = clt.ListResources(ctx, withSearchKeywords)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test expression match.
withExpression := baseRequest
withExpression.PredicateExpression = fmt.Sprintf(`labels.name == "%s"`, testServers[0].GetName())
resources, _, err = clt.ListResources(ctx, withExpression)
resp, err = clt.ListResources(ctx, withExpression)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// deny user to get the first app
role.SetAppLabels(types.Deny, types.Labels{"name": {testServers[0].GetName()}})
@@ -1391,10 +1392,10 @@ func TestGetAndList_ApplicationServers(t *testing.T) {
require.NoError(t, err)
require.EqualValues(t, len(testServers[1:]), len(servers))
require.Empty(t, cmp.Diff(testServers[1:], servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources[1:]))
require.Empty(t, cmp.Diff(testResources[1:], resources))
require.Len(t, resp.Resources, len(testResources[1:]))
require.Empty(t, cmp.Diff(testResources[1:], resp.Resources))
// deny user to get all apps
role.SetAppLabels(types.Deny, types.Labels{types.Wildcard: {types.Wildcard}})
@@ -1402,10 +1403,10 @@ func TestGetAndList_ApplicationServers(t *testing.T) {
servers, err = clt.GetApplicationServers(ctx, defaults.Namespace)
require.NoError(t, err)
require.EqualValues(t, 0, len(servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, 0)
require.Empty(t, cmp.Diff([]types.ResourceWithLabels{}, resources))
require.Len(t, resp.Resources, 0)
require.Empty(t, cmp.Diff([]types.ResourceWithLabels{}, resp.Resources))
}
func TestGetAppServers(t *testing.T) {
@@ -2045,10 +2046,10 @@ func TestGetAndList_KubeServices(t *testing.T) {
require.NoError(t, err)
require.Len(t, testServers, len(testServers))
require.Empty(t, cmp.Diff(testServers, servers))
resources, _, err := clt.ListResources(ctx, listRequest)
resp, err := clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resources))
require.Len(t, resp.Resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resp.Resources))
// Test various filtering.
baseRequest := proto.ListResourcesRequest{
@@ -2060,26 +2061,26 @@ func TestGetAndList_KubeServices(t *testing.T) {
// Test label match.
withLabels := baseRequest
withLabels.Labels = map[string]string{"name": testServers[0].GetName()}
resources, _, err = clt.ListResources(ctx, withLabels)
resp, err = clt.ListResources(ctx, withLabels)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test search keywords match.
withSearchKeywords := baseRequest
withSearchKeywords.SearchKeywords = []string{"name", testServers[0].GetName()}
resources, _, err = clt.ListResources(ctx, withSearchKeywords)
resp, err = clt.ListResources(ctx, withSearchKeywords)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// Test expression match.
withExpression := baseRequest
withExpression.PredicateExpression = fmt.Sprintf(`labels.name == "%s"`, testServers[0].GetName())
resources, _, err = clt.ListResources(ctx, withExpression)
resp, err = clt.ListResources(ctx, withExpression)
require.NoError(t, err)
require.Len(t, resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resources))
require.Len(t, resp.Resources, 1)
require.Empty(t, cmp.Diff(testResources[0:1], resp.Resources))
// deny user to get the first kubernetes service
role.SetKubernetesLabels(types.Deny, types.Labels{"name": {testServers[0].GetName()}})
@@ -2089,10 +2090,10 @@ func TestGetAndList_KubeServices(t *testing.T) {
require.NoError(t, err)
require.Len(t, testServers, len(testServers))
require.Empty(t, cmp.Diff(testServers, servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resources))
require.Len(t, resp.Resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resp.Resources))
// deny user to get all databases
role.SetKubernetesLabels(types.Deny, types.Labels{types.Wildcard: {types.Wildcard}})
@@ -2104,8 +2105,60 @@ func TestGetAndList_KubeServices(t *testing.T) {
require.NoError(t, err)
require.Len(t, testServers, len(testServers))
require.Empty(t, cmp.Diff(testServers, servers))
resources, _, err = clt.ListResources(ctx, listRequest)
resp, err = clt.ListResources(ctx, listRequest)
require.NoError(t, err)
require.Len(t, resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resources))
require.Len(t, resp.Resources, len(testResources))
require.Empty(t, cmp.Diff(testResources, resp.Resources))
}
func TestListResources_NeedTotalCountFlag(t *testing.T) {
t.Parallel()
ctx := context.Background()
srv := newTestTLSServer(t)
// Create test nodes.
for i := 0; i < 3; i++ {
name := uuid.New().String()
node, err := types.NewServerWithLabels(
name,
types.KindNode,
types.ServerSpecV2{},
map[string]string{"name": name},
)
require.NoError(t, err)
_, err = srv.Auth().UpsertNode(ctx, node)
require.NoError(t, err)
}
testNodes, err := srv.Auth().GetNodes(ctx, defaults.Namespace)
require.NoError(t, err)
require.Len(t, testNodes, 3)
// create user and client
user, _, err := CreateUserAndRole(srv.Auth(), "user", nil)
require.NoError(t, err)
clt, err := srv.NewClient(TestUser(user.GetName()))
require.NoError(t, err)
// Total returned.
resp, err := clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Limit: 2,
NeedTotalCount: true,
})
require.NoError(t, err)
require.Len(t, resp.Resources, 2)
require.NotEmpty(t, resp.NextKey)
require.Equal(t, len(testNodes), resp.TotalCount)
// No total.
resp, err = clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Limit: 2,
})
require.NoError(t, err)
require.Len(t, resp.Resources, 2)
require.NotEmpty(t, resp.NextKey)
require.Empty(t, resp.TotalCount)
}
lib/auth/grpcserver.go
+8 -8
View File
@@ -2081,7 +2081,6 @@ func (g *GRPCServer) DeleteMFADevice(stream proto.AuthService_DeleteMFADeviceSer
return trace.Wrap(stream.Send(&proto.DeleteMFADeviceResponse{
Response: &proto.DeleteMFADeviceResponse_Ack{Ack: &proto.DeleteMFADeviceResponseAck{}},
}))
}
func deleteMFADeviceAuthChallenge(gctx *grpcContext, stream proto.AuthService_DeleteMFADeviceServer) error {
@@ -3677,17 +3676,18 @@ func (g *GRPCServer) ListResources(ctx context.Context, req *proto.ListResources
return nil, trace.Wrap(err)
}
resources, nextKey, err := auth.ListResources(ctx, *req)
resp, err := auth.ListResources(ctx, *req)
if err != nil {
return nil, trace.Wrap(err)
}
resp := &proto.ListResourcesResponse{
NextKey: nextKey,
Resources: make([]*proto.PaginatedResource, len(resources)),
protoResp := &proto.ListResourcesResponse{
NextKey: resp.NextKey,
Resources: make([]*proto.PaginatedResource, len(resp.Resources)),
TotalCount: int32(resp.TotalCount),
}
for i, resource := range resources {
for i, resource := range resp.Resources {
var protoResource *proto.PaginatedResource
switch req.ResourceType {
case types.KindDatabaseServer:
@@ -3722,10 +3722,10 @@ func (g *GRPCServer) ListResources(ctx context.Context, req *proto.ListResources
return nil, trace.NotImplemented("resource type %s doesn't support pagination", req.ResourceType)
}
resp.Resources[i] = protoResource
protoResp.Resources[i] = protoResource
}
return resp, nil
return protoResp, nil
}
// CreateSessionTracker creates a tracker resource for an active session.
lib/auth/grpcserver_test.go
+21 -7
View File
@@ -1913,14 +1913,14 @@ func TestListResources(t *testing.T) {
for name, test := range testCases {
t.Run(name, func(t *testing.T) {
resources, nextKey, err := clt.ListResources(ctx, proto.ListResourcesRequest{
resp, err := clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: test.resourceType,
Namespace: apidefaults.Namespace,
Limit: 100,
})
require.NoError(t, err)
require.Len(t, resources, 0)
require.Empty(t, nextKey)
require.Len(t, resp.Resources, 0)
require.Empty(t, resp.NextKey)
// create two resources
err = test.createResource("foo")
@@ -1928,19 +1928,33 @@ func TestListResources(t *testing.T) {
err = test.createResource("bar")
require.NoError(t, err)
resources, nextKey, err = clt.ListResources(ctx, proto.ListResourcesRequest{
resp, err = clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: test.resourceType,
Namespace: apidefaults.Namespace,
Limit: 100,
})
require.NoError(t, err)
require.Len(t, resources, 2)
require.Empty(t, nextKey)
require.Len(t, resp.Resources, 2)
require.Empty(t, resp.NextKey)
require.Empty(t, resp.TotalCount)
// Test listing with NeedTotalCount flag.
if test.resourceType != types.KindKubeService {
resp, err = clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: test.resourceType,
Limit: 100,
NeedTotalCount: true,
})
require.NoError(t, err)
require.Len(t, resp.Resources, 2)
require.Empty(t, resp.NextKey)
require.Equal(t, 2, resp.TotalCount)
}
})
}
t.Run("InvalidResourceType", func(t *testing.T) {
_, _, err := clt.ListResources(ctx, proto.ListResourcesRequest{
_, err := clt.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: "",
Namespace: apidefaults.Namespace,
Limit: 100,
lib/cache/cache.go
Vendored
+6 -6
View File
@@ -1933,10 +1933,10 @@ func (c *Cache) GetWindowsDesktops(ctx context.Context, filter types.WindowsDesk
}
// ListResources is a part of auth.Cache implementation
func (c *Cache) ListResources(ctx context.Context, req proto.ListResourcesRequest) (resources []types.ResourceWithLabels, nextKey string, err error) {
func (c *Cache) ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
rg, err := c.read()
if err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
defer rg.Release()
@@ -1958,14 +1958,14 @@ func (c *Cache) ListResources(ctx context.Context, req proto.ListResourcesReques
// sorting.
//
// NOTE: currently only types.KindNode supports TTL caching.
func (c *Cache) listResourcesFromTTLCache(ctx context.Context, rg readGuard, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func (c *Cache) listResourcesFromTTLCache(ctx context.Context, rg readGuard, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
var resources []types.ResourceWithLabels
switch req.ResourceType {
case types.KindNode:
// Retrieve all nodes.
cachedNodes, err := c.getNodesWithTTLCache(ctx, rg, req.Namespace)
if err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
// Nodes returned from the TTL caching layer
@@ -1977,13 +1977,13 @@ func (c *Cache) listResourcesFromTTLCache(ctx context.Context, rg readGuard, req
servers := types.Servers(clonedNodes)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
default:
return nil, "", trace.NotImplemented("resource type %q does not support TTL caching", req.ResourceType)
return nil, trace.NotImplemented("resource type %q does not support TTL caching", req.ResourceType)
}
return local.FakePaginate(resources, req)
lib/cache/cache_test.go
Vendored
+3 -3
View File
@@ -926,7 +926,7 @@ func TestListResources_NodesTTLVariant(t *testing.T) {
IsDesc: true,
}
require.Eventually(t, func() bool {
page, nextKey, err := p.cache.ListResources(ctx, proto.ListResourcesRequest{
resp, err := p.cache.ListResources(ctx, proto.ListResourcesRequest{
Namespace: apidefaults.Namespace,
ResourceType: types.KindNode,
StartKey: listResourcesStartKey,
@@ -934,8 +934,8 @@ func TestListResources_NodesTTLVariant(t *testing.T) {
SortBy: sortBy,
})
require.NoError(t, err)
resources = append(resources, page...)
listResourcesStartKey = nextKey
resources = append(resources, resp.Resources...)
listResourcesStartKey = resp.NextKey
return len(resources) == nodeCount
}, 5*time.Second, 100*time.Millisecond)
lib/client/client.go
+2 -2
View File
@@ -682,7 +682,7 @@ func (proxy *ProxyClient) ListResources(ctx context.Context, namespace, resource
if err != nil {
return nil, "", trace.Wrap(err)
}
resources, nextKey, err := authClient.ListResources(ctx, proto.ListResourcesRequest{
resp, err := authClient.ListResources(ctx, proto.ListResourcesRequest{
Namespace: namespace,
ResourceType: resource,
StartKey: startKey,
@@ -691,7 +691,7 @@ func (proxy *ProxyClient) ListResources(ctx context.Context, namespace, resource
if err != nil {
return nil, "", trace.Wrap(err)
}
return resources, nextKey, nil
return resp.Resources, resp.NextKey, nil
}
// CurrentClusterAccessPoint returns cluster access point to the currently
lib/services/local/presence.go
+58 -27
View File
@@ -1537,18 +1537,18 @@ func (s *PresenceService) DeleteAllWindowsDesktopServices(ctx context.Context) e
// ListResources returns a paginated list of resources.
// It implements various filtering for scenarios where the call comes directly
// here (without passing through the RBAC).
func (s *PresenceService) ListResources(ctx context.Context, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func (s *PresenceService) ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
switch {
case req.SortBy.Field != "":
case req.RequiresFakePagination():
return s.listResourcesWithSort(ctx, req)
default:
return s.listResources(ctx, req)
}
}
func (s *PresenceService) listResources(ctx context.Context, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func (s *PresenceService) listResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
if err := req.CheckAndSetDefaults(); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
var keyPrefix []string
@@ -1568,7 +1568,7 @@ func (s *PresenceService) listResources(ctx context.Context, req proto.ListResou
keyPrefix = []string{kubeServicesPrefix}
unmarshalItemFunc = backendItemToServer(types.KindKubeService)
default:
return nil, "", trace.NotImplemented("%s not implemented at ListResources", req.ResourceType)
return nil, trace.NotImplemented("%s not implemented at ListResources", req.ResourceType)
}
rangeStart := backend.Key(append(keyPrefix, req.StartKey)...)
@@ -1605,7 +1605,7 @@ func (s *PresenceService) listResources(ctx context.Context, req proto.ListResou
return len(resources) == maxLimit, nil
}); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
var nextKey string
@@ -1615,14 +1615,17 @@ func (s *PresenceService) listResources(ctx context.Context, req proto.ListResou
resources = resources[:reqLimit]
}
return resources, nextKey, nil
return &types.ListResourcesResponse{
Resources: resources,
NextKey: nextKey,
}, nil
}
// listResourcesWithSort supports sorting by falling back to retrieving all resources
// with GetXXXs, filter, and then fake pagination.
func (s *PresenceService) listResourcesWithSort(ctx context.Context, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func (s *PresenceService) listResourcesWithSort(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
if err := req.CheckAndSetDefaults(); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
var resources []types.ResourceWithLabels
@@ -1630,66 +1633,76 @@ func (s *PresenceService) listResourcesWithSort(ctx context.Context, req proto.L
case types.KindNode:
nodes, err := s.GetNodes(ctx, req.Namespace)
if err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
servers := types.Servers(nodes)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
case types.KindAppServer:
appservers, err := s.GetApplicationServers(ctx, req.Namespace)
if err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
servers := types.AppServers(appservers)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
case types.KindDatabaseServer:
dbservers, err := s.GetDatabaseServers(ctx, req.Namespace)
if err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
servers := types.DatabaseServers(dbservers)
if err := servers.SortByCustom(req.SortBy); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
resources = servers.AsResources()
default:
return nil, "", trace.NotImplemented("resource type %q is not supported for ListResourcesWithSort", req.ResourceType)
return nil, trace.NotImplemented("resource type %q is not supported for ListResourcesWithSort", req.ResourceType)
}
return FakePaginate(resources, req)
}
// FakePaginate is used when we are working with an entire list of resources upfront but still requires pagination.
func FakePaginate(resources []types.ResourceWithLabels, req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
func FakePaginate(resources []types.ResourceWithLabels, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
if err := req.CheckAndSetDefaults(); err != nil {
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
}
// Trim resources that precede start key.
// filterAll is a flag to continue matching even after we found limit,
// to determine the total count.
filterAll := req.NeedTotalCount
// Trim resources that precede start key, except when total count
// was requested, then we have to filter all to get accurate count.
pageStart := 0
if req.StartKey != "" {
pageStart := 0
for i, resource := range resources {
if backend.GetPaginationKey(resource) == req.StartKey {
pageStart = i
break
}
}
resources = resources[pageStart:]
if !filterAll {
resources = resources[pageStart:]
}
}
// Iterate and filter resources, finding match up to limit+1 (+1 to determine next key),
// and halting when we reach page limit.
// and if total count is not required, we halt matching when we reach page limit, else
// we continue to match (but not include into list of filtered) to determine the total count.
matchCount := 0
limit := int(req.Limit)
var nextKey string
var filtered []types.ResourceWithLabels
@@ -1700,23 +1713,41 @@ func FakePaginate(resources []types.ResourceWithLabels, req proto.ListResourcesR
PredicateExpression: req.PredicateExpression,
}
for _, resource := range resources {
for currIndex, resource := range resources {
switch match, err := services.MatchResourceByFilters(resource, filter); {
case err != nil:
return nil, "", trace.Wrap(err)
return nil, trace.Wrap(err)
case !match:
continue
}
matchCount++
if filterAll && nextKey != "" {
continue
}
if len(filtered) == limit {
nextKey = backend.GetPaginationKey(resource)
break
if !filterAll {
break
}
continue
}
filtered = append(filtered, resource)
if !filterAll || currIndex >= pageStart {
filtered = append(filtered, resource)
}
}
return filtered, nextKey, nil
if !filterAll {
matchCount = 0
}
return &types.ListResourcesResponse{
Resources: filtered,
NextKey: nextKey,
TotalCount: matchCount,
}, nil
}
// backendItemToDatabaseServer unmarshals `backend.Item` into a
lib/services/local/presence_test.go
+211 -55
View File
@@ -632,14 +632,15 @@ func TestListResources(t *testing.T) {
presence := NewPresenceService(backend)
resources, nextKey, err := presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err := presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: 1,
ResourceType: test.resourceType,
StartKey: "",
})
require.NoError(t, err)
require.Empty(t, resources)
require.Empty(t, nextKey)
require.Empty(t, resp.Resources)
require.Empty(t, resp.NextKey)
require.Empty(t, resp.TotalCount)
resourcesPerPage := 4
totalWithLabels := 7
@@ -661,17 +662,18 @@ func TestListResources(t *testing.T) {
resultResourcesLen := 0
require.Eventually(t, func() bool {
resources, nextKey, err = presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err = presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: int32(resourcesPerPage),
Namespace: apidefaults.Namespace,
ResourceType: test.resourceType,
StartKey: nextKey,
StartKey: resp.NextKey,
})
require.NoError(t, err)
require.Empty(t, resp.TotalCount)
resultResourcesLen += len(resources)
resultResourcesLen += len(resp.Resources)
if resultResourcesLen == totalResources {
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
}
return resultResourcesLen == totalResources
}, time.Second, 100*time.Millisecond)
@@ -679,18 +681,19 @@ func TestListResources(t *testing.T) {
// list resources only with matching labels
resultResourcesWithLabelsLen := 0
require.Eventually(t, func() bool {
resources, nextKey, err = presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err = presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: int32(resourcesPerPage),
Namespace: apidefaults.Namespace,
ResourceType: test.resourceType,
StartKey: nextKey,
StartKey: resp.NextKey,
Labels: labels,
})
require.NoError(t, err)
require.Empty(t, resp.TotalCount)
resultResourcesWithLabelsLen += len(resources)
resultResourcesWithLabelsLen += len(resp.Resources)
if resultResourcesWithLabelsLen == totalWithLabels {
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
}
return resultResourcesWithLabelsLen == totalWithLabels
}, time.Second, 100*time.Millisecond)
@@ -698,18 +701,19 @@ func TestListResources(t *testing.T) {
// list resources only with matching search keywords
resultResourcesWithSearchKeywordsLen := 0
require.Eventually(t, func() bool {
resources, nextKey, err = presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err = presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: int32(resourcesPerPage),
Namespace: apidefaults.Namespace,
ResourceType: test.resourceType,
StartKey: nextKey,
StartKey: resp.NextKey,
SearchKeywords: []string{"env", "test"},
})
require.NoError(t, err)
require.Empty(t, resp.TotalCount)
resultResourcesWithSearchKeywordsLen += len(resources)
resultResourcesWithSearchKeywordsLen += len(resp.Resources)
if resultResourcesWithSearchKeywordsLen == totalWithLabels {
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
}
return resultResourcesWithSearchKeywordsLen == totalWithLabels
}, time.Second, 100*time.Millisecond)
@@ -717,18 +721,19 @@ func TestListResources(t *testing.T) {
// list resources only with matching expression
resultResourcesWithMatchExprsLen := 0
require.Eventually(t, func() bool {
resources, nextKey, err = presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err = presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: int32(resourcesPerPage),
Namespace: apidefaults.Namespace,
ResourceType: test.resourceType,
StartKey: nextKey,
StartKey: resp.NextKey,
PredicateExpression: `labels.env == "test"`,
})
require.NoError(t, err)
require.Empty(t, resp.TotalCount)
resultResourcesWithMatchExprsLen += len(resources)
resultResourcesWithMatchExprsLen += len(resp.Resources)
if resultResourcesWithMatchExprsLen == totalWithLabels {
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
}
return resultResourcesWithMatchExprsLen == totalWithLabels
}, time.Second, 100*time.Millisecond)
@@ -739,19 +744,32 @@ func TestListResources(t *testing.T) {
switch test.resourceType {
case types.KindNode, types.KindAppServer, types.KindDatabaseServer:
// Test NeedTotalCount flag.
res, err := presence.ListResources(ctx, proto.ListResourcesRequest{
ResourceType: test.resourceType,
NeedTotalCount: true,
Limit: 1,
})
require.NoError(t, err)
require.Len(t, res.Resources, 1)
require.NotEmpty(t, res.NextKey)
require.Equal(t, totalResources, res.TotalCount)
// Test sorting.
require.Eventually(t, func() bool {
resources, nextKey, err = presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err = presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: int32(resourcesPerPage),
Namespace: apidefaults.Namespace,
ResourceType: test.resourceType,
StartKey: nextKey,
StartKey: resp.NextKey,
SortBy: sortBy,
})
require.NoError(t, err)
require.Empty(t, resp.TotalCount)
sortedResources = append(sortedResources, resources...)
sortedResources = append(sortedResources, resp.Resources...)
if len(sortedResources) == totalResources {
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
}
return len(sortedResources) == totalResources
}, time.Second, 100*time.Millisecond)
@@ -783,15 +801,16 @@ func TestListResources(t *testing.T) {
err = test.deleteAllResourcesFunc(ctx, presence)
require.NoError(t, err)
resources, nextKey, err = presence.ListResources(ctx, proto.ListResourcesRequest{
resp, err = presence.ListResources(ctx, proto.ListResourcesRequest{
Limit: 1,
Namespace: apidefaults.Namespace,
ResourceType: test.resourceType,
StartKey: "",
})
require.NoError(t, err)
require.Empty(t, nextKey)
require.Empty(t, resources)
require.Empty(t, resp.NextKey)
require.Empty(t, resp.Resources)
require.Empty(t, resp.TotalCount)
})
}
}
@@ -810,23 +829,23 @@ func TestListResources_Helpers(t *testing.T) {
tests := []struct {
name string
fetch func(proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error)
fetch func(proto.ListResourcesRequest) (*types.ListResourcesResponse, error)
}{
{
name: "listResources",
fetch: func(req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
fetch: func(req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
return presence.listResources(ctx, req)
},
},
{
name: "listResourcesWithSort",
fetch: func(req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
fetch: func(req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
return presence.listResourcesWithSort(ctx, req)
},
},
{
name: "FakePaginate",
fetch: func(req proto.ListResourcesRequest) ([]types.ResourceWithLabels, string, error) {
fetch: func(req proto.ListResourcesRequest) (*types.ListResourcesResponse, error) {
nodes, err := presence.GetNodes(ctx, namespace)
require.NoError(t, err)
@@ -844,10 +863,11 @@ func TestListResources_Helpers(t *testing.T) {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
page, nextKey, err := tc.fetch(req)
resp, err := tc.fetch(req)
require.NoError(t, err)
require.Empty(t, nextKey)
require.Empty(t, page)
require.Empty(t, resp.NextKey)
require.Empty(t, resp.Resources)
require.Empty(t, resp.TotalCount)
})
}
})
@@ -873,7 +893,7 @@ func TestListResources_Helpers(t *testing.T) {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
_, _, err := tc.fetch(req)
_, err := tc.fetch(req)
require.True(t, trace.IsBadParameter(err))
})
}
@@ -889,11 +909,12 @@ func TestListResources_Helpers(t *testing.T) {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
page, nextKey, err := tc.fetch(req)
resp, err := tc.fetch(req)
require.NoError(t, err)
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
require.Empty(t, resp.TotalCount)
fetchedNodes, err := types.ResourcesWithLabels(page).AsServers()
fetchedNodes, err := types.ResourcesWithLabels(resp.Resources).AsServers()
require.NoError(t, err)
require.Equal(t, nodes, fetchedNodes)
})
@@ -906,48 +927,51 @@ func TestListResources_Helpers(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
// First fetch.
page, nextKey, err := tc.fetch(proto.ListResourcesRequest{
resp, err := tc.fetch(proto.ListResourcesRequest{
ResourceType: types.KindNode,
Namespace: namespace,
Limit: 10,
})
require.NoError(t, err)
require.Len(t, page, 10)
require.Len(t, resp.Resources, 10)
require.Empty(t, resp.TotalCount)
fetchedNodes, err := types.ResourcesWithLabels(page).AsServers()
fetchedNodes, err := types.ResourcesWithLabels(resp.Resources).AsServers()
require.NoError(t, err)
require.Equal(t, nodes[:10], fetchedNodes)
require.Equal(t, backend.GetPaginationKey(nodes[10]), nextKey) // 11th item
require.Equal(t, backend.GetPaginationKey(nodes[10]), resp.NextKey) // 11th item
// Middle fetch.
page, nextKey, err = tc.fetch(proto.ListResourcesRequest{
resp, err = tc.fetch(proto.ListResourcesRequest{
ResourceType: types.KindNode,
Namespace: namespace,
StartKey: nextKey,
StartKey: resp.NextKey,
Limit: 5,
})
require.NoError(t, err)
require.Len(t, page, 5)
require.Len(t, resp.Resources, 5)
require.Empty(t, resp.TotalCount)
fetchedNodes, err = types.ResourcesWithLabels(page).AsServers()
fetchedNodes, err = types.ResourcesWithLabels(resp.Resources).AsServers()
require.NoError(t, err)
require.Equal(t, nodes[10:15], fetchedNodes)
require.Equal(t, backend.GetPaginationKey(nodes[15]), nextKey) // 16th item
require.Equal(t, backend.GetPaginationKey(nodes[15]), resp.NextKey) // 16th item
// Last fetch.
page, nextKey, err = presence.listResources(ctx, proto.ListResourcesRequest{
resp, err = presence.listResources(ctx, proto.ListResourcesRequest{
ResourceType: types.KindNode,
Namespace: namespace,
StartKey: nextKey,
StartKey: resp.NextKey,
Limit: 5,
})
require.NoError(t, err)
require.Len(t, page, 5)
require.Len(t, resp.Resources, 5)
require.Empty(t, resp.TotalCount)
fetchedNodes, err = types.ResourcesWithLabels(page).AsServers()
fetchedNodes, err = types.ResourcesWithLabels(resp.Resources).AsServers()
require.NoError(t, err)
require.Equal(t, nodes[15:20], fetchedNodes)
require.Empty(t, nextKey)
require.Empty(t, resp.NextKey)
})
}
})
@@ -965,12 +989,144 @@ func TestListResources_Helpers(t *testing.T) {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
page, nextKey, err := tc.fetch(req)
resp, err := tc.fetch(req)
require.NoError(t, err)
require.Len(t, page, 1)
require.Equal(t, targetVal, page[0].GetName())
require.Empty(t, nextKey)
require.Len(t, resp.Resources, 1)
require.Equal(t, targetVal, resp.Resources[0].GetName())
require.Empty(t, resp.NextKey)
require.Empty(t, resp.TotalCount)
})
}
})
}
func TestFakePaginate_TotalCount(t *testing.T) {
t.Parallel()
ctx := context.Background()
clock := clockwork.NewFakeClock()
namespace := apidefaults.Namespace
bend, err := lite.NewWithConfig(ctx, lite.Config{
Path: t.TempDir(),
Clock: clock,
})
require.NoError(t, err)
presence := NewPresenceService(bend)
// Add some control servers.
server := suite.NewServer(types.KindNode, "foo-bar", "127.0.0.1:2022", namespace)
_, err = presence.UpsertNode(ctx, server)
require.NoError(t, err)
server = suite.NewServer(types.KindNode, "foo-baz", "127.0.0.1:2022", namespace)
_, err = presence.UpsertNode(ctx, server)
require.NoError(t, err)
server = suite.NewServer(types.KindNode, "foo-qux", "127.0.0.1:2022", namespace)
_, err = presence.UpsertNode(ctx, server)
require.NoError(t, err)
// Add some test servers.
for i := 0; i < 10; i++ {
server := suite.NewServer(types.KindNode, uuid.New().String(), "127.0.0.1:2022", namespace)
_, err = presence.UpsertNode(ctx, server)
require.NoError(t, err)
}
// Test servers have been inserted.
nodes, err := presence.GetNodes(ctx, namespace)
require.NoError(t, err)
require.Len(t, nodes, 13)
// Convert to resources.
resources := types.Servers(nodes).AsResources()
t.Run("total count without filter", func(t *testing.T) {
t.Parallel()
tests := []struct {
name string
limit int
}{
{
name: "single",
limit: 1,
},
{
name: "even",
limit: 4,
},
{
name: "odd",
limit: 5,
},
{
name: "max",
limit: len(nodes),
},
}
for _, tc := range tests {
tc := tc
t.Run(tc.name, func(t *testing.T) {
t.Parallel()
req := proto.ListResourcesRequest{
ResourceType: types.KindNode,
Limit: int32(tc.limit),
NeedTotalCount: true,
}
// First fetch.
resp, err := FakePaginate(resources, req)
require.NoError(t, err)
require.Len(t, resp.Resources, tc.limit)
require.Equal(t, resources[0:tc.limit], resp.Resources)
require.Equal(t, len(nodes), resp.TotalCount)
// Next fetch should return same amount of totals.
if tc.limit != len(nodes) {
require.NotEmpty(t, resp.NextKey)
req.StartKey = resp.NextKey
resp, err = FakePaginate(resources, req)
require.NoError(t, err)
require.Len(t, resp.Resources, tc.limit)
require.Equal(t, resources[tc.limit:tc.limit*2], resp.Resources)
require.Equal(t, len(nodes), resp.TotalCount)
} else {
require.Empty(t, resp.NextKey)
require.Equal(t, resources, resp.Resources)
require.Equal(t, len(nodes), resp.TotalCount)
}
})
}
})
t.Run("total count with no match", func(t *testing.T) {
t.Parallel()
req := proto.ListResourcesRequest{
ResourceType: types.KindNode,
Limit: 5,
NeedTotalCount: true,
SearchKeywords: []string{"not-found"},
}
resp, err := FakePaginate(resources, req)
require.NoError(t, err)
require.Empty(t, resp.Resources)
require.Empty(t, resp.NextKey)
require.Empty(t, resp.TotalCount)
})
t.Run("total count with all matches", func(t *testing.T) {
t.Parallel()
req := proto.ListResourcesRequest{
ResourceType: types.KindNode,
Limit: 5,
NeedTotalCount: true,
SearchKeywords: []string{"foo"},
}
resp, err := FakePaginate(resources, req)
require.NoError(t, err)
require.Len(t, resp.Resources, 3)
require.Empty(t, resp.NextKey)
require.Equal(t, 3, resp.TotalCount)
})
}
lib/services/presence.go
+1 -1
View File
@@ -240,5 +240,5 @@ type Presence interface {
DeleteAllWindowsDesktopServices(context.Context) error
// ListResoures returns a paginated list of resources.
ListResources(ctx context.Context, req proto.ListResourcesRequest) (resources []types.ResourceWithLabels, nextKey string, err error)
ListResources(ctx context.Context, req proto.ListResourcesRequest) (*types.ListResourcesResponse, error)
}