/* Copyright 2018-2019 Gravitational, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cache import ( "context" "fmt" "os" "sync" "testing" "time" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" "github.com/google/uuid" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/require" "golang.org/x/exp/slices" "github.com/gravitational/teleport/api/client" "github.com/gravitational/teleport/api/client/proto" apidefaults "github.com/gravitational/teleport/api/defaults" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/lib/backend" "github.com/gravitational/teleport/lib/backend/lite" "github.com/gravitational/teleport/lib/backend/memory" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/services/local" "github.com/gravitational/teleport/lib/services/suite" "github.com/gravitational/teleport/lib/utils" ) const eventBufferSize = 1024 func TestMain(m *testing.M) { utils.InitLoggerForTests() os.Exit(m.Run()) } // testPack contains pack of // services used for test run type testPack struct { dataDir string backend *backend.Wrapper eventsC chan Event cache *Cache cacheBackend backend.Backend eventsS *proxyEvents trustS services.Trust provisionerS services.Provisioner clusterConfigS services.ClusterConfiguration usersS services.UsersService accessS services.Access dynamicAccessS services.DynamicAccessCore presenceS services.Presence appSessionS services.AppSession snowflakeSessionS services.SnowflakeSession samlIdPSessionsS services.SAMLIdPSession //nolint:revive // Because we want this to be IdP. restrictions services.Restrictions apps services.Apps kubernetes services.Kubernetes databases services.Databases databaseServices services.DatabaseServices webSessionS types.WebSessionInterface webTokenS types.WebTokenInterface windowsDesktops services.WindowsDesktops samlIDPServiceProviders services.SAMLIdPServiceProviders userGroups services.UserGroups } // testFuncs are functions to support testing an object in a cache. type testFuncs[T types.Resource] struct { newResource func(string) (T, error) create func(context.Context, T) error list func(context.Context) ([]T, error) cacheGet func(context.Context, string) (T, error) cacheList func(context.Context) ([]T, error) update func(context.Context, T) error deleteAll func(context.Context) error } func (t *testPack) Close() { var errors []error if t.backend != nil { errors = append(errors, t.backend.Close()) } if t.cache != nil { errors = append(errors, t.cache.Close()) } if err := trace.NewAggregate(errors...); err != nil { log.Warningf("Failed to close %v", err) } } func newPackForAuth(t *testing.T) *testPack { return newTestPack(t, ForAuth) } func newPackForProxy(t *testing.T) *testPack { return newTestPack(t, ForProxy) } func newTestPack(t *testing.T, setupConfig SetupConfigFn) *testPack { pack, err := newPack(t.TempDir(), setupConfig) require.NoError(t, err) return pack } func newTestPackWithoutCache(t *testing.T) *testPack { pack, err := newPackWithoutCache(t.TempDir()) require.NoError(t, err) return pack } type packCfg struct { memoryBackend bool } type packOption func(cfg *packCfg) func memoryBackend(bool) packOption { return func(cfg *packCfg) { cfg.memoryBackend = true } } // newPackWithoutCache returns a new test pack without creating cache func newPackWithoutCache(dir string, opts ...packOption) (*testPack, error) { ctx := context.Background() var cfg packCfg for _, opt := range opts { opt(&cfg) } p := &testPack{ dataDir: dir, } var bk backend.Backend var err error if cfg.memoryBackend { bk, err = memory.New(memory.Config{ Context: ctx, Mirror: true, }) } else { bk, err = lite.NewWithConfig(ctx, lite.Config{ Path: p.dataDir, PollStreamPeriod: 200 * time.Millisecond, }) } if err != nil { return nil, trace.Wrap(err) } p.backend = backend.NewWrapper(bk) p.cacheBackend, err = memory.New( memory.Config{ Context: ctx, Mirror: true, }) if err != nil { return nil, trace.Wrap(err) } p.eventsC = make(chan Event, eventBufferSize) clusterConfig, err := local.NewClusterConfigurationService(p.backend) if err != nil { return nil, trace.Wrap(err) } p.trustS = local.NewCAService(p.backend) p.clusterConfigS = clusterConfig p.provisionerS = local.NewProvisioningService(p.backend) p.eventsS = &proxyEvents{events: local.NewEventsService(p.backend)} p.presenceS = local.NewPresenceService(p.backend) p.usersS = local.NewIdentityService(p.backend) p.accessS = local.NewAccessService(p.backend) p.dynamicAccessS = local.NewDynamicAccessService(p.backend) p.appSessionS = local.NewIdentityService(p.backend) p.webSessionS = local.NewIdentityService(p.backend).WebSessions() p.snowflakeSessionS = local.NewIdentityService(p.backend) p.samlIdPSessionsS = local.NewIdentityService(p.backend) p.webTokenS = local.NewIdentityService(p.backend).WebTokens() p.restrictions = local.NewRestrictionsService(p.backend) p.apps = local.NewAppService(p.backend) p.kubernetes = local.NewKubernetesService(p.backend) p.databases = local.NewDatabasesService(p.backend) p.databaseServices = local.NewDatabaseServicesService(p.backend) p.windowsDesktops = local.NewWindowsDesktopService(p.backend) p.samlIDPServiceProviders, err = local.NewSAMLIdPServiceProviderService(p.backend) if err != nil { return nil, trace.Wrap(err) } p.userGroups = local.NewUserGroupService(p.backend) return p, nil } // newPack returns a new test pack or fails the test on error func newPack(dir string, setupConfig func(c Config) Config, opts ...packOption) (*testPack, error) { ctx := context.Background() p, err := newPackWithoutCache(dir, opts...) if err != nil { return nil, trace.Wrap(err) } p.cache, err = New(setupConfig(Config{ Context: ctx, Backend: p.cacheBackend, Events: p.eventsS, ClusterConfig: p.clusterConfigS, Provisioner: p.provisionerS, Trust: p.trustS, Users: p.usersS, Access: p.accessS, DynamicAccess: p.dynamicAccessS, Presence: p.presenceS, AppSession: p.appSessionS, WebSession: p.webSessionS, WebToken: p.webTokenS, SnowflakeSession: p.snowflakeSessionS, SAMLIdPSession: p.samlIdPSessionsS, Restrictions: p.restrictions, Apps: p.apps, Kubernetes: p.kubernetes, DatabaseServices: p.databaseServices, Databases: p.databases, WindowsDesktops: p.windowsDesktops, SAMLIdPServiceProviders: p.samlIDPServiceProviders, UserGroups: p.userGroups, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, })) if err != nil { return nil, trace.Wrap(err) } select { case event := <-p.eventsC: if event.Type != WatcherStarted { return nil, trace.CompareFailed("%q != %q %s", event.Type, WatcherStarted, event) } case <-time.After(time.Second): return nil, trace.ConnectionProblem(nil, "wait for the watcher to start") } return p, nil } // TestCA tests certificate authorities func TestCA(t *testing.T) { t.Parallel() p := newPackForAuth(t) t.Cleanup(p.Close) ctx := context.Background() ca := suite.NewTestCA(types.UserCA, "example.com") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) select { case <-p.eventsC: case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } out, err := p.cache.GetCertAuthority(ctx, ca.GetID(), true) require.NoError(t, err) ca.SetResourceID(out.GetResourceID()) require.Empty(t, cmp.Diff(ca, out)) err = p.trustS.DeleteCertAuthority(ctx, ca.GetID()) require.NoError(t, err) select { case <-p.eventsC: case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } _, err = p.cache.GetCertAuthority(ctx, ca.GetID(), false) require.True(t, trace.IsNotFound(err)) } // TestWatchers tests watchers connected to the cache, // verifies that all watchers of the cache will be closed // if the underlying watcher to the target backend is closed func TestWatchers(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) w, err := p.cache.NewWatcher(ctx, types.Watch{Kinds: []types.WatchKind{ { Kind: types.KindCertAuthority, Filter: types.CertAuthorityFilter{ types.HostCA: "example.com", types.UserCA: types.Wildcard, }.IntoMap(), }, { Kind: types.KindAccessRequest, Filter: map[string]string{ "user": "alice", }, }, }}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, w.Close()) }) select { case e := <-w.Events(): require.Equal(t, types.OpInit, e.Type) case <-time.After(100 * time.Millisecond): t.Fatalf("Timeout waiting for event.") } ca := suite.NewTestCA(types.UserCA, "example.com") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) select { case e := <-w.Events(): require.Equal(t, types.OpPut, e.Type) require.Equal(t, types.KindCertAuthority, e.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("Timeout waiting for event.") } // create an access request that matches the supplied filter req, err := services.NewAccessRequest("alice", "dictator") require.NoError(t, err) require.NoError(t, p.dynamicAccessS.CreateAccessRequest(ctx, req)) select { case e := <-w.Events(): require.Equal(t, types.OpPut, e.Type) require.Equal(t, types.KindAccessRequest, e.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("Timeout waiting for event.") } require.NoError(t, p.dynamicAccessS.DeleteAccessRequest(ctx, req.GetName())) select { case e := <-w.Events(): require.Equal(t, types.OpDelete, e.Type) require.Equal(t, types.KindAccessRequest, e.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("Timeout waiting for event.") } // create an access request that does not match the supplied filter req2, err := services.NewAccessRequest("bob", "dictator") require.NoError(t, err) // create and then delete the non-matching request. require.NoError(t, p.dynamicAccessS.CreateAccessRequest(ctx, req2)) require.NoError(t, p.dynamicAccessS.DeleteAccessRequest(ctx, req2.GetName())) // because our filter did not match the request, the create event should never // have been created, meaning that the next event on the pipe is the delete // event (which cannot be filtered out because username is not visible inside // a delete event). select { case e := <-w.Events(): require.Equal(t, types.OpDelete, e.Type) require.Equal(t, types.KindAccessRequest, e.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("Timeout waiting for event.") } // this ca will not be matched by our filter, so the same reasoning applies // as we upsert it and delete it filteredCa := suite.NewTestCA(types.HostCA, "example.net") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, filteredCa)) require.NoError(t, p.trustS.DeleteCertAuthority(ctx, filteredCa.GetID())) select { case e := <-w.Events(): require.Equal(t, types.OpDelete, e.Type) require.Equal(t, types.KindCertAuthority, e.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("Timeout waiting for event.") } // event has arrived, now close the watchers p.backend.CloseWatchers() // make sure watcher has been closed select { case <-w.Done(): case <-time.After(time.Second): t.Fatalf("Timeout waiting for close event.") } } func TestNodeCAFiltering(t *testing.T) { t.Parallel() ctx := context.Background() p := newTestPack(t, ForAuth) t.Cleanup(p.Close) clusterName, err := services.NewClusterNameWithRandomID(types.ClusterNameSpecV2{ ClusterName: "example.com", }) require.NoError(t, err) err = p.cache.clusterConfigCache.UpsertClusterName(clusterName) require.NoError(t, err) nodeCacheBackend, err := memory.New(memory.Config{}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, nodeCacheBackend.Close()) }) // this mimics a cache for a node pulling events from the auth server via WatchEvents nodeCache, err := New(ForNode(Config{ Events: p.cache, Trust: p.cache.trustCache, ClusterConfig: p.cache.clusterConfigCache, Provisioner: p.cache.provisionerCache, Users: p.cache.usersCache, Access: p.cache.accessCache, DynamicAccess: p.cache.dynamicAccessCache, Presence: p.cache.presenceCache, Restrictions: p.cache.restrictionsCache, Apps: p.cache.appsCache, Kubernetes: p.cache.kubernetesCache, Databases: p.cache.databasesCache, DatabaseServices: p.cache.databaseServicesCache, AppSession: p.cache.appSessionCache, WebSession: p.cache.webSessionCache, WebToken: p.cache.webTokenCache, WindowsDesktops: p.cache.windowsDesktopsCache, SAMLIdPServiceProviders: p.samlIDPServiceProviders, UserGroups: p.userGroups, Backend: nodeCacheBackend, })) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, nodeCache.Close()) }) cacheWatcher, err := nodeCache.NewWatcher(ctx, types.Watch{Kinds: []types.WatchKind{{Kind: types.KindCertAuthority}}}) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, cacheWatcher.Close()) }) fetchEvent := func() types.Event { var ev types.Event select { case ev = <-cacheWatcher.Events(): case <-time.After(time.Second * 5): t.Fatal("watcher timeout") } return ev } require.Equal(t, types.OpInit, fetchEvent().Type) // upsert and delete a local host CA, we expect to see a Put and a Delete event localCA := suite.NewTestCA(types.HostCA, "example.com") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, localCA)) require.NoError(t, p.trustS.DeleteCertAuthority(ctx, localCA.GetID())) ev := fetchEvent() require.Equal(t, types.OpPut, ev.Type) require.Equal(t, types.KindCertAuthority, ev.Resource.GetKind()) require.Equal(t, "example.com", ev.Resource.GetName()) ev = fetchEvent() require.Equal(t, types.OpDelete, ev.Type) require.Equal(t, types.KindCertAuthority, ev.Resource.GetKind()) require.Equal(t, "example.com", ev.Resource.GetName()) // upsert and delete a nonlocal host CA, we expect to only see the Delete event nonlocalCA := suite.NewTestCA(types.HostCA, "example.net") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, nonlocalCA)) require.NoError(t, p.trustS.DeleteCertAuthority(ctx, nonlocalCA.GetID())) ev = fetchEvent() require.Equal(t, types.OpDelete, ev.Type) require.Equal(t, types.KindCertAuthority, ev.Resource.GetKind()) require.Equal(t, "example.net", ev.Resource.GetName()) // whereas we expect to see the Put and Delete for a trusted *user* CA trustedUserCA := suite.NewTestCA(types.UserCA, "example.net") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, trustedUserCA)) require.NoError(t, p.trustS.DeleteCertAuthority(ctx, trustedUserCA.GetID())) ev = fetchEvent() require.Equal(t, types.OpPut, ev.Type) require.Equal(t, types.KindCertAuthority, ev.Resource.GetKind()) require.Equal(t, "example.net", ev.Resource.GetName()) ev = fetchEvent() require.Equal(t, types.OpDelete, ev.Type) require.Equal(t, types.KindCertAuthority, ev.Resource.GetKind()) require.Equal(t, "example.net", ev.Resource.GetName()) } func waitForRestart(t *testing.T, eventsC <-chan Event) { expectEvent(t, eventsC, WatcherStarted) } func drainEvents(eventsC <-chan Event) { for { select { case <-eventsC: default: return } } } func expectEvent(t *testing.T, eventsC <-chan Event, expectedEvent string) { timeC := time.After(5 * time.Second) for { select { case event := <-eventsC: if event.Type == expectedEvent { return } case <-timeC: t.Fatalf("Timeout waiting for expected event: %s", expectedEvent) } } } func unexpectedEvent(t *testing.T, eventsC <-chan Event, unexpectedEvent string) { timeC := time.After(time.Second) for { select { case event := <-eventsC: if event.Type == unexpectedEvent { t.Fatalf("Received unexpected event: %s", unexpectedEvent) } case <-timeC: return } } } func expectNextEvent(t *testing.T, eventsC <-chan Event, expectedEvent string, skipEvents ...string) { timeC := time.After(5 * time.Second) for { // wait for watcher to restart select { case event := <-eventsC: if slices.Contains(skipEvents, event.Type) { continue } require.Equal(t, expectedEvent, event.Type) return case <-timeC: t.Fatalf("Timeout waiting for expected event: %s", expectedEvent) } } } // TestCompletenessInit verifies that flaky backends don't cause // the cache to return partial results during init. func TestCompletenessInit(t *testing.T) { t.Parallel() ctx := context.Background() const caCount = 100 const inits = 20 p := newTestPackWithoutCache(t) t.Cleanup(p.Close) // put lots of CAs in the backend for i := 0; i < caCount; i++ { ca := suite.NewTestCA(types.UserCA, fmt.Sprintf("%d.example.com", i)) require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) } for i := 0; i < inits; i++ { var err error p.cacheBackend, err = memory.New( memory.Config{ Context: ctx, Mirror: true, }) require.NoError(t, err) // simulate bad connection to auth server p.backend.SetReadError(trace.ConnectionProblem(nil, "backend is unavailable")) p.eventsS.closeWatchers() p.cache, err = New(ForAuth(Config{ Context: ctx, Backend: p.cacheBackend, Events: p.eventsS, ClusterConfig: p.clusterConfigS, Provisioner: p.provisionerS, Trust: p.trustS, Users: p.usersS, Access: p.accessS, DynamicAccess: p.dynamicAccessS, Presence: p.presenceS, AppSession: p.appSessionS, WebSession: p.webSessionS, SnowflakeSession: p.snowflakeSessionS, SAMLIdPSession: p.samlIdPSessionsS, WebToken: p.webTokenS, Restrictions: p.restrictions, Apps: p.apps, Kubernetes: p.kubernetes, DatabaseServices: p.databaseServices, Databases: p.databases, WindowsDesktops: p.windowsDesktops, SAMLIdPServiceProviders: p.samlIDPServiceProviders, UserGroups: p.userGroups, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, })) require.NoError(t, err) p.backend.SetReadError(nil) cas, err := p.cache.GetCertAuthorities(ctx, types.UserCA, false) // we don't actually care whether the cache ever fully constructed // the CA list. for the purposes of this test, we just care that it // doesn't return the CA list *unless* it was successfully constructed. if err == nil { require.Len(t, cas, caCount) } else { require.True(t, trace.IsConnectionProblem(err)) } require.NoError(t, p.cache.Close()) p.cache = nil require.NoError(t, p.cacheBackend.Close()) p.cacheBackend = nil } } // TestCompletenessReset verifies that flaky backends don't cause // the cache to return partial results during reset. func TestCompletenessReset(t *testing.T) { t.Parallel() ctx := context.Background() const caCount = 100 const resets = 20 p := newTestPackWithoutCache(t) t.Cleanup(p.Close) // put lots of CAs in the backend for i := 0; i < caCount; i++ { ca := suite.NewTestCA(types.UserCA, fmt.Sprintf("%d.example.com", i)) require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) } var err error p.cache, err = New(ForAuth(Config{ Context: ctx, Backend: p.cacheBackend, Events: p.eventsS, ClusterConfig: p.clusterConfigS, Provisioner: p.provisionerS, Trust: p.trustS, Users: p.usersS, Access: p.accessS, DynamicAccess: p.dynamicAccessS, Presence: p.presenceS, AppSession: p.appSessionS, WebSession: p.webSessionS, SnowflakeSession: p.snowflakeSessionS, SAMLIdPSession: p.samlIdPSessionsS, WebToken: p.webTokenS, Restrictions: p.restrictions, Apps: p.apps, Kubernetes: p.kubernetes, DatabaseServices: p.databaseServices, Databases: p.databases, WindowsDesktops: p.windowsDesktops, SAMLIdPServiceProviders: p.samlIDPServiceProviders, UserGroups: p.userGroups, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, })) require.NoError(t, err) // verify that CAs are immediately available cas, err := p.cache.GetCertAuthorities(ctx, types.UserCA, false) require.NoError(t, err) require.Len(t, cas, caCount) for i := 0; i < resets; i++ { // simulate bad connection to auth server p.backend.SetReadError(trace.ConnectionProblem(nil, "backend is unavailable")) p.eventsS.closeWatchers() p.backend.SetReadError(nil) // load CAs while connection is bad cas, err := p.cache.GetCertAuthorities(ctx, types.UserCA, false) // we don't actually care whether the cache ever fully constructed // the CA list. for the purposes of this test, we just care that it // doesn't return the CA list *unless* it was successfully constructed. if err == nil { require.Len(t, cas, caCount) } else { require.True(t, trace.IsConnectionProblem(err)) } } } // TestInitStrategy verifies that cache uses expected init strategy // of serving backend state when init is taking too long. func TestInitStrategy(t *testing.T) { t.Parallel() for i := 0; i < utils.GetIterations(); i++ { initStrategy(t) } } /* goos: linux goarch: amd64 pkg: github.com/gravitational/teleport/lib/cache cpu: Intel(R) Core(TM) i9-10885H CPU @ 2.40GHz BenchmarkGetMaxNodes-16 1 1029199093 ns/op */ func BenchmarkGetMaxNodes(b *testing.B) { benchGetNodes(b, backend.DefaultRangeLimit) } func benchGetNodes(b *testing.B, nodeCount int) { p, err := newPack(b.TempDir(), ForAuth, memoryBackend(true)) require.NoError(b, err) defer p.Close() ctx := context.Background() for i := 0; i < nodeCount; i++ { server := suite.NewServer(types.KindNode, uuid.New().String(), "127.0.0.1:2022", apidefaults.Namespace) _, err := p.presenceS.UpsertNode(ctx, server) require.NoError(b, err) select { case event := <-p.eventsC: require.Equal(b, EventProcessed, event.Type) case <-time.After(200 * time.Millisecond): b.Fatalf("timeout waiting for event, iteration=%d", i) } } b.ResetTimer() for n := 0; n < b.N; n++ { nodes, err := p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(b, err) require.Len(b, nodes, nodeCount) } } /* goos: linux goarch: amd64 pkg: github.com/gravitational/teleport/lib/cache cpu: Intel(R) Core(TM) i7-8550U CPU @ 1.80GHz BenchmarkListResourcesWithSort-8 1 2351035036 ns/op */ func BenchmarkListResourcesWithSort(b *testing.B) { p, err := newPack(b.TempDir(), ForAuth, memoryBackend(true)) require.NoError(b, err) defer p.Close() ctx := context.Background() count := 100000 for i := 0; i < count; i++ { server := suite.NewServer(types.KindNode, uuid.New().String(), "127.0.0.1:2022", apidefaults.Namespace) // Set some static and dynamic labels. server.Metadata.Labels = map[string]string{"os": "mac", "env": "prod", "country": "us", "tier": "frontend"} server.Spec.CmdLabels = map[string]types.CommandLabelV2{ "version": {Result: "v8"}, "time": {Result: "now"}, } _, err := p.presenceS.UpsertNode(ctx, server) require.NoError(b, err) select { case event := <-p.eventsC: require.Equal(b, EventProcessed, event.Type) case <-time.After(200 * time.Millisecond): b.Fatalf("timeout waiting for event, iteration=%d", i) } } b.ResetTimer() for _, limit := range []int32{100, 1_000, 10_000, 100_000} { for _, totalCount := range []bool{true, false} { b.Run(fmt.Sprintf("limit=%d,needTotal=%t", limit, totalCount), func(b *testing.B) { for n := 0; n < b.N; n++ { resp, err := p.cache.ListResources(ctx, proto.ListResourcesRequest{ ResourceType: types.KindNode, Namespace: apidefaults.Namespace, SortBy: types.SortBy{ IsDesc: true, Field: types.ResourceSpecHostname, }, // Predicate is the more expensive filter. PredicateExpression: `search("mac", "frontend") && labels.version == "v8"`, Limit: limit, NeedTotalCount: totalCount, }) require.NoError(b, err) require.Len(b, resp.Resources, int(limit)) } }) } } } // TestListResources_NodesTTLVariant verifies that the custom ListNodes impl that we fallback to when // using ttl-based caching works as expected. func TestListResources_NodesTTLVariant(t *testing.T) { t.Parallel() const nodeCount = 100 const pageSize = 10 var err error ctx := context.Background() p, err := newPackWithoutCache(t.TempDir()) require.NoError(t, err) t.Cleanup(p.Close) p.cache, err = New(ForAuth(Config{ Context: ctx, Backend: p.cacheBackend, Events: p.eventsS, ClusterConfig: p.clusterConfigS, Provisioner: p.provisionerS, Trust: p.trustS, Users: p.usersS, Access: p.accessS, DynamicAccess: p.dynamicAccessS, Presence: p.presenceS, AppSession: p.appSessionS, WebSession: p.webSessionS, WebToken: p.webTokenS, SnowflakeSession: p.snowflakeSessionS, SAMLIdPSession: p.samlIdPSessionsS, Restrictions: p.restrictions, Apps: p.apps, Kubernetes: p.kubernetes, DatabaseServices: p.databaseServices, Databases: p.databases, WindowsDesktops: p.windowsDesktops, SAMLIdPServiceProviders: p.samlIDPServiceProviders, UserGroups: p.userGroups, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, neverOK: true, // ensure reads are never healthy })) require.NoError(t, err) for i := 0; i < nodeCount; i++ { server := suite.NewServer(types.KindNode, uuid.New().String(), "127.0.0.1:2022", apidefaults.Namespace) _, err := p.presenceS.UpsertNode(ctx, server) require.NoError(t, err) } time.Sleep(time.Second * 2) allNodes, err := p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.Len(t, allNodes, nodeCount) var resources []types.ResourceWithLabels var listResourcesStartKey string sortBy := types.SortBy{ Field: types.ResourceMetadataName, IsDesc: true, } require.Eventually(t, func() bool { resp, err := p.cache.ListResources(ctx, proto.ListResourcesRequest{ Namespace: apidefaults.Namespace, ResourceType: types.KindNode, StartKey: listResourcesStartKey, Limit: int32(pageSize), SortBy: sortBy, }) require.NoError(t, err) resources = append(resources, resp.Resources...) listResourcesStartKey = resp.NextKey return len(resources) == nodeCount }, 5*time.Second, 100*time.Millisecond) servers, err := types.ResourcesWithLabels(resources).AsServers() require.NoError(t, err) fieldVals, err := types.Servers(servers).GetFieldVals(sortBy.Field) require.NoError(t, err) require.IsDecreasing(t, fieldVals) } func initStrategy(t *testing.T) { ctx := context.Background() p := newTestPackWithoutCache(t) t.Cleanup(p.Close) p.backend.SetReadError(trace.ConnectionProblem(nil, "backend is out")) var err error p.cache, err = New(ForAuth(Config{ Context: ctx, Backend: p.cacheBackend, Events: p.eventsS, ClusterConfig: p.clusterConfigS, Provisioner: p.provisionerS, Trust: p.trustS, Users: p.usersS, Access: p.accessS, DynamicAccess: p.dynamicAccessS, Presence: p.presenceS, AppSession: p.appSessionS, SnowflakeSession: p.snowflakeSessionS, SAMLIdPSession: p.samlIdPSessionsS, WebSession: p.webSessionS, WebToken: p.webTokenS, Restrictions: p.restrictions, Apps: p.apps, Kubernetes: p.kubernetes, DatabaseServices: p.databaseServices, Databases: p.databases, WindowsDesktops: p.windowsDesktops, SAMLIdPServiceProviders: p.samlIDPServiceProviders, UserGroups: p.userGroups, MaxRetryPeriod: 200 * time.Millisecond, EventsC: p.eventsC, })) require.NoError(t, err) _, err = p.cache.GetCertAuthorities(ctx, types.UserCA, false) require.True(t, trace.IsConnectionProblem(err)) ca := suite.NewTestCA(types.UserCA, "example.com") // NOTE 1: this could produce event processed // below, based on whether watcher restarts to get the event // or not, which is normal, but has to be accounted below require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) p.backend.SetReadError(nil) // wait for watcher to restart waitForRestart(t, p.eventsC) normalizeCA := func(ca types.CertAuthority) types.CertAuthority { ca = ca.Clone() ca.SetResourceID(0) ca.SetExpiry(time.Time{}) types.RemoveCASecrets(ca) return ca } _ = normalizeCA out, err := p.cache.GetCertAuthority(ctx, ca.GetID(), false) require.NoError(t, err) require.Empty(t, cmp.Diff(normalizeCA(ca), normalizeCA(out))) // fail again, make sure last recent data is still served // on errors p.backend.SetReadError(trace.ConnectionProblem(nil, "backend is unavailable")) p.eventsS.closeWatchers() // wait for the watcher to fail // there could be optional event processed event, // see NOTE 1 above expectNextEvent(t, p.eventsC, WatcherFailed, EventProcessed, Reloading) // backend is out, but old value is available out2, err := p.cache.GetCertAuthority(ctx, ca.GetID(), false) require.NoError(t, err) require.Equal(t, out.GetResourceID(), out2.GetResourceID()) require.Empty(t, cmp.Diff(normalizeCA(ca), normalizeCA(out))) // add modification and expect the resource to recover ca.SetRoleMap(types.RoleMap{types.RoleMapping{Remote: "test", Local: []string{"local-test"}}}) require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) // now, recover the backend and make sure the // service is back and the new value has propagated p.backend.SetReadError(nil) // wait for watcher to restart successfully; ignoring any failed // attempts which occurred before backend became healthy again. expectEvent(t, p.eventsC, WatcherStarted) // new value is available now out, err = p.cache.GetCertAuthority(ctx, ca.GetID(), false) require.NoError(t, err) require.Empty(t, cmp.Diff(normalizeCA(ca), normalizeCA(out))) } // TestRecovery tests error recovery scenario func TestRecovery(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) ca := suite.NewTestCA(types.UserCA, "example.com") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca)) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } // event has arrived, now close the watchers watchers := p.eventsS.getWatchers() require.Len(t, watchers, 1) p.eventsS.closeWatchers() // wait for watcher to restart waitForRestart(t, p.eventsC) // add modification and expect the resource to recover ca2 := suite.NewTestCA(types.UserCA, "example2.com") require.NoError(t, p.trustS.UpsertCertAuthority(ctx, ca2)) // wait for watcher to receive an event select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } out, err := p.cache.GetCertAuthority(context.Background(), ca2.GetID(), false) require.NoError(t, err) ca2.SetResourceID(out.GetResourceID()) types.RemoveCASecrets(ca2) require.Empty(t, cmp.Diff(ca2, out)) } // TestTokens tests static and dynamic tokens func TestTokens(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) staticTokens, err := types.NewStaticTokens(types.StaticTokensSpecV2{ StaticTokens: []types.ProvisionTokenV1{ { Token: "static1", Roles: types.SystemRoles{types.RoleAuth, types.RoleNode}, Expires: time.Now().UTC().Add(time.Hour), }, }, }) require.NoError(t, err) err = p.clusterConfigS.SetStaticTokens(staticTokens) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } out, err := p.cache.GetStaticTokens() require.NoError(t, err) staticTokens.SetResourceID(out.GetResourceID()) require.Empty(t, cmp.Diff(staticTokens, out)) expires := time.Now().Add(10 * time.Hour).Truncate(time.Second).UTC() token, err := types.NewProvisionToken("token", types.SystemRoles{types.RoleAuth, types.RoleNode}, expires) require.NoError(t, err) err = p.provisionerS.UpsertToken(ctx, token) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } tout, err := p.cache.GetToken(ctx, token.GetName()) require.NoError(t, err) token.SetResourceID(tout.GetResourceID()) require.Empty(t, cmp.Diff(token, tout)) err = p.provisionerS.DeleteToken(ctx, token.GetName()) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } _, err = p.cache.GetToken(ctx, token.GetName()) require.True(t, trace.IsNotFound(err)) } func TestAuthPreference(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) authPref, err := types.NewAuthPreferenceFromConfigFile(types.AuthPreferenceSpecV2{ AllowLocalAuth: types.NewBoolOption(true), MessageOfTheDay: "test MOTD", }) require.NoError(t, err) err = p.clusterConfigS.SetAuthPreference(ctx, authPref) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) require.Equal(t, types.KindClusterAuthPreference, event.Event.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } outAuthPref, err := p.cache.GetAuthPreference(ctx) require.NoError(t, err) authPref.SetResourceID(outAuthPref.GetResourceID()) require.Empty(t, cmp.Diff(outAuthPref, authPref)) } func TestClusterNetworkingConfig(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) netConfig, err := types.NewClusterNetworkingConfigFromConfigFile(types.ClusterNetworkingConfigSpecV2{ ClientIdleTimeout: types.Duration(time.Minute), ClientIdleTimeoutMessage: "test idle timeout message", }) require.NoError(t, err) err = p.clusterConfigS.SetClusterNetworkingConfig(ctx, netConfig) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) require.Equal(t, types.KindClusterNetworkingConfig, event.Event.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } outNetConfig, err := p.cache.GetClusterNetworkingConfig(ctx) require.NoError(t, err) netConfig.SetResourceID(outNetConfig.GetResourceID()) require.Empty(t, cmp.Diff(outNetConfig, netConfig)) } func TestSessionRecordingConfig(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) recConfig, err := types.NewSessionRecordingConfigFromConfigFile(types.SessionRecordingConfigSpecV2{ Mode: types.RecordAtProxySync, ProxyChecksHostKeys: types.NewBoolOption(true), }) require.NoError(t, err) err = p.clusterConfigS.SetSessionRecordingConfig(ctx, recConfig) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) require.Equal(t, types.KindSessionRecordingConfig, event.Event.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } outRecConfig, err := p.cache.GetSessionRecordingConfig(ctx) require.NoError(t, err) recConfig.SetResourceID(outRecConfig.GetResourceID()) require.Empty(t, cmp.Diff(outRecConfig, recConfig)) } func TestClusterAuditConfig(t *testing.T) { t.Parallel() ctx := context.Background() p := newPackForAuth(t) t.Cleanup(p.Close) auditConfig, err := types.NewClusterAuditConfig(types.ClusterAuditConfigSpecV2{ AuditEventsURI: []string{"dynamodb://audit_table_name", "file:///home/log"}, }) require.NoError(t, err) err = p.clusterConfigS.SetClusterAuditConfig(ctx, auditConfig) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) require.Equal(t, types.KindClusterAuditConfig, event.Event.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } outAuditConfig, err := p.cache.GetClusterAuditConfig(ctx) require.NoError(t, err) auditConfig.SetResourceID(outAuditConfig.GetResourceID()) require.Empty(t, cmp.Diff(outAuditConfig, auditConfig)) } func TestClusterName(t *testing.T) { t.Parallel() p := newPackForAuth(t) t.Cleanup(p.Close) clusterName, err := services.NewClusterNameWithRandomID(types.ClusterNameSpecV2{ ClusterName: "example.com", }) require.NoError(t, err) err = p.clusterConfigS.SetClusterName(clusterName) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) require.Equal(t, types.KindClusterName, event.Event.Resource.GetKind()) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } outName, err := p.cache.GetClusterName() require.NoError(t, err) clusterName.SetResourceID(outName.GetResourceID()) require.Empty(t, cmp.Diff(outName, clusterName)) } // TestNamespaces tests caching of namespaces func TestNamespaces(t *testing.T) { t.Parallel() p := newPackForProxy(t) t.Cleanup(p.Close) v, err := types.NewNamespace("universe") require.NoError(t, err) ns := &v err = p.presenceS.UpsertNamespace(*ns) require.NoError(t, err) ns, err = p.presenceS.GetNamespace(ns.GetName()) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } out, err := p.cache.GetNamespace(ns.GetName()) require.NoError(t, err) ns.SetResourceID(out.GetResourceID()) require.Empty(t, cmp.Diff(ns, out)) // update namespace metadata ns.Metadata.Labels = map[string]string{"a": "b"} require.NoError(t, err) err = p.presenceS.UpsertNamespace(*ns) require.NoError(t, err) ns, err = p.presenceS.GetNamespace(ns.GetName()) require.NoError(t, err) select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } out, err = p.cache.GetNamespace(ns.GetName()) require.NoError(t, err) ns.SetResourceID(out.GetResourceID()) require.Empty(t, cmp.Diff(ns, out)) err = p.presenceS.DeleteNamespace(ns.GetName()) require.NoError(t, err) select { case <-p.eventsC: case <-time.After(time.Second): t.Fatalf("timeout waiting for event") } _, err = p.cache.GetNamespace(ns.GetName()) require.True(t, trace.IsNotFound(err)) } // TestUsers tests caching of users func TestUsers(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.User]{ newResource: func(name string) (types.User, error) { return types.NewUser("bob") }, create: modifyNoContext(p.usersS.UpsertUser), list: func(ctx context.Context) ([]types.User, error) { return p.usersS.GetUsers(false) }, cacheList: func(ctx context.Context) ([]types.User, error) { return p.cache.GetUsers(false) }, update: modifyNoContext(p.usersS.UpsertUser), deleteAll: func(_ context.Context) error { return p.usersS.DeleteAllUsers() }, }) } // TestRoles tests caching of roles func TestRoles(t *testing.T) { t.Parallel() p := newTestPack(t, ForNode) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Role]{ newResource: func(name string) (types.Role, error) { return types.NewRole("role1", types.RoleSpecV6{ Options: types.RoleOptions{ MaxSessionTTL: types.Duration(time.Hour), }, Allow: types.RoleConditions{ Logins: []string{"root", "bob"}, NodeLabels: types.Labels{types.Wildcard: []string{types.Wildcard}}, }, Deny: types.RoleConditions{}, }) }, create: p.accessS.UpsertRole, list: p.accessS.GetRoles, cacheGet: p.cache.GetRole, cacheList: p.cache.GetRoles, update: p.accessS.UpsertRole, deleteAll: func(_ context.Context) error { return p.accessS.DeleteAllRoles() }, }) } // TestReverseTunnels tests reverse tunnels caching func TestReverseTunnels(t *testing.T) { t.Parallel() p, err := newPack(t.TempDir(), ForProxy) require.NoError(t, err) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.ReverseTunnel]{ newResource: func(name string) (types.ReverseTunnel, error) { return types.NewReverseTunnel(name, []string{"example.com:2023"}) }, create: modifyNoContext(p.presenceS.UpsertReverseTunnel), list: func(ctx context.Context) ([]types.ReverseTunnel, error) { return p.presenceS.GetReverseTunnels(ctx) }, cacheList: func(ctx context.Context) ([]types.ReverseTunnel, error) { return p.cache.GetReverseTunnels(ctx) }, update: modifyNoContext(p.presenceS.UpsertReverseTunnel), deleteAll: func(ctx context.Context) error { return p.presenceS.DeleteAllReverseTunnels() }, }) } // TestTunnelConnections tests tunnel connections caching func TestTunnelConnections(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) clusterName := "example.com" testResources(t, p, testFuncs[types.TunnelConnection]{ newResource: func(name string) (types.TunnelConnection, error) { return types.NewTunnelConnection(name, types.TunnelConnectionSpecV2{ ClusterName: clusterName, ProxyName: "p1", LastHeartbeat: time.Now().UTC(), }) }, create: modifyNoContext(p.presenceS.UpsertTunnelConnection), list: func(ctx context.Context) ([]types.TunnelConnection, error) { return p.presenceS.GetTunnelConnections(clusterName) }, cacheList: func(ctx context.Context) ([]types.TunnelConnection, error) { return p.cache.GetTunnelConnections(clusterName) }, update: modifyNoContext(p.presenceS.UpsertTunnelConnection), deleteAll: func(ctx context.Context) error { return p.presenceS.DeleteAllTunnelConnections() }, }) } // TestNodes tests nodes cache func TestNodes(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Server]{ newResource: func(name string) (types.Server, error) { return suite.NewServer(types.KindNode, name, "127.0.0.1:2022", apidefaults.Namespace), nil }, create: withKeepalive(p.presenceS.UpsertNode), list: func(ctx context.Context) ([]types.Server, error) { return p.presenceS.GetNodes(ctx, apidefaults.Namespace) }, cacheGet: func(ctx context.Context, name string) (types.Server, error) { return p.cache.GetNode(ctx, apidefaults.Namespace, name) }, cacheList: func(ctx context.Context) ([]types.Server, error) { return p.cache.GetNodes(ctx, apidefaults.Namespace) }, update: withKeepalive(p.presenceS.UpsertNode), deleteAll: func(ctx context.Context) error { return p.presenceS.DeleteAllNodes(ctx, apidefaults.Namespace) }, }) } // TestProxies tests proxies cache func TestProxies(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Server]{ newResource: func(name string) (types.Server, error) { return suite.NewServer(types.KindProxy, name, "127.0.0.1:2022", apidefaults.Namespace), nil }, create: modifyNoContext(p.presenceS.UpsertProxy), list: func(_ context.Context) ([]types.Server, error) { return p.presenceS.GetProxies() }, cacheList: func(_ context.Context) ([]types.Server, error) { return p.cache.GetProxies() }, update: modifyNoContext(p.presenceS.UpsertProxy), deleteAll: func(_ context.Context) error { return p.presenceS.DeleteAllProxies() }, }) } // TestAuthServers tests auth servers cache func TestAuthServers(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Server]{ newResource: func(name string) (types.Server, error) { return suite.NewServer(types.KindAuthServer, name, "127.0.0.1:2022", apidefaults.Namespace), nil }, create: modifyNoContext(p.presenceS.UpsertAuthServer), list: func(_ context.Context) ([]types.Server, error) { return p.presenceS.GetAuthServers() }, cacheList: func(_ context.Context) ([]types.Server, error) { return p.cache.GetAuthServers() }, update: modifyNoContext(p.presenceS.UpsertAuthServer), deleteAll: func(_ context.Context) error { return p.presenceS.DeleteAllAuthServers() }, }) } // TestRemoteClusters tests remote clusters caching func TestRemoteClusters(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.RemoteCluster]{ newResource: func(name string) (types.RemoteCluster, error) { return types.NewRemoteCluster(name) }, create: modifyNoContext(p.presenceS.CreateRemoteCluster), list: func(ctx context.Context) ([]types.RemoteCluster, error) { return p.presenceS.GetRemoteClusters() }, cacheGet: func(ctx context.Context, name string) (types.RemoteCluster, error) { return p.cache.GetRemoteCluster(name) }, cacheList: func(_ context.Context) ([]types.RemoteCluster, error) { return p.cache.GetRemoteClusters() }, update: p.presenceS.UpdateRemoteCluster, deleteAll: func(_ context.Context) error { return p.presenceS.DeleteAllRemoteClusters() }, }) } // TestKubernetes tests that CRUD operations on kubernetes clusters resources are // replicated from the backend to the cache. func TestKubernetes(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.KubeCluster]{ newResource: func(name string) (types.KubeCluster, error) { return types.NewKubernetesClusterV3(types.Metadata{ Name: name, }, types.KubernetesClusterSpecV3{}) }, create: p.kubernetes.CreateKubernetesCluster, list: p.kubernetes.GetKubernetesClusters, cacheGet: p.cache.GetKubernetesCluster, cacheList: p.cache.GetKubernetesClusters, update: p.kubernetes.UpdateKubernetesCluster, deleteAll: p.kubernetes.DeleteAllKubernetesClusters, }) } // TestApplicationServers tests that CRUD operations on app servers are // replicated from the backend to the cache. func TestApplicationServers(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.AppServer]{ newResource: func(name string) (types.AppServer, error) { app, err := types.NewAppV3(types.Metadata{Name: name}, types.AppSpecV3{URI: "localhost"}) require.NoError(t, err) return types.NewAppServerV3FromApp(app, "host", uuid.New().String()) }, create: withKeepalive(p.presenceS.UpsertApplicationServer), list: func(ctx context.Context) ([]types.AppServer, error) { return p.presenceS.GetApplicationServers(ctx, apidefaults.Namespace) }, cacheList: func(ctx context.Context) ([]types.AppServer, error) { return p.cache.GetApplicationServers(ctx, apidefaults.Namespace) }, update: withKeepalive(p.presenceS.UpsertApplicationServer), deleteAll: func(ctx context.Context) error { return p.presenceS.DeleteAllApplicationServers(ctx, apidefaults.Namespace) }, }) } // TestApps tests that CRUD operations on application resources are // replicated from the backend to the cache. func TestApps(t *testing.T) { t.Parallel() p, err := newPack(t.TempDir(), ForProxy) require.NoError(t, err) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Application]{ newResource: func(name string) (types.Application, error) { return types.NewAppV3(types.Metadata{ Name: "foo", }, types.AppSpecV3{ URI: "localhost", }) }, create: p.apps.CreateApp, list: p.apps.GetApps, cacheGet: p.cache.GetApp, cacheList: p.cache.GetApps, update: p.apps.UpdateApp, deleteAll: p.apps.DeleteAllApps, }) } // TestDatabaseServers tests that CRUD operations on database servers are // replicated from the backend to the cache. func TestDatabaseServers(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.DatabaseServer]{ newResource: func(name string) (types.DatabaseServer, error) { return types.NewDatabaseServerV3(types.Metadata{ Name: name, }, types.DatabaseServerSpecV3{ Protocol: defaults.ProtocolPostgres, URI: "localhost:5432", Hostname: "localhost", HostID: uuid.New().String(), }) }, create: withKeepalive(p.presenceS.UpsertDatabaseServer), list: func(ctx context.Context) ([]types.DatabaseServer, error) { return p.presenceS.GetDatabaseServers(ctx, apidefaults.Namespace) }, cacheList: func(ctx context.Context) ([]types.DatabaseServer, error) { return p.cache.GetDatabaseServers(ctx, apidefaults.Namespace) }, update: withKeepalive(p.presenceS.UpsertDatabaseServer), deleteAll: func(ctx context.Context) error { return p.presenceS.DeleteAllDatabaseServers(ctx, apidefaults.Namespace) }, }) } // TestDatabaseServices tests that CRUD operations on DatabaseServices are // replicated from the backend to the cache. func TestDatabaseServices(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.DatabaseService]{ newResource: func(name string) (types.DatabaseService, error) { return types.NewDatabaseServiceV1(types.Metadata{ Name: uuid.NewString(), }, types.DatabaseServiceSpecV1{ ResourceMatchers: []*types.DatabaseResourceMatcher{ {Labels: &types.Labels{"env": []string{"prod"}}}, }, }) }, create: withKeepalive(p.databaseServices.UpsertDatabaseService), list: func(ctx context.Context) ([]types.DatabaseService, error) { listServicesResp, err := p.presenceS.ListResources(ctx, proto.ListResourcesRequest{ ResourceType: types.KindDatabaseService, Limit: apidefaults.DefaultChunkSize, }) require.NoError(t, err) return types.ResourcesWithLabels(listServicesResp.Resources).AsDatabaseServices() }, cacheList: func(ctx context.Context) ([]types.DatabaseService, error) { listServicesResp, err := p.cache.ListResources(ctx, proto.ListResourcesRequest{ ResourceType: types.KindDatabaseService, Limit: apidefaults.DefaultChunkSize, }) require.NoError(t, err) return types.ResourcesWithLabels(listServicesResp.Resources).AsDatabaseServices() }, update: withKeepalive(p.databaseServices.UpsertDatabaseService), deleteAll: p.databaseServices.DeleteAllDatabaseServices, }) } // TestDatabases tests that CRUD operations on database resources are // replicated from the backend to the cache. func TestDatabases(t *testing.T) { t.Parallel() p := newTestPack(t, ForProxy) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Database]{ newResource: func(name string) (types.Database, error) { return types.NewDatabaseV3(types.Metadata{ Name: name, }, types.DatabaseSpecV3{ Protocol: defaults.ProtocolPostgres, URI: "localhost:5432", }) }, create: p.databases.CreateDatabase, list: p.databases.GetDatabases, cacheGet: p.cache.GetDatabase, cacheList: p.cache.GetDatabases, update: p.databases.UpdateDatabase, deleteAll: p.databases.DeleteAllDatabases, }) } // TestSAMLIdPServiceProviders tests that CRUD operations on SAML IdP service provider resources are // replicated from the backend to the cache. func TestSAMLIdPServiceProviders(t *testing.T) { t.Parallel() p := newTestPack(t, ForAuth) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.SAMLIdPServiceProvider]{ newResource: func(name string) (types.SAMLIdPServiceProvider, error) { return types.NewSAMLIdPServiceProvider( types.Metadata{ Name: name, }, types.SAMLIdPServiceProviderSpecV1{ EntityDescriptor: testEntityDescriptor, EntityID: "IAMShowcase", }) }, create: p.samlIDPServiceProviders.CreateSAMLIdPServiceProvider, list: func(ctx context.Context) ([]types.SAMLIdPServiceProvider, error) { results, _, err := p.samlIDPServiceProviders.ListSAMLIdPServiceProviders(ctx, 0, "") return results, err }, cacheGet: p.cache.GetSAMLIdPServiceProvider, cacheList: func(ctx context.Context) ([]types.SAMLIdPServiceProvider, error) { results, _, err := p.cache.ListSAMLIdPServiceProviders(ctx, 0, "") return results, err }, update: p.samlIDPServiceProviders.UpdateSAMLIdPServiceProvider, deleteAll: p.samlIDPServiceProviders.DeleteAllSAMLIdPServiceProviders, }) } // TestUserGroups tests that CRUD operations on user group resources are // replicated from the backend to the cache. func TestUserGroups(t *testing.T) { t.Parallel() p := newTestPack(t, ForAuth) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.UserGroup]{ newResource: func(name string) (types.UserGroup, error) { return types.NewUserGroup( types.Metadata{ Name: name, }, ) }, create: p.userGroups.CreateUserGroup, list: func(ctx context.Context) ([]types.UserGroup, error) { results, _, err := p.userGroups.ListUserGroups(ctx, 0, "") return results, err }, cacheGet: p.cache.GetUserGroup, cacheList: func(ctx context.Context) ([]types.UserGroup, error) { results, _, err := p.cache.ListUserGroups(ctx, 0, "") return results, err }, update: p.userGroups.UpdateUserGroup, deleteAll: p.userGroups.DeleteAllUserGroups, }) } // TestLocks tests that CRUD operations on lock resources are // replicated from the backend to the cache. func TestLocks(t *testing.T) { t.Parallel() p := newTestPack(t, ForAuth) t.Cleanup(p.Close) testResources(t, p, testFuncs[types.Lock]{ newResource: func(name string) (types.Lock, error) { return types.NewLock( name, types.LockSpecV2{ Target: types.LockTarget{ Role: "target-role", }, }, ) }, create: p.accessS.UpsertLock, list: func(ctx context.Context) ([]types.Lock, error) { results, err := p.accessS.GetLocks(ctx, false) return results, err }, cacheGet: p.cache.GetLock, cacheList: func(ctx context.Context) ([]types.Lock, error) { results, err := p.cache.GetLocks(ctx, false) return results, err }, update: p.accessS.UpsertLock, deleteAll: p.accessS.DeleteAllLocks, }) } // testResources is a generic tester for resources. func testResources[T types.Resource](t *testing.T, p *testPack, funcs testFuncs[T]) { ctx := context.Background() // Create a resource. r, err := funcs.newResource("test-sp") r.SetExpiry(time.Now().Add(30 * time.Minute)) require.NoError(t, err) err = funcs.create(ctx, r) require.NoError(t, err) // Check that the resource is now in the backend. out, err := funcs.list(ctx) require.NoError(t, err) require.Empty(t, cmp.Diff([]T{r}, out, cmpopts.IgnoreFields(types.Metadata{}, "ID"))) // Wait until the information has been replicated to the cache. select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatal("timeout waiting for event") } // Make sure the cache has a single resource in it. out, err = funcs.cacheList(ctx) require.NoError(t, err) require.Empty(t, cmp.Diff([]T{r}, out, cmpopts.IgnoreFields(types.Metadata{}, "ID"))) // cacheGet is optional as not every resource implements it if funcs.cacheGet != nil { // Make sure a single cache get works. getR, err := funcs.cacheGet(ctx, r.GetName()) require.NoError(t, err) require.Empty(t, cmp.Diff(r, getR, cmpopts.IgnoreFields(types.Metadata{}, "ID"))) } // Update the resource and upsert it into the backend again. r.SetExpiry(r.Expiry().Add(30 * time.Minute)) err = funcs.update(ctx, r) require.NoError(t, err) // Check that the resource is in the backend and only one exists (so an // update occurred). out, err = funcs.list(ctx) require.NoError(t, err) require.Empty(t, cmp.Diff([]T{r}, out, cmpopts.IgnoreFields(types.Metadata{}, "ID"))) // Check that information has been replicated to the cache. select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatal("timeout waiting for event") } // Make sure the cache has a single resource in it. out, err = funcs.cacheList(ctx) require.NoError(t, err) require.Empty(t, cmp.Diff([]T{r}, out, cmpopts.IgnoreFields(types.Metadata{}, "ID"))) // Remove all service providers from the backend. err = funcs.deleteAll(ctx) require.NoError(t, err) // Check that information has been replicated to the cache. select { case event := <-p.eventsC: require.Equal(t, EventProcessed, event.Type) case <-time.After(time.Second): t.Fatal("timeout waiting for event") } // Check that the cache is now empty. out, err = funcs.cacheList(ctx) require.NoError(t, err) require.Equal(t, 0, len(out)) } func TestRelativeExpiry(t *testing.T) { t.Parallel() const checkInterval = time.Second const nodeCount = int64(100) // make sure the event buffer is much larger than node count // so that we can batch create nodes without waiting on each event require.True(t, int(nodeCount*3) < eventBufferSize) ctx := context.Background() clock := clockwork.NewFakeClockAt(time.Now().Add(time.Hour)) p := newTestPack(t, func(c Config) Config { c.RelativeExpiryCheckInterval = checkInterval c.Clock = clock return ForAuth(c) }) t.Cleanup(p.Close) // add servers that expire at a range of times now := clock.Now() for i := int64(0); i < nodeCount; i++ { exp := now.Add(time.Minute * time.Duration(i)) server := suite.NewServer(types.KindNode, uuid.New().String(), "127.0.0.1:2022", apidefaults.Namespace) server.SetExpiry(exp) _, err := p.presenceS.UpsertNode(ctx, server) require.NoError(t, err) } // wait for nodes to reach cache (we batch insert first for performance reasons) for i := int64(0); i < nodeCount; i++ { expectEvent(t, p.eventsC, EventProcessed) } nodes, err := p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.Len(t, nodes, 100) clock.Advance(time.Minute * 25) // get rid of events that were emitted before clock advanced drainEvents(p.eventsC) // wait for next relative expiry check to run expectEvent(t, p.eventsC, RelativeExpiry) // verify that roughly expected proportion of nodes was removed. nodes, err = p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.True(t, len(nodes) < 100 && len(nodes) > 75, "node_count=%d", len(nodes)) clock.Advance(time.Minute * 25) // get rid of events that were emitted before clock advanced drainEvents(p.eventsC) // wait for next relative expiry check to run expectEvent(t, p.eventsC, RelativeExpiry) // verify that roughly expected proportion of nodes was removed. nodes, err = p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.True(t, len(nodes) < 75 && len(nodes) > 50, "node_count=%d", len(nodes)) // finally, we check the "sliding window" by verifying that we don't remove all nodes // even if we advance well past the latest expiry time. clock.Advance(time.Hour * 24) // get rid of events that were emitted before clock advanced drainEvents(p.eventsC) // wait for next relative expiry check to run expectEvent(t, p.eventsC, RelativeExpiry) // verify that sliding window has preserved most recent nodes nodes, err = p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.True(t, len(nodes) > 0, "node_count=%d", len(nodes)) } func TestRelativeExpiryLimit(t *testing.T) { const ( checkInterval = time.Second nodeCount = 100 expiryLimit = 10 ) // make sure the event buffer is much larger than node count // so that we can batch create nodes without waiting on each event require.True(t, int(nodeCount*3) < eventBufferSize) ctx := context.Background() clock := clockwork.NewFakeClockAt(time.Now().Add(time.Hour)) p := newTestPack(t, func(c Config) Config { c.RelativeExpiryCheckInterval = checkInterval c.RelativeExpiryLimit = expiryLimit c.Clock = clock return ForProxy(c) }) t.Cleanup(p.Close) // add servers that expire at a range of times now := clock.Now() for i := 0; i < nodeCount; i++ { exp := now.Add(time.Minute * time.Duration(i)) server := suite.NewServer(types.KindNode, uuid.New().String(), "127.0.0.1:2022", apidefaults.Namespace) server.SetExpiry(exp) _, err := p.presenceS.UpsertNode(ctx, server) require.NoError(t, err) } // wait for nodes to reach cache (we batch insert first for performance reasons) for i := 0; i < nodeCount; i++ { expectEvent(t, p.eventsC, EventProcessed) } nodes, err := p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.Len(t, nodes, nodeCount) clock.Advance(time.Hour * 24) for expired := nodeCount - expiryLimit; expired > 0; expired -= expiryLimit { // get rid of events that were emitted before clock advanced drainEvents(p.eventsC) // wait for next relative expiry check to run expectEvent(t, p.eventsC, RelativeExpiry) // verify that the limit is respected. nodes, err = p.cache.GetNodes(ctx, apidefaults.Namespace) require.NoError(t, err) require.Len(t, nodes, expired) // advance clock to trigger next relative expiry check clock.Advance(time.Hour * 24) } } func TestRelativeExpiryOnlyForNodeWatches(t *testing.T) { clock := clockwork.NewFakeClockAt(time.Now().Add(time.Hour)) p := newTestPack(t, func(c Config) Config { c.RelativeExpiryCheckInterval = time.Second c.Clock = clock c.Watches = []types.WatchKind{{Kind: types.KindNode}} return c }) t.Cleanup(p.Close) p2 := newTestPack(t, func(c Config) Config { c.RelativeExpiryCheckInterval = time.Second c.Clock = clock c.Watches = []types.WatchKind{ {Kind: types.KindNamespace}, {Kind: types.KindNamespace}, {Kind: types.KindCertAuthority}, } return c }) t.Cleanup(p2.Close) for i := 0; i < 2; i++ { clock.Advance(time.Hour * 24) drainEvents(p.eventsC) expectEvent(t, p.eventsC, RelativeExpiry) drainEvents(p2.eventsC) unexpectedEvent(t, p2.eventsC, RelativeExpiry) } } func TestCache_Backoff(t *testing.T) { t.Parallel() clock := clockwork.NewFakeClock() p := newTestPack(t, func(c Config) Config { c.MaxRetryPeriod = defaults.MaxWatcherBackoff c.Clock = clock return ForNode(c) }) t.Cleanup(p.Close) // close watchers to trigger a reload event watchers := p.eventsS.getWatchers() require.Len(t, watchers, 1) p.eventsS.closeWatchers() p.backend.SetReadError(trace.ConnectionProblem(nil, "backend is unavailable")) step := p.cache.Config.MaxRetryPeriod / 5.0 for i := 0; i < 5; i++ { // wait for cache to reload select { case event := <-p.eventsC: require.Equal(t, Reloading, event.Type) duration, err := time.ParseDuration(event.Event.Resource.GetKind()) require.NoError(t, err) stepMin := step * time.Duration(i) / 2 stepMax := step * time.Duration(i+1) require.GreaterOrEqual(t, duration, stepMin) require.LessOrEqual(t, duration, stepMax) // wait for cache to get to retry.After clock.BlockUntil(1) // add some extra to the duration to ensure the retry occurs clock.Advance(p.cache.MaxRetryPeriod) case <-time.After(time.Minute): t.Fatalf("timeout waiting for event") } // wait for cache to fail again - backend will still produce a ConnectionProblem error select { case event := <-p.eventsC: require.Equal(t, WatcherFailed, event.Type) case <-time.After(30 * time.Second): t.Fatalf("timeout waiting for event") } } } type proxyEvents struct { sync.Mutex watchers []types.Watcher events types.Events } func (p *proxyEvents) getWatchers() []types.Watcher { p.Lock() defer p.Unlock() out := make([]types.Watcher, len(p.watchers)) copy(out, p.watchers) return out } func (p *proxyEvents) closeWatchers() { p.Lock() defer p.Unlock() for i := range p.watchers { p.watchers[i].Close() } p.watchers = nil } func (p *proxyEvents) NewWatcher(ctx context.Context, watch types.Watch) (types.Watcher, error) { w, err := p.events.NewWatcher(ctx, watch) if err != nil { return nil, trace.Wrap(err) } p.Lock() defer p.Unlock() p.watchers = append(p.watchers, w) return w, nil } // TestCacheWatchKindExistsInEvents ensures that the watch kinds for each cache component are in sync // with proto Events delivered via WatchEvents. If a watch kind is added to a cache component, but it // doesn't exist in the proto Events definition, an error will cause the WatchEvents stream to be closed. // This causes the cache to reinitialize every time that an unknown message is received and can lead to // a permanently unhealthy cache. // // While this test will ensure that there are no issues for the current release, it does not guarantee // that this issue won't arise across releases. func TestCacheWatchKindExistsInEvents(t *testing.T) { t.Parallel() cases := map[string]Config{ "ForAuth": ForAuth(Config{}), "ForProxy": ForProxy(Config{}), "ForRemoteProxy": ForRemoteProxy(Config{}), "ForOldRemoteProxy": ForOldRemoteProxy(Config{}), "ForNode": ForNode(Config{}), "ForKubernetes": ForKubernetes(Config{}), "ForApps": ForApps(Config{}), "ForDatabases": ForDatabases(Config{}), } events := map[string]types.Resource{ types.KindCertAuthority: &types.CertAuthorityV2{}, types.KindClusterName: &types.ClusterNameV2{}, types.KindClusterAuditConfig: types.DefaultClusterAuditConfig(), types.KindClusterNetworkingConfig: types.DefaultClusterNetworkingConfig(), types.KindClusterAuthPreference: types.DefaultAuthPreference(), types.KindSessionRecordingConfig: types.DefaultSessionRecordingConfig(), types.KindUIConfig: &types.UIConfigV1{}, types.KindStaticTokens: &types.StaticTokensV2{}, types.KindToken: &types.ProvisionTokenV2{}, types.KindUser: &types.UserV2{}, types.KindRole: &types.RoleV6{Version: types.V4}, types.KindNamespace: &types.Namespace{}, types.KindNode: &types.ServerV2{}, types.KindProxy: &types.ServerV2{}, types.KindAuthServer: &types.ServerV2{}, types.KindReverseTunnel: &types.ReverseTunnelV2{}, types.KindTunnelConnection: &types.TunnelConnectionV2{}, types.KindAccessRequest: &types.AccessRequestV3{}, types.KindAppServer: &types.AppServerV3{}, types.KindApp: &types.AppV3{}, types.KindWebSession: &types.WebSessionV2{SubKind: types.KindWebSession}, types.KindAppSession: &types.WebSessionV2{SubKind: types.KindAppSession}, types.KindSnowflakeSession: &types.WebSessionV2{SubKind: types.KindSnowflakeSession}, types.KindSAMLIdPSession: &types.WebSessionV2{SubKind: types.KindSAMLIdPServiceProvider}, types.KindWebToken: &types.WebTokenV3{}, types.KindRemoteCluster: &types.RemoteClusterV3{}, types.KindKubeServer: &types.KubernetesServerV3{}, types.KindDatabaseService: &types.DatabaseServiceV1{}, types.KindDatabaseServer: &types.DatabaseServerV3{}, types.KindDatabase: &types.DatabaseV3{}, types.KindNetworkRestrictions: &types.NetworkRestrictionsV4{}, types.KindLock: &types.LockV2{}, types.KindWindowsDesktopService: &types.WindowsDesktopServiceV3{}, types.KindWindowsDesktop: &types.WindowsDesktopV3{}, types.KindInstaller: &types.InstallerV1{}, types.KindKubernetesCluster: &types.KubernetesClusterV3{}, types.KindSAMLIdPServiceProvider: &types.SAMLIdPServiceProviderV1{}, types.KindUserGroup: &types.UserGroupV1{}, } for name, cfg := range cases { t.Run(name, func(t *testing.T) { for _, watch := range cfg.Watches { resource, ok := events[watch.Kind] require.True(t, ok, "missing event for kind %q", watch.Kind) protoEvent, err := client.EventToGRPC(types.Event{ Type: types.OpPut, Resource: resource, }) require.NoError(t, err) event, err := client.EventFromGRPC(*protoEvent) require.NoError(t, err) require.Empty(t, cmp.Diff(resource, event.Resource)) } }) } } func withKeepalive[T any](fn func(context.Context, T) (*types.KeepAlive, error)) func(context.Context, T) error { return func(ctx context.Context, resource T) error { _, err := fn(ctx, resource) return err } } func modifyNoContext[T any](fn func(T) error) func(context.Context, T) error { return func(_ context.Context, resource T) error { return fn(resource) } } // A test entity descriptor from https://sptest.iamshowcase.com/testsp_metadata.xml. const testEntityDescriptor = ` urn:oasis:names:tc:SAML:1.1:nameid-format:unspecified urn:oasis:names:tc:SAML:1.1:nameid-format:emailAddress `