From 948417257f755375983fe4d42f0103061376c460 Mon Sep 17 00:00:00 2001 From: Trent Clarke Date: Fri, 9 Sep 2022 00:27:51 +1000 Subject: [PATCH] Split out appaccess and proxy integration tests (#16232) * Proxy tests running * rollback * whitespace fix * Rollback port fix * Linter appeasement * License fix * Update signals.go --- integration/app_integration_test.go | 1796 ----------------- integration/appaccess/appaccess_test.go | 679 +++++++ integration/appaccess/fixtures.go | 395 ++++ integration/appaccess/jwt.go | 55 + integration/appaccess/pack.go | 804 ++++++++ integration/client_test.go | 2 +- integration/helpers/helpers.go | 6 + integration/helpers/instance.go | 3 +- integration/helpers/ports.go | 2 + integration/helpers/trustedclusters.go | 142 ++ integration/integration_test.go | 244 +-- integration/kube/fixtures.go | 134 ++ integration/kube_integration_test.go | 278 +-- integration/proxy/main_test.go | 28 + .../proxy_helpers.go} | 118 +- integration/{ => proxy}/proxy_test.go | 119 +- .../{ => proxy}/proxy_tunnel_strategy_test.go | 33 +- 17 files changed, 2525 insertions(+), 2313 deletions(-) delete mode 100644 integration/app_integration_test.go create mode 100644 integration/appaccess/appaccess_test.go create mode 100644 integration/appaccess/fixtures.go create mode 100644 integration/appaccess/jwt.go create mode 100644 integration/appaccess/pack.go create mode 100644 integration/helpers/trustedclusters.go create mode 100644 integration/kube/fixtures.go create mode 100644 integration/proxy/main_test.go rename integration/{proxy_helpers_test.go => proxy/proxy_helpers.go} (80%) rename integration/{ => proxy}/proxy_test.go (92%) rename integration/{ => proxy}/proxy_tunnel_strategy_test.go (93%) diff --git a/integration/app_integration_test.go b/integration/app_integration_test.go deleted file mode 100644 index c5d0b07aa8c..00000000000 --- a/integration/app_integration_test.go +++ /dev/null @@ -1,1796 +0,0 @@ -/* -Copyright 2020-2022 Gravitational, Inc. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package integration - -import ( - "bufio" - "bytes" - "context" - "crypto/tls" - "crypto/x509" - "encoding/json" - "errors" - "fmt" - "io" - "net" - "net/http" - "net/http/httptest" - "net/http/httputil" - "net/url" - "strings" - "testing" - "time" - - "github.com/gravitational/teleport" - "github.com/gravitational/teleport/api/breaker" - apidefaults "github.com/gravitational/teleport/api/defaults" - "github.com/gravitational/teleport/api/types" - apievents "github.com/gravitational/teleport/api/types/events" - "github.com/gravitational/teleport/integration/helpers" - "github.com/gravitational/teleport/lib" - "github.com/gravitational/teleport/lib/auth" - "github.com/gravitational/teleport/lib/auth/native" - "github.com/gravitational/teleport/lib/auth/testauthority" - "github.com/gravitational/teleport/lib/client" - "github.com/gravitational/teleport/lib/defaults" - "github.com/gravitational/teleport/lib/events" - "github.com/gravitational/teleport/lib/httplib/csrf" - "github.com/gravitational/teleport/lib/jwt" - "github.com/gravitational/teleport/lib/service" - "github.com/gravitational/teleport/lib/services" - "github.com/gravitational/teleport/lib/srv/alpnproxy" - alpncommon "github.com/gravitational/teleport/lib/srv/alpnproxy/common" - "github.com/gravitational/teleport/lib/utils" - "github.com/gravitational/teleport/lib/web" - "github.com/gravitational/teleport/lib/web/app" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "github.com/google/uuid" - "github.com/gorilla/websocket" - "github.com/gravitational/oxy/forward" - "github.com/gravitational/trace" - "github.com/stretchr/testify/require" -) - -// TestAppAccess runs the full application access integration test suite. -// -// It allows to make the entire cluster set up once, instead of per test, -// which speeds things up significantly. -func TestAppAccess(t *testing.T) { - pack := setup(t) - - t.Run("TestAppAccessForward", pack.appAccessForward) - t.Run("TestAppAccessWebsockets", pack.appAccessWebsockets) - t.Run("TestAppAccessClientCert", pack.appAccessClientCert) - t.Run("TestAppAccessFlush", pack.appAccessFlush) - t.Run("TestAppAccessForwardModes", pack.appAccessForwardModes) - t.Run("TestAppAccessRewriteHeadersRoot", pack.appAccessRewriteHeadersRoot) - t.Run("TestAppAccessRewriteHeadersLeaf", pack.appAccessRewriteHeadersLeaf) - t.Run("TestAppAccessLogout", pack.appAccessLogout) - t.Run("TestAppAccessJWT", pack.appAccessJWT) - t.Run("TestAppAccessNoHeaderOverrides", pack.appAccessNoHeaderOverrides) - t.Run("TestAppAuditEvents", pack.appAuditEvents) - t.Run("TestAppInvalidateAppSessionsOnLogout", pack.appInvalidateAppSessionsOnLogout) - t.Run("TestAppAccessTCP", pack.appAccessTCP) - - // This test should go last because it stops/starts app servers. - t.Run("TestAppServersHA", pack.appServersHA) -} - -// appAccessForward tests that requests get forwarded to the target application -// within a single cluster and trusted cluster. -func (p *pack) appAccessForward(t *testing.T) { - tests := []struct { - desc string - inCookie string - outStatusCode int - outMessage string - }{ - { - desc: "root cluster, valid application session cookie, success", - inCookie: p.createAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName), - outStatusCode: http.StatusOK, - outMessage: p.rootMessage, - }, - { - desc: "leaf cluster, valid application session cookie, success", - inCookie: p.createAppSession(t, p.leafAppPublicAddr, p.leafAppClusterName), - outStatusCode: http.StatusOK, - outMessage: p.leafMessage, - }, - { - desc: "invalid application session cookie, redirect to login", - inCookie: "D25C463CD27861559CC6A0A6AE54818079809AA8731CB18037B4B37A80C4FC6C", - outStatusCode: http.StatusFound, - outMessage: "", - }, - } - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - tt := tt - status, body, err := p.makeRequest(tt.inCookie, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, tt.outStatusCode, status) - require.Contains(t, body, tt.outMessage) - }) - } -} - -// appAccessWebsockets makes sure that websocket requests get forwarded. -func (p *pack) appAccessWebsockets(t *testing.T) { - tests := []struct { - desc string - inCookie string - outMessage string - err error - }{ - { - desc: "root cluster, valid application session cookie, successful websocket (ws://) request", - inCookie: p.createAppSession(t, p.rootWSPublicAddr, p.rootAppClusterName), - outMessage: p.rootWSMessage, - }, - { - desc: "root cluster, valid application session cookie, successful secure websocket (wss://) request", - inCookie: p.createAppSession(t, p.rootWSSPublicAddr, p.rootAppClusterName), - outMessage: p.rootWSSMessage, - }, - { - desc: "leaf cluster, valid application session cookie, successful websocket (ws://) request", - inCookie: p.createAppSession(t, p.leafWSPublicAddr, p.leafAppClusterName), - outMessage: p.leafWSMessage, - }, - { - desc: "leaf cluster, valid application session cookie, successful secure websocket (wss://) request", - inCookie: p.createAppSession(t, p.leafWSSPublicAddr, p.leafAppClusterName), - outMessage: p.leafWSSMessage, - }, - { - desc: "invalid application session cookie, websocket request fails to dial", - inCookie: "ABCDEFGHIJKLMNOPQRSTUVWXYZ", - err: errors.New(""), - }, - } - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - tt := tt - body, err := p.makeWebsocketRequest(tt.inCookie, "/") - if tt.err != nil { - require.IsType(t, tt.err, trace.Unwrap(err)) - } else { - require.NoError(t, err) - require.Equal(t, tt.outMessage, body) - } - }) - } -} - -// appAccessTCP tests proxying of plain TCP applications through app access. -func (p *pack) appAccessTCP(t *testing.T) { - pack := setup(t) - - tests := []struct { - description string - address string - outMessage string - }{ - { - description: "TCP app in root cluster", - address: pack.startLocalProxy(t, pack.rootTCPPublicAddr, pack.rootAppClusterName), - outMessage: pack.rootTCPMessage, - }, - { - description: "TCP app in leaf cluster", - address: pack.startLocalProxy(t, pack.leafTCPPublicAddr, pack.leafAppClusterName), - outMessage: pack.leafTCPMessage, - }, - } - - for _, test := range tests { - t.Run(test.description, func(t *testing.T) { - conn, err := net.Dial("tcp", test.address) - require.NoError(t, err) - - buf := make([]byte, 1024) - n, err := conn.Read(buf) - require.NoError(t, err) - - resp := strings.TrimSpace(string(buf[:n])) - require.Equal(t, test.outMessage, resp) - }) - } -} - -// appAccessClientCert tests mutual TLS authentication flow with application -// access typically used in CLI by curl and other clients. -func (p *pack) appAccessClientCert(t *testing.T) { - tests := []struct { - desc string - inTLSConfig *tls.Config - outStatusCode int - outMessage string - }{ - { - desc: "root cluster, valid TLS config, success", - inTLSConfig: p.makeTLSConfig(t, p.rootAppPublicAddr, p.rootAppClusterName), - outStatusCode: http.StatusOK, - outMessage: p.rootMessage, - }, - { - desc: "leaf cluster, valid TLS config, success", - inTLSConfig: p.makeTLSConfig(t, p.leafAppPublicAddr, p.leafAppClusterName), - outStatusCode: http.StatusOK, - outMessage: p.leafMessage, - }, - { - desc: "root cluster, invalid session ID", - inTLSConfig: p.makeTLSConfigNoSession(t, p.rootAppPublicAddr, p.rootAppClusterName), - outStatusCode: http.StatusFound, - }, - } - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - tt := tt - status, body, err := p.makeRequestWithClientCert(tt.inTLSConfig, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, tt.outStatusCode, status) - require.Contains(t, body, tt.outMessage) - }) - } -} - -// appAccessFlush makes sure that application access periodically flushes -// buffered data to the response. -func (p *pack) appAccessFlush(t *testing.T) { - req, err := http.NewRequest("GET", p.assembleRootProxyURL("/"), nil) - require.NoError(t, err) - - cookie := p.createAppSession(t, p.flushAppPublicAddr, p.flushAppClusterName) - req.AddCookie(&http.Cookie{ - Name: app.CookieName, - Value: cookie, - }) - - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - resp, err := client.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - - // The "flush server" will send 2 messages, "hello" and "world", with a - // 500ms delay between them. They should arrive as 2 different frames - // due to the periodic flushing. - frames := []string{"hello", "world"} - for _, frame := range frames { - buffer := make([]byte, 1024) - n, err := resp.Body.Read(buffer) - if err != nil { - require.ErrorIs(t, err, io.EOF) - } - require.Equal(t, frame, strings.TrimSpace(string(buffer[:n]))) - } -} - -// appAccessForwardModes ensures that requests are forwarded to applications -// even when the cluster is in proxy recording mode. -func (p *pack) appAccessForwardModes(t *testing.T) { - // Create cluster, user, sessions, and credentials package. - ctx := context.Background() - - // Update root and leaf clusters to record sessions at the proxy. - recConfig, err := types.NewSessionRecordingConfigFromConfigFile(types.SessionRecordingConfigSpecV2{ - Mode: types.RecordAtProxy, - }) - require.NoError(t, err) - err = p.rootCluster.Process.GetAuthServer().SetSessionRecordingConfig(ctx, recConfig) - require.NoError(t, err) - err = p.leafCluster.Process.GetAuthServer().SetSessionRecordingConfig(ctx, recConfig) - require.NoError(t, err) - - // Requests to root and leaf cluster are successful. - tests := []struct { - desc string - inCookie string - outStatusCode int - outMessage string - }{ - { - desc: "root cluster, valid application session cookie, success", - inCookie: p.createAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName), - outStatusCode: http.StatusOK, - outMessage: p.rootMessage, - }, - { - desc: "leaf cluster, valid application session cookie, success", - inCookie: p.createAppSession(t, p.leafAppPublicAddr, p.leafAppClusterName), - outStatusCode: http.StatusOK, - outMessage: p.leafMessage, - }, - } - for _, tt := range tests { - t.Run(tt.desc, func(t *testing.T) { - tt := tt - status, body, err := p.makeRequest(tt.inCookie, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, tt.outStatusCode, status) - require.Contains(t, body, tt.outMessage) - }) - } -} - -// appAccessLogout verifies the session is removed from the backend when the user logs out. -func (p *pack) appAccessLogout(t *testing.T) { - // Create an application session. - appCookie := p.createAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName) - - // Log user out of session. - status, _, err := p.makeRequest(appCookie, http.MethodGet, "/teleport-logout") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - - // Wait until requests using the session cookie have failed. - status, err = p.waitForLogout(appCookie) - require.NoError(t, err) - require.Equal(t, http.StatusFound, status) -} - -// appAccessJWT ensures a JWT token is attached to requests and the JWT token can -// be validated. -func (p *pack) appAccessJWT(t *testing.T) { - // Create an application session. - appCookie := p.createAppSession(t, p.jwtAppPublicAddr, p.jwtAppClusterName) - - // Get JWT. - status, token, err := p.makeRequest(appCookie, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - - // Verify JWT token. - verifyJWT(t, p, token, p.jwtAppURI) - - // Connect to websocket application that dumps the upgrade request. - wsCookie := p.createAppSession(t, p.wsHeaderAppPublicAddr, p.wsHeaderAppClusterName) - body, err := p.makeWebsocketRequest(wsCookie, "/") - require.NoError(t, err) - - // Parse the upgrade request the websocket application received. - req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(body))) - require.NoError(t, err) - - // Extract JWT token from header and verify it. - wsToken := req.Header.Get(teleport.AppJWTHeader) - require.NotEmpty(t, wsToken, "websocket upgrade request doesn't contain JWT header") - verifyJWT(t, p, wsToken, p.wsHeaderAppURI) -} - -func verifyJWT(t *testing.T, pack *pack, token, appURI string) { - // Get and unmarshal JWKs - status, body, err := pack.makeRequest("", http.MethodGet, "/.well-known/jwks.json") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - var jwks web.JWKSResponse - err = json.Unmarshal([]byte(body), &jwks) - require.NoError(t, err) - require.Len(t, jwks.Keys, 1) - publicKey, err := jwt.UnmarshalJWK(jwks.Keys[0]) - require.NoError(t, err) - - // Verify JWT. - key, err := jwt.New(&jwt.Config{ - PublicKey: publicKey, - Algorithm: defaults.ApplicationTokenAlgorithm, - ClusterName: pack.jwtAppClusterName, - }) - require.NoError(t, err) - claims, err := key.Verify(jwt.VerifyParams{ - Username: pack.username, - RawToken: token, - URI: appURI, - }) - require.NoError(t, err) - require.Equal(t, pack.username, claims.Username) - require.Equal(t, pack.user.GetRoles(), claims.Roles) -} - -// appAccessNoHeaderOverrides ensures that AAP-specific headers cannot be overridden -// by values passed in by the user. -func (p *pack) appAccessNoHeaderOverrides(t *testing.T) { - // Create an application session. - appCookie := p.createAppSession(t, p.headerAppPublicAddr, p.headerAppClusterName) - - // Get HTTP headers forwarded to the application. - status, origHeaderResp, err := p.makeRequest(appCookie, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - origHeaders := strings.Split(origHeaderResp, "\n") - require.Equal(t, len(origHeaders), len(forwardedHeaderNames)+1) - - // Construct HTTP request with custom headers. - req, err := http.NewRequest(http.MethodGet, p.assembleRootProxyURL("/"), nil) - require.NoError(t, err) - req.AddCookie(&http.Cookie{ - Name: app.CookieName, - Value: appCookie, - }) - for _, headerName := range forwardedHeaderNames { - req.Header.Set(headerName, uuid.New().String()) - } - - // Issue the request. - status, newHeaderResp, err := p.sendRequest(req, nil) - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - newHeaders := strings.Split(newHeaderResp, "\n") - require.Equal(t, len(newHeaders), len(forwardedHeaderNames)+1) - - // Headers sent to the application should not be affected. - for i := range forwardedHeaderNames { - require.Equal(t, origHeaders[i], newHeaders[i]) - } -} - -// appAccessRewriteHeadersRoot validates that http headers from application -// rewrite configuration are correctly passed to proxied applications in root. -func (p *pack) appAccessRewriteHeadersRoot(t *testing.T) { - // Create an application session for dumper app in root cluster. - appCookie := p.createAppSession(t, "dumper-root.example.com", "example.com") - - // Get headers response and make sure headers were passed. - status, resp, err := p.makeRequest(appCookie, http.MethodGet, "/", service.Header{ - Name: "X-Existing", Value: "existing", - }) - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - - // Dumper app just dumps HTTP request so we should be able to read it back. - req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(resp))) - require.NoError(t, err) - require.Equal(t, req.Host, "example.com") - require.Equal(t, req.Header.Get("X-Teleport-Cluster"), "root") - require.Equal(t, req.Header.Get("X-External-Env"), "production") - require.Equal(t, req.Header.Get("X-Existing"), "rewritten-existing-header") - require.NotEqual(t, req.Header.Get(teleport.AppJWTHeader), "rewritten-app-jwt-header") - require.NotEqual(t, req.Header.Get(teleport.AppCFHeader), "rewritten-app-cf-header") - require.NotEqual(t, req.Header.Get(forward.XForwardedFor), "rewritten-x-forwarded-for-header") - require.NotEqual(t, req.Header.Get(forward.XForwardedHost), "rewritten-x-forwarded-host-header") - require.NotEqual(t, req.Header.Get(forward.XForwardedProto), "rewritten-x-forwarded-proto-header") - require.NotEqual(t, req.Header.Get(forward.XForwardedServer), "rewritten-x-forwarded-server-header") - - // Verify JWT tokens. - for _, header := range []string{teleport.AppJWTHeader, teleport.AppCFHeader, "X-JWT"} { - verifyJWT(t, p, req.Header.Get(header), p.dumperAppURI) - } -} - -// appAccessRewriteHeadersLeaf validates that http headers from application -// rewrite configuration are correctly passed to proxied applications in leaf. -func (p *pack) appAccessRewriteHeadersLeaf(t *testing.T) { - // Create an application session for dumper app in leaf cluster. - appCookie := p.createAppSession(t, "dumper-leaf.example.com", "leaf.example.com") - - // Get headers response and make sure headers were passed. - status, resp, err := p.makeRequest(appCookie, http.MethodGet, "/", service.Header{ - Name: "X-Existing", Value: "existing", - }) - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - require.Contains(t, resp, "X-Teleport-Cluster: leaf") - require.Contains(t, resp, "X-Teleport-Login: root") - require.Contains(t, resp, "X-Teleport-Login: ubuntu") - require.Contains(t, resp, "X-External-Env: production") - require.Contains(t, resp, "Host: example.com") - require.Contains(t, resp, "X-Existing: rewritten-existing-header") - require.NotContains(t, resp, "X-Existing: existing") - require.NotContains(t, resp, "rewritten-app-jwt-header") - require.NotContains(t, resp, "rewritten-app-cf-header") - require.NotContains(t, resp, "rewritten-x-forwarded-for-header") - require.NotContains(t, resp, "rewritten-x-forwarded-host-header") - require.NotContains(t, resp, "rewritten-x-forwarded-proto-header") - require.NotContains(t, resp, "rewritten-x-forwarded-server-header") -} - -func (p *pack) appAuditEvents(t *testing.T) { - inCookie := p.createAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName) - - status, body, err := p.makeRequest(inCookie, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - require.Contains(t, body, p.rootMessage) - - // session start event - p.ensureAuditEvent(t, events.AppSessionStartEvent, func(event apievents.AuditEvent) { - expectedEvent := &apievents.AppSessionStart{ - Metadata: apievents.Metadata{ - Type: events.AppSessionStartEvent, - Code: events.AppSessionStartCode, - ClusterName: p.rootAppClusterName, - }, - AppMetadata: apievents.AppMetadata{ - AppURI: p.rootAppURI, - AppPublicAddr: p.rootAppPublicAddr, - AppName: p.rootAppName, - }, - PublicAddr: p.rootAppPublicAddr, - } - require.Empty(t, cmp.Diff( - expectedEvent, - event, - cmpopts.IgnoreTypes(apievents.ServerMetadata{}, apievents.SessionMetadata{}, apievents.UserMetadata{}, apievents.ConnectionMetadata{}), - cmpopts.IgnoreFields(apievents.Metadata{}, "ID", "Time"), - )) - }) - - // session chunk event - p.ensureAuditEvent(t, events.AppSessionChunkEvent, func(event apievents.AuditEvent) { - expectedEvent := &apievents.AppSessionChunk{ - Metadata: apievents.Metadata{ - Type: events.AppSessionChunkEvent, - Code: events.AppSessionChunkCode, - ClusterName: p.rootAppClusterName, - }, - AppMetadata: apievents.AppMetadata{ - AppURI: p.rootAppURI, - AppPublicAddr: p.rootAppPublicAddr, - AppName: p.rootAppName, - }, - } - require.Empty(t, cmp.Diff( - expectedEvent, - event, - cmpopts.IgnoreTypes(apievents.ServerMetadata{}, apievents.SessionMetadata{}, apievents.UserMetadata{}, apievents.ConnectionMetadata{}), - cmpopts.IgnoreFields(apievents.Metadata{}, "ID", "Time"), - cmpopts.IgnoreFields(apievents.AppSessionChunk{}, "SessionChunkID"), - )) - }) -} - -func (p *pack) appServersHA(t *testing.T) { - type packInfo struct { - clusterName string - publicHTTPAddr string - publicWSAddr string - appServers []*service.TeleportProcess - } - - testCases := map[string]struct { - packInfo func(pack *pack) packInfo - startAppServers func(pack *pack, count int) []*service.TeleportProcess - waitForTunnelConn func(t *testing.T, pack *pack, count int) - }{ - "RootServer": { - packInfo: func(pack *pack) packInfo { - return packInfo{ - clusterName: pack.rootAppClusterName, - publicHTTPAddr: pack.rootAppPublicAddr, - publicWSAddr: pack.rootWSPublicAddr, - appServers: pack.rootAppServers, - } - }, - startAppServers: func(pack *pack, count int) []*service.TeleportProcess { - return pack.startRootAppServers(t, count, []service.App{}) - }, - waitForTunnelConn: func(t *testing.T, pack *pack, count int) { - waitForActiveTunnelConnections(t, pack.rootCluster.Tunnel, pack.rootCluster.Secrets.SiteName, count) - }, - }, - "LeafServer": { - packInfo: func(pack *pack) packInfo { - return packInfo{ - clusterName: pack.leafAppClusterName, - publicHTTPAddr: pack.leafAppPublicAddr, - publicWSAddr: pack.leafWSPublicAddr, - appServers: pack.leafAppServers, - } - }, - startAppServers: func(pack *pack, count int) []*service.TeleportProcess { - return pack.startLeafAppServers(t, count, []service.App{}) - }, - waitForTunnelConn: func(t *testing.T, pack *pack, count int) { - waitForActiveTunnelConnections(t, pack.leafCluster.Tunnel, pack.leafCluster.Secrets.SiteName, count) - }, - }, - } - - // asserts that the response has error. - responseWithError := func(t *testing.T, status int, err error) { - if status > 0 { - require.NoError(t, err) - require.Equal(t, http.StatusInternalServerError, status) - return - } - - require.Error(t, err) - } - // asserts that the response has no errors. - responseWithoutError := func(t *testing.T, status int, err error) { - if status > 0 { - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - return - } - - require.NoError(t, err) - } - - makeRequests := func(t *testing.T, pack *pack, httpCookie, wsCookie string, responseAssertion func(*testing.T, int, error)) { - status, _, err := pack.makeRequest(httpCookie, http.MethodGet, "/") - responseAssertion(t, status, err) - - _, err = pack.makeWebsocketRequest(wsCookie, "/") - responseAssertion(t, 0, err) - } - - for name, test := range testCases { - name, test := name, test - t.Run(name, func(t *testing.T) { - info := test.packInfo(p) - httpCookie := p.createAppSession(t, info.publicHTTPAddr, info.clusterName) - wsCookie := p.createAppSession(t, info.publicWSAddr, info.clusterName) - - makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) - - // Stop all root app servers. - for i, appServer := range info.appServers { - require.NoError(t, appServer.Close()) - require.NoError(t, appServer.Wait()) - - if i == len(info.appServers)-1 { - // fails only when the last one is closed. - makeRequests(t, p, httpCookie, wsCookie, responseWithError) - } else { - // otherwise the request should be handled by another - // server. - makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) - } - } - - servers := test.startAppServers(p, 1) - test.waitForTunnelConn(t, p, 1) - makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) - - // Start an additional app server and stop all current running - // ones. - test.startAppServers(p, 1) - test.waitForTunnelConn(t, p, 2) - - for _, appServer := range servers { - require.NoError(t, appServer.Close()) - require.NoError(t, appServer.Wait()) - - // Everytime an app server stops we issue a request to - // guarantee that the requests are going to be resolved by - // the remaining app servers. - makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) - } - }) - } -} - -func (p *pack) appInvalidateAppSessionsOnLogout(t *testing.T) { - t.Cleanup(func() { - // This test will invalidate the web session so init it again after the - // test, otherwise tests that run after this one will be getting 403's. - p.initWebSession(t) - }) - - // Create an application session. - appCookie := p.createAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName) - - // Issue a request to the application to guarantee everything is working correctly. - status, _, err := p.makeRequest(appCookie, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - - // Generates TLS config for making app requests. - reqTLS := p.makeTLSConfig(t, p.rootAppPublicAddr, p.rootAppClusterName) - require.NotNil(t, reqTLS) - - // Issue a request to the application to guarantee everything is working correctly. - status, _, err = p.makeRequestWithClientCert(reqTLS, http.MethodGet, "/") - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - - // Logout from Teleport. - status, _, err = p.makeWebapiRequest(http.MethodDelete, "sessions", []byte{}) - require.NoError(t, err) - require.Equal(t, http.StatusOK, status) - - // As deleting WebSessions might not happen immediately, run the next request - // in an `Eventually` block. - require.Eventually(t, func() bool { - // Issue another request to the application. Now, it should receive a - // redirect because the application sessions are gone. - status, _, err = p.makeRequest(appCookie, http.MethodGet, "/") - require.NoError(t, err) - return status == http.StatusFound - }, time.Second, 250*time.Millisecond) - - // Check the same for the client certificate. - require.Eventually(t, func() bool { - // Issue another request to the application. Now, it should receive a - // redirect because the application sessions are gone. - status, _, err = p.makeRequestWithClientCert(reqTLS, http.MethodGet, "/") - require.NoError(t, err) - return status == http.StatusFound - }, time.Second, 250*time.Millisecond) -} - -// pack contains identity as well as initialized Teleport clusters and instances. -type pack struct { - username string - password string - - tc *client.TeleportClient - - user types.User - - webCookie string - webToken string - - rootCluster *helpers.TeleInstance - rootAppServers []*service.TeleportProcess - rootCertPool *x509.CertPool - - rootAppName string - rootAppPublicAddr string - rootAppClusterName string - rootMessage string - rootAppURI string - - rootWSAppName string - rootWSPublicAddr string - rootWSMessage string - rootWSAppURI string - - rootWSSAppName string - rootWSSPublicAddr string - rootWSSMessage string - rootWSSAppURI string - - rootTCPAppName string - rootTCPPublicAddr string - rootTCPMessage string - rootTCPAppURI string - - jwtAppName string - jwtAppPublicAddr string - jwtAppClusterName string - jwtAppURI string - - dumperAppURI string - - leafCluster *helpers.TeleInstance - leafAppServers []*service.TeleportProcess - - leafAppName string - leafAppPublicAddr string - leafAppClusterName string - leafMessage string - leafAppURI string - - leafWSAppName string - leafWSPublicAddr string - leafWSMessage string - leafWSAppURI string - - leafWSSAppName string - leafWSSPublicAddr string - leafWSSMessage string - leafWSSAppURI string - - leafTCPAppName string - leafTCPPublicAddr string - leafTCPMessage string - leafTCPAppURI string - - headerAppName string - headerAppPublicAddr string - headerAppClusterName string - headerAppURI string - - wsHeaderAppName string - wsHeaderAppPublicAddr string - wsHeaderAppClusterName string - wsHeaderAppURI string - - flushAppName string - flushAppPublicAddr string - flushAppClusterName string - flushAppURI string -} - -type appTestOptions struct { - extraRootApps []service.App - extraLeafApps []service.App - rootClusterListeners helpers.InstanceListenerSetupFunc - leafClusterListeners helpers.InstanceListenerSetupFunc - - rootConfig func(config *service.Config) - leafConfig func(config *service.Config) -} - -// setup configures all clusters and servers needed for a test. -func setup(t *testing.T) *pack { - return setupWithOptions(t, appTestOptions{}) -} - -// newTCPServer starts accepting TCP connections and serving them using the -// provided handler. Handlers are expected to close client connections. -// Returns the TCP listener. -func newTCPServer(t *testing.T, handleConn func(net.Conn)) net.Listener { - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - - go func() { - for { - conn, err := listener.Accept() - if err == nil { - go handleConn(conn) - } - if err != nil && !utils.IsOKNetworkError(err) { - t.Error(err) - return - } - } - }() - - return listener -} - -// setupWithOptions configures app access test with custom options. -func setupWithOptions(t *testing.T, opts appTestOptions) *pack { - tr := utils.NewTracer(utils.ThisFunction()).Start() - defer tr.Stop() - - log := utils.NewLoggerForTests() - - // Insecure development mode needs to be set because the web proxy uses a - // self-signed certificate during tests. - lib.SetInsecureDevMode(true) - - p := &pack{ - rootAppName: "app-01", - rootAppPublicAddr: "app-01.example.com", - rootAppClusterName: "example.com", - rootMessage: uuid.New().String(), - - rootWSAppName: "ws-01", - rootWSPublicAddr: "ws-01.example.com", - rootWSMessage: uuid.New().String(), - - rootWSSAppName: "wss-01", - rootWSSPublicAddr: "wss-01.example.com", - rootWSSMessage: uuid.New().String(), - - rootTCPAppName: "tcp-01", - rootTCPPublicAddr: "tcp-01.example.com", - rootTCPMessage: uuid.New().String(), - - leafAppName: "app-02", - leafAppPublicAddr: "app-02.example.com", - leafAppClusterName: "leaf.example.com", - leafMessage: uuid.New().String(), - - leafWSAppName: "ws-02", - leafWSPublicAddr: "ws-02.example.com", - leafWSMessage: uuid.New().String(), - - leafWSSAppName: "wss-02", - leafWSSPublicAddr: "wss-02.example.com", - leafWSSMessage: uuid.New().String(), - - leafTCPAppName: "tcp-02", - leafTCPPublicAddr: "tcp-02.example.com", - leafTCPMessage: uuid.New().String(), - - jwtAppName: "app-03", - jwtAppPublicAddr: "app-03.example.com", - jwtAppClusterName: "example.com", - - headerAppName: "app-04", - headerAppPublicAddr: "app-04.example.com", - headerAppClusterName: "example.com", - - wsHeaderAppName: "ws-header", - wsHeaderAppPublicAddr: "ws-header.example.com", - wsHeaderAppClusterName: "example.com", - - flushAppName: "app-05", - flushAppPublicAddr: "app-05.example.com", - flushAppClusterName: "example.com", - } - - createHandler := func(handler func(conn *websocket.Conn)) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - upgrader := websocket.Upgrader{ - ReadBufferSize: 1024, - WriteBufferSize: 1024, - } - conn, err := upgrader.Upgrade(w, r, nil) - require.NoError(t, err) - handler(conn) - } - } - - // Start a few different HTTP server that will be acting like a proxied application. - rootServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, p.rootMessage) - })) - t.Cleanup(rootServer.Close) - // Websockets server in root cluster (ws://). - rootWSServer := httptest.NewServer(createHandler(func(conn *websocket.Conn) { - conn.WriteMessage(websocket.BinaryMessage, []byte(p.rootWSMessage)) - conn.Close() - })) - t.Cleanup(rootWSServer.Close) - // Secure websockets server in root cluster (wss://). - rootWSSServer := httptest.NewTLSServer(createHandler(func(conn *websocket.Conn) { - conn.WriteMessage(websocket.BinaryMessage, []byte(p.rootWSSMessage)) - conn.Close() - })) - t.Cleanup(rootWSSServer.Close) - // Plain TCP application in root cluster (tcp://). - rootTCPServer := newTCPServer(t, func(c net.Conn) { - c.Write([]byte(p.rootTCPMessage)) - c.Close() - }) - t.Cleanup(func() { rootTCPServer.Close() }) - // HTTP server in leaf cluster. - leafServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, p.leafMessage) - })) - t.Cleanup(leafServer.Close) - // Websockets server in leaf cluster (ws://). - leafWSServer := httptest.NewServer(createHandler(func(conn *websocket.Conn) { - conn.WriteMessage(websocket.BinaryMessage, []byte(p.leafWSMessage)) - conn.Close() - })) - t.Cleanup(leafWSServer.Close) - // Secure websockets server in leaf cluster (wss://). - leafWSSServer := httptest.NewTLSServer(createHandler(func(conn *websocket.Conn) { - conn.WriteMessage(websocket.BinaryMessage, []byte(p.leafWSSMessage)) - conn.Close() - })) - t.Cleanup(leafWSSServer.Close) - // Plain TCP application in leaf cluster (tcp://). - leafTCPServer := newTCPServer(t, func(c net.Conn) { - c.Write([]byte(p.leafTCPMessage)) - c.Close() - }) - t.Cleanup(func() { leafTCPServer.Close() }) - // JWT server writes generated JWT token in the response. - jwtServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - fmt.Fprintln(w, r.Header.Get(teleport.AppJWTHeader)) - })) - t.Cleanup(jwtServer.Close) - // Websocket header server dumps initial HTTP upgrade request in the response. - wsHeaderServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) - require.NoError(t, err) - reqDump, err := httputil.DumpRequest(r, false) - require.NoError(t, err) - require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, reqDump)) - require.NoError(t, conn.Close()) - })) - t.Cleanup(wsHeaderServer.Close) - headerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - for _, headerName := range forwardedHeaderNames { - fmt.Fprintln(w, r.Header.Get(headerName)) - } - })) - t.Cleanup(headerServer.Close) - // Start test server that will dump all request headers in the response. - dumperServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - r.Write(w) - })) - t.Cleanup(dumperServer.Close) - flushServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - h := w.(http.Hijacker) - conn, _, err := h.Hijack() - require.NoError(t, err) - defer conn.Close() - data := "HTTP/1.1 200 OK\r\n" + - "Transfer-Encoding: chunked\r\n" + - "\r\n" + - "05\r\n" + - "hello\r\n" - fmt.Fprint(conn, data) - time.Sleep(500 * time.Millisecond) - data = "05\r\n" + - "world\r\n" + - "0\r\n" + - "\r\n" - fmt.Fprint(conn, data) - })) - t.Cleanup(flushServer.Close) - - p.rootAppURI = rootServer.URL - p.rootWSAppURI = rootWSServer.URL - p.rootWSSAppURI = rootWSSServer.URL - p.rootTCPAppURI = fmt.Sprintf("tcp://%v", rootTCPServer.Addr().String()) - p.leafAppURI = leafServer.URL - p.leafWSAppURI = leafWSServer.URL - p.leafWSSAppURI = leafWSSServer.URL - p.leafTCPAppURI = fmt.Sprintf("tcp://%v", leafTCPServer.Addr().String()) - p.jwtAppURI = jwtServer.URL - p.headerAppURI = headerServer.URL - p.wsHeaderAppURI = wsHeaderServer.URL - p.flushAppURI = flushServer.URL - p.dumperAppURI = dumperServer.URL - - privateKey, publicKey, err := testauthority.New().GenerateKeyPair() - require.NoError(t, err) - - // Create a new Teleport instance with passed in configuration. - rootCfg := helpers.InstanceConfig{ - ClusterName: "example.com", - HostID: uuid.New().String(), - NodeName: Host, - Priv: privateKey, - Pub: publicKey, - Log: log, - } - if opts.rootClusterListeners != nil { - rootCfg.Listeners = opts.rootClusterListeners(t, &rootCfg.Fds) - } - p.rootCluster = helpers.NewInstance(t, rootCfg) - - // Create a new Teleport instance with passed in configuration. - leafCfg := helpers.InstanceConfig{ - ClusterName: "leaf.example.com", - HostID: uuid.New().String(), - NodeName: Host, - Priv: privateKey, - Pub: publicKey, - Log: log, - } - if opts.leafClusterListeners != nil { - leafCfg.Listeners = opts.leafClusterListeners(t, &leafCfg.Fds) - } - p.leafCluster = helpers.NewInstance(t, leafCfg) - - rcConf := service.MakeDefaultConfig() - rcConf.Console = nil - rcConf.Log = log - rcConf.DataDir = t.TempDir() - rcConf.Auth.Enabled = true - rcConf.Auth.Preference.SetSecondFactor("off") - rcConf.Proxy.Enabled = true - rcConf.Proxy.DisableWebService = false - rcConf.Proxy.DisableWebInterface = true - rcConf.SSH.Enabled = false - rcConf.Apps.Enabled = false - rcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() - if opts.rootConfig != nil { - opts.rootConfig(rcConf) - } - - lcConf := service.MakeDefaultConfig() - lcConf.Console = nil - lcConf.Log = log - lcConf.DataDir = t.TempDir() - lcConf.Auth.Enabled = true - lcConf.Auth.Preference.SetSecondFactor("off") - lcConf.Proxy.Enabled = true - lcConf.Proxy.DisableWebService = false - lcConf.Proxy.DisableWebInterface = true - lcConf.SSH.Enabled = false - lcConf.Apps.Enabled = false - lcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() - if opts.rootConfig != nil { - opts.rootConfig(lcConf) - } - - err = p.leafCluster.CreateEx(t, p.rootCluster.Secrets.AsSlice(), lcConf) - require.NoError(t, err) - err = p.rootCluster.CreateEx(t, p.leafCluster.Secrets.AsSlice(), rcConf) - require.NoError(t, err) - - err = p.leafCluster.Start() - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, p.leafCluster.StopAll()) }) - err = p.rootCluster.Start() - require.NoError(t, err) - t.Cleanup(func() { require.NoError(t, p.rootCluster.StopAll()) }) - - // At least one rootAppServer should start during the setup - rootAppServersCount := 1 - p.rootAppServers = p.startRootAppServers(t, rootAppServersCount, opts.extraRootApps) - - // At least one leafAppServer should start during the setup - leafAppServersCount := 1 - p.leafAppServers = p.startLeafAppServers(t, leafAppServersCount, opts.extraLeafApps) - - // Create user for tests. - p.initUser(t, opts) - - // Create Web UI session. - p.initWebSession(t) - - // Initialize cert pool with root CA's. - p.initCertPool(t) - - // Initialize Teleport client with the user's credentials. - p.initTeleportClient(t) - - return p -} - -// initUser will create a user within the root cluster. -func (p *pack) initUser(t *testing.T, opts appTestOptions) { - p.username = uuid.New().String() - p.password = uuid.New().String() - - user, err := types.NewUser(p.username) - require.NoError(t, err) - - role := services.RoleForUser(user) - role.SetLogins(types.Allow, []string{p.username, "root", "ubuntu"}) - err = p.rootCluster.Process.GetAuthServer().UpsertRole(context.Background(), role) - require.NoError(t, err) - - user.AddRole(role.GetName()) - user.SetTraits(map[string][]string{"env": {"production"}}) - err = p.rootCluster.Process.GetAuthServer().CreateUser(context.Background(), user) - require.NoError(t, err) - - err = p.rootCluster.Process.GetAuthServer().UpsertPassword(user.GetName(), []byte(p.password)) - require.NoError(t, err) - - p.user = user -} - -// initWebSession creates a Web UI session within the root cluster. -func (p *pack) initWebSession(t *testing.T) { - csReq, err := json.Marshal(web.CreateSessionReq{ - User: p.username, - Pass: p.password, - }) - require.NoError(t, err) - - // Create POST request to create session. - u := url.URL{ - Scheme: "https", - Host: p.rootCluster.Web, - Path: "/v1/webapi/sessions/web", - } - req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(csReq)) - require.NoError(t, err) - - // Attach CSRF token in cookie and header. - csrfToken, err := utils.CryptoRandomHex(32) - require.NoError(t, err) - req.AddCookie(&http.Cookie{ - Name: csrf.CookieName, - Value: csrfToken, - }) - req.Header.Set("Content-Type", "application/json; charset=utf-8") - req.Header.Set(csrf.HeaderName, csrfToken) - - // Issue request. - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - resp, err := client.Do(req) - require.NoError(t, err) - defer resp.Body.Close() - require.Equal(t, http.StatusOK, resp.StatusCode) - - // Read in response. - var csResp *web.CreateSessionResponse - err = json.NewDecoder(resp.Body).Decode(&csResp) - require.NoError(t, err) - - // Extract session cookie and bearer token. - require.Len(t, resp.Cookies(), 1) - cookie := resp.Cookies()[0] - require.Equal(t, cookie.Name, web.CookieName) - - p.webCookie = cookie.Value - p.webToken = csResp.Token -} - -// initTeleportClient initializes a Teleport client with this pack's user -// credentials. -func (p *pack) initTeleportClient(t *testing.T) { - creds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{ - Process: p.rootCluster.Process, - Username: p.user.GetName(), - }) - require.NoError(t, err) - - tc, err := p.rootCluster.NewClientWithCreds(helpers.ClientConfig{ - Login: p.user.GetName(), - Cluster: p.rootCluster.Secrets.SiteName, - Host: Loopback, - Port: helpers.Port(t, p.rootCluster.SSH), - }, *creds) - require.NoError(t, err) - - p.tc = tc -} - -// createAppSession creates an application session with the root cluster. The -// application that the user connects to may be running in a leaf cluster. -func (p *pack) createAppSession(t *testing.T, publicAddr, clusterName string) string { - require.NotEmpty(t, p.webCookie) - require.NotEmpty(t, p.webToken) - - casReq, err := json.Marshal(web.CreateAppSessionRequest{ - FQDNHint: publicAddr, - PublicAddr: publicAddr, - ClusterName: clusterName, - }) - require.NoError(t, err) - statusCode, body, err := p.makeWebapiRequest(http.MethodPost, "sessions/app", casReq) - require.NoError(t, err) - require.Equal(t, http.StatusOK, statusCode) - - var casResp *web.CreateAppSessionResponse - err = json.Unmarshal(body, &casResp) - require.NoError(t, err) - - return casResp.CookieValue -} - -// makeWebapiRequest makes a request to the root cluster Web API. -func (p *pack) makeWebapiRequest(method, endpoint string, payload []byte) (int, []byte, error) { - u := url.URL{ - Scheme: "https", - Host: p.rootCluster.Web, - Path: fmt.Sprintf("/v1/webapi/%s", endpoint), - } - - req, err := http.NewRequest(method, u.String(), bytes.NewBuffer(payload)) - if err != nil { - return 0, nil, trace.Wrap(err) - } - - req.AddCookie(&http.Cookie{ - Name: web.CookieName, - Value: p.webCookie, - }) - req.Header.Add("Authorization", fmt.Sprintf("Bearer %v", p.webToken)) - req.Header.Add("Content-Type", "application/json") - - statusCode, body, err := p.sendRequest(req, nil) - return statusCode, []byte(body), trace.Wrap(err) -} - -func (p *pack) ensureAuditEvent(t *testing.T, eventType string, checkEvent func(event apievents.AuditEvent)) { - require.Eventuallyf(t, func() bool { - events, _, err := p.rootCluster.Process.GetAuthServer().SearchEvents( - time.Now().Add(-time.Hour), - time.Now().Add(time.Hour), - apidefaults.Namespace, - []string{eventType}, - 1, - types.EventOrderDescending, - "", - ) - require.NoError(t, err) - if len(events) == 0 { - return false - } - - checkEvent(events[0]) - return true - }, 500*time.Millisecond, 50*time.Millisecond, "failed to fetch audit event \"%s\"", eventType) -} - -// initCertPool initializes root cluster CA pool. -func (p *pack) initCertPool(t *testing.T) { - authClient := p.rootCluster.GetSiteAPI(p.rootCluster.Secrets.SiteName) - ca, err := authClient.GetCertAuthority(context.Background(), types.CertAuthID{ - Type: types.HostCA, - DomainName: p.rootCluster.Secrets.SiteName, - }, false) - require.NoError(t, err) - - pool, err := services.CertPool(ca) - require.NoError(t, err) - - p.rootCertPool = pool -} - -// startLocalProxy starts a local ALPN proxy for the specified application. -func (p *pack) startLocalProxy(t *testing.T, publicAddr, clusterName string) string { - tlsConfig := p.makeTLSConfig(t, publicAddr, clusterName) - - listener, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - - proxy, err := alpnproxy.NewLocalProxy(alpnproxy.LocalProxyConfig{ - RemoteProxyAddr: p.rootCluster.Web, - Protocols: []alpncommon.Protocol{alpncommon.ProtocolTCP}, - InsecureSkipVerify: true, - Listener: listener, - ParentContext: context.Background(), - Certs: tlsConfig.Certificates, - }) - require.NoError(t, err) - t.Cleanup(func() { proxy.Close() }) - - go proxy.Start(context.Background()) - - return proxy.GetAddr() -} - -// makeTLSConfig returns TLS config suitable for making an app access request. -func (p *pack) makeTLSConfig(t *testing.T, publicAddr, clusterName string) *tls.Config { - privateKey, publicKey, err := native.GenerateKeyPair() - require.NoError(t, err) - - ws, err := p.tc.CreateAppSession(context.Background(), types.CreateAppSessionRequest{ - Username: p.user.GetName(), - PublicAddr: publicAddr, - ClusterName: clusterName, - }) - require.NoError(t, err) - - certificate, err := p.rootCluster.Process.GetAuthServer().GenerateUserAppTestCert( - auth.AppTestCertRequest{ - PublicKey: publicKey, - Username: p.user.GetName(), - TTL: time.Hour, - PublicAddr: publicAddr, - ClusterName: clusterName, - SessionID: ws.GetName(), - }) - require.NoError(t, err) - - tlsCert, err := tls.X509KeyPair(certificate, privateKey) - require.NoError(t, err) - - return &tls.Config{ - RootCAs: p.rootCertPool, - Certificates: []tls.Certificate{tlsCert}, - InsecureSkipVerify: true, - } -} - -// makeTLSConfigNoSession returns TLS config for application access without -// creating session to simulate nonexistent session scenario. -func (p *pack) makeTLSConfigNoSession(t *testing.T, publicAddr, clusterName string) *tls.Config { - privateKey, publicKey, err := native.GenerateKeyPair() - require.NoError(t, err) - - certificate, err := p.rootCluster.Process.GetAuthServer().GenerateUserAppTestCert( - auth.AppTestCertRequest{ - PublicKey: publicKey, - Username: p.user.GetName(), - TTL: time.Hour, - PublicAddr: publicAddr, - ClusterName: clusterName, - // Use arbitrary session ID - SessionID: uuid.New().String(), - }) - require.NoError(t, err) - - tlsCert, err := tls.X509KeyPair(certificate, privateKey) - require.NoError(t, err) - - return &tls.Config{ - RootCAs: p.rootCertPool, - Certificates: []tls.Certificate{tlsCert}, - InsecureSkipVerify: true, - } -} - -// makeRequest makes a request to the root cluster with the given session cookie. -func (p *pack) makeRequest(sessionCookie string, method string, endpoint string, headers ...service.Header) (int, string, error) { - req, err := http.NewRequest(method, p.assembleRootProxyURL(endpoint), nil) - if err != nil { - return 0, "", trace.Wrap(err) - } - - // Only attach session cookie if passed in. - if sessionCookie != "" { - req.AddCookie(&http.Cookie{ - Name: app.CookieName, - Value: sessionCookie, - }) - } - - for _, h := range headers { - req.Header.Add(h.Name, h.Value) - } - - return p.sendRequest(req, nil) -} - -// makeRequestWithClientCert makes a request to the root cluster using the -// client certificate authentication from the provided tls config. -func (p *pack) makeRequestWithClientCert(tlsConfig *tls.Config, method, endpoint string) (int, string, error) { - req, err := http.NewRequest(method, p.assembleRootProxyURL(endpoint), nil) - if err != nil { - return 0, "", trace.Wrap(err) - } - return p.sendRequest(req, tlsConfig) -} - -// makeWebsocketRequest makes a websocket request with the given session cookie. -func (p *pack) makeWebsocketRequest(sessionCookie, endpoint string) (string, error) { - header := http.Header{} - dialer := websocket.Dialer{} - - if sessionCookie != "" { - header.Set("Cookie", (&http.Cookie{ - Name: app.CookieName, - Value: sessionCookie, - }).String()) - } - dialer.TLSClientConfig = &tls.Config{ - InsecureSkipVerify: true, - } - conn, resp, err := dialer.Dial(fmt.Sprintf("wss://%s%s", p.rootCluster.Web, endpoint), header) - if err != nil { - return "", err - } - defer conn.Close() - defer resp.Body.Close() - stream := &web.WebsocketIO{Conn: conn} - data, err := io.ReadAll(stream) - if err != nil && websocket.IsUnexpectedCloseError(err, websocket.CloseAbnormalClosure) { - return "", err - } - return string(data), nil -} - -// assembleRootProxyURL returns the URL string of an endpoint at the root -// cluster's proxy web. -func (p *pack) assembleRootProxyURL(endpoint string) string { - u := url.URL{ - Scheme: "https", - Host: p.rootCluster.Web, - Path: endpoint, - } - return u.String() -} - -// sendReqeust sends the request to the root cluster. -func (p *pack) sendRequest(req *http.Request, tlsConfig *tls.Config) (int, string, error) { - if tlsConfig == nil { - tlsConfig = &tls.Config{ - InsecureSkipVerify: true, - } - } - - client := &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: tlsConfig, - }, - CheckRedirect: func(req *http.Request, via []*http.Request) error { - return http.ErrUseLastResponse - }, - } - resp, err := client.Do(req) - if err != nil { - return 0, "", trace.Wrap(err) - } - defer resp.Body.Close() - - // Read in response body. - body, err := io.ReadAll(resp.Body) - if err != nil { - return 0, "", trace.Wrap(err) - } - - return resp.StatusCode, string(body), nil -} - -// waitForLogout keeps making request with the passed in session cookie until -// they return a non-200 status. -func (p *pack) waitForLogout(appCookie string) (int, error) { - ticker := time.NewTicker(500 * time.Millisecond) - defer ticker.Stop() - timeout := time.NewTimer(5 * time.Second) - defer timeout.Stop() - - for { - select { - case <-ticker.C: - status, _, err := p.makeRequest(appCookie, http.MethodGet, "/") - if err != nil { - return 0, trace.Wrap(err) - } - if status != http.StatusOK { - return status, nil - } - case <-timeout.C: - return 0, trace.BadParameter("timed out waiting for logout") - } - } -} - -func (p *pack) startRootAppServers(t *testing.T, count int, extraApps []service.App) []*service.TeleportProcess { - log := utils.NewLoggerForTests() - - configs := make([]*service.Config, count) - - for i := 0; i < count; i++ { - raConf := service.MakeDefaultConfig() - raConf.Console = nil - raConf.Log = log - raConf.DataDir = t.TempDir() - raConf.SetToken("static-token-value") - raConf.AuthServers = []utils.NetAddr{ - { - AddrNetwork: "tcp", - Addr: p.rootCluster.Web, - }, - } - raConf.Auth.Enabled = false - raConf.Proxy.Enabled = false - raConf.SSH.Enabled = false - raConf.Apps.Enabled = true - raConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() - raConf.Apps.Apps = append([]service.App{ - { - Name: p.rootAppName, - URI: p.rootAppURI, - PublicAddr: p.rootAppPublicAddr, - }, - { - Name: p.rootWSAppName, - URI: p.rootWSAppURI, - PublicAddr: p.rootWSPublicAddr, - }, - { - Name: p.rootWSSAppName, - URI: p.rootWSSAppURI, - PublicAddr: p.rootWSSPublicAddr, - }, - { - Name: p.rootTCPAppName, - URI: p.rootTCPAppURI, - PublicAddr: p.rootTCPPublicAddr, - }, - { - Name: p.jwtAppName, - URI: p.jwtAppURI, - PublicAddr: p.jwtAppPublicAddr, - }, - { - Name: p.headerAppName, - URI: p.headerAppURI, - PublicAddr: p.headerAppPublicAddr, - }, - { - Name: p.wsHeaderAppName, - URI: p.wsHeaderAppURI, - PublicAddr: p.wsHeaderAppPublicAddr, - }, - { - Name: p.flushAppName, - URI: p.flushAppURI, - PublicAddr: p.flushAppPublicAddr, - }, - { - Name: "dumper-root", - URI: p.dumperAppURI, - PublicAddr: "dumper-root.example.com", - Rewrite: &service.Rewrite{ - Headers: []service.Header{ - { - Name: "X-Teleport-Cluster", - Value: "root", - }, - { - Name: "X-External-Env", - Value: "{{external.env}}", - }, - // Make sure can rewrite Host header. - { - Name: "Host", - Value: "example.com", - }, - // Make sure can rewrite existing header. - { - Name: "X-Existing", - Value: "rewritten-existing-header", - }, - // Make sure can't rewrite Teleport headers. - { - Name: teleport.AppJWTHeader, - Value: "rewritten-app-jwt-header", - }, - { - Name: teleport.AppCFHeader, - Value: "rewritten-app-cf-header", - }, - { - Name: forward.XForwardedFor, - Value: "rewritten-x-forwarded-for-header", - }, - { - Name: forward.XForwardedHost, - Value: "rewritten-x-forwarded-host-header", - }, - { - Name: forward.XForwardedProto, - Value: "rewritten-x-forwarded-proto-header", - }, - { - Name: forward.XForwardedServer, - Value: "rewritten-x-forwarded-server-header", - }, - // Make sure we can insert JWT token in custom header. - { - Name: "X-JWT", - Value: teleport.TraitInternalJWTVariable, - }, - }, - }, - }, - }, extraApps...) - - configs[i] = raConf - } - - servers, err := p.rootCluster.StartApps(configs) - require.NoError(t, err) - require.Equal(t, len(configs), len(servers)) - - for _, appServer := range servers { - srv := appServer - t.Cleanup(func() { - require.NoError(t, srv.Close()) - }) - waitAppServerTunnel(t, p.rootCluster.Tunnel, p.rootAppClusterName, srv.Config.HostUUID) - } - - return servers -} - -func (p *pack) startLeafAppServers(t *testing.T, count int, extraApps []service.App) []*service.TeleportProcess { - log := utils.NewLoggerForTests() - configs := make([]*service.Config, count) - - for i := 0; i < count; i++ { - laConf := service.MakeDefaultConfig() - laConf.Console = nil - laConf.Log = log - laConf.DataDir = t.TempDir() - laConf.SetToken("static-token-value") - laConf.AuthServers = []utils.NetAddr{ - { - AddrNetwork: "tcp", - Addr: p.leafCluster.Web, - }, - } - laConf.Auth.Enabled = false - laConf.Proxy.Enabled = false - laConf.SSH.Enabled = false - laConf.Apps.Enabled = true - laConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() - laConf.Apps.Apps = append([]service.App{ - { - Name: p.leafAppName, - URI: p.leafAppURI, - PublicAddr: p.leafAppPublicAddr, - }, - { - Name: p.leafWSAppName, - URI: p.leafWSAppURI, - PublicAddr: p.leafWSPublicAddr, - }, - { - Name: p.leafWSSAppName, - URI: p.leafWSSAppURI, - PublicAddr: p.leafWSSPublicAddr, - }, - { - Name: p.leafTCPAppName, - URI: p.leafTCPAppURI, - PublicAddr: p.leafTCPPublicAddr, - }, - { - Name: "dumper-leaf", - URI: p.dumperAppURI, - PublicAddr: "dumper-leaf.example.com", - Rewrite: &service.Rewrite{ - Headers: []service.Header{ - { - Name: "X-Teleport-Cluster", - Value: "leaf", - }, - // In leaf clusters internal.logins variable is - // populated with the user's root role logins. - { - Name: "X-Teleport-Login", - Value: "{{internal.logins}}", - }, - { - Name: "X-External-Env", - Value: "{{external.env}}", - }, - // Make sure can rewrite Host header. - { - Name: "Host", - Value: "example.com", - }, - // Make sure can rewrite existing header. - { - Name: "X-Existing", - Value: "rewritten-existing-header", - }, - // Make sure can't rewrite Teleport headers. - { - Name: teleport.AppJWTHeader, - Value: "rewritten-app-jwt-header", - }, - { - Name: teleport.AppCFHeader, - Value: "rewritten-app-cf-header", - }, - { - Name: forward.XForwardedFor, - Value: "rewritten-x-forwarded-for-header", - }, - { - Name: forward.XForwardedHost, - Value: "rewritten-x-forwarded-host-header", - }, - { - Name: forward.XForwardedProto, - Value: "rewritten-x-forwarded-proto-header", - }, - { - Name: forward.XForwardedServer, - Value: "rewritten-x-forwarded-server-header", - }, - }, - }, - }, - }, extraApps...) - - configs[i] = laConf - } - - servers, err := p.leafCluster.StartApps(configs) - require.NoError(t, err) - require.Equal(t, len(configs), len(servers)) - - for _, appServer := range servers { - srv := appServer - t.Cleanup(func() { - require.NoError(t, srv.Close()) - }) - waitAppServerTunnel(t, p.rootCluster.Tunnel, p.leafAppClusterName, srv.Config.HostUUID) - } - - return servers -} - -var forwardedHeaderNames = []string{ - teleport.AppJWTHeader, - teleport.AppCFHeader, - "X-Forwarded-Proto", - "X-Forwarded-Host", - "X-Forwarded-Server", - "X-Forwarded-For", -} diff --git a/integration/appaccess/appaccess_test.go b/integration/appaccess/appaccess_test.go new file mode 100644 index 00000000000..6cfcd598886 --- /dev/null +++ b/integration/appaccess/appaccess_test.go @@ -0,0 +1,679 @@ +// Copyright 2022 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appaccess + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "io" + "net" + "net/http" + "strings" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/google/uuid" + "github.com/gravitational/oxy/forward" + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/types" + apievents "github.com/gravitational/teleport/api/types/events" + "github.com/gravitational/teleport/integration/helpers" + "github.com/gravitational/teleport/lib/events" + "github.com/gravitational/teleport/lib/service" + "github.com/gravitational/teleport/lib/web/app" + "github.com/gravitational/trace" + "github.com/stretchr/testify/require" +) + +// TestAppAccess runs the full application access integration test suite. +// +// It allows to make the entire cluster set up once, instead of per test, +// which speeds things up significantly. +func TestAppAccess(t *testing.T) { + pack := Setup(t) + + t.Run("Forward", bind(pack, testForward)) + t.Run("Websockets", bind(pack, testWebsockets)) + t.Run("ClientCert", bind(pack, testClientCert)) + t.Run("Flush", bind(pack, testFlush)) + t.Run("ForwardModes", bind(pack, testForwardModes)) + t.Run("RewriteHeadersRoot", bind(pack, testRewriteHeadersRoot)) + t.Run("RewriteHeadersLeaf", bind(pack, testRewriteHeadersLeaf)) + t.Run("Logout", bind(pack, testLogout)) + t.Run("JWT", bind(pack, testJWT)) + t.Run("NoHeaderOverrides", bind(pack, testNoHeaderOverrides)) + t.Run("AuditEvents", bind(pack, testAuditEvents)) + t.Run("TestAppInvalidateAppSessionsOnLogout", bind(pack, testInvalidateAppSessionsOnLogout)) + + // This test should go last because it stops/starts app servers. + t.Run("TestAppServersHA", bind(pack, testServersHA)) +} + +// testForward tests that requests get forwarded to the target application +// within a single cluster and trusted cluster. +func testForward(p *Pack, t *testing.T) { + tests := []struct { + desc string + inCookie string + outStatusCode int + outMessage string + }{ + { + desc: "root cluster, valid application session cookie, success", + inCookie: p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName), + outStatusCode: http.StatusOK, + outMessage: p.rootMessage, + }, + { + desc: "leaf cluster, valid application session cookie, success", + inCookie: p.CreateAppSession(t, p.leafAppPublicAddr, p.leafAppClusterName), + outStatusCode: http.StatusOK, + outMessage: p.leafMessage, + }, + { + desc: "invalid application session cookie, redirect to login", + inCookie: "D25C463CD27861559CC6A0A6AE54818079809AA8731CB18037B4B37A80C4FC6C", + outStatusCode: http.StatusFound, + outMessage: "", + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + tt := tt + status, body, err := p.MakeRequest(tt.inCookie, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, tt.outStatusCode, status) + require.Contains(t, body, tt.outMessage) + }) + } +} + +// TestWebsockets makes sure that websocket requests get forwarded. +func testWebsockets(p *Pack, t *testing.T) { + tests := []struct { + desc string + inCookie string + outMessage string + err error + }{ + { + desc: "root cluster, valid application session cookie, successful websocket (ws://) request", + inCookie: p.CreateAppSession(t, p.rootWSPublicAddr, p.rootAppClusterName), + outMessage: p.rootWSMessage, + }, + { + desc: "root cluster, valid application session cookie, successful secure websocket (wss://) request", + inCookie: p.CreateAppSession(t, p.rootWSSPublicAddr, p.rootAppClusterName), + outMessage: p.rootWSSMessage, + }, + { + desc: "leaf cluster, valid application session cookie, successful websocket (ws://) request", + inCookie: p.CreateAppSession(t, p.leafWSPublicAddr, p.leafAppClusterName), + outMessage: p.leafWSMessage, + }, + { + desc: "leaf cluster, valid application session cookie, successful secure websocket (wss://) request", + inCookie: p.CreateAppSession(t, p.leafWSSPublicAddr, p.leafAppClusterName), + outMessage: p.leafWSSMessage, + }, + { + desc: "invalid application session cookie, websocket request fails to dial", + inCookie: "ABCDEFGHIJKLMNOPQRSTUVWXYZ", + err: errors.New(""), + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + tt := tt + body, err := p.makeWebsocketRequest(tt.inCookie, "/") + if tt.err != nil { + require.IsType(t, tt.err, trace.Unwrap(err)) + } else { + require.NoError(t, err) + require.Equal(t, tt.outMessage, body) + } + }) + } +} + +// testForwardModes ensures that requests are forwarded to applications +// even when the cluster is in proxy recording mode. +func testForwardModes(p *Pack, t *testing.T) { + // Create cluster, user, sessions, and credentials package. + ctx := context.Background() + + // Update root and leaf clusters to record sessions at the proxy. + recConfig, err := types.NewSessionRecordingConfigFromConfigFile(types.SessionRecordingConfigSpecV2{ + Mode: types.RecordAtProxy, + }) + require.NoError(t, err) + err = p.rootCluster.Process.GetAuthServer().SetSessionRecordingConfig(ctx, recConfig) + require.NoError(t, err) + err = p.leafCluster.Process.GetAuthServer().SetSessionRecordingConfig(ctx, recConfig) + require.NoError(t, err) + + // Requests to root and leaf cluster are successful. + tests := []struct { + desc string + inCookie string + outStatusCode int + outMessage string + }{ + { + desc: "root cluster, valid application session cookie, success", + inCookie: p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName), + outStatusCode: http.StatusOK, + outMessage: p.rootMessage, + }, + { + desc: "leaf cluster, valid application session cookie, success", + inCookie: p.CreateAppSession(t, p.leafAppPublicAddr, p.leafAppClusterName), + outStatusCode: http.StatusOK, + outMessage: p.leafMessage, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + tt := tt + status, body, err := p.MakeRequest(tt.inCookie, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, tt.outStatusCode, status) + require.Contains(t, body, tt.outMessage) + }) + } +} + +// testClientCert tests mutual TLS authentication flow with application +// access typically used in CLI by curl and other clients. +func testClientCert(p *Pack, t *testing.T) { + tests := []struct { + desc string + inTLSConfig *tls.Config + outStatusCode int + outMessage string + }{ + { + desc: "root cluster, valid TLS config, success", + inTLSConfig: p.makeTLSConfig(t, p.rootAppPublicAddr, p.rootAppClusterName), + outStatusCode: http.StatusOK, + outMessage: p.rootMessage, + }, + { + desc: "leaf cluster, valid TLS config, success", + inTLSConfig: p.makeTLSConfig(t, p.leafAppPublicAddr, p.leafAppClusterName), + outStatusCode: http.StatusOK, + outMessage: p.leafMessage, + }, + { + desc: "root cluster, invalid session ID", + inTLSConfig: p.makeTLSConfigNoSession(t, p.rootAppPublicAddr, p.rootAppClusterName), + outStatusCode: http.StatusFound, + }, + } + for _, tt := range tests { + t.Run(tt.desc, func(t *testing.T) { + tt := tt + status, body, err := p.makeRequestWithClientCert(tt.inTLSConfig, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, tt.outStatusCode, status) + require.Contains(t, body, tt.outMessage) + }) + } +} + +// appAccessFlush makes sure that application access periodically flushes +// buffered data to the response. +func testFlush(p *Pack, t *testing.T) { + req, err := http.NewRequest("GET", p.assembleRootProxyURL("/"), nil) + require.NoError(t, err) + + cookie := p.CreateAppSession(t, p.flushAppPublicAddr, p.flushAppClusterName) + req.AddCookie(&http.Cookie{ + Name: app.CookieName, + Value: cookie, + }) + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + // The "flush server" will send 2 messages, "hello" and "world", with a + // 500ms delay between them. They should arrive as 2 different frames + // due to the periodic flushing. + frames := []string{"hello", "world"} + for _, frame := range frames { + buffer := make([]byte, 1024) + n, err := resp.Body.Read(buffer) + if err != nil { + require.ErrorIs(t, err, io.EOF) + } + require.Equal(t, frame, strings.TrimSpace(string(buffer[:n]))) + } +} + +// testRewriteHeadersRoot validates that http headers from application +// rewrite configuration are correctly passed to proxied applications in root. +func testRewriteHeadersRoot(p *Pack, t *testing.T) { + // Create an application session for dumper app in root cluster. + appCookie := p.CreateAppSession(t, "dumper-root.example.com", "example.com") + + // Get headers response and make sure headers were passed. + status, resp, err := p.MakeRequest(appCookie, http.MethodGet, "/", service.Header{ + Name: "X-Existing", Value: "existing", + }) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + // Dumper app just dumps HTTP request so we should be able to read it back. + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(resp))) + require.NoError(t, err) + require.Equal(t, req.Host, "example.com") + require.Equal(t, req.Header.Get("X-Teleport-Cluster"), "root") + require.Equal(t, req.Header.Get("X-External-Env"), "production") + require.Equal(t, req.Header.Get("X-Existing"), "rewritten-existing-header") + require.NotEqual(t, req.Header.Get(teleport.AppJWTHeader), "rewritten-app-jwt-header") + require.NotEqual(t, req.Header.Get(teleport.AppCFHeader), "rewritten-app-cf-header") + require.NotEqual(t, req.Header.Get(forward.XForwardedFor), "rewritten-x-forwarded-for-header") + require.NotEqual(t, req.Header.Get(forward.XForwardedHost), "rewritten-x-forwarded-host-header") + require.NotEqual(t, req.Header.Get(forward.XForwardedProto), "rewritten-x-forwarded-proto-header") + require.NotEqual(t, req.Header.Get(forward.XForwardedServer), "rewritten-x-forwarded-server-header") + + // Verify JWT tokens. + for _, header := range []string{teleport.AppJWTHeader, teleport.AppCFHeader, "X-JWT"} { + verifyJWT(t, p, req.Header.Get(header), p.dumperAppURI) + } +} + +// testRewriteHeadersLeaf validates that http headers from application +// rewrite configuration are correctly passed to proxied applications in leaf. +func testRewriteHeadersLeaf(p *Pack, t *testing.T) { + // Create an application session for dumper app in leaf cluster. + appCookie := p.CreateAppSession(t, "dumper-leaf.example.com", "leaf.example.com") + + // Get headers response and make sure headers were passed. + status, resp, err := p.MakeRequest(appCookie, http.MethodGet, "/", service.Header{ + Name: "X-Existing", Value: "existing", + }) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.Contains(t, resp, "X-Teleport-Cluster: leaf") + require.Contains(t, resp, "X-Teleport-Login: root") + require.Contains(t, resp, "X-Teleport-Login: ubuntu") + require.Contains(t, resp, "X-External-Env: production") + require.Contains(t, resp, "Host: example.com") + require.Contains(t, resp, "X-Existing: rewritten-existing-header") + require.NotContains(t, resp, "X-Existing: existing") + require.NotContains(t, resp, "rewritten-app-jwt-header") + require.NotContains(t, resp, "rewritten-app-cf-header") + require.NotContains(t, resp, "rewritten-x-forwarded-for-header") + require.NotContains(t, resp, "rewritten-x-forwarded-host-header") + require.NotContains(t, resp, "rewritten-x-forwarded-proto-header") + require.NotContains(t, resp, "rewritten-x-forwarded-server-header") +} + +// testLogout verifies the session is removed from the backend when the user logs out. +func testLogout(p *Pack, t *testing.T) { + // Create an application session. + appCookie := p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName) + + // Log user out of session. + status, _, err := p.MakeRequest(appCookie, http.MethodGet, "/teleport-logout") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + // Wait until requests using the session cookie have failed. + status, err = p.waitForLogout(appCookie) + require.NoError(t, err) + require.Equal(t, http.StatusFound, status) +} + +// testJWT ensures a JWT token is attached to requests and the JWT token can +// be validated. +func testJWT(p *Pack, t *testing.T) { + // Create an application session. + appCookie := p.CreateAppSession(t, p.jwtAppPublicAddr, p.jwtAppClusterName) + + // Get JWT. + status, token, err := p.MakeRequest(appCookie, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + // Verify JWT token. + verifyJWT(t, p, token, p.jwtAppURI) + + // Connect to websocket application that dumps the upgrade request. + wsCookie := p.CreateAppSession(t, p.wsHeaderAppPublicAddr, p.wsHeaderAppClusterName) + body, err := p.makeWebsocketRequest(wsCookie, "/") + require.NoError(t, err) + + // Parse the upgrade request the websocket application received. + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(body))) + require.NoError(t, err) + + // Extract JWT token from header and verify it. + wsToken := req.Header.Get(teleport.AppJWTHeader) + require.NotEmpty(t, wsToken, "websocket upgrade request doesn't contain JWT header") + verifyJWT(t, p, wsToken, p.wsHeaderAppURI) +} + +// testNoHeaderOverrides ensures that AAP-specific headers cannot be overridden +// by values passed in by the user. +func testNoHeaderOverrides(p *Pack, t *testing.T) { + // Create an application session. + appCookie := p.CreateAppSession(t, p.headerAppPublicAddr, p.headerAppClusterName) + + // Get HTTP headers forwarded to the application. + status, origHeaderResp, err := p.MakeRequest(appCookie, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + origHeaders := strings.Split(origHeaderResp, "\n") + require.Equal(t, len(origHeaders), len(forwardedHeaderNames)+1) + + // Construct HTTP request with custom headers. + req, err := http.NewRequest(http.MethodGet, p.assembleRootProxyURL("/"), nil) + require.NoError(t, err) + req.AddCookie(&http.Cookie{ + Name: app.CookieName, + Value: appCookie, + }) + for _, headerName := range forwardedHeaderNames { + req.Header.Set(headerName, uuid.New().String()) + } + + // Issue the request. + status, newHeaderResp, err := p.sendRequest(req, nil) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + newHeaders := strings.Split(newHeaderResp, "\n") + require.Equal(t, len(newHeaders), len(forwardedHeaderNames)+1) + + // Headers sent to the application should not be affected. + for i := range forwardedHeaderNames { + require.Equal(t, origHeaders[i], newHeaders[i]) + } +} + +func testAuditEvents(p *Pack, t *testing.T) { + inCookie := p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName) + + status, body, err := p.MakeRequest(inCookie, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + require.Contains(t, body, p.rootMessage) + + // session start event + p.ensureAuditEvent(t, events.AppSessionStartEvent, func(event apievents.AuditEvent) { + expectedEvent := &apievents.AppSessionStart{ + Metadata: apievents.Metadata{ + Type: events.AppSessionStartEvent, + Code: events.AppSessionStartCode, + ClusterName: p.rootAppClusterName, + }, + AppMetadata: apievents.AppMetadata{ + AppURI: p.rootAppURI, + AppPublicAddr: p.rootAppPublicAddr, + AppName: p.rootAppName, + }, + PublicAddr: p.rootAppPublicAddr, + } + require.Empty(t, cmp.Diff( + expectedEvent, + event, + cmpopts.IgnoreTypes(apievents.ServerMetadata{}, apievents.SessionMetadata{}, apievents.UserMetadata{}, apievents.ConnectionMetadata{}), + cmpopts.IgnoreFields(apievents.Metadata{}, "ID", "Time"), + )) + }) + + // session chunk event + p.ensureAuditEvent(t, events.AppSessionChunkEvent, func(event apievents.AuditEvent) { + expectedEvent := &apievents.AppSessionChunk{ + Metadata: apievents.Metadata{ + Type: events.AppSessionChunkEvent, + Code: events.AppSessionChunkCode, + ClusterName: p.rootAppClusterName, + }, + AppMetadata: apievents.AppMetadata{ + AppURI: p.rootAppURI, + AppPublicAddr: p.rootAppPublicAddr, + AppName: p.rootAppName, + }, + } + require.Empty(t, cmp.Diff( + expectedEvent, + event, + cmpopts.IgnoreTypes(apievents.ServerMetadata{}, apievents.SessionMetadata{}, apievents.UserMetadata{}, apievents.ConnectionMetadata{}), + cmpopts.IgnoreFields(apievents.Metadata{}, "ID", "Time"), + cmpopts.IgnoreFields(apievents.AppSessionChunk{}, "SessionChunkID"), + )) + }) +} + +func testInvalidateAppSessionsOnLogout(p *Pack, t *testing.T) { + t.Cleanup(func() { + // This test will invalidate the web session so init it again after the + // test, otherwise tests that run after this one will be getting 403's. + p.initWebSession(t) + }) + + // Create an application session. + appCookie := p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName) + + // Issue a request to the application to guarantee everything is working correctly. + status, _, err := p.MakeRequest(appCookie, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + // Generates TLS config for making app requests. + reqTLS := p.makeTLSConfig(t, p.rootAppPublicAddr, p.rootAppClusterName) + require.NotNil(t, reqTLS) + + // Issue a request to the application to guarantee everything is working correctly. + status, _, err = p.makeRequestWithClientCert(reqTLS, http.MethodGet, "/") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + // Logout from Teleport. + status, _, err = p.makeWebapiRequest(http.MethodDelete, "sessions", []byte{}) + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + + // As deleting WebSessions might not happen immediately, run the next request + // in an `Eventually` block. + require.Eventually(t, func() bool { + // Issue another request to the application. Now, it should receive a + // redirect because the application sessions are gone. + status, _, err = p.MakeRequest(appCookie, http.MethodGet, "/") + require.NoError(t, err) + return status == http.StatusFound + }, time.Second, 250*time.Millisecond) + + // Check the same for the client certificate. + require.Eventually(t, func() bool { + // Issue another request to the application. Now, it should receive a + // redirect because the application sessions are gone. + status, _, err = p.makeRequestWithClientCert(reqTLS, http.MethodGet, "/") + require.NoError(t, err) + return status == http.StatusFound + }, time.Second, 250*time.Millisecond) +} + +// TestTCP tests proxying of plain TCP applications through app access. +func TestTCP(t *testing.T) { + pack := Setup(t) + + tests := []struct { + description string + address string + outMessage string + }{ + { + description: "TCP app in root cluster", + address: pack.startLocalProxy(t, pack.rootTCPPublicAddr, pack.rootAppClusterName), + outMessage: pack.rootTCPMessage, + }, + { + description: "TCP app in leaf cluster", + address: pack.startLocalProxy(t, pack.leafTCPPublicAddr, pack.leafAppClusterName), + outMessage: pack.leafTCPMessage, + }, + } + + for _, test := range tests { + t.Run(test.description, func(t *testing.T) { + conn, err := net.Dial("tcp", test.address) + require.NoError(t, err) + + buf := make([]byte, 1024) + n, err := conn.Read(buf) + require.NoError(t, err) + + resp := strings.TrimSpace(string(buf[:n])) + require.Equal(t, test.outMessage, resp) + }) + } +} + +func testServersHA(p *Pack, t *testing.T) { + type packInfo struct { + clusterName string + publicHTTPAddr string + publicWSAddr string + appServers []*service.TeleportProcess + } + + testCases := map[string]struct { + packInfo func(pack *Pack) packInfo + startAppServers func(pack *Pack, count int) []*service.TeleportProcess + waitForTunnelConn func(t *testing.T, pack *Pack, count int) + }{ + "RootServer": { + packInfo: func(pack *Pack) packInfo { + return packInfo{ + clusterName: pack.rootAppClusterName, + publicHTTPAddr: pack.rootAppPublicAddr, + publicWSAddr: pack.rootWSPublicAddr, + appServers: pack.rootAppServers, + } + }, + startAppServers: func(pack *Pack, count int) []*service.TeleportProcess { + return pack.startRootAppServers(t, count, []service.App{}) + }, + waitForTunnelConn: func(t *testing.T, pack *Pack, count int) { + helpers.WaitForActiveTunnelConnections(t, pack.rootCluster.Tunnel, pack.rootCluster.Secrets.SiteName, count) + }, + }, + "LeafServer": { + packInfo: func(pack *Pack) packInfo { + return packInfo{ + clusterName: pack.leafAppClusterName, + publicHTTPAddr: pack.leafAppPublicAddr, + publicWSAddr: pack.leafWSPublicAddr, + appServers: pack.leafAppServers, + } + }, + startAppServers: func(pack *Pack, count int) []*service.TeleportProcess { + return pack.startLeafAppServers(t, count, []service.App{}) + }, + waitForTunnelConn: func(t *testing.T, pack *Pack, count int) { + helpers.WaitForActiveTunnelConnections(t, pack.leafCluster.Tunnel, pack.leafCluster.Secrets.SiteName, count) + }, + }, + } + + // asserts that the response has error. + responseWithError := func(t *testing.T, status int, err error) { + if status > 0 { + require.NoError(t, err) + require.Equal(t, http.StatusInternalServerError, status) + return + } + + require.Error(t, err) + } + // asserts that the response has no errors. + responseWithoutError := func(t *testing.T, status int, err error) { + if status > 0 { + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + return + } + + require.NoError(t, err) + } + + makeRequests := func(t *testing.T, pack *Pack, httpCookie, wsCookie string, responseAssertion func(*testing.T, int, error)) { + status, _, err := pack.MakeRequest(httpCookie, http.MethodGet, "/") + responseAssertion(t, status, err) + + _, err = pack.makeWebsocketRequest(wsCookie, "/") + responseAssertion(t, 0, err) + } + + for name, test := range testCases { + name, test := name, test + t.Run(name, func(t *testing.T) { + info := test.packInfo(p) + httpCookie := p.CreateAppSession(t, info.publicHTTPAddr, info.clusterName) + wsCookie := p.CreateAppSession(t, info.publicWSAddr, info.clusterName) + + makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) + + // Stop all root app servers. + for i, appServer := range info.appServers { + require.NoError(t, appServer.Close()) + require.NoError(t, appServer.Wait()) + + if i == len(info.appServers)-1 { + // fails only when the last one is closed. + makeRequests(t, p, httpCookie, wsCookie, responseWithError) + } else { + // otherwise the request should be handled by another + // server. + makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) + } + } + + servers := test.startAppServers(p, 1) + test.waitForTunnelConn(t, p, 1) + makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) + + // Start an additional app server and stop all current running + // ones. + test.startAppServers(p, 1) + test.waitForTunnelConn(t, p, 2) + + for _, appServer := range servers { + require.NoError(t, appServer.Close()) + require.NoError(t, appServer.Wait()) + + // Everytime an app server stops we issue a request to + // guarantee that the requests are going to be resolved by + // the remaining app servers. + makeRequests(t, p, httpCookie, wsCookie, responseWithoutError) + } + }) + } +} diff --git a/integration/appaccess/fixtures.go b/integration/appaccess/fixtures.go new file mode 100644 index 00000000000..c02d8f5fa93 --- /dev/null +++ b/integration/appaccess/fixtures.go @@ -0,0 +1,395 @@ +// Copyright 2022 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appaccess + +import ( + "fmt" + "net" + "net/http" + "net/http/httptest" + "net/http/httputil" + "testing" + "time" + + "github.com/google/uuid" + "github.com/gorilla/websocket" + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/breaker" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/integration/helpers" + "github.com/gravitational/teleport/lib" + "github.com/gravitational/teleport/lib/auth/testauthority" + "github.com/gravitational/teleport/lib/reversetunnel" + "github.com/gravitational/teleport/lib/service" + "github.com/gravitational/teleport/lib/utils" + + "github.com/stretchr/testify/require" +) + +type AppTestOptions struct { + ExtraRootApps []service.App + ExtraLeafApps []service.App + RootClusterListeners helpers.InstanceListenerSetupFunc + LeafClusterListeners helpers.InstanceListenerSetupFunc + + RootConfig func(config *service.Config) + LeafConfig func(config *service.Config) +} + +// Setup configures all clusters and servers needed for a test. +func Setup(t *testing.T) *Pack { + return SetupWithOptions(t, AppTestOptions{}) +} + +// SetupWithOptions configures app access test with custom options. +func SetupWithOptions(t *testing.T, opts AppTestOptions) *Pack { + tr := utils.NewTracer(utils.ThisFunction()).Start() + defer tr.Stop() + + log := utils.NewLoggerForTests() + + // Insecure development mode needs to be set because the web proxy uses a + // self-signed certificate during tests. + lib.SetInsecureDevMode(true) + + p := &Pack{ + rootAppName: "app-01", + rootAppPublicAddr: "app-01.example.com", + rootAppClusterName: "example.com", + rootMessage: uuid.New().String(), + + rootWSAppName: "ws-01", + rootWSPublicAddr: "ws-01.example.com", + rootWSMessage: uuid.New().String(), + + rootWSSAppName: "wss-01", + rootWSSPublicAddr: "wss-01.example.com", + rootWSSMessage: uuid.New().String(), + + rootTCPAppName: "tcp-01", + rootTCPPublicAddr: "tcp-01.example.com", + rootTCPMessage: uuid.New().String(), + + leafAppName: "app-02", + leafAppPublicAddr: "app-02.example.com", + leafAppClusterName: "leaf.example.com", + leafMessage: uuid.New().String(), + + leafWSAppName: "ws-02", + leafWSPublicAddr: "ws-02.example.com", + leafWSMessage: uuid.New().String(), + + leafWSSAppName: "wss-02", + leafWSSPublicAddr: "wss-02.example.com", + leafWSSMessage: uuid.New().String(), + + leafTCPAppName: "tcp-02", + leafTCPPublicAddr: "tcp-02.example.com", + leafTCPMessage: uuid.New().String(), + + jwtAppName: "app-03", + jwtAppPublicAddr: "app-03.example.com", + jwtAppClusterName: "example.com", + + headerAppName: "app-04", + headerAppPublicAddr: "app-04.example.com", + headerAppClusterName: "example.com", + + wsHeaderAppName: "ws-header", + wsHeaderAppPublicAddr: "ws-header.example.com", + wsHeaderAppClusterName: "example.com", + + flushAppName: "app-05", + flushAppPublicAddr: "app-05.example.com", + flushAppClusterName: "example.com", + } + + createHandler := func(handler func(conn *websocket.Conn)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + upgrader := websocket.Upgrader{ + ReadBufferSize: 1024, + WriteBufferSize: 1024, + } + conn, err := upgrader.Upgrade(w, r, nil) + require.NoError(t, err) + handler(conn) + } + } + + // Start a few different HTTP server that will be acting like a proxied application. + rootServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, p.rootMessage) + })) + t.Cleanup(rootServer.Close) + // Websockets server in root cluster (ws://). + rootWSServer := httptest.NewServer(createHandler(func(conn *websocket.Conn) { + conn.WriteMessage(websocket.BinaryMessage, []byte(p.rootWSMessage)) + conn.Close() + })) + t.Cleanup(rootWSServer.Close) + // Secure websockets server in root cluster (wss://). + rootWSSServer := httptest.NewTLSServer(createHandler(func(conn *websocket.Conn) { + conn.WriteMessage(websocket.BinaryMessage, []byte(p.rootWSSMessage)) + conn.Close() + })) + t.Cleanup(rootWSSServer.Close) + // Plain TCP application in root cluster (tcp://). + rootTCPServer := newTCPServer(t, func(c net.Conn) { + c.Write([]byte(p.rootTCPMessage)) + c.Close() + }) + t.Cleanup(func() { rootTCPServer.Close() }) + // HTTP server in leaf cluster. + leafServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, p.leafMessage) + })) + t.Cleanup(leafServer.Close) + // Websockets server in leaf cluster (ws://). + leafWSServer := httptest.NewServer(createHandler(func(conn *websocket.Conn) { + conn.WriteMessage(websocket.BinaryMessage, []byte(p.leafWSMessage)) + conn.Close() + })) + t.Cleanup(leafWSServer.Close) + // Secure websockets server in leaf cluster (wss://). + leafWSSServer := httptest.NewTLSServer(createHandler(func(conn *websocket.Conn) { + conn.WriteMessage(websocket.BinaryMessage, []byte(p.leafWSSMessage)) + conn.Close() + })) + t.Cleanup(leafWSSServer.Close) + // Plain TCP application in leaf cluster (tcp://). + leafTCPServer := newTCPServer(t, func(c net.Conn) { + c.Write([]byte(p.leafTCPMessage)) + c.Close() + }) + t.Cleanup(func() { leafTCPServer.Close() }) + // JWT server writes generated JWT token in the response. + jwtServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, r.Header.Get(teleport.AppJWTHeader)) + })) + t.Cleanup(jwtServer.Close) + // Websocket header server dumps initial HTTP upgrade request in the response. + wsHeaderServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil) + require.NoError(t, err) + reqDump, err := httputil.DumpRequest(r, false) + require.NoError(t, err) + require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, reqDump)) + require.NoError(t, conn.Close()) + })) + t.Cleanup(wsHeaderServer.Close) + headerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + for _, headerName := range forwardedHeaderNames { + fmt.Fprintln(w, r.Header.Get(headerName)) + } + })) + t.Cleanup(headerServer.Close) + // Start test server that will dump all request headers in the response. + dumperServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.Write(w) + })) + t.Cleanup(dumperServer.Close) + flushServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h := w.(http.Hijacker) + conn, _, err := h.Hijack() + require.NoError(t, err) + defer conn.Close() + data := "HTTP/1.1 200 OK\r\n" + + "Transfer-Encoding: chunked\r\n" + + "\r\n" + + "05\r\n" + + "hello\r\n" + fmt.Fprint(conn, data) + time.Sleep(500 * time.Millisecond) + data = "05\r\n" + + "world\r\n" + + "0\r\n" + + "\r\n" + fmt.Fprint(conn, data) + })) + t.Cleanup(flushServer.Close) + + p.rootAppURI = rootServer.URL + p.rootWSAppURI = rootWSServer.URL + p.rootWSSAppURI = rootWSSServer.URL + p.rootTCPAppURI = fmt.Sprintf("tcp://%v", rootTCPServer.Addr().String()) + p.leafAppURI = leafServer.URL + p.leafWSAppURI = leafWSServer.URL + p.leafWSSAppURI = leafWSSServer.URL + p.leafTCPAppURI = fmt.Sprintf("tcp://%v", leafTCPServer.Addr().String()) + p.jwtAppURI = jwtServer.URL + p.headerAppURI = headerServer.URL + p.wsHeaderAppURI = wsHeaderServer.URL + p.flushAppURI = flushServer.URL + p.dumperAppURI = dumperServer.URL + + privateKey, publicKey, err := testauthority.New().GenerateKeyPair() + require.NoError(t, err) + + // Create a new Teleport instance with passed in configuration. + rootCfg := helpers.InstanceConfig{ + ClusterName: "example.com", + HostID: uuid.New().String(), + NodeName: helpers.Host, + Priv: privateKey, + Pub: publicKey, + Log: log, + } + if opts.RootClusterListeners != nil { + rootCfg.Listeners = opts.RootClusterListeners(t, &rootCfg.Fds) + } + p.rootCluster = helpers.NewInstance(t, rootCfg) + + // Create a new Teleport instance with passed in configuration. + leafCfg := helpers.InstanceConfig{ + ClusterName: "leaf.example.com", + HostID: uuid.New().String(), + NodeName: helpers.Host, + Priv: privateKey, + Pub: publicKey, + Log: log, + } + if opts.LeafClusterListeners != nil { + leafCfg.Listeners = opts.LeafClusterListeners(t, &leafCfg.Fds) + } + p.leafCluster = helpers.NewInstance(t, leafCfg) + + rcConf := service.MakeDefaultConfig() + rcConf.Console = nil + rcConf.Log = log + rcConf.DataDir = t.TempDir() + rcConf.Auth.Enabled = true + rcConf.Auth.Preference.SetSecondFactor("off") + rcConf.Proxy.Enabled = true + rcConf.Proxy.DisableWebService = false + rcConf.Proxy.DisableWebInterface = true + rcConf.SSH.Enabled = false + rcConf.Apps.Enabled = false + rcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() + if opts.RootConfig != nil { + opts.RootConfig(rcConf) + } + + lcConf := service.MakeDefaultConfig() + lcConf.Console = nil + lcConf.Log = log + lcConf.DataDir = t.TempDir() + lcConf.Auth.Enabled = true + lcConf.Auth.Preference.SetSecondFactor("off") + lcConf.Proxy.Enabled = true + lcConf.Proxy.DisableWebService = false + lcConf.Proxy.DisableWebInterface = true + lcConf.SSH.Enabled = false + lcConf.Apps.Enabled = false + lcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() + if opts.RootConfig != nil { + opts.RootConfig(lcConf) + } + + err = p.leafCluster.CreateEx(t, p.rootCluster.Secrets.AsSlice(), lcConf) + require.NoError(t, err) + err = p.rootCluster.CreateEx(t, p.leafCluster.Secrets.AsSlice(), rcConf) + require.NoError(t, err) + + err = p.leafCluster.Start() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, p.leafCluster.StopAll()) }) + err = p.rootCluster.Start() + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, p.rootCluster.StopAll()) }) + + // At least one rootAppServer should start during the setup + rootAppServersCount := 1 + p.rootAppServers = p.startRootAppServers(t, rootAppServersCount, opts.ExtraRootApps) + + // At least one leafAppServer should start during the setup + leafAppServersCount := 1 + p.leafAppServers = p.startLeafAppServers(t, leafAppServersCount, opts.ExtraLeafApps) + + // Create user for tests. + p.initUser(t, opts) + + // Create Web UI session. + p.initWebSession(t) + + // Initialize cert pool with root CA's. + p.initCertPool(t) + + // Initialize Teleport client with the user's credentials. + p.initTeleportClient(t) + + return p +} + +var forwardedHeaderNames = []string{ + teleport.AppJWTHeader, + teleport.AppCFHeader, + "X-Forwarded-Proto", + "X-Forwarded-Host", + "X-Forwarded-Server", + "X-Forwarded-For", +} + +// waitAppServerTunnel waits for application server tunnel connections. +func waitAppServerTunnel(t *testing.T, tunnel reversetunnel.Server, clusterName, serverUUID string) { + t.Helper() + cluster, err := tunnel.GetSite(clusterName) + require.NoError(t, err) + + require.Eventually(t, func() bool { + conn, err := cluster.Dial(reversetunnel.DialParams{ + From: &utils.NetAddr{AddrNetwork: "tcp", Addr: "@web-proxy"}, + To: &utils.NetAddr{AddrNetwork: "tcp", Addr: reversetunnel.LocalNode}, + ServerID: fmt.Sprintf("%v.%v", serverUUID, clusterName), + ConnType: types.AppTunnel, + }) + if err != nil { + return false + } + + require.NoError(t, conn.Close()) + return true + }, 10*time.Second, time.Second) +} + +type appAccessTestFunc func(*Pack, *testing.T) + +func bind(p *Pack, fn appAccessTestFunc) func(*testing.T) { + return func(t *testing.T) { + fn(p, t) + } +} + +// newTCPServer starts accepting TCP connections and serving them using the +// provided handler. Handlers are expected to close client connections. +// Returns the TCP listener. +func newTCPServer(t *testing.T, handleConn func(net.Conn)) net.Listener { + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + go func() { + for { + conn, err := listener.Accept() + if err == nil { + go handleConn(conn) + } + if err != nil && !utils.IsOKNetworkError(err) { + t.Error(err) + return + } + } + }() + + return listener +} diff --git a/integration/appaccess/jwt.go b/integration/appaccess/jwt.go new file mode 100644 index 00000000000..1b0ca0ebee9 --- /dev/null +++ b/integration/appaccess/jwt.go @@ -0,0 +1,55 @@ +// Copyright 2022 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appaccess + +import ( + "encoding/json" + "net/http" + "testing" + + "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/jwt" + "github.com/gravitational/teleport/lib/web" + "github.com/stretchr/testify/require" +) + +func verifyJWT(t *testing.T, pack *Pack, token, appURI string) { + // Get and unmarshal JWKs + status, body, err := pack.MakeRequest("", http.MethodGet, "/.well-known/jwks.json") + require.NoError(t, err) + require.Equal(t, http.StatusOK, status) + var jwks web.JWKSResponse + err = json.Unmarshal([]byte(body), &jwks) + require.NoError(t, err) + require.Len(t, jwks.Keys, 1) + publicKey, err := jwt.UnmarshalJWK(jwks.Keys[0]) + require.NoError(t, err) + + // Verify JWT. + key, err := jwt.New(&jwt.Config{ + PublicKey: publicKey, + Algorithm: defaults.ApplicationTokenAlgorithm, + ClusterName: pack.jwtAppClusterName, + }) + require.NoError(t, err) + claims, err := key.Verify(jwt.VerifyParams{ + Username: pack.username, + RawToken: token, + URI: appURI, + }) + require.NoError(t, err) + require.Equal(t, pack.username, claims.Username) + require.Equal(t, pack.user.GetRoles(), claims.Roles) +} diff --git a/integration/appaccess/pack.go b/integration/appaccess/pack.go new file mode 100644 index 00000000000..ed2afc4f475 --- /dev/null +++ b/integration/appaccess/pack.go @@ -0,0 +1,804 @@ +// Copyright 2022 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package appaccess + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "testing" + "time" + + "github.com/google/uuid" + "github.com/gorilla/websocket" + "github.com/gravitational/oxy/forward" + "github.com/gravitational/teleport" + "github.com/gravitational/teleport/api/breaker" + apidefaults "github.com/gravitational/teleport/api/defaults" + "github.com/gravitational/teleport/api/types" + apievents "github.com/gravitational/teleport/api/types/events" + "github.com/gravitational/teleport/integration/helpers" + "github.com/gravitational/teleport/lib/auth" + "github.com/gravitational/teleport/lib/auth/native" + "github.com/gravitational/teleport/lib/client" + "github.com/gravitational/teleport/lib/httplib/csrf" + "github.com/gravitational/teleport/lib/service" + "github.com/gravitational/teleport/lib/services" + "github.com/gravitational/teleport/lib/srv/alpnproxy" + alpncommon "github.com/gravitational/teleport/lib/srv/alpnproxy/common" + "github.com/gravitational/teleport/lib/utils" + "github.com/gravitational/teleport/lib/web" + "github.com/gravitational/teleport/lib/web/app" + "github.com/gravitational/trace" + "github.com/stretchr/testify/require" +) + +// Pack contains identity as well as initialized Teleport clusters and instances. +type Pack struct { + username string + password string + + tc *client.TeleportClient + + user types.User + + webCookie string + webToken string + + rootCluster *helpers.TeleInstance + rootAppServers []*service.TeleportProcess + rootCertPool *x509.CertPool + + rootAppName string + rootAppPublicAddr string + rootAppClusterName string + rootMessage string + rootAppURI string + + rootWSAppName string + rootWSPublicAddr string + rootWSMessage string + rootWSAppURI string + + rootWSSAppName string + rootWSSPublicAddr string + rootWSSMessage string + rootWSSAppURI string + + rootTCPAppName string + rootTCPPublicAddr string + rootTCPMessage string + rootTCPAppURI string + + jwtAppName string + jwtAppPublicAddr string + jwtAppClusterName string + jwtAppURI string + + dumperAppURI string + + leafCluster *helpers.TeleInstance + leafAppServers []*service.TeleportProcess + + leafAppName string + leafAppPublicAddr string + leafAppClusterName string + leafMessage string + leafAppURI string + + leafWSAppName string + leafWSPublicAddr string + leafWSMessage string + leafWSAppURI string + + leafWSSAppName string + leafWSSPublicAddr string + leafWSSMessage string + leafWSSAppURI string + + leafTCPAppName string + leafTCPPublicAddr string + leafTCPMessage string + leafTCPAppURI string + + headerAppName string + headerAppPublicAddr string + headerAppClusterName string + headerAppURI string + + wsHeaderAppName string + wsHeaderAppPublicAddr string + wsHeaderAppClusterName string + wsHeaderAppURI string + + flushAppName string + flushAppPublicAddr string + flushAppClusterName string + flushAppURI string +} + +func (p *Pack) RootAppClusterName() string { + return p.rootAppClusterName +} + +func (p *Pack) RootAppPublicAddr() string { + return p.rootAppPublicAddr +} + +func (p *Pack) LeafAppClusterName() string { + return p.leafAppClusterName +} + +func (p *Pack) LeafAppPublicAddr() string { + return p.leafAppPublicAddr +} + +// initUser will create a user within the root cluster. +func (p *Pack) initUser(t *testing.T, opts AppTestOptions) { + p.username = uuid.New().String() + p.password = uuid.New().String() + + user, err := types.NewUser(p.username) + require.NoError(t, err) + + role := services.RoleForUser(user) + role.SetLogins(types.Allow, []string{p.username, "root", "ubuntu"}) + err = p.rootCluster.Process.GetAuthServer().UpsertRole(context.Background(), role) + require.NoError(t, err) + + user.AddRole(role.GetName()) + user.SetTraits(map[string][]string{"env": {"production"}}) + err = p.rootCluster.Process.GetAuthServer().CreateUser(context.Background(), user) + require.NoError(t, err) + + err = p.rootCluster.Process.GetAuthServer().UpsertPassword(user.GetName(), []byte(p.password)) + require.NoError(t, err) + + p.user = user +} + +// initWebSession creates a Web UI session within the root cluster. +func (p *Pack) initWebSession(t *testing.T) { + csReq, err := json.Marshal(web.CreateSessionReq{ + User: p.username, + Pass: p.password, + }) + require.NoError(t, err) + + // Create POST request to create session. + u := url.URL{ + Scheme: "https", + Host: p.rootCluster.Web, + Path: "/v1/webapi/sessions/web", + } + req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(csReq)) + require.NoError(t, err) + + // Attach CSRF token in cookie and header. + csrfToken, err := utils.CryptoRandomHex(32) + require.NoError(t, err) + req.AddCookie(&http.Cookie{ + Name: csrf.CookieName, + Value: csrfToken, + }) + req.Header.Set("Content-Type", "application/json; charset=utf-8") + req.Header.Set(csrf.HeaderName, csrfToken) + + // Issue request. + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + resp, err := client.Do(req) + require.NoError(t, err) + defer resp.Body.Close() + require.Equal(t, http.StatusOK, resp.StatusCode) + + // Read in response. + var csResp *web.CreateSessionResponse + err = json.NewDecoder(resp.Body).Decode(&csResp) + require.NoError(t, err) + + // Extract session cookie and bearer token. + require.Len(t, resp.Cookies(), 1) + cookie := resp.Cookies()[0] + require.Equal(t, cookie.Name, web.CookieName) + + p.webCookie = cookie.Value + p.webToken = csResp.Token +} + +// initTeleportClient initializes a Teleport client with this pack's user +// credentials. +func (p *Pack) initTeleportClient(t *testing.T) { + creds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{ + Process: p.rootCluster.Process, + Username: p.user.GetName(), + }) + require.NoError(t, err) + + tc, err := p.rootCluster.NewClientWithCreds(helpers.ClientConfig{ + Login: p.user.GetName(), + Cluster: p.rootCluster.Secrets.SiteName, + Host: helpers.Loopback, + Port: helpers.Port(t, p.rootCluster.SSH), + }, *creds) + require.NoError(t, err) + + p.tc = tc +} + +// CreateAppSession creates an application session with the root cluster. The +// application that the user connects to may be running in a leaf cluster. +func (p *Pack) CreateAppSession(t *testing.T, publicAddr, clusterName string) string { + require.NotEmpty(t, p.webCookie) + require.NotEmpty(t, p.webToken) + + casReq, err := json.Marshal(web.CreateAppSessionRequest{ + FQDNHint: publicAddr, + PublicAddr: publicAddr, + ClusterName: clusterName, + }) + require.NoError(t, err) + statusCode, body, err := p.makeWebapiRequest(http.MethodPost, "sessions/app", casReq) + require.NoError(t, err) + require.Equal(t, http.StatusOK, statusCode) + + var casResp *web.CreateAppSessionResponse + err = json.Unmarshal(body, &casResp) + require.NoError(t, err) + + return casResp.CookieValue +} + +// makeWebapiRequest makes a request to the root cluster Web API. +func (p *Pack) makeWebapiRequest(method, endpoint string, payload []byte) (int, []byte, error) { + u := url.URL{ + Scheme: "https", + Host: p.rootCluster.Web, + Path: fmt.Sprintf("/v1/webapi/%s", endpoint), + } + + req, err := http.NewRequest(method, u.String(), bytes.NewBuffer(payload)) + if err != nil { + return 0, nil, trace.Wrap(err) + } + + req.AddCookie(&http.Cookie{ + Name: web.CookieName, + Value: p.webCookie, + }) + req.Header.Add("Authorization", fmt.Sprintf("Bearer %v", p.webToken)) + req.Header.Add("Content-Type", "application/json") + + statusCode, body, err := p.sendRequest(req, nil) + return statusCode, []byte(body), trace.Wrap(err) +} + +func (p *Pack) ensureAuditEvent(t *testing.T, eventType string, checkEvent func(event apievents.AuditEvent)) { + require.Eventuallyf(t, func() bool { + events, _, err := p.rootCluster.Process.GetAuthServer().SearchEvents( + time.Now().Add(-time.Hour), + time.Now().Add(time.Hour), + apidefaults.Namespace, + []string{eventType}, + 1, + types.EventOrderDescending, + "", + ) + require.NoError(t, err) + if len(events) == 0 { + return false + } + + checkEvent(events[0]) + return true + }, 500*time.Millisecond, 50*time.Millisecond, "failed to fetch audit event \"%s\"", eventType) +} + +// initCertPool initializes root cluster CA pool. +func (p *Pack) initCertPool(t *testing.T) { + authClient := p.rootCluster.GetSiteAPI(p.rootCluster.Secrets.SiteName) + ca, err := authClient.GetCertAuthority(context.Background(), types.CertAuthID{ + Type: types.HostCA, + DomainName: p.rootCluster.Secrets.SiteName, + }, false) + require.NoError(t, err) + + pool, err := services.CertPool(ca) + require.NoError(t, err) + + p.rootCertPool = pool +} + +// startLocalProxy starts a local ALPN proxy for the specified application. +func (p *Pack) startLocalProxy(t *testing.T, publicAddr, clusterName string) string { + tlsConfig := p.makeTLSConfig(t, publicAddr, clusterName) + + listener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + + proxy, err := alpnproxy.NewLocalProxy(alpnproxy.LocalProxyConfig{ + RemoteProxyAddr: p.rootCluster.Web, + Protocols: []alpncommon.Protocol{alpncommon.ProtocolTCP}, + InsecureSkipVerify: true, + Listener: listener, + ParentContext: context.Background(), + Certs: tlsConfig.Certificates, + }) + require.NoError(t, err) + t.Cleanup(func() { proxy.Close() }) + + go proxy.Start(context.Background()) + + return proxy.GetAddr() +} + +// makeTLSConfig returns TLS config suitable for making an app access request. +func (p *Pack) makeTLSConfig(t *testing.T, publicAddr, clusterName string) *tls.Config { + privateKey, publicKey, err := native.GenerateKeyPair() + require.NoError(t, err) + + ws, err := p.tc.CreateAppSession(context.Background(), types.CreateAppSessionRequest{ + Username: p.user.GetName(), + PublicAddr: publicAddr, + ClusterName: clusterName, + }) + require.NoError(t, err) + + certificate, err := p.rootCluster.Process.GetAuthServer().GenerateUserAppTestCert( + auth.AppTestCertRequest{ + PublicKey: publicKey, + Username: p.user.GetName(), + TTL: time.Hour, + PublicAddr: publicAddr, + ClusterName: clusterName, + SessionID: ws.GetName(), + }) + require.NoError(t, err) + + tlsCert, err := tls.X509KeyPair(certificate, privateKey) + require.NoError(t, err) + + return &tls.Config{ + RootCAs: p.rootCertPool, + Certificates: []tls.Certificate{tlsCert}, + InsecureSkipVerify: true, + } +} + +// makeTLSConfigNoSession returns TLS config for application access without +// creating session to simulate nonexistent session scenario. +func (p *Pack) makeTLSConfigNoSession(t *testing.T, publicAddr, clusterName string) *tls.Config { + privateKey, publicKey, err := native.GenerateKeyPair() + require.NoError(t, err) + + certificate, err := p.rootCluster.Process.GetAuthServer().GenerateUserAppTestCert( + auth.AppTestCertRequest{ + PublicKey: publicKey, + Username: p.user.GetName(), + TTL: time.Hour, + PublicAddr: publicAddr, + ClusterName: clusterName, + // Use arbitrary session ID + SessionID: uuid.New().String(), + }) + require.NoError(t, err) + + tlsCert, err := tls.X509KeyPair(certificate, privateKey) + require.NoError(t, err) + + return &tls.Config{ + RootCAs: p.rootCertPool, + Certificates: []tls.Certificate{tlsCert}, + InsecureSkipVerify: true, + } +} + +// MakeRequest makes a request to the root cluster with the given session cookie. +func (p *Pack) MakeRequest(sessionCookie string, method string, endpoint string, headers ...service.Header) (int, string, error) { + req, err := http.NewRequest(method, p.assembleRootProxyURL(endpoint), nil) + if err != nil { + return 0, "", trace.Wrap(err) + } + + // Only attach session cookie if passed in. + if sessionCookie != "" { + req.AddCookie(&http.Cookie{ + Name: app.CookieName, + Value: sessionCookie, + }) + } + + for _, h := range headers { + req.Header.Add(h.Name, h.Value) + } + + return p.sendRequest(req, nil) +} + +// makeRequestWithClientCert makes a request to the root cluster using the +// client certificate authentication from the provided tls config. +func (p *Pack) makeRequestWithClientCert(tlsConfig *tls.Config, method, endpoint string) (int, string, error) { + req, err := http.NewRequest(method, p.assembleRootProxyURL(endpoint), nil) + if err != nil { + return 0, "", trace.Wrap(err) + } + return p.sendRequest(req, tlsConfig) +} + +// makeWebsocketRequest makes a websocket request with the given session cookie. +func (p *Pack) makeWebsocketRequest(sessionCookie, endpoint string) (string, error) { + header := http.Header{} + dialer := websocket.Dialer{} + + if sessionCookie != "" { + header.Set("Cookie", (&http.Cookie{ + Name: app.CookieName, + Value: sessionCookie, + }).String()) + } + dialer.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, + } + conn, resp, err := dialer.Dial(fmt.Sprintf("wss://%s%s", p.rootCluster.Web, endpoint), header) + if err != nil { + return "", err + } + defer conn.Close() + defer resp.Body.Close() + stream := &web.WebsocketIO{Conn: conn} + data, err := io.ReadAll(stream) + if err != nil && websocket.IsUnexpectedCloseError(err, websocket.CloseAbnormalClosure) { + return "", err + } + return string(data), nil +} + +// assembleRootProxyURL returns the URL string of an endpoint at the root +// cluster's proxy web. +func (p *Pack) assembleRootProxyURL(endpoint string) string { + u := url.URL{ + Scheme: "https", + Host: p.rootCluster.Web, + Path: endpoint, + } + return u.String() +} + +// sendReqeust sends the request to the root cluster. +func (p *Pack) sendRequest(req *http.Request, tlsConfig *tls.Config) (int, string, error) { + if tlsConfig == nil { + tlsConfig = &tls.Config{ + InsecureSkipVerify: true, + } + } + + client := &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: tlsConfig, + }, + CheckRedirect: func(req *http.Request, via []*http.Request) error { + return http.ErrUseLastResponse + }, + } + resp, err := client.Do(req) + if err != nil { + return 0, "", trace.Wrap(err) + } + defer resp.Body.Close() + + // Read in response body. + body, err := io.ReadAll(resp.Body) + if err != nil { + return 0, "", trace.Wrap(err) + } + + return resp.StatusCode, string(body), nil +} + +// waitForLogout keeps making request with the passed in session cookie until +// they return a non-200 status. +func (p *Pack) waitForLogout(appCookie string) (int, error) { + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + timeout := time.NewTimer(5 * time.Second) + defer timeout.Stop() + + for { + select { + case <-ticker.C: + status, _, err := p.MakeRequest(appCookie, http.MethodGet, "/") + if err != nil { + return 0, trace.Wrap(err) + } + if status != http.StatusOK { + return status, nil + } + case <-timeout.C: + return 0, trace.BadParameter("timed out waiting for logout") + } + } +} + +func (p *Pack) startRootAppServers(t *testing.T, count int, extraApps []service.App) []*service.TeleportProcess { + log := utils.NewLoggerForTests() + + configs := make([]*service.Config, count) + + for i := 0; i < count; i++ { + raConf := service.MakeDefaultConfig() + raConf.Console = nil + raConf.Log = log + raConf.DataDir = t.TempDir() + raConf.SetToken("static-token-value") + raConf.AuthServers = []utils.NetAddr{ + { + AddrNetwork: "tcp", + Addr: p.rootCluster.Web, + }, + } + raConf.Auth.Enabled = false + raConf.Proxy.Enabled = false + raConf.SSH.Enabled = false + raConf.Apps.Enabled = true + raConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() + raConf.Apps.Apps = append([]service.App{ + { + Name: p.rootAppName, + URI: p.rootAppURI, + PublicAddr: p.rootAppPublicAddr, + }, + { + Name: p.rootWSAppName, + URI: p.rootWSAppURI, + PublicAddr: p.rootWSPublicAddr, + }, + { + Name: p.rootWSSAppName, + URI: p.rootWSSAppURI, + PublicAddr: p.rootWSSPublicAddr, + }, + { + Name: p.rootTCPAppName, + URI: p.rootTCPAppURI, + PublicAddr: p.rootTCPPublicAddr, + }, + { + Name: p.jwtAppName, + URI: p.jwtAppURI, + PublicAddr: p.jwtAppPublicAddr, + }, + { + Name: p.headerAppName, + URI: p.headerAppURI, + PublicAddr: p.headerAppPublicAddr, + }, + { + Name: p.wsHeaderAppName, + URI: p.wsHeaderAppURI, + PublicAddr: p.wsHeaderAppPublicAddr, + }, + { + Name: p.flushAppName, + URI: p.flushAppURI, + PublicAddr: p.flushAppPublicAddr, + }, + { + Name: "dumper-root", + URI: p.dumperAppURI, + PublicAddr: "dumper-root.example.com", + Rewrite: &service.Rewrite{ + Headers: []service.Header{ + { + Name: "X-Teleport-Cluster", + Value: "root", + }, + { + Name: "X-External-Env", + Value: "{{external.env}}", + }, + // Make sure can rewrite Host header. + { + Name: "Host", + Value: "example.com", + }, + // Make sure can rewrite existing header. + { + Name: "X-Existing", + Value: "rewritten-existing-header", + }, + // Make sure can't rewrite Teleport headers. + { + Name: teleport.AppJWTHeader, + Value: "rewritten-app-jwt-header", + }, + { + Name: teleport.AppCFHeader, + Value: "rewritten-app-cf-header", + }, + { + Name: forward.XForwardedFor, + Value: "rewritten-x-forwarded-for-header", + }, + { + Name: forward.XForwardedHost, + Value: "rewritten-x-forwarded-host-header", + }, + { + Name: forward.XForwardedProto, + Value: "rewritten-x-forwarded-proto-header", + }, + { + Name: forward.XForwardedServer, + Value: "rewritten-x-forwarded-server-header", + }, + // Make sure we can insert JWT token in custom header. + { + Name: "X-JWT", + Value: teleport.TraitInternalJWTVariable, + }, + }, + }, + }, + }, extraApps...) + + configs[i] = raConf + } + + servers, err := p.rootCluster.StartApps(configs) + require.NoError(t, err) + require.Equal(t, len(configs), len(servers)) + + for _, appServer := range servers { + srv := appServer + t.Cleanup(func() { + require.NoError(t, srv.Close()) + }) + waitAppServerTunnel(t, p.rootCluster.Tunnel, p.rootAppClusterName, srv.Config.HostUUID) + } + + return servers +} + +func (p *Pack) startLeafAppServers(t *testing.T, count int, extraApps []service.App) []*service.TeleportProcess { + log := utils.NewLoggerForTests() + configs := make([]*service.Config, count) + + for i := 0; i < count; i++ { + laConf := service.MakeDefaultConfig() + laConf.Console = nil + laConf.Log = log + laConf.DataDir = t.TempDir() + laConf.SetToken("static-token-value") + laConf.AuthServers = []utils.NetAddr{ + { + AddrNetwork: "tcp", + Addr: p.leafCluster.Web, + }, + } + laConf.Auth.Enabled = false + laConf.Proxy.Enabled = false + laConf.SSH.Enabled = false + laConf.Apps.Enabled = true + laConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() + laConf.Apps.Apps = append([]service.App{ + { + Name: p.leafAppName, + URI: p.leafAppURI, + PublicAddr: p.leafAppPublicAddr, + }, + { + Name: p.leafWSAppName, + URI: p.leafWSAppURI, + PublicAddr: p.leafWSPublicAddr, + }, + { + Name: p.leafWSSAppName, + URI: p.leafWSSAppURI, + PublicAddr: p.leafWSSPublicAddr, + }, + { + Name: p.leafTCPAppName, + URI: p.leafTCPAppURI, + PublicAddr: p.leafTCPPublicAddr, + }, + { + Name: "dumper-leaf", + URI: p.dumperAppURI, + PublicAddr: "dumper-leaf.example.com", + Rewrite: &service.Rewrite{ + Headers: []service.Header{ + { + Name: "X-Teleport-Cluster", + Value: "leaf", + }, + // In leaf clusters internal.logins variable is + // populated with the user's root role logins. + { + Name: "X-Teleport-Login", + Value: "{{internal.logins}}", + }, + { + Name: "X-External-Env", + Value: "{{external.env}}", + }, + // Make sure can rewrite Host header. + { + Name: "Host", + Value: "example.com", + }, + // Make sure can rewrite existing header. + { + Name: "X-Existing", + Value: "rewritten-existing-header", + }, + // Make sure can't rewrite Teleport headers. + { + Name: teleport.AppJWTHeader, + Value: "rewritten-app-jwt-header", + }, + { + Name: teleport.AppCFHeader, + Value: "rewritten-app-cf-header", + }, + { + Name: forward.XForwardedFor, + Value: "rewritten-x-forwarded-for-header", + }, + { + Name: forward.XForwardedHost, + Value: "rewritten-x-forwarded-host-header", + }, + { + Name: forward.XForwardedProto, + Value: "rewritten-x-forwarded-proto-header", + }, + { + Name: forward.XForwardedServer, + Value: "rewritten-x-forwarded-server-header", + }, + }, + }, + }, + }, extraApps...) + + configs[i] = laConf + } + + servers, err := p.leafCluster.StartApps(configs) + require.NoError(t, err) + require.Equal(t, len(configs), len(servers)) + + for _, appServer := range servers { + srv := appServer + t.Cleanup(func() { + require.NoError(t, srv.Close()) + }) + waitAppServerTunnel(t, p.rootCluster.Tunnel, p.leafAppClusterName, srv.Config.HostUUID) + } + + return servers +} diff --git a/integration/client_test.go b/integration/client_test.go index 09025e4af09..fc3f15fd03c 100644 --- a/integration/client_test.go +++ b/integration/client_test.go @@ -51,7 +51,7 @@ func TestClientWithExpiredCredentialsAndDetailedErrorMessage(t *testing.T) { rcConf.SSH.Enabled = true rcConf.Version = "v2" - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username rc.AddUser(username, []string{username}) err := rc.CreateEx(t, nil, rcConf) diff --git a/integration/helpers/helpers.go b/integration/helpers/helpers.go index 5eeaa77e082..6af92d42997 100644 --- a/integration/helpers/helpers.go +++ b/integration/helpers/helpers.go @@ -257,3 +257,9 @@ func WaitForAuditEventTypeWithBackoff(t *testing.T, cli *auth.Server, startTime } } } + +func MustGetCurrentUser(t *testing.T) *user.User { + user, err := user.Current() + require.NoError(t, err) + return user +} diff --git a/integration/helpers/instance.go b/integration/helpers/instance.go index d3f5ccbda80..2726886c5a9 100644 --- a/integration/helpers/instance.go +++ b/integration/helpers/instance.go @@ -520,7 +520,7 @@ func (i *TeleInstance) GenerateConfig(t *testing.T, trustedSecrets []*InstanceSe tconf.Keygen = testauthority.New() tconf.MaxRetryPeriod = defaults.HighResPollingPeriod tconf.CircuitBreakerConfig = breaker.NoopBreakerConfig() - tconf.FileDescriptors = i.Fds + tconf.FileDescriptors = append(tconf.FileDescriptors, i.Fds...) i.Config = tconf return tconf, nil @@ -963,6 +963,7 @@ func (i *TeleInstance) StartNodeAndProxy(t *testing.T, name string) (sshPort, we } // ProxyConfig is a set of configuration parameters for Proxy +// TODO(tcsc): Add file descriptor slice to inject FDs into proxy process type ProxyConfig struct { // Name is a proxy name Name string diff --git a/integration/helpers/ports.go b/integration/helpers/ports.go index 7098667c5f3..dde00c440c9 100644 --- a/integration/helpers/ports.go +++ b/integration/helpers/ports.go @@ -240,6 +240,8 @@ func NewListenerOn(t *testing.T, hostAddr string, ty service.ListenerType, fds * File: lf, }) + fmt.Printf("New Listener %s %s\n", ty, addr) + return addr } diff --git a/integration/helpers/trustedclusters.go b/integration/helpers/trustedclusters.go new file mode 100644 index 00000000000..9ad9a1cf70a --- /dev/null +++ b/integration/helpers/trustedclusters.go @@ -0,0 +1,142 @@ +// Copyright 2022 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package helpers + +import ( + "context" + "testing" + "time" + + "github.com/gravitational/teleport/api/defaults" + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/lib/auth" + "github.com/gravitational/teleport/lib/reversetunnel" + "github.com/gravitational/teleport/lib/utils" + "github.com/gravitational/trace" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +// WaitForTunnelConnections waits for remote tunnels connections +func WaitForTunnelConnections(t *testing.T, authServer *auth.Server, clusterName string, expectedCount int) { + t.Helper() + var conns []types.TunnelConnection + for i := 0; i < 30; i++ { + // to speed things up a bit, bypass the auth cache + conns, err := authServer.Services.GetTunnelConnections(clusterName) + require.NoError(t, err) + if len(conns) == expectedCount { + return + } + time.Sleep(1 * time.Second) + } + require.Len(t, conns, expectedCount) +} + +// TryCreateTrustedCluster performs several attempts to create a trusted cluster, +// retries on connection problems and access denied errors to let caches +// propagate and services to start +// +// Duplicated in tool/tsh/tsh_test.go +func TryCreateTrustedCluster(t *testing.T, authServer *auth.Server, trustedCluster types.TrustedCluster) { + t.Helper() + ctx := context.TODO() + for i := 0; i < 10; i++ { + log.Debugf("Will create trusted cluster %v, attempt %v.", trustedCluster, i) + _, err := authServer.UpsertTrustedCluster(ctx, trustedCluster) + if err == nil { + return + } + if trace.IsConnectionProblem(err) { + log.Debugf("Retrying on connection problem: %v.", err) + time.Sleep(500 * time.Millisecond) + continue + } + if trace.IsAccessDenied(err) { + log.Debugf("Retrying on access denied: %v.", err) + time.Sleep(500 * time.Millisecond) + continue + } + require.FailNow(t, "Terminating on unexpected problem", "%v.", err) + } + require.FailNow(t, "Timeout creating trusted cluster") +} + +func WaitForClusters(tun reversetunnel.Server, expected int) func() bool { + return func() bool { + clusters, err := tun.GetSites() + if err != nil { + return false + } + + // Check the expected number of clusters are connected, and they have all + // connected with the past 10 seconds. + if len(clusters) >= expected { + for _, cluster := range clusters { + if time.Since(cluster.GetLastConnected()).Seconds() > 10.0 { + return false + } + } + } + + return true + } +} + +// WaitForNodeCount waits for a certain number of nodes to show up in the remote site. +func WaitForNodeCount(ctx context.Context, t *TeleInstance, clusterName string, count int) error { + const ( + deadline = time.Second * 30 + iterWaitTime = time.Second + ) + + err := utils.RetryStaticFor(deadline, iterWaitTime, func() error { + remoteSite, err := t.Tunnel.GetSite(clusterName) + if err != nil { + return trace.Wrap(err) + } + accessPoint, err := remoteSite.CachingAccessPoint() + if err != nil { + return trace.Wrap(err) + } + nodes, err := accessPoint.GetNodes(ctx, defaults.Namespace) + if err != nil { + return trace.Wrap(err) + } + if len(nodes) == count { + return nil + } + return trace.BadParameter("did not find %v nodes", count) + }) + if err != nil { + return trace.Wrap(err) + } + return nil +} + +// WaitForActiveTunnelConnections waits for remote cluster to report a minimum number of active connections +func WaitForActiveTunnelConnections(t *testing.T, tunnel reversetunnel.Server, clusterName string, expectedCount int) { + require.Eventually(t, func() bool { + cluster, err := tunnel.GetSite(clusterName) + if err != nil { + return false + } + return cluster.GetTunnelsCount() >= expectedCount + }, + 30*time.Second, + time.Second, + "Active tunnel connections did not reach %v in the expected time frame", expectedCount, + ) +} diff --git a/integration/integration_test.go b/integration/integration_test.go index 2ecb48d11f1..56e24ac749a 100644 --- a/integration/integration_test.go +++ b/integration/integration_test.go @@ -1839,9 +1839,9 @@ func twoClustersTunnel(t *testing.T, suite *integrationTestSuite, now time.Time, require.NoError(t, err) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(a.Tunnel, 2), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 2), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(b.Tunnel, 2), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 2), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") var ( @@ -1898,7 +1898,7 @@ func twoClustersTunnel(t *testing.T, suite *integrationTestSuite, now time.Time, require.True(t, ok) // wait for active tunnel connections to be established - waitForActiveTunnelConnections(t, b.Tunnel, a.Secrets.SiteName, 1) + helpers.WaitForActiveTunnelConnections(t, b.Tunnel, a.Secrets.SiteName, 1) // via tunnel b->a: tc, err = b.NewClient(helpers.ClientConfig{ @@ -1998,9 +1998,9 @@ func testTwoClustersProxy(t *testing.T, suite *integrationTestSuite) { require.NoError(t, a.Start()) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // make sure the reverse tunnel went through the proxy @@ -2045,9 +2045,9 @@ func testHA(t *testing.T, suite *integrationTestSuite) { sshPort, _, _ := a.StartNodeAndProxy(t, "cluster-a-node") // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") cmd := []string{"echo", "hello world"} @@ -2089,9 +2089,9 @@ func testHA(t *testing.T, suite *integrationTestSuite) { require.NoError(t, a.Start()) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // try to execute an SSH command using the same old client to site-B @@ -2182,15 +2182,15 @@ func testMapRoles(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err) // try and upsert a trusted cluster - tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) - waitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1) + helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) + helpers.WaitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1) sshPort, _, _ := aux.StartNodeAndProxy(t, "aux-node") // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // Make sure that GetNodes returns nodes in the remote site. This makes @@ -2323,34 +2323,6 @@ func testMapRoles(t *testing.T, suite *integrationTestSuite) { require.NoError(t, aux.StopAll()) } -// tryCreateTrustedCluster performs several attempts to create a trusted cluster, -// retries on connection problems and access denied errors to let caches -// propagate and services to start -// -// Duplicated in tool/tsh/tsh_test.go -func tryCreateTrustedCluster(t *testing.T, authServer *auth.Server, trustedCluster types.TrustedCluster) { - ctx := context.TODO() - for i := 0; i < 10; i++ { - log.Debugf("Will create trusted cluster %v, attempt %v.", trustedCluster, i) - _, err := authServer.UpsertTrustedCluster(ctx, trustedCluster) - if err == nil { - return - } - if trace.IsConnectionProblem(err) { - log.Debugf("Retrying on connection problem: %v.", err) - time.Sleep(500 * time.Millisecond) - continue - } - if trace.IsAccessDenied(err) { - log.Debugf("Retrying on access denied: %v.", err) - time.Sleep(500 * time.Millisecond) - continue - } - require.FailNow(t, "Terminating on unexpected problem", "%v.", err) - } - require.FailNow(t, "Timeout creating trusted cluster") -} - // trustedClusterTest is a test setup for trusted clusters tests type trustedClusterTest struct { // multiplex sets up multiplexing of the reversetunnel SSH @@ -2528,15 +2500,15 @@ func trustedClusters(t *testing.T, suite *integrationTestSuite, test trustedClus require.NoError(t, err) // try and upsert a trusted cluster - tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) - waitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1) + helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) + helpers.WaitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1) sshPort, _, _ := aux.StartNodeAndProxy(t, "aux-node") // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") cmd := []string{"echo", "hello world"} @@ -2628,7 +2600,7 @@ func trustedClusters(t *testing.T, suite *integrationTestSuite, test trustedClus require.Equal(t, clusterAux, remoteClusters[0].GetName()) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // connection and client should recover and work again @@ -2649,27 +2621,6 @@ func trustedClusters(t *testing.T, suite *integrationTestSuite, test trustedClus require.NoError(t, aux.StopAll()) } -func waitForClusters(tun reversetunnel.Server, expected int) func() bool { - return func() bool { - clusters, err := tun.GetSites() - if err != nil { - return false - } - - // Check the expected number of clusters are connected, and they have all - // connected with the past 10 seconds. - if len(clusters) >= expected { - for _, cluster := range clusters { - if time.Since(cluster.GetLastConnected()).Seconds() > 10.0 { - return false - } - } - } - - return true - } -} - func testTrustedTunnelNode(t *testing.T, suite *integrationTestSuite) { ctx := context.Background() username := suite.Me.Username @@ -2736,8 +2687,8 @@ func testTrustedTunnelNode(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err) // try and upsert a trusted cluster - tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) - waitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1) + helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) + helpers.WaitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1) // Create a Teleport instance with a node that dials back to the aux cluster. tunnelNodeHostname := "cluster-aux-node" @@ -2760,13 +2711,13 @@ func testTrustedTunnelNode(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // Wait for both nodes to show up before attempting to dial to them. - err = waitForNodeCount(ctx, main, clusterAux, 2) + err = helpers.WaitForNodeCount(ctx, main, clusterAux, 2) require.NoError(t, err) cmd := []string{"echo", "hello world"} @@ -2853,9 +2804,9 @@ func testDiscoveryRecovers(t *testing.T, suite *integrationTestSuite) { require.NoError(t, remote.Start()) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // Helper function for adding a new proxy to "main". @@ -2916,7 +2867,7 @@ func testDiscoveryRecovers(t *testing.T, suite *integrationTestSuite) { } // ensure that initial proxy's tunnel has been established - waitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1) // execute the connection via initial proxy; should not fail testProxyConn(nil, false) @@ -2986,12 +2937,13 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) { require.NoError(t, remote.Start()) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // start second proxy + // TODO(tcsc): Replace use of deprecated NewPortSlice() with preconfigured listeners nodePorts := helpers.NewPortSlice(3) proxyReverseTunnelPort, proxyWebPort, proxySSHPort := nodePorts[0], nodePorts[1], nodePorts[2] proxyConfig := helpers.ProxyConfig{ @@ -3009,8 +2961,8 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) { // At this point the main cluster should observe two tunnels // connected to it from remote cluster - waitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1) - waitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1) + helpers.WaitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1) // execute the connection via first proxy cfg := helpers.ClientConfig{ @@ -3040,7 +2992,7 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) { // Now disconnect the main proxy and make sure it will reconnect eventually. require.NoError(t, lb.RemoveBackend(mainProxyAddr)) - waitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1) + helpers.WaitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1) // Requests going via main proxy should fail. _, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1) @@ -3058,8 +3010,8 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) { lb.AddBackend(mainProxyAddr) // Once the proxy is added a matching tunnel connection should be created. - waitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1) - waitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1) + helpers.WaitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1) // Requests going via main proxy should succeed. output, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 40) @@ -3161,8 +3113,8 @@ func testReverseTunnelCollapse(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err) // Wait for active tunnel connections to be established. - waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0) - waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0) + helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) // Execute the connection via first proxy. cfg := helpers.ClientConfig{ @@ -3186,8 +3138,8 @@ func testReverseTunnelCollapse(t *testing.T, suite *integrationTestSuite) { // stop the proxy to collapse the tunnel require.NoError(t, main.StopProxy()) - waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0) - waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 0) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0) + helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 0) // Requests going via both proxy will fail. _, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1) @@ -3202,8 +3154,8 @@ func testReverseTunnelCollapse(t *testing.T, suite *integrationTestSuite) { // start the proxy again and ensure the tunnel is re-established proxyTunnel, err = main.StartProxy(proxyConfig) require.NoError(t, err) - waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0) - waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0) + helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) // Requests going to the connected proxy should succeed. _, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1) @@ -3300,8 +3252,8 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err) // Wait for active tunnel connections to be established. - waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1) - waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) // Execute the connection via first proxy. cfg := helpers.ClientConfig{ @@ -3330,7 +3282,7 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) { // Remove second proxy from LB. require.NoError(t, lb.RemoveBackend(*proxyTwoBackend)) - waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1) // Requests going via main proxy will succeed. Requests going via second // proxy will fail. @@ -3342,8 +3294,8 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) { // Add second proxy to LB, both should have a connection. lb.AddBackend(*proxyTwoBackend) - waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1) - waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1) + helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1) // Requests going via both proxies will succeed. output, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1) @@ -3360,100 +3312,6 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) { require.NoError(t, err) } -// waitForActiveTunnelConnections waits for remote cluster to report a minimum number of active connections -func waitForActiveTunnelConnections(t *testing.T, tunnel reversetunnel.Server, clusterName string, expectedCount int) { - require.Eventually(t, func() bool { - cluster, err := tunnel.GetSite(clusterName) - if err != nil { - return false - } - return cluster.GetTunnelsCount() >= expectedCount - }, - 30*time.Second, - time.Second, - "Active tunnel connections did not reach %v in the expected time frame", expectedCount, - ) -} - -// waitForActivePeerProxyConnections waits for remote cluster to report a minimum number of active proxy peer connections -func waitForActivePeerProxyConnections(t *testing.T, tunnel reversetunnel.Server, expectedCount int) { - require.Eventually(t, func() bool { - return tunnel.GetProxyPeerClient().GetConnectionsCount() >= expectedCount - }, - 30*time.Second, - time.Second, - "Peer proxy connections did not reach %v in the expected time frame", expectedCount, - ) -} - -// waitForNodeCount waits for a certain number of nodes to show up in the remote site. -func waitForNodeCount(ctx context.Context, t *helpers.TeleInstance, clusterName string, count int) error { - const ( - deadline = time.Second * 30 - iterWaitTime = time.Second - ) - - err := utils.RetryStaticFor(deadline, iterWaitTime, func() error { - remoteSite, err := t.Tunnel.GetSite(clusterName) - if err != nil { - return trace.Wrap(err) - } - accessPoint, err := remoteSite.CachingAccessPoint() - if err != nil { - return trace.Wrap(err) - } - nodes, err := accessPoint.GetNodes(ctx, defaults.Namespace) - if err != nil { - return trace.Wrap(err) - } - if len(nodes) == count { - return nil - } - return trace.BadParameter("did not find %v nodes", count) - }) - if err != nil { - return trace.Wrap(err) - } - return nil -} - -// waitForTunnelConnections waits for remote tunnels connections -func waitForTunnelConnections(t *testing.T, authServer *auth.Server, clusterName string, expectedCount int) { - var conns []types.TunnelConnection - for i := 0; i < 30; i++ { - // to speed things up a bit, bypass the auth cache - conns, err := authServer.Services.GetTunnelConnections(clusterName) - require.NoError(t, err) - if len(conns) == expectedCount { - return - } - time.Sleep(1 * time.Second) - } - require.Len(t, conns, expectedCount) -} - -// waitAppServerTunnel waits for application server tunnel connections. -func waitAppServerTunnel(t *testing.T, tunnel reversetunnel.Server, clusterName, serverUUID string) { - t.Helper() - cluster, err := tunnel.GetSite(clusterName) - require.NoError(t, err) - - require.Eventually(t, func() bool { - conn, err := cluster.Dial(reversetunnel.DialParams{ - From: &utils.NetAddr{AddrNetwork: "tcp", Addr: "@web-proxy"}, - To: &utils.NetAddr{AddrNetwork: "tcp", Addr: reversetunnel.LocalNode}, - ServerID: fmt.Sprintf("%v.%v", serverUUID, clusterName), - ConnType: types.AppTunnel, - }) - if err != nil { - return false - } - - require.NoError(t, conn.Close()) - return true - }, 10*time.Second, time.Second) -} - // TestExternalClient tests if we can connect to a node in a Teleport // cluster. Both normal and recording proxies are tested. func testExternalClient(t *testing.T, suite *integrationTestSuite) { @@ -4406,8 +4264,8 @@ func testRotateTrustedClusters(t *testing.T, suite *integrationTestSuite) { lib.SetInsecureDevMode(true) defer lib.SetInsecureDevMode(false) - tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) - waitForTunnelConnections(t, svc.GetAuthServer(), aux.Secrets.SiteName, 1) + helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster) + helpers.WaitForTunnelConnections(t, svc.GetAuthServer(), aux.Secrets.SiteName, 1) // capture credentials before reload has started to simulate old client initialCreds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{ @@ -6364,8 +6222,8 @@ func createTrustedClusterPair(t *testing.T, suite *integrationTestSuite, extraSe t.Cleanup(func() { leaf.StopAll() }) require.NoError(t, trustedCluster.CheckAndSetDefaults()) - tryCreateTrustedCluster(t, leaf.Process.GetAuthServer(), trustedCluster) - waitForTunnelConnections(t, root.Process.GetAuthServer(), leafName, 1) + helpers.TryCreateTrustedCluster(t, leaf.Process.GetAuthServer(), trustedCluster) + helpers.WaitForTunnelConnections(t, root.Process.GetAuthServer(), leafName, 1) _, _, rootProxySSHPort := root.StartNodeAndProxy(t, "root-zero") _, _, _ = leaf.StartNodeAndProxy(t, "leaf-zero") @@ -6375,8 +6233,8 @@ func createTrustedClusterPair(t *testing.T, suite *integrationTestSuite, extraSe extraServices(t, root, leaf) } - require.Eventually(t, waitForClusters(root.Tunnel, 1), 10*time.Second, 1*time.Second) - require.Eventually(t, waitForClusters(leaf.Tunnel, 1), 10*time.Second, 1*time.Second) + require.Eventually(t, helpers.WaitForClusters(root.Tunnel, 1), 10*time.Second, 1*time.Second) + require.Eventually(t, helpers.WaitForClusters(leaf.Tunnel, 1), 10*time.Second, 1*time.Second) // Create client. creds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{ diff --git a/integration/kube/fixtures.go b/integration/kube/fixtures.go new file mode 100644 index 00000000000..e7e824cbe78 --- /dev/null +++ b/integration/kube/fixtures.go @@ -0,0 +1,134 @@ +// Copyright 2022 Gravitational, Inc +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kube + +import ( + "context" + "time" + + "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/integration/helpers" + "github.com/gravitational/teleport/lib/auth/native" + "github.com/gravitational/teleport/lib/services" + "github.com/gravitational/teleport/lib/tlsca" + "github.com/gravitational/teleport/lib/utils" + "github.com/gravitational/trace" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" +) + +// For this test suite to work, the target Kubernetes cluster must have the +// following RBAC objects configured: +// https://github.com/gravitational/teleport/blob/master/fixtures/ci-teleport-rbac/ci-teleport.yaml +const TestImpersonationGroup = "teleport-ci-test-group" + +type ProxyConfig struct { + T *helpers.TeleInstance + Username string + KubeUsers []string + KubeGroups []string + Impersonation *rest.ImpersonationConfig + RouteToCluster string + CustomTLSServerName string + TargetAddress utils.NetAddr +} + +// ProxyClient returns kubernetes client using local teleport proxy +func ProxyClient(cfg ProxyConfig) (*kubernetes.Clientset, *rest.Config, error) { + authServer := cfg.T.Process.GetAuthServer() + clusterName, err := authServer.GetClusterName() + if err != nil { + return nil, nil, trace.Wrap(err) + } + + // Fetch user info to get roles and max session TTL. + user, err := authServer.GetUser(cfg.Username, false) + if err != nil { + return nil, nil, trace.Wrap(err) + } + roles, err := services.FetchRoles(user.GetRoles(), authServer, user.GetTraits()) + if err != nil { + return nil, nil, trace.Wrap(err) + } + ttl := roles.AdjustSessionTTL(10 * time.Minute) + + ca, err := authServer.GetCertAuthority(context.Background(), types.CertAuthID{ + Type: types.HostCA, + DomainName: clusterName.GetClusterName(), + }, true) + if err != nil { + return nil, nil, trace.Wrap(err) + } + caCert, signer, err := authServer.GetKeyStore().GetTLSCertAndSigner(ca) + if err != nil { + return nil, nil, trace.Wrap(err) + } + tlsCA, err := tlsca.FromCertAndSigner(caCert, signer) + if err != nil { + return nil, nil, trace.Wrap(err) + } + privPEM, _, err := native.GenerateKeyPair() + if err != nil { + return nil, nil, trace.Wrap(err) + } + priv, err := tlsca.ParsePrivateKeyPEM(privPEM) + if err != nil { + return nil, nil, trace.Wrap(err) + } + + id := tlsca.Identity{ + Username: cfg.Username, + Groups: user.GetRoles(), + KubernetesUsers: cfg.KubeUsers, + KubernetesGroups: cfg.KubeGroups, + RouteToCluster: cfg.RouteToCluster, + } + subj, err := id.Subject() + if err != nil { + return nil, nil, trace.Wrap(err) + } + cert, err := tlsCA.GenerateCertificate(tlsca.CertificateRequest{ + Clock: authServer.GetClock(), + PublicKey: priv.Public(), + Subject: subj, + NotAfter: authServer.GetClock().Now().Add(ttl), + }) + if err != nil { + return nil, nil, trace.Wrap(err) + } + + tlsClientConfig := rest.TLSClientConfig{ + CAData: ca.GetActiveKeys().TLS[0].Cert, + CertData: cert, + KeyData: privPEM, + ServerName: cfg.CustomTLSServerName, + } + config := &rest.Config{ + Host: "https://" + cfg.T.Config.Proxy.Kube.ListenAddr.Addr, + TLSClientConfig: tlsClientConfig, + } + if !cfg.TargetAddress.IsEmpty() { + config.Host = "https://" + cfg.TargetAddress.Addr + } + if cfg.Impersonation != nil { + config.Impersonate = *cfg.Impersonation + } + client, err := kubernetes.NewForConfig(config) + if err != nil { + return nil, nil, trace.Wrap(err) + } + + return client, config, nil +} diff --git a/integration/kube_integration_test.go b/integration/kube_integration_test.go index 150c13b7777..346b8ddd41f 100644 --- a/integration/kube_integration_test.go +++ b/integration/kube_integration_test.go @@ -37,8 +37,8 @@ import ( "github.com/gravitational/teleport/api/profile" "github.com/gravitational/teleport/api/types" "github.com/gravitational/teleport/integration/helpers" + "github.com/gravitational/teleport/integration/kube" "github.com/gravitational/teleport/lib" - "github.com/gravitational/teleport/lib/auth/native" "github.com/gravitational/teleport/lib/auth/testauthority" "github.com/gravitational/teleport/lib/client" "github.com/gravitational/teleport/lib/events" @@ -137,11 +137,6 @@ func newKubeSuite(t *testing.T) *KubeSuite { return suite } -// For this test suite to work, the target Kubernetes cluster must have the -// following RBAC objects configured: -// https://github.com/gravitational/teleport/blob/master/fixtures/ci-teleport-rbac/ci-teleport.yaml -const testImpersonationGroup = "teleport-ci-test-group" - type kubeIntegrationTest func(t *testing.T, suite *KubeSuite) func (s *KubeSuite) bind(test kubeIntegrationTest) func(t *testing.T) { @@ -179,7 +174,7 @@ func testKubeExec(t *testing.T, suite *KubeSuite) { }) username := suite.me.Username - kubeGroups := []string{testImpersonationGroup} + kubeGroups := []string{kube.TestImpersonationGroup} kubeUsers := []string{"alice@example.com"} role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{ Allow: types.RoleConditions{ @@ -200,12 +195,12 @@ func testKubeExec(t *testing.T, suite *KubeSuite) { // impersonating client requests will be denied if the headers // are referencing users or groups not allowed by the existing roles - impersonatingProxyClient, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeUsers: kubeUsers, - kubeGroups: kubeGroups, - impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}}, + impersonatingProxyClient, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeUsers: kubeUsers, + KubeGroups: kubeGroups, + Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}}, }) require.NoError(t, err) @@ -216,12 +211,12 @@ func testKubeExec(t *testing.T, suite *KubeSuite) { // scoped client requests will be allowed, as long as the impersonation headers // are referencing users and groups allowed by existing roles - scopedProxyClient, scopedProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeUsers: kubeUsers, - kubeGroups: kubeGroups, - impersonation: &rest.ImpersonationConfig{ + scopedProxyClient, scopedProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeUsers: kubeUsers, + KubeGroups: kubeGroups, + Impersonation: &rest.ImpersonationConfig{ UserName: role.GetKubeUsers(types.Allow)[0], Groups: role.GetKubeGroups(types.Allow), }, @@ -232,11 +227,11 @@ func testKubeExec(t *testing.T, suite *KubeSuite) { require.NoError(t, err) // set up kube configuration using proxy - proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeUsers: kubeUsers, - kubeGroups: kubeGroups, + proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeUsers: kubeUsers, + KubeGroups: kubeGroups, }) require.NoError(t, err) @@ -348,7 +343,7 @@ func testKubeDeny(t *testing.T, suite *KubeSuite) { }) username := suite.me.Username - kubeGroups := []string{testImpersonationGroup} + kubeGroups := []string{kube.TestImpersonationGroup} kubeUsers := []string{"alice@example.com"} role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{ Allow: types.RoleConditions{ @@ -372,11 +367,11 @@ func testKubeDeny(t *testing.T, suite *KubeSuite) { defer teleport.StopAll() // set up kube configuration using proxy - proxyClient, _, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeUsers: kubeUsers, - kubeGroups: kubeGroups, + proxyClient, _, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeUsers: kubeUsers, + KubeGroups: kubeGroups, }) require.NoError(t, err) @@ -400,7 +395,7 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) { }) username := suite.me.Username - kubeGroups := []string{testImpersonationGroup} + kubeGroups := []string{kube.TestImpersonationGroup} role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{ Allow: types.RoleConditions{ Logins: []string{username}, @@ -418,10 +413,10 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) { defer teleport.StopAll() // set up kube configuration using proxy - _, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeGroups: kubeGroups, + _, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeGroups: kubeGroups, }) require.NoError(t, err) @@ -453,12 +448,12 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) { require.Equal(t, http.StatusOK, resp.StatusCode) require.NoError(t, resp.Body.Close()) - // impersonating client requests will be denied - _, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeGroups: kubeGroups, - impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}}, + // impersonating client requests will bse denied + _, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeGroups: kubeGroups, + Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}}, }) require.NoError(t, err) @@ -496,7 +491,7 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) { // main cluster has a role and user called main-kube username := suite.me.Username - mainKubeGroups := []string{testImpersonationGroup} + mainKubeGroups := []string{kube.TestImpersonationGroup} mainRole, err := types.NewRoleV3("main-kube", types.RoleSpecV5{ Allow: types.RoleConditions{ Logins: []string{username}, @@ -581,18 +576,18 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) { require.True(t, upsertSuccess) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // impersonating client requests will be denied - impersonatingProxyClient, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: main, - username: username, - kubeGroups: mainKubeGroups, - impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}}, - routeToCluster: clusterAux, + impersonatingProxyClient, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: main, + Username: username, + KubeGroups: mainKubeGroups, + Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}}, + RouteToCluster: clusterAux, }) require.NoError(t, err) @@ -601,11 +596,11 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) { require.Error(t, err) // set up kube configuration using main proxy - proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: main, - username: username, - kubeGroups: mainKubeGroups, - routeToCluster: clusterAux, + proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: main, + Username: username, + KubeGroups: mainKubeGroups, + RouteToCluster: clusterAux, }) require.NoError(t, err) @@ -747,7 +742,7 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) { // main cluster has a role and user called main-kube username := suite.me.Username - mainKubeGroups := []string{testImpersonationGroup} + mainKubeGroups := []string{kube.TestImpersonationGroup} mainRole, err := types.NewRoleV3("main-kube", types.RoleSpecV5{ Allow: types.RoleConditions{ Logins: []string{username}, @@ -836,17 +831,17 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) { require.True(t, upsertSuccess) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // impersonating client requests will be denied - impersonatingProxyClient, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: main, - username: username, - kubeGroups: mainKubeGroups, - impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}}, + impersonatingProxyClient, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: main, + Username: username, + KubeGroups: mainKubeGroups, + Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}}, }) require.NoError(t, err) @@ -855,10 +850,10 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) { require.Error(t, err) // set up kube configuration using main proxy - proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: main, - username: username, - kubeGroups: mainKubeGroups, + proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: main, + Username: username, + KubeGroups: mainKubeGroups, }) require.NoError(t, err) @@ -1021,7 +1016,7 @@ func runKubeDisconnectTest(t *testing.T, suite *KubeSuite, tc disconnectTestCase }) username := suite.me.Username - kubeGroups := []string{testImpersonationGroup} + kubeGroups := []string{kube.TestImpersonationGroup} role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{ Options: tc.options, Allow: types.RoleConditions{ @@ -1040,10 +1035,10 @@ func runKubeDisconnectTest(t *testing.T, suite *KubeSuite, tc disconnectTestCase defer teleport.StopAll() // set up kube configuration using proxy - proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeGroups: kubeGroups, + proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeGroups: kubeGroups, }) require.NoError(t, err) @@ -1107,7 +1102,7 @@ func testKubeTransportProtocol(t *testing.T, suite *KubeSuite) { }) username := suite.me.Username - kubeGroups := []string{testImpersonationGroup} + kubeGroups := []string{kube.TestImpersonationGroup} role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{ Allow: types.RoleConditions{ Logins: []string{username}, @@ -1125,10 +1120,10 @@ func testKubeTransportProtocol(t *testing.T, suite *KubeSuite) { defer teleport.StopAll() // set up kube configuration using proxy - proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: username, - kubeGroups: kubeGroups, + proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: username, + KubeGroups: kubeGroups, }) require.NoError(t, err) @@ -1227,20 +1222,9 @@ func tlsClientConfig(cfg *rest.Config) (*tls.Config, error) { return tlsConfig, nil } -type kubeProxyConfig struct { - t *helpers.TeleInstance - username string - kubeUsers []string - kubeGroups []string - impersonation *rest.ImpersonationConfig - routeToCluster string - customTLSServerName string - targetAddress utils.NetAddr -} - -func kubeProxyTLSConfig(cfg kubeProxyConfig) (*tls.Config, error) { +func kubeProxyTLSConfig(cfg kube.ProxyConfig) (*tls.Config, error) { tlsConfig := &tls.Config{} - _, kubeConfig, err := kubeProxyClient(cfg) + _, kubeConfig, err := kube.ProxyClient(cfg) if err != nil { return nil, trace.Wrap(err) } @@ -1262,94 +1246,6 @@ func kubeProxyTLSConfig(cfg kubeProxyConfig) (*tls.Config, error) { return tlsConfig, nil } -// kubeProxyClient returns kubernetes client using local teleport proxy -func kubeProxyClient(cfg kubeProxyConfig) (*kubernetes.Clientset, *rest.Config, error) { - authServer := cfg.t.Process.GetAuthServer() - clusterName, err := authServer.GetClusterName() - if err != nil { - return nil, nil, trace.Wrap(err) - } - - // Fetch user info to get roles and max session TTL. - user, err := authServer.GetUser(cfg.username, false) - if err != nil { - return nil, nil, trace.Wrap(err) - } - roles, err := services.FetchRoles(user.GetRoles(), authServer, user.GetTraits()) - if err != nil { - return nil, nil, trace.Wrap(err) - } - ttl := roles.AdjustSessionTTL(10 * time.Minute) - - ca, err := authServer.GetCertAuthority(context.Background(), types.CertAuthID{ - Type: types.HostCA, - DomainName: clusterName.GetClusterName(), - }, true) - if err != nil { - return nil, nil, trace.Wrap(err) - } - caCert, signer, err := authServer.GetKeyStore().GetTLSCertAndSigner(ca) - if err != nil { - return nil, nil, trace.Wrap(err) - } - tlsCA, err := tlsca.FromCertAndSigner(caCert, signer) - if err != nil { - return nil, nil, trace.Wrap(err) - } - privPEM, _, err := native.GenerateKeyPair() - if err != nil { - return nil, nil, trace.Wrap(err) - } - priv, err := tlsca.ParsePrivateKeyPEM(privPEM) - if err != nil { - return nil, nil, trace.Wrap(err) - } - - id := tlsca.Identity{ - Username: cfg.username, - Groups: user.GetRoles(), - KubernetesUsers: cfg.kubeUsers, - KubernetesGroups: cfg.kubeGroups, - RouteToCluster: cfg.routeToCluster, - } - subj, err := id.Subject() - if err != nil { - return nil, nil, trace.Wrap(err) - } - cert, err := tlsCA.GenerateCertificate(tlsca.CertificateRequest{ - Clock: authServer.GetClock(), - PublicKey: priv.Public(), - Subject: subj, - NotAfter: authServer.GetClock().Now().Add(ttl), - }) - if err != nil { - return nil, nil, trace.Wrap(err) - } - - tlsClientConfig := rest.TLSClientConfig{ - CAData: ca.GetActiveKeys().TLS[0].Cert, - CertData: cert, - KeyData: privPEM, - ServerName: cfg.customTLSServerName, - } - config := &rest.Config{ - Host: "https://" + cfg.t.Config.Proxy.Kube.ListenAddr.Addr, - TLSClientConfig: tlsClientConfig, - } - if !cfg.targetAddress.IsEmpty() { - config.Host = "https://" + cfg.targetAddress.Addr - } - if cfg.impersonation != nil { - config.Impersonate = *cfg.impersonation - } - client, err := kubernetes.NewForConfig(config) - if err != nil { - return nil, nil, trace.Wrap(err) - } - - return client, config, nil -} - const ( testNamespace = "teletest" testPod = "test-pod" @@ -1481,7 +1377,7 @@ func kubeExec(kubeConfig *rest.Config, args kubeExecArgs) error { return executor.Stream(opts) } -func kubeJoin(kubeConfig kubeProxyConfig, tc *client.TeleportClient, sessionID string) (*client.KubeSession, error) { +func kubeJoin(kubeConfig kube.ProxyConfig, tc *client.TeleportClient, sessionID string) (*client.KubeSession, error) { tlsConfig, err := kubeProxyTLSConfig(kubeConfig) if err != nil { return nil, trace.Wrap(err) @@ -1494,7 +1390,7 @@ func kubeJoin(kubeConfig kubeProxyConfig, tc *client.TeleportClient, sessionID s return nil, trace.Wrap(err) } - sess, err := client.NewKubeSession(context.TODO(), tc, meta, kubeConfig.t.Config.Proxy.Kube.ListenAddr.Addr, "", types.SessionPeerMode, tlsConfig) + sess, err := client.NewKubeSession(context.TODO(), tc, meta, kubeConfig.T.Config.Proxy.Kube.ListenAddr.Addr, "", types.SessionPeerMode, tlsConfig) if err != nil { return nil, trace.Wrap(err) } @@ -1517,7 +1413,7 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) { hostUsername := suite.me.Username participantUsername := suite.me.Username + "-participant" - kubeGroups := []string{testImpersonationGroup} + kubeGroups := []string{kube.TestImpersonationGroup} kubeUsers := []string{"alice@example.com"} role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{ Allow: types.RoleConditions{ @@ -1551,11 +1447,11 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) { ctx := context.Background() // set up kube configuration using proxy - proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{ - t: teleport, - username: hostUsername, - kubeUsers: kubeUsers, - kubeGroups: kubeGroups, + proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{ + T: teleport, + Username: hostUsername, + KubeUsers: kubeUsers, + KubeGroups: kubeGroups, }) require.NoError(t, err) @@ -1595,11 +1491,11 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) { tc.Stdin = participantStdinR tc.Stdout = participantStdoutW - stream, err := kubeJoin(kubeProxyConfig{ - t: teleport, - username: participantUsername, - kubeUsers: kubeUsers, - kubeGroups: kubeGroups, + stream, err := kubeJoin(kube.ProxyConfig{ + T: teleport, + Username: participantUsername, + KubeUsers: kubeUsers, + KubeGroups: kubeGroups, }, tc, "") require.NoError(t, err) defer stream.Close() diff --git a/integration/proxy/main_test.go b/integration/proxy/main_test.go new file mode 100644 index 00000000000..827524c36cf --- /dev/null +++ b/integration/proxy/main_test.go @@ -0,0 +1,28 @@ +/* +Copyright 2022 Gravitational, Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package proxy + +import ( + "testing" + + "github.com/gravitational/teleport/integration/helpers" +) + +// TestMain will re-execute Teleport to run a command if "exec" is passed to +// it as an argument. Otherwise, it will run tests as normal. +func TestMain(m *testing.M) { + helpers.TestMainImplementation(m) +} diff --git a/integration/proxy_helpers_test.go b/integration/proxy/proxy_helpers.go similarity index 80% rename from integration/proxy_helpers_test.go rename to integration/proxy/proxy_helpers.go index c60741c9acb..fbecf93e59d 100644 --- a/integration/proxy_helpers_test.go +++ b/integration/proxy/proxy_helpers.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration +package proxy import ( "bytes" @@ -25,7 +25,6 @@ import ( "net" "net/http" "net/http/httptest" - "os/user" "path/filepath" "testing" "time" @@ -36,6 +35,7 @@ import ( apiutils "github.com/gravitational/teleport/api/utils" "github.com/gravitational/teleport/integration/helpers" "github.com/gravitational/teleport/lib/defaults" + "github.com/gravitational/teleport/lib/reversetunnel" "github.com/gravitational/teleport/lib/service" "github.com/gravitational/teleport/lib/services" "github.com/gravitational/teleport/lib/srv/alpnproxy" @@ -52,14 +52,14 @@ import ( clientcmdapi "k8s.io/client-go/tools/clientcmd/api" ) -type ProxySuite struct { +type Suite struct { root *helpers.TeleInstance leaf *helpers.TeleInstance } -type proxySuiteOptions struct { - rootConfigFunc func(suite *ProxySuite) *service.Config - leafConfigFunc func(suite *ProxySuite) *service.Config +type suiteOptions struct { + rootConfigFunc func(suite *Suite) *service.Config + leafConfigFunc func(suite *Suite) *service.Config rootConfigModFunc []func(config *service.Config) leafConfigModFunc []func(config *service.Config) @@ -70,22 +70,22 @@ type proxySuiteOptions struct { rootClusterListeners helpers.InstanceListenerSetupFunc leafClusterListeners helpers.InstanceListenerSetupFunc - rootTrustedSecretFunc func(suite *ProxySuite) []*helpers.InstanceSecrets - leafTrustedFunc func(suite *ProxySuite) []*helpers.InstanceSecrets + rootTrustedSecretFunc func(suite *Suite) []*helpers.InstanceSecrets + leafTrustedFunc func(suite *Suite) []*helpers.InstanceSecrets rootClusterRoles []types.Role leafClusterRoles []types.Role - updateRoleMappingFunc func(t *testing.T, suite *ProxySuite) + updateRoleMappingFunc func(t *testing.T, suite *Suite) trustedCluster types.TrustedCluster } -func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite { - options := proxySuiteOptions{ - rootClusterNodeName: Host, - leafClusterNodeName: Host, - rootClusterListeners: helpers.SingleProxyPortSetupOn(Host), - leafClusterListeners: helpers.SingleProxyPortSetupOn(Host), +func newSuite(t *testing.T, opts ...proxySuiteOptionsFunc) *Suite { + options := suiteOptions{ + rootClusterNodeName: helpers.Host, + leafClusterNodeName: helpers.Host, + rootClusterListeners: helpers.SingleProxyPortSetupOn(helpers.Host), + leafClusterListeners: helpers.SingleProxyPortSetupOn(helpers.Host), } for _, opt := range opts { opt(&options) @@ -111,12 +111,12 @@ func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite { } lCfg.Listeners = options.leafClusterListeners(t, &lCfg.Fds) lc := helpers.NewInstance(t, lCfg) - suite := &ProxySuite{ + suite := &Suite{ root: rc, leaf: lc, } - user := mustGetCurrentUser(t) + user := helpers.MustGetCurrentUser(t) for _, role := range options.rootClusterRoles { rc.AddUserWithRole(user.Username, role) } @@ -162,14 +162,14 @@ func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite { } if options.trustedCluster != nil { - tryCreateTrustedCluster(t, suite.leaf.Process.GetAuthServer(), options.trustedCluster) - waitForTunnelConnections(t, suite.root.Process.GetAuthServer(), suite.leaf.Secrets.SiteName, 1) + helpers.TryCreateTrustedCluster(t, suite.leaf.Process.GetAuthServer(), options.trustedCluster) + helpers.WaitForTunnelConnections(t, suite.root.Process.GetAuthServer(), suite.leaf.Secrets.SiteName, 1) } return suite } -func (p *ProxySuite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname string) { +func (p *Suite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname string) { nodeConfig := func() *service.Config { tconf := service.MakeDefaultConfig() tconf.Console = nil @@ -192,17 +192,17 @@ func (p *ProxySuite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname strin require.NoError(t, err) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(p.root.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(p.root.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(p.leaf.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(p.leaf.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") // Wait for both nodes to show up before attempting to dial to them. - err = waitForNodeCount(context.Background(), p.root, p.leaf.Secrets.SiteName, 2) + err = helpers.WaitForNodeCount(context.Background(), p.root, p.leaf.Secrets.SiteName, 2) require.NoError(t, err) } -func (p *ProxySuite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config helpers.ClientConfig) { +func (p *Suite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config helpers.ClientConfig) { const ( deadline = time.Second * 5 nextIterWaitTime = time.Millisecond * 100 @@ -225,73 +225,72 @@ func (p *ProxySuite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config h require.Equal(t, "hello world\n", output.String()) } -type proxySuiteOptionsFunc func(*proxySuiteOptions) +type proxySuiteOptionsFunc func(*suiteOptions) func withRootClusterRoles(roles ...types.Role) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { options.rootClusterRoles = roles } } func withLeafClusterRoles(roles ...types.Role) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { options.leafClusterRoles = roles } } func withRootAndLeafClusterRoles(roles ...types.Role) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { withRootClusterRoles(roles...)(options) withLeafClusterRoles(roles...)(options) - } } -func withLeafClusterConfig(fn func(suite *ProxySuite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { +func withLeafClusterConfig(fn func(suite *Suite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc { + return func(options *suiteOptions) { options.leafConfigFunc = fn options.leafConfigModFunc = append(options.leafConfigModFunc, configModFunctions...) } } -func withRootClusterConfig(fn func(suite *ProxySuite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { +func withRootClusterConfig(fn func(suite *Suite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc { + return func(options *suiteOptions) { options.rootConfigFunc = fn options.rootConfigModFunc = append(options.rootConfigModFunc, configModFunctions...) } } func withRootAndLeafTrustedClusterReset() proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { - options.rootTrustedSecretFunc = func(suite *ProxySuite) []*helpers.InstanceSecrets { + return func(options *suiteOptions) { + options.rootTrustedSecretFunc = func(suite *Suite) []*helpers.InstanceSecrets { return nil } - options.leafTrustedFunc = func(suite *ProxySuite) []*helpers.InstanceSecrets { + options.leafTrustedFunc = func(suite *Suite) []*helpers.InstanceSecrets { return nil } } } func withRootClusterNodeName(nodeName string) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { options.rootClusterNodeName = nodeName } } func withLeafClusterNodeName(nodeName string) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { options.leafClusterNodeName = nodeName } } func withRootClusterListeners(fn helpers.InstanceListenerSetupFunc) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { options.rootClusterListeners = fn } } func withLeafClusterListeners(fn helpers.InstanceListenerSetupFunc) proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { + return func(options *suiteOptions) { options.leafClusterListeners = fn } } @@ -306,8 +305,8 @@ func newRole(t *testing.T, roleName string, username string) types.Role { return role } -func rootClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Config { - return func(suite *ProxySuite) *service.Config { +func rootClusterStandardConfig(t *testing.T) func(suite *Suite) *service.Config { + return func(suite *Suite) *service.Config { rc := suite.root config := service.MakeDefaultConfig() config.DataDir = t.TempDir() @@ -326,8 +325,8 @@ func rootClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Co } } -func leafClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Config { - return func(suite *ProxySuite) *service.Config { +func leafClusterStandardConfig(t *testing.T) func(suite *Suite) *service.Config { + return func(suite *Suite) *service.Config { lc := suite.leaf config := service.MakeDefaultConfig() config.DataDir = t.TempDir() @@ -355,11 +354,11 @@ func createTestRole(username string) types.Role { } func withStandardRoleMapping() proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { - options.updateRoleMappingFunc = func(t *testing.T, suite *ProxySuite) { + return func(options *suiteOptions) { + options.updateRoleMappingFunc = func(t *testing.T, suite *Suite) { rc := suite.root lc := suite.leaf - role := suite.root.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0] + role := suite.root.Secrets.Users[helpers.MustGetCurrentUser(t).Username].Roles[0] ca, err := lc.Process.GetAuthServer().GetCertAuthority(context.Background(), types.CertAuthID{ Type: types.UserCA, DomainName: rc.Secrets.SiteName, @@ -375,11 +374,11 @@ func withStandardRoleMapping() proxySuiteOptionsFunc { } func withTrustedCluster() proxySuiteOptionsFunc { - return func(options *proxySuiteOptions) { - options.updateRoleMappingFunc = func(t *testing.T, suite *ProxySuite) { + return func(options *suiteOptions) { + options.updateRoleMappingFunc = func(t *testing.T, suite *Suite) { root := suite.root - rootRole := suite.root.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0] - secondRole := suite.leaf.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0] + rootRole := suite.root.Secrets.Users[helpers.MustGetCurrentUser(t).Username].Roles[0] + secondRole := suite.leaf.Secrets.Users[helpers.MustGetCurrentUser(t).Username].Roles[0] trustedClusterToken := "trustedclustertoken" err := root.Process.GetAuthServer().UpsertToken(context.Background(), @@ -397,12 +396,6 @@ func withTrustedCluster() proxySuiteOptionsFunc { } } -func mustGetCurrentUser(t *testing.T) *user.User { - user, err := user.Current() - require.NoError(t, err) - return user -} - func mustRunPostgresQuery(t *testing.T, client *pgconn.PgConn) { result, err := client.Exec(context.Background(), "select 1").ReadAll() require.NoError(t, err) @@ -602,3 +595,14 @@ func mustStartMockALBProxy(t *testing.T, proxyAddr string) *mockAWSALBProxy { go m.serve(ctx, t) return m } + +// waitForActivePeerProxyConnections waits for remote cluster to report a minimum number of active proxy peer connections +func waitForActivePeerProxyConnections(t *testing.T, tunnel reversetunnel.Server, expectedCount int) { + require.Eventually(t, func() bool { + return tunnel.GetProxyPeerClient().GetConnectionsCount() >= expectedCount + }, + 30*time.Second, + time.Second, + "Peer proxy connections did not reach %v in the expected time frame", expectedCount, + ) +} diff --git a/integration/proxy_test.go b/integration/proxy/proxy_test.go similarity index 92% rename from integration/proxy_test.go rename to integration/proxy/proxy_test.go index ee7b3b24055..6b1fac0e0ef 100644 --- a/integration/proxy_test.go +++ b/integration/proxy/proxy_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration +package proxy import ( "bytes" @@ -36,7 +36,9 @@ import ( "github.com/gravitational/teleport/api/breaker" "github.com/gravitational/teleport/api/client" "github.com/gravitational/teleport/api/types" + "github.com/gravitational/teleport/integration/appaccess" "github.com/gravitational/teleport/integration/helpers" + "github.com/gravitational/teleport/integration/kube" "github.com/gravitational/teleport/lib" "github.com/gravitational/teleport/lib/auth/testauthority" "github.com/gravitational/teleport/lib/defaults" @@ -96,9 +98,9 @@ func TestALPNSNIProxyMultiCluster(t *testing.T) { lib.SetInsecureDevMode(true) defer lib.SetInsecureDevMode(false) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t), func(config *service.Config) { config.Proxy.DisableALPNSNIListener = tc.disableALPNListenerOnRoot }), @@ -114,14 +116,14 @@ func TestALPNSNIProxyMultiCluster(t *testing.T) { suite.mustConnectToClusterAndRunSSHCommand(t, helpers.ClientConfig{ Login: username, Cluster: suite.root.Secrets.SiteName, - Host: Loopback, + Host: helpers.Loopback, Port: helpers.Port(t, suite.root.SSH), }) // Run command in leaf. suite.mustConnectToClusterAndRunSSHCommand(t, helpers.ClientConfig{ Login: username, Cluster: suite.leaf.Secrets.SiteName, - Host: Loopback, + Host: helpers.Loopback, Port: helpers.Port(t, suite.leaf.SSH), }) }) @@ -170,9 +172,9 @@ func TestALPNSNIProxyTrustedClusterNode(t *testing.T) { lib.SetInsecureDevMode(true) defer lib.SetInsecureDevMode(false) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t)), withLeafClusterConfig(leafClusterStandardConfig(t)), withRootClusterListeners(tc.mainClusterListenerSetup), @@ -191,7 +193,7 @@ func TestALPNSNIProxyTrustedClusterNode(t *testing.T) { suite.mustConnectToClusterAndRunSSHCommand(t, helpers.ClientConfig{ Login: username, Cluster: suite.leaf.Secrets.SiteName, - Host: Loopback, + Host: helpers.Loopback, Port: helpers.Port(t, suite.leaf.SSH), }) @@ -219,7 +221,7 @@ func TestALPNSNIHTTPSProxy(t *testing.T) { require.NoError(t, err) t.Setenv("http_proxy", u.Host) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username // We need to use the non-loopback address for our Teleport cluster, as the // Go HTTP library will recognize requests to the loopback address and @@ -227,7 +229,7 @@ func TestALPNSNIHTTPSProxy(t *testing.T) { addr, err := helpers.GetLocalIP() require.NoError(t, err) - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t)), withLeafClusterConfig(leafClusterStandardConfig(t)), withRootClusterNodeName(addr), @@ -239,9 +241,9 @@ func TestALPNSNIHTTPSProxy(t *testing.T) { ) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") require.Greater(t, ph.Count(), 0, "proxy did not intercept any connection") @@ -260,7 +262,7 @@ func TestMultiPortHTTPSProxy(t *testing.T) { require.NoError(t, err) t.Setenv("http_proxy", u.Host) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username // We need to use the non-loopback address for our Teleport cluster, as the // Go HTTP library will recognize requests to the loopback address and @@ -268,7 +270,7 @@ func TestMultiPortHTTPSProxy(t *testing.T) { addr, err := helpers.GetLocalIP() require.NoError(t, err) - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t)), withLeafClusterConfig(leafClusterStandardConfig(t)), withRootClusterNodeName(addr), @@ -280,9 +282,9 @@ func TestMultiPortHTTPSProxy(t *testing.T) { ) // Wait for both cluster to see each other via reverse tunnels. - require.Eventually(t, waitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") - require.Eventually(t, waitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second, + require.Eventually(t, helpers.WaitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second, "Two clusters do not see each other: tunnels are not working.") require.Greater(t, ph.Count(), 0, "proxy did not intercept any connection") @@ -300,18 +302,18 @@ func TestALPNSNIProxyKube(t *testing.T) { kubeAPIMockSvr := startKubeAPIMock(t) kubeConfigPath := mustCreateKubeConfigFile(t, k8ClientConfig(kubeAPIMockSvr.URL, localK8SNI)) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username kubeRoleSpec := types.RoleSpecV5{ Allow: types.RoleConditions{ Logins: []string{username}, - KubeGroups: []string{testImpersonationGroup}, + KubeGroups: []string{kube.TestImpersonationGroup}, KubeUsers: []string{k8User}, }, } kubeRole, err := types.NewRoleV3(k8RoleName, kubeRoleSpec) require.NoError(t, err) - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t), func(config *service.Config) { config.Proxy.Kube.Enabled = true config.Proxy.Kube.KubeconfigPath = kubeConfigPath @@ -322,13 +324,13 @@ func TestALPNSNIProxyKube(t *testing.T) { withStandardRoleMapping(), ) - k8Client, _, err := kubeProxyClient(kubeProxyConfig{ - t: suite.root, - username: kubeRoleSpec.Allow.Logins[0], - kubeUsers: kubeRoleSpec.Allow.KubeGroups, - kubeGroups: kubeRoleSpec.Allow.KubeUsers, - customTLSServerName: localK8SNI, - targetAddress: suite.root.Config.Proxy.WebAddr, + k8Client, _, err := kube.ProxyClient(kube.ProxyConfig{ + T: suite.root, + Username: kubeRoleSpec.Allow.Logins[0], + KubeUsers: kubeRoleSpec.Allow.KubeGroups, + KubeGroups: kubeRoleSpec.Allow.KubeUsers, + CustomTLSServerName: localK8SNI, + TargetAddress: suite.root.Config.Proxy.WebAddr, }) require.NoError(t, err) @@ -352,18 +354,18 @@ func TestALPNSNIProxyKubeV2Leaf(t *testing.T) { kubeAPIMockSvr := startKubeAPIMock(t) kubeConfigPath := mustCreateKubeConfigFile(t, k8ClientConfig(kubeAPIMockSvr.URL, localK8SNI)) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username kubeRoleSpec := types.RoleSpecV5{ Allow: types.RoleConditions{ Logins: []string{username}, - KubeGroups: []string{testImpersonationGroup}, + KubeGroups: []string{kube.TestImpersonationGroup}, KubeUsers: []string{k8User}, }, } kubeRole, err := types.NewRoleV3(k8RoleName, kubeRoleSpec) require.NoError(t, err) - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t), func(config *service.Config) { config.Proxy.Kube.Enabled = true config.Version = defaults.TeleportConfigVersionV2 @@ -374,7 +376,8 @@ func TestALPNSNIProxyKubeV2Leaf(t *testing.T) { config.Kube.Enabled = true config.Kube.KubeconfigPath = kubeConfigPath - config.Kube.ListenAddr = utils.MustParseAddr(net.JoinHostPort(Loopback, helpers.NewPortStr())) + config.Kube.ListenAddr = utils.MustParseAddr( + helpers.NewListener(t, service.ListenerKube, &config.FileDescriptors)) }), withRootClusterRoles(kubeRole), withLeafClusterRoles(kubeRole), @@ -382,14 +385,14 @@ func TestALPNSNIProxyKubeV2Leaf(t *testing.T) { withTrustedCluster(), ) - k8Client, _, err := kubeProxyClient(kubeProxyConfig{ - t: suite.root, - username: kubeRoleSpec.Allow.Logins[0], - kubeUsers: kubeRoleSpec.Allow.KubeGroups, - kubeGroups: kubeRoleSpec.Allow.KubeUsers, - customTLSServerName: localK8SNI, - targetAddress: suite.root.Config.Proxy.WebAddr, - routeToCluster: suite.leaf.Secrets.SiteName, + k8Client, _, err := kube.ProxyClient(kube.ProxyConfig{ + T: suite.root, + Username: kubeRoleSpec.Allow.Logins[0], + KubeUsers: kubeRoleSpec.Allow.KubeGroups, + KubeGroups: kubeRoleSpec.Allow.KubeUsers, + CustomTLSServerName: localK8SNI, + TargetAddress: suite.root.Config.Proxy.WebAddr, + RouteToCluster: suite.leaf.Secrets.SiteName, }) require.NoError(t, err) @@ -755,24 +758,24 @@ func TestALPNSNIProxyDatabaseAccess(t *testing.T) { // TestALPNSNIProxyAppAccess tests application access via ALPN SNI proxy service. func TestALPNSNIProxyAppAccess(t *testing.T) { - pack := setupWithOptions(t, appTestOptions{ - rootClusterListeners: helpers.SingleProxyPortSetup, - leafClusterListeners: helpers.SingleProxyPortSetup, - rootConfig: func(config *service.Config) { + pack := appaccess.SetupWithOptions(t, appaccess.AppTestOptions{ + RootClusterListeners: helpers.SingleProxyPortSetup, + LeafClusterListeners: helpers.SingleProxyPortSetup, + RootConfig: func(config *service.Config) { config.Auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex) }, - leafConfig: func(config *service.Config) { + LeafConfig: func(config *service.Config) { config.Auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex) }, }) - sess := pack.createAppSession(t, pack.rootAppPublicAddr, pack.rootAppClusterName) - status, _, err := pack.makeRequest(sess, http.MethodGet, "/") + sess := pack.CreateAppSession(t, pack.RootAppPublicAddr(), pack.RootAppClusterName()) + status, _, err := pack.MakeRequest(sess, http.MethodGet, "/") require.NoError(t, err) require.Equal(t, http.StatusOK, status) - sess = pack.createAppSession(t, pack.leafAppPublicAddr, pack.leafAppClusterName) - status, _, err = pack.makeRequest(sess, http.MethodGet, "/") + sess = pack.CreateAppSession(t, pack.LeafAppPublicAddr(), pack.LeafAppClusterName()) + status, _, err = pack.MakeRequest(sess, http.MethodGet, "/") require.NoError(t, err) require.Equal(t, http.StatusOK, status) } @@ -783,9 +786,9 @@ func TestALPNProxyRootLeafAuthDial(t *testing.T) { lib.SetInsecureDevMode(true) defer lib.SetInsecureDevMode(false) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username - suite := newProxySuite(t, + suite := newSuite(t, withRootClusterConfig(rootClusterStandardConfig(t)), withLeafClusterConfig(leafClusterStandardConfig(t)), withRootClusterListeners(helpers.SingleProxyPortSetup), @@ -834,7 +837,7 @@ func TestALPNProxyAuthClientConnectWithUserIdentity(t *testing.T) { cfg := helpers.InstanceConfig{ ClusterName: "root.example.com", HostID: uuid.New().String(), - NodeName: Loopback, + NodeName: helpers.Loopback, Log: utils.NewLoggerForTests(), } cfg.Listeners = helpers.SingleProxyPortSetup(t, &cfg.Fds) @@ -851,7 +854,7 @@ func TestALPNProxyAuthClientConnectWithUserIdentity(t *testing.T) { rcConf.Version = "v2" rcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig() - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username rc.AddUser(username, []string{username}) err := rc.CreateEx(t, nil, rcConf) @@ -889,14 +892,14 @@ func TestALPNProxyDialProxySSHWithoutInsecureMode(t *testing.T) { rootCfg := helpers.InstanceConfig{ ClusterName: "root.example.com", HostID: uuid.New().String(), - NodeName: Loopback, + NodeName: helpers.Loopback, Priv: privateKey, Pub: publicKey, Log: utils.NewLoggerForTests(), } rootCfg.Listeners = helpers.StandardListenerSetup(t, &rootCfg.Fds) rc := helpers.NewInstance(t, rootCfg) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username rc.AddUser(username, []string{username}) // Make root cluster config. @@ -966,7 +969,7 @@ func TestALPNProxyHTTPProxyNoProxyDial(t *testing.T) { } instanceCfg.Listeners = helpers.SingleProxyPortSetupOn(addr)(t, &instanceCfg.Fds) rc := helpers.NewInstance(t, instanceCfg) - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username rc.AddUser(username, []string{username}) rcConf := service.MakeDefaultConfig() @@ -1007,7 +1010,7 @@ func TestALPNProxyHTTPProxyNoProxyDial(t *testing.T) { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() - err = waitForNodeCount(ctx, rc, "root.example.com", 1) + err = helpers.WaitForNodeCount(ctx, rc, "root.example.com", 1) require.NoError(t, err) require.Zero(t, ph.Count()) @@ -1017,7 +1020,7 @@ func TestALPNProxyHTTPProxyNoProxyDial(t *testing.T) { require.NoError(t, os.Unsetenv("no_proxy")) _, err = rc.StartNode(makeNodeConfig("second-root-node", rcProxyAddr)) require.NoError(t, err) - err = waitForNodeCount(ctx, rc, "root.example.com", 2) + err = helpers.WaitForNodeCount(ctx, rc, "root.example.com", 2) require.NoError(t, err) require.NotZero(t, ph.Count()) @@ -1048,7 +1051,7 @@ func TestALPNProxyHTTPProxyBasicAuthDial(t *testing.T) { rc := helpers.NewInstance(t, cfg) log.Info("Teleport root cluster instance created") - username := mustGetCurrentUser(t).Username + username := helpers.MustGetCurrentUser(t).Username rc.AddUser(username, []string{username}) rcConf := service.MakeDefaultConfig() @@ -1109,7 +1112,7 @@ func TestALPNProxyHTTPProxyBasicAuthDial(t *testing.T) { t.Setenv("http_proxy", helpers.MakeProxyAddr(validUser, validPass, proxyURL.Host)) _, err = rc.StartNode(makeNodeConfig("third-root-node", rcProxyAddr)) require.NoError(t, err) - err = waitForNodeCount(ctx, rc, "root.example.com", 1) + err = helpers.WaitForNodeCount(ctx, rc, "root.example.com", 1) require.NoError(t, err) require.NoError(t, authorizer.LastError()) require.NotZero(t, ph.Count()) diff --git a/integration/proxy_tunnel_strategy_test.go b/integration/proxy/proxy_tunnel_strategy_test.go similarity index 93% rename from integration/proxy_tunnel_strategy_test.go rename to integration/proxy/proxy_tunnel_strategy_test.go index 55f24452102..b8bf11aa7b8 100644 --- a/integration/proxy_tunnel_strategy_test.go +++ b/integration/proxy/proxy_tunnel_strategy_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package integration +package proxy import ( "bytes" @@ -65,7 +65,7 @@ type proxyTunnelStrategy struct { func newProxyTunnelStrategy(t *testing.T, cluster string, strategy *types.TunnelStrategyV1) *proxyTunnelStrategy { p := &proxyTunnelStrategy{ cluster: cluster, - username: mustGetCurrentUser(t).Username, + username: helpers.MustGetCurrentUser(t).Username, strategy: strategy, log: utils.NewLoggerForTests(), } @@ -100,8 +100,8 @@ func testProxyTunnelStrategyAgentMesh(t *testing.T) { p.makeNode(t) // wait for the node to be connected to both proxies - waitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1) - waitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1) + helpers.WaitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1) + helpers.WaitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1) // make sure we can connect to the node going through any proxy. p.waitForNodeToBeReachable(t) @@ -114,8 +114,8 @@ func testProxyTunnelStrategyAgentMesh(t *testing.T) { p.makeDatabase(t) // wait for the node to be connected to both proxies - waitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1) - waitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1) + helpers.WaitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1) + helpers.WaitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1) // make sure we can connect to the database going through any proxy. p.waitForDatabaseToBeReachable(t) @@ -184,7 +184,7 @@ func testProxyTunnelStrategyProxyPeering(t *testing.T) { p.makeDatabase(t) // wait for the node and db to open reverse tunnels to the first proxy. - waitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 2) + helpers.WaitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 2) // bootstrap the second proxy instance after the node and db have already // established reverse tunnels to the first proxy. @@ -268,7 +268,9 @@ func (p *proxyTunnelStrategy) makeLoadBalancer(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - lbAddr := utils.MustParseAddr(net.JoinHostPort(Loopback, helpers.NewPortStr())) + // TODO(tcsc): fix ports before merging + lbAddr := utils.MustParseAddr(net.JoinHostPort(helpers.Loopback, "0")) + //lbAddr := utils.MustParseAddr(net.JoinHostPort(helpers.Loopback, helpers.NewPortStr())) lb, err := utils.NewLoadBalancer(ctx, *lbAddr) require.NoError(t, err) @@ -293,7 +295,7 @@ func (p *proxyTunnelStrategy) makeAuth(t *testing.T) { auth := helpers.NewInstance(t, helpers.InstanceConfig{ ClusterName: p.cluster, HostID: uuid.New().String(), - NodeName: Loopback, + NodeName: helpers.Loopback, Priv: privateKey, Pub: publicKey, Log: utils.NewLoggerForTests(), @@ -321,7 +323,7 @@ func (p *proxyTunnelStrategy) makeProxy(t *testing.T) { proxy := helpers.NewInstance(t, helpers.InstanceConfig{ ClusterName: p.cluster, HostID: uuid.New().String(), - NodeName: Loopback, + NodeName: helpers.Loopback, Log: utils.NewLoggerForTests(), }) @@ -335,12 +337,11 @@ func (p *proxyTunnelStrategy) makeProxy(t *testing.T) { conf.Auth.Enabled = false conf.SSH.Enabled = false - // TODO: Replace old-style NewPortStr() call with preconfigured listener conf.Proxy.Enabled = true conf.Proxy.ReverseTunnelListenAddr.Addr = proxy.ReverseTunnel conf.Proxy.SSHAddr.Addr = proxy.SSHProxy conf.Proxy.WebAddr.Addr = proxy.Web - conf.Proxy.PeerAddr.Addr = net.JoinHostPort(Loopback, helpers.NewPortStr()) + conf.Proxy.PeerAddr.Addr = helpers.NewListenerOn(t, helpers.Loopback, service.ListenerProxyPeer, &proxy.Fds) conf.Proxy.PeerPublicAddr = conf.Proxy.PeerAddr conf.Proxy.PublicAddrs = append(conf.Proxy.PublicAddrs, utils.FromAddr(p.lb.Addr())) conf.Proxy.DisableWebInterface = true @@ -367,7 +368,7 @@ func (p *proxyTunnelStrategy) makeNode(t *testing.T) { node := helpers.NewInstance(t, helpers.InstanceConfig{ ClusterName: p.cluster, HostID: uuid.New().String(), - NodeName: Loopback, + NodeName: helpers.Loopback, Log: utils.NewLoggerForTests(), }) @@ -397,19 +398,19 @@ func (p *proxyTunnelStrategy) makeDatabase(t *testing.T) { require.Fail(t, "database already initialized") } - dbListener, err := net.Listen("tcp", net.JoinHostPort(Host, "0")) + dbListener, err := net.Listen("tcp", net.JoinHostPort(helpers.Host, "0")) require.NoError(t, err) _, portStr, err := net.SplitHostPort(dbListener.Addr().String()) require.NoError(t, err) - dbAddr := net.JoinHostPort(Host, portStr) + dbAddr := net.JoinHostPort(helpers.Host, portStr) // setup database service db := helpers.NewInstance(t, helpers.InstanceConfig{ ClusterName: p.cluster, HostID: uuid.New().String(), - NodeName: Loopback, + NodeName: helpers.Loopback, Log: utils.NewLoggerForTests(), })