Split out appaccess and proxy integration tests (#16232)

* Proxy tests running

* rollback

* whitespace  fix

* Rollback port fix

* Linter appeasement

* License fix

* Update signals.go

integration/appaccess/appaccess_test.go

@@ -0,0 +1,679 @@
+// Copyright 2022 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package appaccess
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/go-cmp/cmp"
+	"github.com/google/go-cmp/cmp/cmpopts"
+	"github.com/google/uuid"
+	"github.com/gravitational/oxy/forward"
+	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/api/types"
+	apievents "github.com/gravitational/teleport/api/types/events"
+	"github.com/gravitational/teleport/integration/helpers"
+	"github.com/gravitational/teleport/lib/events"
+	"github.com/gravitational/teleport/lib/service"
+	"github.com/gravitational/teleport/lib/web/app"
+	"github.com/gravitational/trace"
+	"github.com/stretchr/testify/require"
+)
+
+// TestAppAccess runs the full application access integration test suite.
+//
+// It allows to make the entire cluster set up once, instead of per test,
+// which speeds things up significantly.
+func TestAppAccess(t *testing.T) {
+	pack := Setup(t)
+
+	t.Run("Forward", bind(pack, testForward))
+	t.Run("Websockets", bind(pack, testWebsockets))
+	t.Run("ClientCert", bind(pack, testClientCert))
+	t.Run("Flush", bind(pack, testFlush))
+	t.Run("ForwardModes", bind(pack, testForwardModes))
+	t.Run("RewriteHeadersRoot", bind(pack, testRewriteHeadersRoot))
+	t.Run("RewriteHeadersLeaf", bind(pack, testRewriteHeadersLeaf))
+	t.Run("Logout", bind(pack, testLogout))
+	t.Run("JWT", bind(pack, testJWT))
+	t.Run("NoHeaderOverrides", bind(pack, testNoHeaderOverrides))
+	t.Run("AuditEvents", bind(pack, testAuditEvents))
+	t.Run("TestAppInvalidateAppSessionsOnLogout", bind(pack, testInvalidateAppSessionsOnLogout))
+
+	// This test should go last because it stops/starts app servers.
+	t.Run("TestAppServersHA", bind(pack, testServersHA))
+}
+
+// testForward tests that requests get forwarded to the target application
+// within a single cluster and trusted cluster.
+func testForward(p *Pack, t *testing.T) {
+	tests := []struct {
+		desc          string
+		inCookie      string
+		outStatusCode int
+		outMessage    string
+	}{
+		{
+			desc:          "root cluster, valid application session cookie, success",
+			inCookie:      p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName),
+			outStatusCode: http.StatusOK,
+			outMessage:    p.rootMessage,
+		},
+		{
+			desc:          "leaf cluster, valid application session cookie, success",
+			inCookie:      p.CreateAppSession(t, p.leafAppPublicAddr, p.leafAppClusterName),
+			outStatusCode: http.StatusOK,
+			outMessage:    p.leafMessage,
+		},
+		{
+			desc:          "invalid application session cookie, redirect to login",
+			inCookie:      "D25C463CD27861559CC6A0A6AE54818079809AA8731CB18037B4B37A80C4FC6C",
+			outStatusCode: http.StatusFound,
+			outMessage:    "",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			tt := tt
+			status, body, err := p.MakeRequest(tt.inCookie, http.MethodGet, "/")
+			require.NoError(t, err)
+			require.Equal(t, tt.outStatusCode, status)
+			require.Contains(t, body, tt.outMessage)
+		})
+	}
+}
+
+// TestWebsockets makes sure that websocket requests get forwarded.
+func testWebsockets(p *Pack, t *testing.T) {
+	tests := []struct {
+		desc       string
+		inCookie   string
+		outMessage string
+		err        error
+	}{
+		{
+			desc:       "root cluster, valid application session cookie, successful websocket (ws://) request",
+			inCookie:   p.CreateAppSession(t, p.rootWSPublicAddr, p.rootAppClusterName),
+			outMessage: p.rootWSMessage,
+		},
+		{
+			desc:       "root cluster, valid application session cookie, successful secure websocket (wss://) request",
+			inCookie:   p.CreateAppSession(t, p.rootWSSPublicAddr, p.rootAppClusterName),
+			outMessage: p.rootWSSMessage,
+		},
+		{
+			desc:       "leaf cluster, valid application session cookie, successful websocket (ws://) request",
+			inCookie:   p.CreateAppSession(t, p.leafWSPublicAddr, p.leafAppClusterName),
+			outMessage: p.leafWSMessage,
+		},
+		{
+			desc:       "leaf cluster, valid application session cookie, successful secure websocket (wss://) request",
+			inCookie:   p.CreateAppSession(t, p.leafWSSPublicAddr, p.leafAppClusterName),
+			outMessage: p.leafWSSMessage,
+		},
+		{
+			desc:     "invalid application session cookie, websocket request fails to dial",
+			inCookie: "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
+			err:      errors.New(""),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			tt := tt
+			body, err := p.makeWebsocketRequest(tt.inCookie, "/")
+			if tt.err != nil {
+				require.IsType(t, tt.err, trace.Unwrap(err))
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tt.outMessage, body)
+			}
+		})
+	}
+}
+
+// testForwardModes ensures that requests are forwarded to applications
+// even when the cluster is in proxy recording mode.
+func testForwardModes(p *Pack, t *testing.T) {
+	// Create cluster, user, sessions, and credentials package.
+	ctx := context.Background()
+
+	// Update root and leaf clusters to record sessions at the proxy.
+	recConfig, err := types.NewSessionRecordingConfigFromConfigFile(types.SessionRecordingConfigSpecV2{
+		Mode: types.RecordAtProxy,
+	})
+	require.NoError(t, err)
+	err = p.rootCluster.Process.GetAuthServer().SetSessionRecordingConfig(ctx, recConfig)
+	require.NoError(t, err)
+	err = p.leafCluster.Process.GetAuthServer().SetSessionRecordingConfig(ctx, recConfig)
+	require.NoError(t, err)
+
+	// Requests to root and leaf cluster are successful.
+	tests := []struct {
+		desc          string
+		inCookie      string
+		outStatusCode int
+		outMessage    string
+	}{
+		{
+			desc:          "root cluster, valid application session cookie, success",
+			inCookie:      p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName),
+			outStatusCode: http.StatusOK,
+			outMessage:    p.rootMessage,
+		},
+		{
+			desc:          "leaf cluster, valid application session cookie, success",
+			inCookie:      p.CreateAppSession(t, p.leafAppPublicAddr, p.leafAppClusterName),
+			outStatusCode: http.StatusOK,
+			outMessage:    p.leafMessage,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			tt := tt
+			status, body, err := p.MakeRequest(tt.inCookie, http.MethodGet, "/")
+			require.NoError(t, err)
+			require.Equal(t, tt.outStatusCode, status)
+			require.Contains(t, body, tt.outMessage)
+		})
+	}
+}
+
+// testClientCert tests mutual TLS authentication flow with application
+// access typically used in CLI by curl and other clients.
+func testClientCert(p *Pack, t *testing.T) {
+	tests := []struct {
+		desc          string
+		inTLSConfig   *tls.Config
+		outStatusCode int
+		outMessage    string
+	}{
+		{
+			desc:          "root cluster, valid TLS config, success",
+			inTLSConfig:   p.makeTLSConfig(t, p.rootAppPublicAddr, p.rootAppClusterName),
+			outStatusCode: http.StatusOK,
+			outMessage:    p.rootMessage,
+		},
+		{
+			desc:          "leaf cluster, valid TLS config, success",
+			inTLSConfig:   p.makeTLSConfig(t, p.leafAppPublicAddr, p.leafAppClusterName),
+			outStatusCode: http.StatusOK,
+			outMessage:    p.leafMessage,
+		},
+		{
+			desc:          "root cluster, invalid session ID",
+			inTLSConfig:   p.makeTLSConfigNoSession(t, p.rootAppPublicAddr, p.rootAppClusterName),
+			outStatusCode: http.StatusFound,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.desc, func(t *testing.T) {
+			tt := tt
+			status, body, err := p.makeRequestWithClientCert(tt.inTLSConfig, http.MethodGet, "/")
+			require.NoError(t, err)
+			require.Equal(t, tt.outStatusCode, status)
+			require.Contains(t, body, tt.outMessage)
+		})
+	}
+}
+
+// appAccessFlush makes sure that application access periodically flushes
+// buffered data to the response.
+func testFlush(p *Pack, t *testing.T) {
+	req, err := http.NewRequest("GET", p.assembleRootProxyURL("/"), nil)
+	require.NoError(t, err)
+
+	cookie := p.CreateAppSession(t, p.flushAppPublicAddr, p.flushAppClusterName)
+	req.AddCookie(&http.Cookie{
+		Name:  app.CookieName,
+		Value: cookie,
+	})
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+
+	// The "flush server" will send 2 messages, "hello" and "world", with a
+	// 500ms delay between them. They should arrive as 2 different frames
+	// due to the periodic flushing.
+	frames := []string{"hello", "world"}
+	for _, frame := range frames {
+		buffer := make([]byte, 1024)
+		n, err := resp.Body.Read(buffer)
+		if err != nil {
+			require.ErrorIs(t, err, io.EOF)
+		}
+		require.Equal(t, frame, strings.TrimSpace(string(buffer[:n])))
+	}
+}
+
+// testRewriteHeadersRoot validates that http headers from application
+// rewrite configuration are correctly passed to proxied applications in root.
+func testRewriteHeadersRoot(p *Pack, t *testing.T) {
+	// Create an application session for dumper app in root cluster.
+	appCookie := p.CreateAppSession(t, "dumper-root.example.com", "example.com")
+
+	// Get headers response and make sure headers were passed.
+	status, resp, err := p.MakeRequest(appCookie, http.MethodGet, "/", service.Header{
+		Name: "X-Existing", Value: "existing",
+	})
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+
+	// Dumper app just dumps HTTP request so we should be able to read it back.
+	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(resp)))
+	require.NoError(t, err)
+	require.Equal(t, req.Host, "example.com")
+	require.Equal(t, req.Header.Get("X-Teleport-Cluster"), "root")
+	require.Equal(t, req.Header.Get("X-External-Env"), "production")
+	require.Equal(t, req.Header.Get("X-Existing"), "rewritten-existing-header")
+	require.NotEqual(t, req.Header.Get(teleport.AppJWTHeader), "rewritten-app-jwt-header")
+	require.NotEqual(t, req.Header.Get(teleport.AppCFHeader), "rewritten-app-cf-header")
+	require.NotEqual(t, req.Header.Get(forward.XForwardedFor), "rewritten-x-forwarded-for-header")
+	require.NotEqual(t, req.Header.Get(forward.XForwardedHost), "rewritten-x-forwarded-host-header")
+	require.NotEqual(t, req.Header.Get(forward.XForwardedProto), "rewritten-x-forwarded-proto-header")
+	require.NotEqual(t, req.Header.Get(forward.XForwardedServer), "rewritten-x-forwarded-server-header")
+
+	// Verify JWT tokens.
+	for _, header := range []string{teleport.AppJWTHeader, teleport.AppCFHeader, "X-JWT"} {
+		verifyJWT(t, p, req.Header.Get(header), p.dumperAppURI)
+	}
+}
+
+// testRewriteHeadersLeaf validates that http headers from application
+// rewrite configuration are correctly passed to proxied applications in leaf.
+func testRewriteHeadersLeaf(p *Pack, t *testing.T) {
+	// Create an application session for dumper app in leaf cluster.
+	appCookie := p.CreateAppSession(t, "dumper-leaf.example.com", "leaf.example.com")
+
+	// Get headers response and make sure headers were passed.
+	status, resp, err := p.MakeRequest(appCookie, http.MethodGet, "/", service.Header{
+		Name: "X-Existing", Value: "existing",
+	})
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+	require.Contains(t, resp, "X-Teleport-Cluster: leaf")
+	require.Contains(t, resp, "X-Teleport-Login: root")
+	require.Contains(t, resp, "X-Teleport-Login: ubuntu")
+	require.Contains(t, resp, "X-External-Env: production")
+	require.Contains(t, resp, "Host: example.com")
+	require.Contains(t, resp, "X-Existing: rewritten-existing-header")
+	require.NotContains(t, resp, "X-Existing: existing")
+	require.NotContains(t, resp, "rewritten-app-jwt-header")
+	require.NotContains(t, resp, "rewritten-app-cf-header")
+	require.NotContains(t, resp, "rewritten-x-forwarded-for-header")
+	require.NotContains(t, resp, "rewritten-x-forwarded-host-header")
+	require.NotContains(t, resp, "rewritten-x-forwarded-proto-header")
+	require.NotContains(t, resp, "rewritten-x-forwarded-server-header")
+}
+
+// testLogout verifies the session is removed from the backend when the user logs out.
+func testLogout(p *Pack, t *testing.T) {
+	// Create an application session.
+	appCookie := p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName)
+
+	// Log user out of session.
+	status, _, err := p.MakeRequest(appCookie, http.MethodGet, "/teleport-logout")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+
+	// Wait until requests using the session cookie have failed.
+	status, err = p.waitForLogout(appCookie)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusFound, status)
+}
+
+// testJWT ensures a JWT token is attached to requests and the JWT token can
+// be validated.
+func testJWT(p *Pack, t *testing.T) {
+	// Create an application session.
+	appCookie := p.CreateAppSession(t, p.jwtAppPublicAddr, p.jwtAppClusterName)
+
+	// Get JWT.
+	status, token, err := p.MakeRequest(appCookie, http.MethodGet, "/")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+
+	// Verify JWT token.
+	verifyJWT(t, p, token, p.jwtAppURI)
+
+	// Connect to websocket application that dumps the upgrade request.
+	wsCookie := p.CreateAppSession(t, p.wsHeaderAppPublicAddr, p.wsHeaderAppClusterName)
+	body, err := p.makeWebsocketRequest(wsCookie, "/")
+	require.NoError(t, err)
+
+	// Parse the upgrade request the websocket application received.
+	req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(body)))
+	require.NoError(t, err)
+
+	// Extract JWT token from header and verify it.
+	wsToken := req.Header.Get(teleport.AppJWTHeader)
+	require.NotEmpty(t, wsToken, "websocket upgrade request doesn't contain JWT header")
+	verifyJWT(t, p, wsToken, p.wsHeaderAppURI)
+}
+
+// testNoHeaderOverrides ensures that AAP-specific headers cannot be overridden
+// by values passed in by the user.
+func testNoHeaderOverrides(p *Pack, t *testing.T) {
+	// Create an application session.
+	appCookie := p.CreateAppSession(t, p.headerAppPublicAddr, p.headerAppClusterName)
+
+	// Get HTTP headers forwarded to the application.
+	status, origHeaderResp, err := p.MakeRequest(appCookie, http.MethodGet, "/")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+	origHeaders := strings.Split(origHeaderResp, "\n")
+	require.Equal(t, len(origHeaders), len(forwardedHeaderNames)+1)
+
+	// Construct HTTP request with custom headers.
+	req, err := http.NewRequest(http.MethodGet, p.assembleRootProxyURL("/"), nil)
+	require.NoError(t, err)
+	req.AddCookie(&http.Cookie{
+		Name:  app.CookieName,
+		Value: appCookie,
+	})
+	for _, headerName := range forwardedHeaderNames {
+		req.Header.Set(headerName, uuid.New().String())
+	}
+
+	// Issue the request.
+	status, newHeaderResp, err := p.sendRequest(req, nil)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+	newHeaders := strings.Split(newHeaderResp, "\n")
+	require.Equal(t, len(newHeaders), len(forwardedHeaderNames)+1)
+
+	// Headers sent to the application should not be affected.
+	for i := range forwardedHeaderNames {
+		require.Equal(t, origHeaders[i], newHeaders[i])
+	}
+}
+
+func testAuditEvents(p *Pack, t *testing.T) {
+	inCookie := p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName)
+
+	status, body, err := p.MakeRequest(inCookie, http.MethodGet, "/")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+	require.Contains(t, body, p.rootMessage)
+
+	// session start event
+	p.ensureAuditEvent(t, events.AppSessionStartEvent, func(event apievents.AuditEvent) {
+		expectedEvent := &apievents.AppSessionStart{
+			Metadata: apievents.Metadata{
+				Type:        events.AppSessionStartEvent,
+				Code:        events.AppSessionStartCode,
+				ClusterName: p.rootAppClusterName,
+			},
+			AppMetadata: apievents.AppMetadata{
+				AppURI:        p.rootAppURI,
+				AppPublicAddr: p.rootAppPublicAddr,
+				AppName:       p.rootAppName,
+			},
+			PublicAddr: p.rootAppPublicAddr,
+		}
+		require.Empty(t, cmp.Diff(
+			expectedEvent,
+			event,
+			cmpopts.IgnoreTypes(apievents.ServerMetadata{}, apievents.SessionMetadata{}, apievents.UserMetadata{}, apievents.ConnectionMetadata{}),
+			cmpopts.IgnoreFields(apievents.Metadata{}, "ID", "Time"),
+		))
+	})
+
+	// session chunk event
+	p.ensureAuditEvent(t, events.AppSessionChunkEvent, func(event apievents.AuditEvent) {
+		expectedEvent := &apievents.AppSessionChunk{
+			Metadata: apievents.Metadata{
+				Type:        events.AppSessionChunkEvent,
+				Code:        events.AppSessionChunkCode,
+				ClusterName: p.rootAppClusterName,
+			},
+			AppMetadata: apievents.AppMetadata{
+				AppURI:        p.rootAppURI,
+				AppPublicAddr: p.rootAppPublicAddr,
+				AppName:       p.rootAppName,
+			},
+		}
+		require.Empty(t, cmp.Diff(
+			expectedEvent,
+			event,
+			cmpopts.IgnoreTypes(apievents.ServerMetadata{}, apievents.SessionMetadata{}, apievents.UserMetadata{}, apievents.ConnectionMetadata{}),
+			cmpopts.IgnoreFields(apievents.Metadata{}, "ID", "Time"),
+			cmpopts.IgnoreFields(apievents.AppSessionChunk{}, "SessionChunkID"),
+		))
+	})
+}
+
+func testInvalidateAppSessionsOnLogout(p *Pack, t *testing.T) {
+	t.Cleanup(func() {
+		// This test will invalidate the web session so init it again after the
+		// test, otherwise tests that run after this one will be getting 403's.
+		p.initWebSession(t)
+	})
+
+	// Create an application session.
+	appCookie := p.CreateAppSession(t, p.rootAppPublicAddr, p.rootAppClusterName)
+
+	// Issue a request to the application to guarantee everything is working correctly.
+	status, _, err := p.MakeRequest(appCookie, http.MethodGet, "/")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+
+	// Generates TLS config for making app requests.
+	reqTLS := p.makeTLSConfig(t, p.rootAppPublicAddr, p.rootAppClusterName)
+	require.NotNil(t, reqTLS)
+
+	// Issue a request to the application to guarantee everything is working correctly.
+	status, _, err = p.makeRequestWithClientCert(reqTLS, http.MethodGet, "/")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+
+	// Logout from Teleport.
+	status, _, err = p.makeWebapiRequest(http.MethodDelete, "sessions", []byte{})
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+
+	// As deleting WebSessions might not happen immediately, run the next request
+	// in an `Eventually` block.
+	require.Eventually(t, func() bool {
+		// Issue another request to the application. Now, it should receive a
+		// redirect because the application sessions are gone.
+		status, _, err = p.MakeRequest(appCookie, http.MethodGet, "/")
+		require.NoError(t, err)
+		return status == http.StatusFound
+	}, time.Second, 250*time.Millisecond)
+
+	// Check the same for the client certificate.
+	require.Eventually(t, func() bool {
+		// Issue another request to the application. Now, it should receive a
+		// redirect because the application sessions are gone.
+		status, _, err = p.makeRequestWithClientCert(reqTLS, http.MethodGet, "/")
+		require.NoError(t, err)
+		return status == http.StatusFound
+	}, time.Second, 250*time.Millisecond)
+}
+
+// TestTCP tests proxying of plain TCP applications through app access.
+func TestTCP(t *testing.T) {
+	pack := Setup(t)
+
+	tests := []struct {
+		description string
+		address     string
+		outMessage  string
+	}{
+		{
+			description: "TCP app in root cluster",
+			address:     pack.startLocalProxy(t, pack.rootTCPPublicAddr, pack.rootAppClusterName),
+			outMessage:  pack.rootTCPMessage,
+		},
+		{
+			description: "TCP app in leaf cluster",
+			address:     pack.startLocalProxy(t, pack.leafTCPPublicAddr, pack.leafAppClusterName),
+			outMessage:  pack.leafTCPMessage,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.description, func(t *testing.T) {
+			conn, err := net.Dial("tcp", test.address)
+			require.NoError(t, err)
+
+			buf := make([]byte, 1024)
+			n, err := conn.Read(buf)
+			require.NoError(t, err)
+
+			resp := strings.TrimSpace(string(buf[:n]))
+			require.Equal(t, test.outMessage, resp)
+		})
+	}
+}
+
+func testServersHA(p *Pack, t *testing.T) {
+	type packInfo struct {
+		clusterName    string
+		publicHTTPAddr string
+		publicWSAddr   string
+		appServers     []*service.TeleportProcess
+	}
+
+	testCases := map[string]struct {
+		packInfo          func(pack *Pack) packInfo
+		startAppServers   func(pack *Pack, count int) []*service.TeleportProcess
+		waitForTunnelConn func(t *testing.T, pack *Pack, count int)
+	}{
+		"RootServer": {
+			packInfo: func(pack *Pack) packInfo {
+				return packInfo{
+					clusterName:    pack.rootAppClusterName,
+					publicHTTPAddr: pack.rootAppPublicAddr,
+					publicWSAddr:   pack.rootWSPublicAddr,
+					appServers:     pack.rootAppServers,
+				}
+			},
+			startAppServers: func(pack *Pack, count int) []*service.TeleportProcess {
+				return pack.startRootAppServers(t, count, []service.App{})
+			},
+			waitForTunnelConn: func(t *testing.T, pack *Pack, count int) {
+				helpers.WaitForActiveTunnelConnections(t, pack.rootCluster.Tunnel, pack.rootCluster.Secrets.SiteName, count)
+			},
+		},
+		"LeafServer": {
+			packInfo: func(pack *Pack) packInfo {
+				return packInfo{
+					clusterName:    pack.leafAppClusterName,
+					publicHTTPAddr: pack.leafAppPublicAddr,
+					publicWSAddr:   pack.leafWSPublicAddr,
+					appServers:     pack.leafAppServers,
+				}
+			},
+			startAppServers: func(pack *Pack, count int) []*service.TeleportProcess {
+				return pack.startLeafAppServers(t, count, []service.App{})
+			},
+			waitForTunnelConn: func(t *testing.T, pack *Pack, count int) {
+				helpers.WaitForActiveTunnelConnections(t, pack.leafCluster.Tunnel, pack.leafCluster.Secrets.SiteName, count)
+			},
+		},
+	}
+
+	// asserts that the response has error.
+	responseWithError := func(t *testing.T, status int, err error) {
+		if status > 0 {
+			require.NoError(t, err)
+			require.Equal(t, http.StatusInternalServerError, status)
+			return
+		}
+
+		require.Error(t, err)
+	}
+	// asserts that the response has no errors.
+	responseWithoutError := func(t *testing.T, status int, err error) {
+		if status > 0 {
+			require.NoError(t, err)
+			require.Equal(t, http.StatusOK, status)
+			return
+		}
+
+		require.NoError(t, err)
+	}
+
+	makeRequests := func(t *testing.T, pack *Pack, httpCookie, wsCookie string, responseAssertion func(*testing.T, int, error)) {
+		status, _, err := pack.MakeRequest(httpCookie, http.MethodGet, "/")
+		responseAssertion(t, status, err)
+
+		_, err = pack.makeWebsocketRequest(wsCookie, "/")
+		responseAssertion(t, 0, err)
+	}
+
+	for name, test := range testCases {
+		name, test := name, test
+		t.Run(name, func(t *testing.T) {
+			info := test.packInfo(p)
+			httpCookie := p.CreateAppSession(t, info.publicHTTPAddr, info.clusterName)
+			wsCookie := p.CreateAppSession(t, info.publicWSAddr, info.clusterName)
+
+			makeRequests(t, p, httpCookie, wsCookie, responseWithoutError)
+
+			// Stop all root app servers.
+			for i, appServer := range info.appServers {
+				require.NoError(t, appServer.Close())
+				require.NoError(t, appServer.Wait())
+
+				if i == len(info.appServers)-1 {
+					// fails only when the last one is closed.
+					makeRequests(t, p, httpCookie, wsCookie, responseWithError)
+				} else {
+					// otherwise the request should be handled by another
+					// server.
+					makeRequests(t, p, httpCookie, wsCookie, responseWithoutError)
+				}
+			}
+
+			servers := test.startAppServers(p, 1)
+			test.waitForTunnelConn(t, p, 1)
+			makeRequests(t, p, httpCookie, wsCookie, responseWithoutError)
+
+			// Start an additional app server and stop all current running
+			// ones.
+			test.startAppServers(p, 1)
+			test.waitForTunnelConn(t, p, 2)
+
+			for _, appServer := range servers {
+				require.NoError(t, appServer.Close())
+				require.NoError(t, appServer.Wait())
+
+				// Everytime an app server stops we issue a request to
+				// guarantee that the requests are going to be resolved by
+				// the remaining app servers.
+				makeRequests(t, p, httpCookie, wsCookie, responseWithoutError)
+			}
+		})
+	}
+}

integration/appaccess/fixtures.go

@@ -0,0 +1,395 @@
+// Copyright 2022 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package appaccess
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/websocket"
+	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/api/breaker"
+	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/integration/helpers"
+	"github.com/gravitational/teleport/lib"
+	"github.com/gravitational/teleport/lib/auth/testauthority"
+	"github.com/gravitational/teleport/lib/reversetunnel"
+	"github.com/gravitational/teleport/lib/service"
+	"github.com/gravitational/teleport/lib/utils"
+
+	"github.com/stretchr/testify/require"
+)
+
+type AppTestOptions struct {
+	ExtraRootApps        []service.App
+	ExtraLeafApps        []service.App
+	RootClusterListeners helpers.InstanceListenerSetupFunc
+	LeafClusterListeners helpers.InstanceListenerSetupFunc
+
+	RootConfig func(config *service.Config)
+	LeafConfig func(config *service.Config)
+}
+
+// Setup configures all clusters and servers needed for a test.
+func Setup(t *testing.T) *Pack {
+	return SetupWithOptions(t, AppTestOptions{})
+}
+
+// SetupWithOptions configures app access test with custom options.
+func SetupWithOptions(t *testing.T, opts AppTestOptions) *Pack {
+	tr := utils.NewTracer(utils.ThisFunction()).Start()
+	defer tr.Stop()
+
+	log := utils.NewLoggerForTests()
+
+	// Insecure development mode needs to be set because the web proxy uses a
+	// self-signed certificate during tests.
+	lib.SetInsecureDevMode(true)
+
+	p := &Pack{
+		rootAppName:        "app-01",
+		rootAppPublicAddr:  "app-01.example.com",
+		rootAppClusterName: "example.com",
+		rootMessage:        uuid.New().String(),
+
+		rootWSAppName:    "ws-01",
+		rootWSPublicAddr: "ws-01.example.com",
+		rootWSMessage:    uuid.New().String(),
+
+		rootWSSAppName:    "wss-01",
+		rootWSSPublicAddr: "wss-01.example.com",
+		rootWSSMessage:    uuid.New().String(),
+
+		rootTCPAppName:    "tcp-01",
+		rootTCPPublicAddr: "tcp-01.example.com",
+		rootTCPMessage:    uuid.New().String(),
+
+		leafAppName:        "app-02",
+		leafAppPublicAddr:  "app-02.example.com",
+		leafAppClusterName: "leaf.example.com",
+		leafMessage:        uuid.New().String(),
+
+		leafWSAppName:    "ws-02",
+		leafWSPublicAddr: "ws-02.example.com",
+		leafWSMessage:    uuid.New().String(),
+
+		leafWSSAppName:    "wss-02",
+		leafWSSPublicAddr: "wss-02.example.com",
+		leafWSSMessage:    uuid.New().String(),
+
+		leafTCPAppName:    "tcp-02",
+		leafTCPPublicAddr: "tcp-02.example.com",
+		leafTCPMessage:    uuid.New().String(),
+
+		jwtAppName:        "app-03",
+		jwtAppPublicAddr:  "app-03.example.com",
+		jwtAppClusterName: "example.com",
+
+		headerAppName:        "app-04",
+		headerAppPublicAddr:  "app-04.example.com",
+		headerAppClusterName: "example.com",
+
+		wsHeaderAppName:        "ws-header",
+		wsHeaderAppPublicAddr:  "ws-header.example.com",
+		wsHeaderAppClusterName: "example.com",
+
+		flushAppName:        "app-05",
+		flushAppPublicAddr:  "app-05.example.com",
+		flushAppClusterName: "example.com",
+	}
+
+	createHandler := func(handler func(conn *websocket.Conn)) http.HandlerFunc {
+		return func(w http.ResponseWriter, r *http.Request) {
+			upgrader := websocket.Upgrader{
+				ReadBufferSize:  1024,
+				WriteBufferSize: 1024,
+			}
+			conn, err := upgrader.Upgrade(w, r, nil)
+			require.NoError(t, err)
+			handler(conn)
+		}
+	}
+
+	// Start a few different HTTP server that will be acting like a proxied application.
+	rootServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, p.rootMessage)
+	}))
+	t.Cleanup(rootServer.Close)
+	// Websockets server in root cluster (ws://).
+	rootWSServer := httptest.NewServer(createHandler(func(conn *websocket.Conn) {
+		conn.WriteMessage(websocket.BinaryMessage, []byte(p.rootWSMessage))
+		conn.Close()
+	}))
+	t.Cleanup(rootWSServer.Close)
+	// Secure websockets server in root cluster (wss://).
+	rootWSSServer := httptest.NewTLSServer(createHandler(func(conn *websocket.Conn) {
+		conn.WriteMessage(websocket.BinaryMessage, []byte(p.rootWSSMessage))
+		conn.Close()
+	}))
+	t.Cleanup(rootWSSServer.Close)
+	// Plain TCP application in root cluster (tcp://).
+	rootTCPServer := newTCPServer(t, func(c net.Conn) {
+		c.Write([]byte(p.rootTCPMessage))
+		c.Close()
+	})
+	t.Cleanup(func() { rootTCPServer.Close() })
+	// HTTP server in leaf cluster.
+	leafServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, p.leafMessage)
+	}))
+	t.Cleanup(leafServer.Close)
+	// Websockets server in leaf cluster (ws://).
+	leafWSServer := httptest.NewServer(createHandler(func(conn *websocket.Conn) {
+		conn.WriteMessage(websocket.BinaryMessage, []byte(p.leafWSMessage))
+		conn.Close()
+	}))
+	t.Cleanup(leafWSServer.Close)
+	// Secure websockets server in leaf cluster (wss://).
+	leafWSSServer := httptest.NewTLSServer(createHandler(func(conn *websocket.Conn) {
+		conn.WriteMessage(websocket.BinaryMessage, []byte(p.leafWSSMessage))
+		conn.Close()
+	}))
+	t.Cleanup(leafWSSServer.Close)
+	// Plain TCP application in leaf cluster (tcp://).
+	leafTCPServer := newTCPServer(t, func(c net.Conn) {
+		c.Write([]byte(p.leafTCPMessage))
+		c.Close()
+	})
+	t.Cleanup(func() { leafTCPServer.Close() })
+	// JWT server writes generated JWT token in the response.
+	jwtServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprintln(w, r.Header.Get(teleport.AppJWTHeader))
+	}))
+	t.Cleanup(jwtServer.Close)
+	// Websocket header server dumps initial HTTP upgrade request in the response.
+	wsHeaderServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		conn, err := (&websocket.Upgrader{}).Upgrade(w, r, nil)
+		require.NoError(t, err)
+		reqDump, err := httputil.DumpRequest(r, false)
+		require.NoError(t, err)
+		require.NoError(t, conn.WriteMessage(websocket.BinaryMessage, reqDump))
+		require.NoError(t, conn.Close())
+	}))
+	t.Cleanup(wsHeaderServer.Close)
+	headerServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		for _, headerName := range forwardedHeaderNames {
+			fmt.Fprintln(w, r.Header.Get(headerName))
+		}
+	}))
+	t.Cleanup(headerServer.Close)
+	// Start test server that will dump all request headers in the response.
+	dumperServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		r.Write(w)
+	}))
+	t.Cleanup(dumperServer.Close)
+	flushServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		h := w.(http.Hijacker)
+		conn, _, err := h.Hijack()
+		require.NoError(t, err)
+		defer conn.Close()
+		data := "HTTP/1.1 200 OK\r\n" +
+			"Transfer-Encoding: chunked\r\n" +
+			"\r\n" +
+			"05\r\n" +
+			"hello\r\n"
+		fmt.Fprint(conn, data)
+		time.Sleep(500 * time.Millisecond)
+		data = "05\r\n" +
+			"world\r\n" +
+			"0\r\n" +
+			"\r\n"
+		fmt.Fprint(conn, data)
+	}))
+	t.Cleanup(flushServer.Close)
+
+	p.rootAppURI = rootServer.URL
+	p.rootWSAppURI = rootWSServer.URL
+	p.rootWSSAppURI = rootWSSServer.URL
+	p.rootTCPAppURI = fmt.Sprintf("tcp://%v", rootTCPServer.Addr().String())
+	p.leafAppURI = leafServer.URL
+	p.leafWSAppURI = leafWSServer.URL
+	p.leafWSSAppURI = leafWSSServer.URL
+	p.leafTCPAppURI = fmt.Sprintf("tcp://%v", leafTCPServer.Addr().String())
+	p.jwtAppURI = jwtServer.URL
+	p.headerAppURI = headerServer.URL
+	p.wsHeaderAppURI = wsHeaderServer.URL
+	p.flushAppURI = flushServer.URL
+	p.dumperAppURI = dumperServer.URL
+
+	privateKey, publicKey, err := testauthority.New().GenerateKeyPair()
+	require.NoError(t, err)
+
+	// Create a new Teleport instance with passed in configuration.
+	rootCfg := helpers.InstanceConfig{
+		ClusterName: "example.com",
+		HostID:      uuid.New().String(),
+		NodeName:    helpers.Host,
+		Priv:        privateKey,
+		Pub:         publicKey,
+		Log:         log,
+	}
+	if opts.RootClusterListeners != nil {
+		rootCfg.Listeners = opts.RootClusterListeners(t, &rootCfg.Fds)
+	}
+	p.rootCluster = helpers.NewInstance(t, rootCfg)
+
+	// Create a new Teleport instance with passed in configuration.
+	leafCfg := helpers.InstanceConfig{
+		ClusterName: "leaf.example.com",
+		HostID:      uuid.New().String(),
+		NodeName:    helpers.Host,
+		Priv:        privateKey,
+		Pub:         publicKey,
+		Log:         log,
+	}
+	if opts.LeafClusterListeners != nil {
+		leafCfg.Listeners = opts.LeafClusterListeners(t, &leafCfg.Fds)
+	}
+	p.leafCluster = helpers.NewInstance(t, leafCfg)
+
+	rcConf := service.MakeDefaultConfig()
+	rcConf.Console = nil
+	rcConf.Log = log
+	rcConf.DataDir = t.TempDir()
+	rcConf.Auth.Enabled = true
+	rcConf.Auth.Preference.SetSecondFactor("off")
+	rcConf.Proxy.Enabled = true
+	rcConf.Proxy.DisableWebService = false
+	rcConf.Proxy.DisableWebInterface = true
+	rcConf.SSH.Enabled = false
+	rcConf.Apps.Enabled = false
+	rcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
+	if opts.RootConfig != nil {
+		opts.RootConfig(rcConf)
+	}
+
+	lcConf := service.MakeDefaultConfig()
+	lcConf.Console = nil
+	lcConf.Log = log
+	lcConf.DataDir = t.TempDir()
+	lcConf.Auth.Enabled = true
+	lcConf.Auth.Preference.SetSecondFactor("off")
+	lcConf.Proxy.Enabled = true
+	lcConf.Proxy.DisableWebService = false
+	lcConf.Proxy.DisableWebInterface = true
+	lcConf.SSH.Enabled = false
+	lcConf.Apps.Enabled = false
+	lcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
+	if opts.RootConfig != nil {
+		opts.RootConfig(lcConf)
+	}
+
+	err = p.leafCluster.CreateEx(t, p.rootCluster.Secrets.AsSlice(), lcConf)
+	require.NoError(t, err)
+	err = p.rootCluster.CreateEx(t, p.leafCluster.Secrets.AsSlice(), rcConf)
+	require.NoError(t, err)
+
+	err = p.leafCluster.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, p.leafCluster.StopAll()) })
+	err = p.rootCluster.Start()
+	require.NoError(t, err)
+	t.Cleanup(func() { require.NoError(t, p.rootCluster.StopAll()) })
+
+	// At least one rootAppServer should start during the setup
+	rootAppServersCount := 1
+	p.rootAppServers = p.startRootAppServers(t, rootAppServersCount, opts.ExtraRootApps)
+
+	// At least one leafAppServer should start during the setup
+	leafAppServersCount := 1
+	p.leafAppServers = p.startLeafAppServers(t, leafAppServersCount, opts.ExtraLeafApps)
+
+	// Create user for tests.
+	p.initUser(t, opts)
+
+	// Create Web UI session.
+	p.initWebSession(t)
+
+	// Initialize cert pool with root CA's.
+	p.initCertPool(t)
+
+	// Initialize Teleport client with the user's credentials.
+	p.initTeleportClient(t)
+
+	return p
+}
+
+var forwardedHeaderNames = []string{
+	teleport.AppJWTHeader,
+	teleport.AppCFHeader,
+	"X-Forwarded-Proto",
+	"X-Forwarded-Host",
+	"X-Forwarded-Server",
+	"X-Forwarded-For",
+}
+
+// waitAppServerTunnel waits for application server tunnel connections.
+func waitAppServerTunnel(t *testing.T, tunnel reversetunnel.Server, clusterName, serverUUID string) {
+	t.Helper()
+	cluster, err := tunnel.GetSite(clusterName)
+	require.NoError(t, err)
+
+	require.Eventually(t, func() bool {
+		conn, err := cluster.Dial(reversetunnel.DialParams{
+			From:     &utils.NetAddr{AddrNetwork: "tcp", Addr: "@web-proxy"},
+			To:       &utils.NetAddr{AddrNetwork: "tcp", Addr: reversetunnel.LocalNode},
+			ServerID: fmt.Sprintf("%v.%v", serverUUID, clusterName),
+			ConnType: types.AppTunnel,
+		})
+		if err != nil {
+			return false
+		}
+
+		require.NoError(t, conn.Close())
+		return true
+	}, 10*time.Second, time.Second)
+}
+
+type appAccessTestFunc func(*Pack, *testing.T)
+
+func bind(p *Pack, fn appAccessTestFunc) func(*testing.T) {
+	return func(t *testing.T) {
+		fn(p, t)
+	}
+}
+
+// newTCPServer starts accepting TCP connections and serving them using the
+// provided handler. Handlers are expected to close client connections.
+// Returns the TCP listener.
+func newTCPServer(t *testing.T, handleConn func(net.Conn)) net.Listener {
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	go func() {
+		for {
+			conn, err := listener.Accept()
+			if err == nil {
+				go handleConn(conn)
+			}
+			if err != nil && !utils.IsOKNetworkError(err) {
+				t.Error(err)
+				return
+			}
+		}
+	}()
+
+	return listener
+}

integration/appaccess/jwt.go

@@ -0,0 +1,55 @@
+// Copyright 2022 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package appaccess
+
+import (
+	"encoding/json"
+	"net/http"
+	"testing"
+
+	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/jwt"
+	"github.com/gravitational/teleport/lib/web"
+	"github.com/stretchr/testify/require"
+)
+
+func verifyJWT(t *testing.T, pack *Pack, token, appURI string) {
+	// Get and unmarshal JWKs
+	status, body, err := pack.MakeRequest("", http.MethodGet, "/.well-known/jwks.json")
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, status)
+	var jwks web.JWKSResponse
+	err = json.Unmarshal([]byte(body), &jwks)
+	require.NoError(t, err)
+	require.Len(t, jwks.Keys, 1)
+	publicKey, err := jwt.UnmarshalJWK(jwks.Keys[0])
+	require.NoError(t, err)
+
+	// Verify JWT.
+	key, err := jwt.New(&jwt.Config{
+		PublicKey:   publicKey,
+		Algorithm:   defaults.ApplicationTokenAlgorithm,
+		ClusterName: pack.jwtAppClusterName,
+	})
+	require.NoError(t, err)
+	claims, err := key.Verify(jwt.VerifyParams{
+		Username: pack.username,
+		RawToken: token,
+		URI:      appURI,
+	})
+	require.NoError(t, err)
+	require.Equal(t, pack.username, claims.Username)
+	require.Equal(t, pack.user.GetRoles(), claims.Roles)
+}

integration/appaccess/pack.go

@@ -0,0 +1,804 @@
+// Copyright 2022 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package appaccess
+
+import (
+	"bytes"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"net/url"
+	"testing"
+	"time"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/websocket"
+	"github.com/gravitational/oxy/forward"
+	"github.com/gravitational/teleport"
+	"github.com/gravitational/teleport/api/breaker"
+	apidefaults "github.com/gravitational/teleport/api/defaults"
+	"github.com/gravitational/teleport/api/types"
+	apievents "github.com/gravitational/teleport/api/types/events"
+	"github.com/gravitational/teleport/integration/helpers"
+	"github.com/gravitational/teleport/lib/auth"
+	"github.com/gravitational/teleport/lib/auth/native"
+	"github.com/gravitational/teleport/lib/client"
+	"github.com/gravitational/teleport/lib/httplib/csrf"
+	"github.com/gravitational/teleport/lib/service"
+	"github.com/gravitational/teleport/lib/services"
+	"github.com/gravitational/teleport/lib/srv/alpnproxy"
+	alpncommon "github.com/gravitational/teleport/lib/srv/alpnproxy/common"
+	"github.com/gravitational/teleport/lib/utils"
+	"github.com/gravitational/teleport/lib/web"
+	"github.com/gravitational/teleport/lib/web/app"
+	"github.com/gravitational/trace"
+	"github.com/stretchr/testify/require"
+)
+
+// Pack contains identity as well as initialized Teleport clusters and instances.
+type Pack struct {
+	username string
+	password string
+
+	tc *client.TeleportClient
+
+	user types.User
+
+	webCookie string
+	webToken  string
+
+	rootCluster    *helpers.TeleInstance
+	rootAppServers []*service.TeleportProcess
+	rootCertPool   *x509.CertPool
+
+	rootAppName        string
+	rootAppPublicAddr  string
+	rootAppClusterName string
+	rootMessage        string
+	rootAppURI         string
+
+	rootWSAppName    string
+	rootWSPublicAddr string
+	rootWSMessage    string
+	rootWSAppURI     string
+
+	rootWSSAppName    string
+	rootWSSPublicAddr string
+	rootWSSMessage    string
+	rootWSSAppURI     string
+
+	rootTCPAppName    string
+	rootTCPPublicAddr string
+	rootTCPMessage    string
+	rootTCPAppURI     string
+
+	jwtAppName        string
+	jwtAppPublicAddr  string
+	jwtAppClusterName string
+	jwtAppURI         string
+
+	dumperAppURI string
+
+	leafCluster    *helpers.TeleInstance
+	leafAppServers []*service.TeleportProcess
+
+	leafAppName        string
+	leafAppPublicAddr  string
+	leafAppClusterName string
+	leafMessage        string
+	leafAppURI         string
+
+	leafWSAppName    string
+	leafWSPublicAddr string
+	leafWSMessage    string
+	leafWSAppURI     string
+
+	leafWSSAppName    string
+	leafWSSPublicAddr string
+	leafWSSMessage    string
+	leafWSSAppURI     string
+
+	leafTCPAppName    string
+	leafTCPPublicAddr string
+	leafTCPMessage    string
+	leafTCPAppURI     string
+
+	headerAppName        string
+	headerAppPublicAddr  string
+	headerAppClusterName string
+	headerAppURI         string
+
+	wsHeaderAppName        string
+	wsHeaderAppPublicAddr  string
+	wsHeaderAppClusterName string
+	wsHeaderAppURI         string
+
+	flushAppName        string
+	flushAppPublicAddr  string
+	flushAppClusterName string
+	flushAppURI         string
+}
+
+func (p *Pack) RootAppClusterName() string {
+	return p.rootAppClusterName
+}
+
+func (p *Pack) RootAppPublicAddr() string {
+	return p.rootAppPublicAddr
+}
+
+func (p *Pack) LeafAppClusterName() string {
+	return p.leafAppClusterName
+}
+
+func (p *Pack) LeafAppPublicAddr() string {
+	return p.leafAppPublicAddr
+}
+
+// initUser will create a user within the root cluster.
+func (p *Pack) initUser(t *testing.T, opts AppTestOptions) {
+	p.username = uuid.New().String()
+	p.password = uuid.New().String()
+
+	user, err := types.NewUser(p.username)
+	require.NoError(t, err)
+
+	role := services.RoleForUser(user)
+	role.SetLogins(types.Allow, []string{p.username, "root", "ubuntu"})
+	err = p.rootCluster.Process.GetAuthServer().UpsertRole(context.Background(), role)
+	require.NoError(t, err)
+
+	user.AddRole(role.GetName())
+	user.SetTraits(map[string][]string{"env": {"production"}})
+	err = p.rootCluster.Process.GetAuthServer().CreateUser(context.Background(), user)
+	require.NoError(t, err)
+
+	err = p.rootCluster.Process.GetAuthServer().UpsertPassword(user.GetName(), []byte(p.password))
+	require.NoError(t, err)
+
+	p.user = user
+}
+
+// initWebSession creates a Web UI session within the root cluster.
+func (p *Pack) initWebSession(t *testing.T) {
+	csReq, err := json.Marshal(web.CreateSessionReq{
+		User: p.username,
+		Pass: p.password,
+	})
+	require.NoError(t, err)
+
+	// Create POST request to create session.
+	u := url.URL{
+		Scheme: "https",
+		Host:   p.rootCluster.Web,
+		Path:   "/v1/webapi/sessions/web",
+	}
+	req, err := http.NewRequest(http.MethodPost, u.String(), bytes.NewBuffer(csReq))
+	require.NoError(t, err)
+
+	// Attach CSRF token in cookie and header.
+	csrfToken, err := utils.CryptoRandomHex(32)
+	require.NoError(t, err)
+	req.AddCookie(&http.Cookie{
+		Name:  csrf.CookieName,
+		Value: csrfToken,
+	})
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set(csrf.HeaderName, csrfToken)
+
+	// Issue request.
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				InsecureSkipVerify: true,
+			},
+		},
+	}
+	resp, err := client.Do(req)
+	require.NoError(t, err)
+	defer resp.Body.Close()
+	require.Equal(t, http.StatusOK, resp.StatusCode)
+
+	// Read in response.
+	var csResp *web.CreateSessionResponse
+	err = json.NewDecoder(resp.Body).Decode(&csResp)
+	require.NoError(t, err)
+
+	// Extract session cookie and bearer token.
+	require.Len(t, resp.Cookies(), 1)
+	cookie := resp.Cookies()[0]
+	require.Equal(t, cookie.Name, web.CookieName)
+
+	p.webCookie = cookie.Value
+	p.webToken = csResp.Token
+}
+
+// initTeleportClient initializes a Teleport client with this pack's user
+// credentials.
+func (p *Pack) initTeleportClient(t *testing.T) {
+	creds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{
+		Process:  p.rootCluster.Process,
+		Username: p.user.GetName(),
+	})
+	require.NoError(t, err)
+
+	tc, err := p.rootCluster.NewClientWithCreds(helpers.ClientConfig{
+		Login:   p.user.GetName(),
+		Cluster: p.rootCluster.Secrets.SiteName,
+		Host:    helpers.Loopback,
+		Port:    helpers.Port(t, p.rootCluster.SSH),
+	}, *creds)
+	require.NoError(t, err)
+
+	p.tc = tc
+}
+
+// CreateAppSession creates an application session with the root cluster. The
+// application that the user connects to may be running in a leaf cluster.
+func (p *Pack) CreateAppSession(t *testing.T, publicAddr, clusterName string) string {
+	require.NotEmpty(t, p.webCookie)
+	require.NotEmpty(t, p.webToken)
+
+	casReq, err := json.Marshal(web.CreateAppSessionRequest{
+		FQDNHint:    publicAddr,
+		PublicAddr:  publicAddr,
+		ClusterName: clusterName,
+	})
+	require.NoError(t, err)
+	statusCode, body, err := p.makeWebapiRequest(http.MethodPost, "sessions/app", casReq)
+	require.NoError(t, err)
+	require.Equal(t, http.StatusOK, statusCode)
+
+	var casResp *web.CreateAppSessionResponse
+	err = json.Unmarshal(body, &casResp)
+	require.NoError(t, err)
+
+	return casResp.CookieValue
+}
+
+// makeWebapiRequest makes a request to the root cluster Web API.
+func (p *Pack) makeWebapiRequest(method, endpoint string, payload []byte) (int, []byte, error) {
+	u := url.URL{
+		Scheme: "https",
+		Host:   p.rootCluster.Web,
+		Path:   fmt.Sprintf("/v1/webapi/%s", endpoint),
+	}
+
+	req, err := http.NewRequest(method, u.String(), bytes.NewBuffer(payload))
+	if err != nil {
+		return 0, nil, trace.Wrap(err)
+	}
+
+	req.AddCookie(&http.Cookie{
+		Name:  web.CookieName,
+		Value: p.webCookie,
+	})
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %v", p.webToken))
+	req.Header.Add("Content-Type", "application/json")
+
+	statusCode, body, err := p.sendRequest(req, nil)
+	return statusCode, []byte(body), trace.Wrap(err)
+}
+
+func (p *Pack) ensureAuditEvent(t *testing.T, eventType string, checkEvent func(event apievents.AuditEvent)) {
+	require.Eventuallyf(t, func() bool {
+		events, _, err := p.rootCluster.Process.GetAuthServer().SearchEvents(
+			time.Now().Add(-time.Hour),
+			time.Now().Add(time.Hour),
+			apidefaults.Namespace,
+			[]string{eventType},
+			1,
+			types.EventOrderDescending,
+			"",
+		)
+		require.NoError(t, err)
+		if len(events) == 0 {
+			return false
+		}
+
+		checkEvent(events[0])
+		return true
+	}, 500*time.Millisecond, 50*time.Millisecond, "failed to fetch audit event \"%s\"", eventType)
+}
+
+// initCertPool initializes root cluster CA pool.
+func (p *Pack) initCertPool(t *testing.T) {
+	authClient := p.rootCluster.GetSiteAPI(p.rootCluster.Secrets.SiteName)
+	ca, err := authClient.GetCertAuthority(context.Background(), types.CertAuthID{
+		Type:       types.HostCA,
+		DomainName: p.rootCluster.Secrets.SiteName,
+	}, false)
+	require.NoError(t, err)
+
+	pool, err := services.CertPool(ca)
+	require.NoError(t, err)
+
+	p.rootCertPool = pool
+}
+
+// startLocalProxy starts a local ALPN proxy for the specified application.
+func (p *Pack) startLocalProxy(t *testing.T, publicAddr, clusterName string) string {
+	tlsConfig := p.makeTLSConfig(t, publicAddr, clusterName)
+
+	listener, err := net.Listen("tcp", "127.0.0.1:0")
+	require.NoError(t, err)
+
+	proxy, err := alpnproxy.NewLocalProxy(alpnproxy.LocalProxyConfig{
+		RemoteProxyAddr:    p.rootCluster.Web,
+		Protocols:          []alpncommon.Protocol{alpncommon.ProtocolTCP},
+		InsecureSkipVerify: true,
+		Listener:           listener,
+		ParentContext:      context.Background(),
+		Certs:              tlsConfig.Certificates,
+	})
+	require.NoError(t, err)
+	t.Cleanup(func() { proxy.Close() })
+
+	go proxy.Start(context.Background())
+
+	return proxy.GetAddr()
+}
+
+// makeTLSConfig returns TLS config suitable for making an app access request.
+func (p *Pack) makeTLSConfig(t *testing.T, publicAddr, clusterName string) *tls.Config {
+	privateKey, publicKey, err := native.GenerateKeyPair()
+	require.NoError(t, err)
+
+	ws, err := p.tc.CreateAppSession(context.Background(), types.CreateAppSessionRequest{
+		Username:    p.user.GetName(),
+		PublicAddr:  publicAddr,
+		ClusterName: clusterName,
+	})
+	require.NoError(t, err)
+
+	certificate, err := p.rootCluster.Process.GetAuthServer().GenerateUserAppTestCert(
+		auth.AppTestCertRequest{
+			PublicKey:   publicKey,
+			Username:    p.user.GetName(),
+			TTL:         time.Hour,
+			PublicAddr:  publicAddr,
+			ClusterName: clusterName,
+			SessionID:   ws.GetName(),
+		})
+	require.NoError(t, err)
+
+	tlsCert, err := tls.X509KeyPair(certificate, privateKey)
+	require.NoError(t, err)
+
+	return &tls.Config{
+		RootCAs:            p.rootCertPool,
+		Certificates:       []tls.Certificate{tlsCert},
+		InsecureSkipVerify: true,
+	}
+}
+
+// makeTLSConfigNoSession returns TLS config for application access without
+// creating session to simulate nonexistent session scenario.
+func (p *Pack) makeTLSConfigNoSession(t *testing.T, publicAddr, clusterName string) *tls.Config {
+	privateKey, publicKey, err := native.GenerateKeyPair()
+	require.NoError(t, err)
+
+	certificate, err := p.rootCluster.Process.GetAuthServer().GenerateUserAppTestCert(
+		auth.AppTestCertRequest{
+			PublicKey:   publicKey,
+			Username:    p.user.GetName(),
+			TTL:         time.Hour,
+			PublicAddr:  publicAddr,
+			ClusterName: clusterName,
+			// Use arbitrary session ID
+			SessionID: uuid.New().String(),
+		})
+	require.NoError(t, err)
+
+	tlsCert, err := tls.X509KeyPair(certificate, privateKey)
+	require.NoError(t, err)
+
+	return &tls.Config{
+		RootCAs:            p.rootCertPool,
+		Certificates:       []tls.Certificate{tlsCert},
+		InsecureSkipVerify: true,
+	}
+}
+
+// MakeRequest makes a request to the root cluster with the given session cookie.
+func (p *Pack) MakeRequest(sessionCookie string, method string, endpoint string, headers ...service.Header) (int, string, error) {
+	req, err := http.NewRequest(method, p.assembleRootProxyURL(endpoint), nil)
+	if err != nil {
+		return 0, "", trace.Wrap(err)
+	}
+
+	// Only attach session cookie if passed in.
+	if sessionCookie != "" {
+		req.AddCookie(&http.Cookie{
+			Name:  app.CookieName,
+			Value: sessionCookie,
+		})
+	}
+
+	for _, h := range headers {
+		req.Header.Add(h.Name, h.Value)
+	}
+
+	return p.sendRequest(req, nil)
+}
+
+// makeRequestWithClientCert makes a request to the root cluster using the
+// client certificate authentication from the provided tls config.
+func (p *Pack) makeRequestWithClientCert(tlsConfig *tls.Config, method, endpoint string) (int, string, error) {
+	req, err := http.NewRequest(method, p.assembleRootProxyURL(endpoint), nil)
+	if err != nil {
+		return 0, "", trace.Wrap(err)
+	}
+	return p.sendRequest(req, tlsConfig)
+}
+
+// makeWebsocketRequest makes a websocket request with the given session cookie.
+func (p *Pack) makeWebsocketRequest(sessionCookie, endpoint string) (string, error) {
+	header := http.Header{}
+	dialer := websocket.Dialer{}
+
+	if sessionCookie != "" {
+		header.Set("Cookie", (&http.Cookie{
+			Name:  app.CookieName,
+			Value: sessionCookie,
+		}).String())
+	}
+	dialer.TLSClientConfig = &tls.Config{
+		InsecureSkipVerify: true,
+	}
+	conn, resp, err := dialer.Dial(fmt.Sprintf("wss://%s%s", p.rootCluster.Web, endpoint), header)
+	if err != nil {
+		return "", err
+	}
+	defer conn.Close()
+	defer resp.Body.Close()
+	stream := &web.WebsocketIO{Conn: conn}
+	data, err := io.ReadAll(stream)
+	if err != nil && websocket.IsUnexpectedCloseError(err, websocket.CloseAbnormalClosure) {
+		return "", err
+	}
+	return string(data), nil
+}
+
+// assembleRootProxyURL returns the URL string of an endpoint at the root
+// cluster's proxy web.
+func (p *Pack) assembleRootProxyURL(endpoint string) string {
+	u := url.URL{
+		Scheme: "https",
+		Host:   p.rootCluster.Web,
+		Path:   endpoint,
+	}
+	return u.String()
+}
+
+// sendReqeust sends the request to the root cluster.
+func (p *Pack) sendRequest(req *http.Request, tlsConfig *tls.Config) (int, string, error) {
+	if tlsConfig == nil {
+		tlsConfig = &tls.Config{
+			InsecureSkipVerify: true,
+		}
+	}
+
+	client := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return 0, "", trace.Wrap(err)
+	}
+	defer resp.Body.Close()
+
+	// Read in response body.
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return 0, "", trace.Wrap(err)
+	}
+
+	return resp.StatusCode, string(body), nil
+}
+
+// waitForLogout keeps making request with the passed in session cookie until
+// they return a non-200 status.
+func (p *Pack) waitForLogout(appCookie string) (int, error) {
+	ticker := time.NewTicker(500 * time.Millisecond)
+	defer ticker.Stop()
+	timeout := time.NewTimer(5 * time.Second)
+	defer timeout.Stop()
+
+	for {
+		select {
+		case <-ticker.C:
+			status, _, err := p.MakeRequest(appCookie, http.MethodGet, "/")
+			if err != nil {
+				return 0, trace.Wrap(err)
+			}
+			if status != http.StatusOK {
+				return status, nil
+			}
+		case <-timeout.C:
+			return 0, trace.BadParameter("timed out waiting for logout")
+		}
+	}
+}
+
+func (p *Pack) startRootAppServers(t *testing.T, count int, extraApps []service.App) []*service.TeleportProcess {
+	log := utils.NewLoggerForTests()
+
+	configs := make([]*service.Config, count)
+
+	for i := 0; i < count; i++ {
+		raConf := service.MakeDefaultConfig()
+		raConf.Console = nil
+		raConf.Log = log
+		raConf.DataDir = t.TempDir()
+		raConf.SetToken("static-token-value")
+		raConf.AuthServers = []utils.NetAddr{
+			{
+				AddrNetwork: "tcp",
+				Addr:        p.rootCluster.Web,
+			},
+		}
+		raConf.Auth.Enabled = false
+		raConf.Proxy.Enabled = false
+		raConf.SSH.Enabled = false
+		raConf.Apps.Enabled = true
+		raConf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
+		raConf.Apps.Apps = append([]service.App{
+			{
+				Name:       p.rootAppName,
+				URI:        p.rootAppURI,
+				PublicAddr: p.rootAppPublicAddr,
+			},
+			{
+				Name:       p.rootWSAppName,
+				URI:        p.rootWSAppURI,
+				PublicAddr: p.rootWSPublicAddr,
+			},
+			{
+				Name:       p.rootWSSAppName,
+				URI:        p.rootWSSAppURI,
+				PublicAddr: p.rootWSSPublicAddr,
+			},
+			{
+				Name:       p.rootTCPAppName,
+				URI:        p.rootTCPAppURI,
+				PublicAddr: p.rootTCPPublicAddr,
+			},
+			{
+				Name:       p.jwtAppName,
+				URI:        p.jwtAppURI,
+				PublicAddr: p.jwtAppPublicAddr,
+			},
+			{
+				Name:       p.headerAppName,
+				URI:        p.headerAppURI,
+				PublicAddr: p.headerAppPublicAddr,
+			},
+			{
+				Name:       p.wsHeaderAppName,
+				URI:        p.wsHeaderAppURI,
+				PublicAddr: p.wsHeaderAppPublicAddr,
+			},
+			{
+				Name:       p.flushAppName,
+				URI:        p.flushAppURI,
+				PublicAddr: p.flushAppPublicAddr,
+			},
+			{
+				Name:       "dumper-root",
+				URI:        p.dumperAppURI,
+				PublicAddr: "dumper-root.example.com",
+				Rewrite: &service.Rewrite{
+					Headers: []service.Header{
+						{
+							Name:  "X-Teleport-Cluster",
+							Value: "root",
+						},
+						{
+							Name:  "X-External-Env",
+							Value: "{{external.env}}",
+						},
+						// Make sure can rewrite Host header.
+						{
+							Name:  "Host",
+							Value: "example.com",
+						},
+						// Make sure can rewrite existing header.
+						{
+							Name:  "X-Existing",
+							Value: "rewritten-existing-header",
+						},
+						// Make sure can't rewrite Teleport headers.
+						{
+							Name:  teleport.AppJWTHeader,
+							Value: "rewritten-app-jwt-header",
+						},
+						{
+							Name:  teleport.AppCFHeader,
+							Value: "rewritten-app-cf-header",
+						},
+						{
+							Name:  forward.XForwardedFor,
+							Value: "rewritten-x-forwarded-for-header",
+						},
+						{
+							Name:  forward.XForwardedHost,
+							Value: "rewritten-x-forwarded-host-header",
+						},
+						{
+							Name:  forward.XForwardedProto,
+							Value: "rewritten-x-forwarded-proto-header",
+						},
+						{
+							Name:  forward.XForwardedServer,
+							Value: "rewritten-x-forwarded-server-header",
+						},
+						// Make sure we can insert JWT token in custom header.
+						{
+							Name:  "X-JWT",
+							Value: teleport.TraitInternalJWTVariable,
+						},
+					},
+				},
+			},
+		}, extraApps...)
+
+		configs[i] = raConf
+	}
+
+	servers, err := p.rootCluster.StartApps(configs)
+	require.NoError(t, err)
+	require.Equal(t, len(configs), len(servers))
+
+	for _, appServer := range servers {
+		srv := appServer
+		t.Cleanup(func() {
+			require.NoError(t, srv.Close())
+		})
+		waitAppServerTunnel(t, p.rootCluster.Tunnel, p.rootAppClusterName, srv.Config.HostUUID)
+	}
+
+	return servers
+}
+
+func (p *Pack) startLeafAppServers(t *testing.T, count int, extraApps []service.App) []*service.TeleportProcess {
+	log := utils.NewLoggerForTests()
+	configs := make([]*service.Config, count)
+
+	for i := 0; i < count; i++ {
+		laConf := service.MakeDefaultConfig()
+		laConf.Console = nil
+		laConf.Log = log
+		laConf.DataDir = t.TempDir()
+		laConf.SetToken("static-token-value")
+		laConf.AuthServers = []utils.NetAddr{
+			{
+				AddrNetwork: "tcp",
+				Addr:        p.leafCluster.Web,
+			},
+		}
+		laConf.Auth.Enabled = false
+		laConf.Proxy.Enabled = false
+		laConf.SSH.Enabled = false
+		laConf.Apps.Enabled = true
+		laConf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
+		laConf.Apps.Apps = append([]service.App{
+			{
+				Name:       p.leafAppName,
+				URI:        p.leafAppURI,
+				PublicAddr: p.leafAppPublicAddr,
+			},
+			{
+				Name:       p.leafWSAppName,
+				URI:        p.leafWSAppURI,
+				PublicAddr: p.leafWSPublicAddr,
+			},
+			{
+				Name:       p.leafWSSAppName,
+				URI:        p.leafWSSAppURI,
+				PublicAddr: p.leafWSSPublicAddr,
+			},
+			{
+				Name:       p.leafTCPAppName,
+				URI:        p.leafTCPAppURI,
+				PublicAddr: p.leafTCPPublicAddr,
+			},
+			{
+				Name:       "dumper-leaf",
+				URI:        p.dumperAppURI,
+				PublicAddr: "dumper-leaf.example.com",
+				Rewrite: &service.Rewrite{
+					Headers: []service.Header{
+						{
+							Name:  "X-Teleport-Cluster",
+							Value: "leaf",
+						},
+						// In leaf clusters internal.logins variable is
+						// populated with the user's root role logins.
+						{
+							Name:  "X-Teleport-Login",
+							Value: "{{internal.logins}}",
+						},
+						{
+							Name:  "X-External-Env",
+							Value: "{{external.env}}",
+						},
+						// Make sure can rewrite Host header.
+						{
+							Name:  "Host",
+							Value: "example.com",
+						},
+						// Make sure can rewrite existing header.
+						{
+							Name:  "X-Existing",
+							Value: "rewritten-existing-header",
+						},
+						// Make sure can't rewrite Teleport headers.
+						{
+							Name:  teleport.AppJWTHeader,
+							Value: "rewritten-app-jwt-header",
+						},
+						{
+							Name:  teleport.AppCFHeader,
+							Value: "rewritten-app-cf-header",
+						},
+						{
+							Name:  forward.XForwardedFor,
+							Value: "rewritten-x-forwarded-for-header",
+						},
+						{
+							Name:  forward.XForwardedHost,
+							Value: "rewritten-x-forwarded-host-header",
+						},
+						{
+							Name:  forward.XForwardedProto,
+							Value: "rewritten-x-forwarded-proto-header",
+						},
+						{
+							Name:  forward.XForwardedServer,
+							Value: "rewritten-x-forwarded-server-header",
+						},
+					},
+				},
+			},
+		}, extraApps...)
+
+		configs[i] = laConf
+	}
+
+	servers, err := p.leafCluster.StartApps(configs)
+	require.NoError(t, err)
+	require.Equal(t, len(configs), len(servers))
+
+	for _, appServer := range servers {
+		srv := appServer
+		t.Cleanup(func() {
+			require.NoError(t, srv.Close())
+		})
+		waitAppServerTunnel(t, p.rootCluster.Tunnel, p.leafAppClusterName, srv.Config.HostUUID)
+	}
+
+	return servers
+}

integration/client_test.go

@@ -51,7 +51,7 @@ func TestClientWithExpiredCredentialsAndDetailedErrorMessage(t *testing.T) {
 	rcConf.SSH.Enabled = true
 	rcConf.Version = "v2"
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	rc.AddUser(username, []string{username})
 
 	err := rc.CreateEx(t, nil, rcConf)

integration/helpers/helpers.go

@@ -257,3 +257,9 @@ func WaitForAuditEventTypeWithBackoff(t *testing.T, cli *auth.Server, startTime
 		}
 	}
 }
+
+func MustGetCurrentUser(t *testing.T) *user.User {
+	user, err := user.Current()
+	require.NoError(t, err)
+	return user
+}

integration/helpers/instance.go

@@ -520,7 +520,7 @@ func (i *TeleInstance) GenerateConfig(t *testing.T, trustedSecrets []*InstanceSe
 	tconf.Keygen = testauthority.New()
 	tconf.MaxRetryPeriod = defaults.HighResPollingPeriod
 	tconf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
-	tconf.FileDescriptors = i.Fds
+	tconf.FileDescriptors = append(tconf.FileDescriptors, i.Fds...)
 
 	i.Config = tconf
 	return tconf, nil
@@ -963,6 +963,7 @@ func (i *TeleInstance) StartNodeAndProxy(t *testing.T, name string) (sshPort, we
 }
 
 // ProxyConfig is a set of configuration parameters for Proxy
+// TODO(tcsc): Add file descriptor slice to inject FDs into proxy process
 type ProxyConfig struct {
 	// Name is a proxy name
 	Name string

integration/helpers/ports.go

@@ -240,6 +240,8 @@ func NewListenerOn(t *testing.T, hostAddr string, ty service.ListenerType, fds *
 		File:    lf,
 	})
 
+	fmt.Printf("New Listener %s %s\n", ty, addr)
+
 	return addr
 }
 

integration/helpers/trustedclusters.go

@@ -0,0 +1,142 @@
+// Copyright 2022 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package helpers
+
+import (
+	"context"
+	"testing"
+	"time"
+
+	"github.com/gravitational/teleport/api/defaults"
+	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/lib/auth"
+	"github.com/gravitational/teleport/lib/reversetunnel"
+	"github.com/gravitational/teleport/lib/utils"
+	"github.com/gravitational/trace"
+	log "github.com/sirupsen/logrus"
+	"github.com/stretchr/testify/require"
+)
+
+// WaitForTunnelConnections waits for remote tunnels connections
+func WaitForTunnelConnections(t *testing.T, authServer *auth.Server, clusterName string, expectedCount int) {
+	t.Helper()
+	var conns []types.TunnelConnection
+	for i := 0; i < 30; i++ {
+		// to speed things up a bit, bypass the auth cache
+		conns, err := authServer.Services.GetTunnelConnections(clusterName)
+		require.NoError(t, err)
+		if len(conns) == expectedCount {
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+	require.Len(t, conns, expectedCount)
+}
+
+// TryCreateTrustedCluster performs several attempts to create a trusted cluster,
+// retries on connection problems and access denied errors to let caches
+// propagate and services to start
+//
+// Duplicated in tool/tsh/tsh_test.go
+func TryCreateTrustedCluster(t *testing.T, authServer *auth.Server, trustedCluster types.TrustedCluster) {
+	t.Helper()
+	ctx := context.TODO()
+	for i := 0; i < 10; i++ {
+		log.Debugf("Will create trusted cluster %v, attempt %v.", trustedCluster, i)
+		_, err := authServer.UpsertTrustedCluster(ctx, trustedCluster)
+		if err == nil {
+			return
+		}
+		if trace.IsConnectionProblem(err) {
+			log.Debugf("Retrying on connection problem: %v.", err)
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		if trace.IsAccessDenied(err) {
+			log.Debugf("Retrying on access denied: %v.", err)
+			time.Sleep(500 * time.Millisecond)
+			continue
+		}
+		require.FailNow(t, "Terminating on unexpected problem", "%v.", err)
+	}
+	require.FailNow(t, "Timeout creating trusted cluster")
+}
+
+func WaitForClusters(tun reversetunnel.Server, expected int) func() bool {
+	return func() bool {
+		clusters, err := tun.GetSites()
+		if err != nil {
+			return false
+		}
+
+		// Check the expected number of clusters are connected, and they have all
+		// connected with the past 10 seconds.
+		if len(clusters) >= expected {
+			for _, cluster := range clusters {
+				if time.Since(cluster.GetLastConnected()).Seconds() > 10.0 {
+					return false
+				}
+			}
+		}
+
+		return true
+	}
+}
+
+// WaitForNodeCount waits for a certain number of nodes to show up in the remote site.
+func WaitForNodeCount(ctx context.Context, t *TeleInstance, clusterName string, count int) error {
+	const (
+		deadline     = time.Second * 30
+		iterWaitTime = time.Second
+	)
+
+	err := utils.RetryStaticFor(deadline, iterWaitTime, func() error {
+		remoteSite, err := t.Tunnel.GetSite(clusterName)
+		if err != nil {
+			return trace.Wrap(err)
+		}
+		accessPoint, err := remoteSite.CachingAccessPoint()
+		if err != nil {
+			return trace.Wrap(err)
+		}
+		nodes, err := accessPoint.GetNodes(ctx, defaults.Namespace)
+		if err != nil {
+			return trace.Wrap(err)
+		}
+		if len(nodes) == count {
+			return nil
+		}
+		return trace.BadParameter("did not find %v nodes", count)
+	})
+	if err != nil {
+		return trace.Wrap(err)
+	}
+	return nil
+}
+
+// WaitForActiveTunnelConnections waits for remote cluster to report a minimum number of active connections
+func WaitForActiveTunnelConnections(t *testing.T, tunnel reversetunnel.Server, clusterName string, expectedCount int) {
+	require.Eventually(t, func() bool {
+		cluster, err := tunnel.GetSite(clusterName)
+		if err != nil {
+			return false
+		}
+		return cluster.GetTunnelsCount() >= expectedCount
+	},
+		30*time.Second,
+		time.Second,
+		"Active tunnel connections did not reach %v in the expected time frame", expectedCount,
+	)
+}

integration/integration_test.go

@@ -1839,9 +1839,9 @@ func twoClustersTunnel(t *testing.T, suite *integrationTestSuite, now time.Time,
 	require.NoError(t, err)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(a.Tunnel, 2), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 2), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(b.Tunnel, 2), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 2), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	var (
@@ -1898,7 +1898,7 @@ func twoClustersTunnel(t *testing.T, suite *integrationTestSuite, now time.Time,
 	require.True(t, ok)
 
 	// wait for active tunnel connections to be established
-	waitForActiveTunnelConnections(t, b.Tunnel, a.Secrets.SiteName, 1)
+	helpers.WaitForActiveTunnelConnections(t, b.Tunnel, a.Secrets.SiteName, 1)
 
 	// via tunnel b->a:
 	tc, err = b.NewClient(helpers.ClientConfig{
@@ -1998,9 +1998,9 @@ func testTwoClustersProxy(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, a.Start())
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// make sure the reverse tunnel went through the proxy
@@ -2045,9 +2045,9 @@ func testHA(t *testing.T, suite *integrationTestSuite) {
 	sshPort, _, _ := a.StartNodeAndProxy(t, "cluster-a-node")
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	cmd := []string{"echo", "hello world"}
@@ -2089,9 +2089,9 @@ func testHA(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, a.Start())
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(a.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(b.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// try to execute an SSH command using the same old client to site-B
@@ -2182,15 +2182,15 @@ func testMapRoles(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, err)
 
 	// try and upsert a trusted cluster
-	tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
-	waitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1)
+	helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
+	helpers.WaitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1)
 
 	sshPort, _, _ := aux.StartNodeAndProxy(t, "aux-node")
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// Make sure that GetNodes returns nodes in the remote site. This makes
@@ -2323,34 +2323,6 @@ func testMapRoles(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, aux.StopAll())
 }
 
-// tryCreateTrustedCluster performs several attempts to create a trusted cluster,
-// retries on connection problems and access denied errors to let caches
-// propagate and services to start
-//
-// Duplicated in tool/tsh/tsh_test.go
-func tryCreateTrustedCluster(t *testing.T, authServer *auth.Server, trustedCluster types.TrustedCluster) {
-	ctx := context.TODO()
-	for i := 0; i < 10; i++ {
-		log.Debugf("Will create trusted cluster %v, attempt %v.", trustedCluster, i)
-		_, err := authServer.UpsertTrustedCluster(ctx, trustedCluster)
-		if err == nil {
-			return
-		}
-		if trace.IsConnectionProblem(err) {
-			log.Debugf("Retrying on connection problem: %v.", err)
-			time.Sleep(500 * time.Millisecond)
-			continue
-		}
-		if trace.IsAccessDenied(err) {
-			log.Debugf("Retrying on access denied: %v.", err)
-			time.Sleep(500 * time.Millisecond)
-			continue
-		}
-		require.FailNow(t, "Terminating on unexpected problem", "%v.", err)
-	}
-	require.FailNow(t, "Timeout creating trusted cluster")
-}
-
 // trustedClusterTest is a test setup for trusted clusters tests
 type trustedClusterTest struct {
 	// multiplex sets up multiplexing of the reversetunnel SSH
@@ -2528,15 +2500,15 @@ func trustedClusters(t *testing.T, suite *integrationTestSuite, test trustedClus
 	require.NoError(t, err)
 
 	// try and upsert a trusted cluster
-	tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
-	waitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1)
+	helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
+	helpers.WaitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1)
 
 	sshPort, _, _ := aux.StartNodeAndProxy(t, "aux-node")
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	cmd := []string{"echo", "hello world"}
@@ -2628,7 +2600,7 @@ func trustedClusters(t *testing.T, suite *integrationTestSuite, test trustedClus
 	require.Equal(t, clusterAux, remoteClusters[0].GetName())
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// connection and client should recover and work again
@@ -2649,27 +2621,6 @@ func trustedClusters(t *testing.T, suite *integrationTestSuite, test trustedClus
 	require.NoError(t, aux.StopAll())
 }
 
-func waitForClusters(tun reversetunnel.Server, expected int) func() bool {
-	return func() bool {
-		clusters, err := tun.GetSites()
-		if err != nil {
-			return false
-		}
-
-		// Check the expected number of clusters are connected, and they have all
-		// connected with the past 10 seconds.
-		if len(clusters) >= expected {
-			for _, cluster := range clusters {
-				if time.Since(cluster.GetLastConnected()).Seconds() > 10.0 {
-					return false
-				}
-			}
-		}
-
-		return true
-	}
-}
-
 func testTrustedTunnelNode(t *testing.T, suite *integrationTestSuite) {
 	ctx := context.Background()
 	username := suite.Me.Username
@@ -2736,8 +2687,8 @@ func testTrustedTunnelNode(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, err)
 
 	// try and upsert a trusted cluster
-	tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
-	waitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1)
+	helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
+	helpers.WaitForTunnelConnections(t, main.Process.GetAuthServer(), clusterAux, 1)
 
 	// Create a Teleport instance with a node that dials back to the aux cluster.
 	tunnelNodeHostname := "cluster-aux-node"
@@ -2760,13 +2711,13 @@ func testTrustedTunnelNode(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, err)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// Wait for both nodes to show up before attempting to dial to them.
-	err = waitForNodeCount(ctx, main, clusterAux, 2)
+	err = helpers.WaitForNodeCount(ctx, main, clusterAux, 2)
 	require.NoError(t, err)
 
 	cmd := []string{"echo", "hello world"}
@@ -2853,9 +2804,9 @@ func testDiscoveryRecovers(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, remote.Start())
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// Helper function for adding a new proxy to "main".
@@ -2916,7 +2867,7 @@ func testDiscoveryRecovers(t *testing.T, suite *integrationTestSuite) {
 	}
 
 	// ensure that initial proxy's tunnel has been established
-	waitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1)
 	// execute the connection via initial proxy; should not fail
 	testProxyConn(nil, false)
 
@@ -2986,12 +2937,13 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, remote.Start())
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(remote.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// start second proxy
+	// TODO(tcsc): Replace use of deprecated NewPortSlice() with preconfigured listeners
 	nodePorts := helpers.NewPortSlice(3)
 	proxyReverseTunnelPort, proxyWebPort, proxySSHPort := nodePorts[0], nodePorts[1], nodePorts[2]
 	proxyConfig := helpers.ProxyConfig{
@@ -3009,8 +2961,8 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) {
 
 	// At this point the main cluster should observe two tunnels
 	// connected to it from remote cluster
-	waitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1)
-	waitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1)
+	helpers.WaitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1)
 
 	// execute the connection via first proxy
 	cfg := helpers.ClientConfig{
@@ -3040,7 +2992,7 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) {
 
 	// Now disconnect the main proxy and make sure it will reconnect eventually.
 	require.NoError(t, lb.RemoveBackend(mainProxyAddr))
-	waitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1)
+	helpers.WaitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1)
 
 	// Requests going via main proxy should fail.
 	_, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1)
@@ -3058,8 +3010,8 @@ func testDiscovery(t *testing.T, suite *integrationTestSuite) {
 	lb.AddBackend(mainProxyAddr)
 
 	// Once the proxy is added a matching tunnel connection should be created.
-	waitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1)
-	waitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, "cluster-remote", 1)
+	helpers.WaitForActiveTunnelConnections(t, secondProxy, "cluster-remote", 1)
 
 	// Requests going via main proxy should succeed.
 	output, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 40)
@@ -3161,8 +3113,8 @@ func testReverseTunnelCollapse(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, err)
 
 	// Wait for active tunnel connections to be established.
-	waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0)
-	waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0)
+	helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
 
 	// Execute the connection via first proxy.
 	cfg := helpers.ClientConfig{
@@ -3186,8 +3138,8 @@ func testReverseTunnelCollapse(t *testing.T, suite *integrationTestSuite) {
 
 	// stop the proxy to collapse the tunnel
 	require.NoError(t, main.StopProxy())
-	waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0)
-	waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 0)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0)
+	helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 0)
 
 	// Requests going via both proxy will fail.
 	_, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1)
@@ -3202,8 +3154,8 @@ func testReverseTunnelCollapse(t *testing.T, suite *integrationTestSuite) {
 	// start the proxy again and ensure the tunnel is re-established
 	proxyTunnel, err = main.StartProxy(proxyConfig)
 	require.NoError(t, err)
-	waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0)
-	waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 0)
+	helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
 
 	// Requests going to the connected proxy should succeed.
 	_, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1)
@@ -3300,8 +3252,8 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, err)
 
 	// Wait for active tunnel connections to be established.
-	waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1)
-	waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
 
 	// Execute the connection via first proxy.
 	cfg := helpers.ClientConfig{
@@ -3330,7 +3282,7 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) {
 
 	// Remove second proxy from LB.
 	require.NoError(t, lb.RemoveBackend(*proxyTwoBackend))
-	waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1)
 
 	// Requests going via main proxy will succeed. Requests going via second
 	// proxy will fail.
@@ -3342,8 +3294,8 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) {
 
 	// Add second proxy to LB, both should have a connection.
 	lb.AddBackend(*proxyTwoBackend)
-	waitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1)
-	waitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, main.Tunnel, helpers.Site, 1)
+	helpers.WaitForActiveTunnelConnections(t, proxyTunnel, helpers.Site, 1)
 
 	// Requests going via both proxies will succeed.
 	output, err = runCommand(t, main, []string{"echo", "hello world"}, cfg, 1)
@@ -3360,100 +3312,6 @@ func testDiscoveryNode(t *testing.T, suite *integrationTestSuite) {
 	require.NoError(t, err)
 }
 
-// waitForActiveTunnelConnections waits for remote cluster to report a minimum number of active connections
-func waitForActiveTunnelConnections(t *testing.T, tunnel reversetunnel.Server, clusterName string, expectedCount int) {
-	require.Eventually(t, func() bool {
-		cluster, err := tunnel.GetSite(clusterName)
-		if err != nil {
-			return false
-		}
-		return cluster.GetTunnelsCount() >= expectedCount
-	},
-		30*time.Second,
-		time.Second,
-		"Active tunnel connections did not reach %v in the expected time frame", expectedCount,
-	)
-}
-
-// waitForActivePeerProxyConnections waits for remote cluster to report a minimum number of active proxy peer connections
-func waitForActivePeerProxyConnections(t *testing.T, tunnel reversetunnel.Server, expectedCount int) {
-	require.Eventually(t, func() bool {
-		return tunnel.GetProxyPeerClient().GetConnectionsCount() >= expectedCount
-	},
-		30*time.Second,
-		time.Second,
-		"Peer proxy connections did not reach %v in the expected time frame", expectedCount,
-	)
-}
-
-// waitForNodeCount waits for a certain number of nodes to show up in the remote site.
-func waitForNodeCount(ctx context.Context, t *helpers.TeleInstance, clusterName string, count int) error {
-	const (
-		deadline     = time.Second * 30
-		iterWaitTime = time.Second
-	)
-
-	err := utils.RetryStaticFor(deadline, iterWaitTime, func() error {
-		remoteSite, err := t.Tunnel.GetSite(clusterName)
-		if err != nil {
-			return trace.Wrap(err)
-		}
-		accessPoint, err := remoteSite.CachingAccessPoint()
-		if err != nil {
-			return trace.Wrap(err)
-		}
-		nodes, err := accessPoint.GetNodes(ctx, defaults.Namespace)
-		if err != nil {
-			return trace.Wrap(err)
-		}
-		if len(nodes) == count {
-			return nil
-		}
-		return trace.BadParameter("did not find %v nodes", count)
-	})
-	if err != nil {
-		return trace.Wrap(err)
-	}
-	return nil
-}
-
-// waitForTunnelConnections waits for remote tunnels connections
-func waitForTunnelConnections(t *testing.T, authServer *auth.Server, clusterName string, expectedCount int) {
-	var conns []types.TunnelConnection
-	for i := 0; i < 30; i++ {
-		// to speed things up a bit, bypass the auth cache
-		conns, err := authServer.Services.GetTunnelConnections(clusterName)
-		require.NoError(t, err)
-		if len(conns) == expectedCount {
-			return
-		}
-		time.Sleep(1 * time.Second)
-	}
-	require.Len(t, conns, expectedCount)
-}
-
-// waitAppServerTunnel waits for application server tunnel connections.
-func waitAppServerTunnel(t *testing.T, tunnel reversetunnel.Server, clusterName, serverUUID string) {
-	t.Helper()
-	cluster, err := tunnel.GetSite(clusterName)
-	require.NoError(t, err)
-
-	require.Eventually(t, func() bool {
-		conn, err := cluster.Dial(reversetunnel.DialParams{
-			From:     &utils.NetAddr{AddrNetwork: "tcp", Addr: "@web-proxy"},
-			To:       &utils.NetAddr{AddrNetwork: "tcp", Addr: reversetunnel.LocalNode},
-			ServerID: fmt.Sprintf("%v.%v", serverUUID, clusterName),
-			ConnType: types.AppTunnel,
-		})
-		if err != nil {
-			return false
-		}
-
-		require.NoError(t, conn.Close())
-		return true
-	}, 10*time.Second, time.Second)
-}
-
 // TestExternalClient tests if we can connect to a node in a Teleport
 // cluster. Both normal and recording proxies are tested.
 func testExternalClient(t *testing.T, suite *integrationTestSuite) {
@@ -4406,8 +4264,8 @@ func testRotateTrustedClusters(t *testing.T, suite *integrationTestSuite) {
 	lib.SetInsecureDevMode(true)
 	defer lib.SetInsecureDevMode(false)
 
-	tryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
-	waitForTunnelConnections(t, svc.GetAuthServer(), aux.Secrets.SiteName, 1)
+	helpers.TryCreateTrustedCluster(t, aux.Process.GetAuthServer(), trustedCluster)
+	helpers.WaitForTunnelConnections(t, svc.GetAuthServer(), aux.Secrets.SiteName, 1)
 
 	// capture credentials before reload has started to simulate old client
 	initialCreds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{
@@ -6364,8 +6222,8 @@ func createTrustedClusterPair(t *testing.T, suite *integrationTestSuite, extraSe
 	t.Cleanup(func() { leaf.StopAll() })
 
 	require.NoError(t, trustedCluster.CheckAndSetDefaults())
-	tryCreateTrustedCluster(t, leaf.Process.GetAuthServer(), trustedCluster)
-	waitForTunnelConnections(t, root.Process.GetAuthServer(), leafName, 1)
+	helpers.TryCreateTrustedCluster(t, leaf.Process.GetAuthServer(), trustedCluster)
+	helpers.WaitForTunnelConnections(t, root.Process.GetAuthServer(), leafName, 1)
 
 	_, _, rootProxySSHPort := root.StartNodeAndProxy(t, "root-zero")
 	_, _, _ = leaf.StartNodeAndProxy(t, "leaf-zero")
@@ -6375,8 +6233,8 @@ func createTrustedClusterPair(t *testing.T, suite *integrationTestSuite, extraSe
 		extraServices(t, root, leaf)
 	}
 
-	require.Eventually(t, waitForClusters(root.Tunnel, 1), 10*time.Second, 1*time.Second)
-	require.Eventually(t, waitForClusters(leaf.Tunnel, 1), 10*time.Second, 1*time.Second)
+	require.Eventually(t, helpers.WaitForClusters(root.Tunnel, 1), 10*time.Second, 1*time.Second)
+	require.Eventually(t, helpers.WaitForClusters(leaf.Tunnel, 1), 10*time.Second, 1*time.Second)
 
 	// Create client.
 	creds, err := helpers.GenerateUserCreds(helpers.UserCredsRequest{

integration/kube/fixtures.go

@@ -0,0 +1,134 @@
+// Copyright 2022 Gravitational, Inc
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kube
+
+import (
+	"context"
+	"time"
+
+	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/integration/helpers"
+	"github.com/gravitational/teleport/lib/auth/native"
+	"github.com/gravitational/teleport/lib/services"
+	"github.com/gravitational/teleport/lib/tlsca"
+	"github.com/gravitational/teleport/lib/utils"
+	"github.com/gravitational/trace"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+)
+
+// For this test suite to work, the target Kubernetes cluster must have the
+// following RBAC objects configured:
+// https://github.com/gravitational/teleport/blob/master/fixtures/ci-teleport-rbac/ci-teleport.yaml
+const TestImpersonationGroup = "teleport-ci-test-group"
+
+type ProxyConfig struct {
+	T                   *helpers.TeleInstance
+	Username            string
+	KubeUsers           []string
+	KubeGroups          []string
+	Impersonation       *rest.ImpersonationConfig
+	RouteToCluster      string
+	CustomTLSServerName string
+	TargetAddress       utils.NetAddr
+}
+
+// ProxyClient returns kubernetes client using local teleport proxy
+func ProxyClient(cfg ProxyConfig) (*kubernetes.Clientset, *rest.Config, error) {
+	authServer := cfg.T.Process.GetAuthServer()
+	clusterName, err := authServer.GetClusterName()
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+
+	// Fetch user info to get roles and max session TTL.
+	user, err := authServer.GetUser(cfg.Username, false)
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	roles, err := services.FetchRoles(user.GetRoles(), authServer, user.GetTraits())
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	ttl := roles.AdjustSessionTTL(10 * time.Minute)
+
+	ca, err := authServer.GetCertAuthority(context.Background(), types.CertAuthID{
+		Type:       types.HostCA,
+		DomainName: clusterName.GetClusterName(),
+	}, true)
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	caCert, signer, err := authServer.GetKeyStore().GetTLSCertAndSigner(ca)
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	tlsCA, err := tlsca.FromCertAndSigner(caCert, signer)
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	privPEM, _, err := native.GenerateKeyPair()
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	priv, err := tlsca.ParsePrivateKeyPEM(privPEM)
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+
+	id := tlsca.Identity{
+		Username:         cfg.Username,
+		Groups:           user.GetRoles(),
+		KubernetesUsers:  cfg.KubeUsers,
+		KubernetesGroups: cfg.KubeGroups,
+		RouteToCluster:   cfg.RouteToCluster,
+	}
+	subj, err := id.Subject()
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+	cert, err := tlsCA.GenerateCertificate(tlsca.CertificateRequest{
+		Clock:     authServer.GetClock(),
+		PublicKey: priv.Public(),
+		Subject:   subj,
+		NotAfter:  authServer.GetClock().Now().Add(ttl),
+	})
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+
+	tlsClientConfig := rest.TLSClientConfig{
+		CAData:     ca.GetActiveKeys().TLS[0].Cert,
+		CertData:   cert,
+		KeyData:    privPEM,
+		ServerName: cfg.CustomTLSServerName,
+	}
+	config := &rest.Config{
+		Host:            "https://" + cfg.T.Config.Proxy.Kube.ListenAddr.Addr,
+		TLSClientConfig: tlsClientConfig,
+	}
+	if !cfg.TargetAddress.IsEmpty() {
+		config.Host = "https://" + cfg.TargetAddress.Addr
+	}
+	if cfg.Impersonation != nil {
+		config.Impersonate = *cfg.Impersonation
+	}
+	client, err := kubernetes.NewForConfig(config)
+	if err != nil {
+		return nil, nil, trace.Wrap(err)
+	}
+
+	return client, config, nil
+}

integration/kube_integration_test.go

@@ -37,8 +37,8 @@ import (
 	"github.com/gravitational/teleport/api/profile"
 	"github.com/gravitational/teleport/api/types"
 	"github.com/gravitational/teleport/integration/helpers"
+	"github.com/gravitational/teleport/integration/kube"
 	"github.com/gravitational/teleport/lib"
-	"github.com/gravitational/teleport/lib/auth/native"
 	"github.com/gravitational/teleport/lib/auth/testauthority"
 	"github.com/gravitational/teleport/lib/client"
 	"github.com/gravitational/teleport/lib/events"
@@ -137,11 +137,6 @@ func newKubeSuite(t *testing.T) *KubeSuite {
 	return suite
 }
 
-// For this test suite to work, the target Kubernetes cluster must have the
-// following RBAC objects configured:
-// https://github.com/gravitational/teleport/blob/master/fixtures/ci-teleport-rbac/ci-teleport.yaml
-const testImpersonationGroup = "teleport-ci-test-group"
-
 type kubeIntegrationTest func(t *testing.T, suite *KubeSuite)
 
 func (s *KubeSuite) bind(test kubeIntegrationTest) func(t *testing.T) {
@@ -179,7 +174,7 @@ func testKubeExec(t *testing.T, suite *KubeSuite) {
 	})
 
 	username := suite.me.Username
-	kubeGroups := []string{testImpersonationGroup}
+	kubeGroups := []string{kube.TestImpersonationGroup}
 	kubeUsers := []string{"alice@example.com"}
 	role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{
 		Allow: types.RoleConditions{
@@ -200,12 +195,12 @@ func testKubeExec(t *testing.T, suite *KubeSuite) {
 
 	// impersonating client requests will be denied if the headers
 	// are referencing users or groups not allowed by the existing roles
-	impersonatingProxyClient, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:             teleport,
-		username:      username,
-		kubeUsers:     kubeUsers,
-		kubeGroups:    kubeGroups,
-		impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}},
+	impersonatingProxyClient, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:             teleport,
+		Username:      username,
+		KubeUsers:     kubeUsers,
+		KubeGroups:    kubeGroups,
+		Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
 	})
 	require.NoError(t, err)
 
@@ -216,12 +211,12 @@ func testKubeExec(t *testing.T, suite *KubeSuite) {
 
 	// scoped client requests will be allowed, as long as the impersonation headers
 	// are referencing users and groups allowed by existing roles
-	scopedProxyClient, scopedProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   username,
-		kubeUsers:  kubeUsers,
-		kubeGroups: kubeGroups,
-		impersonation: &rest.ImpersonationConfig{
+	scopedProxyClient, scopedProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   username,
+		KubeUsers:  kubeUsers,
+		KubeGroups: kubeGroups,
+		Impersonation: &rest.ImpersonationConfig{
 			UserName: role.GetKubeUsers(types.Allow)[0],
 			Groups:   role.GetKubeGroups(types.Allow),
 		},
@@ -232,11 +227,11 @@ func testKubeExec(t *testing.T, suite *KubeSuite) {
 	require.NoError(t, err)
 
 	// set up kube configuration using proxy
-	proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   username,
-		kubeUsers:  kubeUsers,
-		kubeGroups: kubeGroups,
+	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   username,
+		KubeUsers:  kubeUsers,
+		KubeGroups: kubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -348,7 +343,7 @@ func testKubeDeny(t *testing.T, suite *KubeSuite) {
 	})
 
 	username := suite.me.Username
-	kubeGroups := []string{testImpersonationGroup}
+	kubeGroups := []string{kube.TestImpersonationGroup}
 	kubeUsers := []string{"alice@example.com"}
 	role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{
 		Allow: types.RoleConditions{
@@ -372,11 +367,11 @@ func testKubeDeny(t *testing.T, suite *KubeSuite) {
 	defer teleport.StopAll()
 
 	// set up kube configuration using proxy
-	proxyClient, _, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   username,
-		kubeUsers:  kubeUsers,
-		kubeGroups: kubeGroups,
+	proxyClient, _, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   username,
+		KubeUsers:  kubeUsers,
+		KubeGroups: kubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -400,7 +395,7 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) {
 	})
 
 	username := suite.me.Username
-	kubeGroups := []string{testImpersonationGroup}
+	kubeGroups := []string{kube.TestImpersonationGroup}
 	role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{
 		Allow: types.RoleConditions{
 			Logins:     []string{username},
@@ -418,10 +413,10 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) {
 	defer teleport.StopAll()
 
 	// set up kube configuration using proxy
-	_, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   username,
-		kubeGroups: kubeGroups,
+	_, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   username,
+		KubeGroups: kubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -453,12 +448,12 @@ func testKubePortForward(t *testing.T, suite *KubeSuite) {
 	require.Equal(t, http.StatusOK, resp.StatusCode)
 	require.NoError(t, resp.Body.Close())
 
-	// impersonating client requests will be denied
-	_, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:             teleport,
-		username:      username,
-		kubeGroups:    kubeGroups,
-		impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}},
+	// impersonating client requests will bse denied
+	_, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:             teleport,
+		Username:      username,
+		KubeGroups:    kubeGroups,
+		Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
 	})
 	require.NoError(t, err)
 
@@ -496,7 +491,7 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) {
 
 	// main cluster has a role and user called main-kube
 	username := suite.me.Username
-	mainKubeGroups := []string{testImpersonationGroup}
+	mainKubeGroups := []string{kube.TestImpersonationGroup}
 	mainRole, err := types.NewRoleV3("main-kube", types.RoleSpecV5{
 		Allow: types.RoleConditions{
 			Logins:     []string{username},
@@ -581,18 +576,18 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) {
 	require.True(t, upsertSuccess)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// impersonating client requests will be denied
-	impersonatingProxyClient, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:              main,
-		username:       username,
-		kubeGroups:     mainKubeGroups,
-		impersonation:  &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}},
-		routeToCluster: clusterAux,
+	impersonatingProxyClient, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:              main,
+		Username:       username,
+		KubeGroups:     mainKubeGroups,
+		Impersonation:  &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
+		RouteToCluster: clusterAux,
 	})
 	require.NoError(t, err)
 
@@ -601,11 +596,11 @@ func testKubeTrustedClustersClientCert(t *testing.T, suite *KubeSuite) {
 	require.Error(t, err)
 
 	// set up kube configuration using main proxy
-	proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:              main,
-		username:       username,
-		kubeGroups:     mainKubeGroups,
-		routeToCluster: clusterAux,
+	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:              main,
+		Username:       username,
+		KubeGroups:     mainKubeGroups,
+		RouteToCluster: clusterAux,
 	})
 	require.NoError(t, err)
 
@@ -747,7 +742,7 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 
 	// main cluster has a role and user called main-kube
 	username := suite.me.Username
-	mainKubeGroups := []string{testImpersonationGroup}
+	mainKubeGroups := []string{kube.TestImpersonationGroup}
 	mainRole, err := types.NewRoleV3("main-kube", types.RoleSpecV5{
 		Allow: types.RoleConditions{
 			Logins:     []string{username},
@@ -836,17 +831,17 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 	require.True(t, upsertSuccess)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(main.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(aux.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// impersonating client requests will be denied
-	impersonatingProxyClient, impersonatingProxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:             main,
-		username:      username,
-		kubeGroups:    mainKubeGroups,
-		impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{testImpersonationGroup}},
+	impersonatingProxyClient, impersonatingProxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:             main,
+		Username:      username,
+		KubeGroups:    mainKubeGroups,
+		Impersonation: &rest.ImpersonationConfig{UserName: "bob", Groups: []string{kube.TestImpersonationGroup}},
 	})
 	require.NoError(t, err)
 
@@ -855,10 +850,10 @@ func testKubeTrustedClustersSNI(t *testing.T, suite *KubeSuite) {
 	require.Error(t, err)
 
 	// set up kube configuration using main proxy
-	proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          main,
-		username:   username,
-		kubeGroups: mainKubeGroups,
+	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          main,
+		Username:   username,
+		KubeGroups: mainKubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -1021,7 +1016,7 @@ func runKubeDisconnectTest(t *testing.T, suite *KubeSuite, tc disconnectTestCase
 	})
 
 	username := suite.me.Username
-	kubeGroups := []string{testImpersonationGroup}
+	kubeGroups := []string{kube.TestImpersonationGroup}
 	role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{
 		Options: tc.options,
 		Allow: types.RoleConditions{
@@ -1040,10 +1035,10 @@ func runKubeDisconnectTest(t *testing.T, suite *KubeSuite, tc disconnectTestCase
 	defer teleport.StopAll()
 
 	// set up kube configuration using proxy
-	proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   username,
-		kubeGroups: kubeGroups,
+	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   username,
+		KubeGroups: kubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -1107,7 +1102,7 @@ func testKubeTransportProtocol(t *testing.T, suite *KubeSuite) {
 	})
 
 	username := suite.me.Username
-	kubeGroups := []string{testImpersonationGroup}
+	kubeGroups := []string{kube.TestImpersonationGroup}
 	role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{
 		Allow: types.RoleConditions{
 			Logins:     []string{username},
@@ -1125,10 +1120,10 @@ func testKubeTransportProtocol(t *testing.T, suite *KubeSuite) {
 	defer teleport.StopAll()
 
 	// set up kube configuration using proxy
-	proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   username,
-		kubeGroups: kubeGroups,
+	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   username,
+		KubeGroups: kubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -1227,20 +1222,9 @@ func tlsClientConfig(cfg *rest.Config) (*tls.Config, error) {
 	return tlsConfig, nil
 }
 
-type kubeProxyConfig struct {
-	t                   *helpers.TeleInstance
-	username            string
-	kubeUsers           []string
-	kubeGroups          []string
-	impersonation       *rest.ImpersonationConfig
-	routeToCluster      string
-	customTLSServerName string
-	targetAddress       utils.NetAddr
-}
-
-func kubeProxyTLSConfig(cfg kubeProxyConfig) (*tls.Config, error) {
+func kubeProxyTLSConfig(cfg kube.ProxyConfig) (*tls.Config, error) {
 	tlsConfig := &tls.Config{}
-	_, kubeConfig, err := kubeProxyClient(cfg)
+	_, kubeConfig, err := kube.ProxyClient(cfg)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1262,94 +1246,6 @@ func kubeProxyTLSConfig(cfg kubeProxyConfig) (*tls.Config, error) {
 	return tlsConfig, nil
 }
 
-// kubeProxyClient returns kubernetes client using local teleport proxy
-func kubeProxyClient(cfg kubeProxyConfig) (*kubernetes.Clientset, *rest.Config, error) {
-	authServer := cfg.t.Process.GetAuthServer()
-	clusterName, err := authServer.GetClusterName()
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-
-	// Fetch user info to get roles and max session TTL.
-	user, err := authServer.GetUser(cfg.username, false)
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	roles, err := services.FetchRoles(user.GetRoles(), authServer, user.GetTraits())
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	ttl := roles.AdjustSessionTTL(10 * time.Minute)
-
-	ca, err := authServer.GetCertAuthority(context.Background(), types.CertAuthID{
-		Type:       types.HostCA,
-		DomainName: clusterName.GetClusterName(),
-	}, true)
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	caCert, signer, err := authServer.GetKeyStore().GetTLSCertAndSigner(ca)
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	tlsCA, err := tlsca.FromCertAndSigner(caCert, signer)
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	privPEM, _, err := native.GenerateKeyPair()
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	priv, err := tlsca.ParsePrivateKeyPEM(privPEM)
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-
-	id := tlsca.Identity{
-		Username:         cfg.username,
-		Groups:           user.GetRoles(),
-		KubernetesUsers:  cfg.kubeUsers,
-		KubernetesGroups: cfg.kubeGroups,
-		RouteToCluster:   cfg.routeToCluster,
-	}
-	subj, err := id.Subject()
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-	cert, err := tlsCA.GenerateCertificate(tlsca.CertificateRequest{
-		Clock:     authServer.GetClock(),
-		PublicKey: priv.Public(),
-		Subject:   subj,
-		NotAfter:  authServer.GetClock().Now().Add(ttl),
-	})
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-
-	tlsClientConfig := rest.TLSClientConfig{
-		CAData:     ca.GetActiveKeys().TLS[0].Cert,
-		CertData:   cert,
-		KeyData:    privPEM,
-		ServerName: cfg.customTLSServerName,
-	}
-	config := &rest.Config{
-		Host:            "https://" + cfg.t.Config.Proxy.Kube.ListenAddr.Addr,
-		TLSClientConfig: tlsClientConfig,
-	}
-	if !cfg.targetAddress.IsEmpty() {
-		config.Host = "https://" + cfg.targetAddress.Addr
-	}
-	if cfg.impersonation != nil {
-		config.Impersonate = *cfg.impersonation
-	}
-	client, err := kubernetes.NewForConfig(config)
-	if err != nil {
-		return nil, nil, trace.Wrap(err)
-	}
-
-	return client, config, nil
-}
-
 const (
 	testNamespace = "teletest"
 	testPod       = "test-pod"
@@ -1481,7 +1377,7 @@ func kubeExec(kubeConfig *rest.Config, args kubeExecArgs) error {
 	return executor.Stream(opts)
 }
 
-func kubeJoin(kubeConfig kubeProxyConfig, tc *client.TeleportClient, sessionID string) (*client.KubeSession, error) {
+func kubeJoin(kubeConfig kube.ProxyConfig, tc *client.TeleportClient, sessionID string) (*client.KubeSession, error) {
 	tlsConfig, err := kubeProxyTLSConfig(kubeConfig)
 	if err != nil {
 		return nil, trace.Wrap(err)
@@ -1494,7 +1390,7 @@ func kubeJoin(kubeConfig kubeProxyConfig, tc *client.TeleportClient, sessionID s
 		return nil, trace.Wrap(err)
 	}
 
-	sess, err := client.NewKubeSession(context.TODO(), tc, meta, kubeConfig.t.Config.Proxy.Kube.ListenAddr.Addr, "", types.SessionPeerMode, tlsConfig)
+	sess, err := client.NewKubeSession(context.TODO(), tc, meta, kubeConfig.T.Config.Proxy.Kube.ListenAddr.Addr, "", types.SessionPeerMode, tlsConfig)
 	if err != nil {
 		return nil, trace.Wrap(err)
 	}
@@ -1517,7 +1413,7 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) {
 
 	hostUsername := suite.me.Username
 	participantUsername := suite.me.Username + "-participant"
-	kubeGroups := []string{testImpersonationGroup}
+	kubeGroups := []string{kube.TestImpersonationGroup}
 	kubeUsers := []string{"alice@example.com"}
 	role, err := types.NewRoleV3("kubemaster", types.RoleSpecV5{
 		Allow: types.RoleConditions{
@@ -1551,11 +1447,11 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) {
 	ctx := context.Background()
 
 	// set up kube configuration using proxy
-	proxyClient, proxyClientConfig, err := kubeProxyClient(kubeProxyConfig{
-		t:          teleport,
-		username:   hostUsername,
-		kubeUsers:  kubeUsers,
-		kubeGroups: kubeGroups,
+	proxyClient, proxyClientConfig, err := kube.ProxyClient(kube.ProxyConfig{
+		T:          teleport,
+		Username:   hostUsername,
+		KubeUsers:  kubeUsers,
+		KubeGroups: kubeGroups,
 	})
 	require.NoError(t, err)
 
@@ -1595,11 +1491,11 @@ func testKubeJoin(t *testing.T, suite *KubeSuite) {
 	tc.Stdin = participantStdinR
 	tc.Stdout = participantStdoutW
 
-	stream, err := kubeJoin(kubeProxyConfig{
-		t:          teleport,
-		username:   participantUsername,
-		kubeUsers:  kubeUsers,
-		kubeGroups: kubeGroups,
+	stream, err := kubeJoin(kube.ProxyConfig{
+		T:          teleport,
+		Username:   participantUsername,
+		KubeUsers:  kubeUsers,
+		KubeGroups: kubeGroups,
 	}, tc, "")
 	require.NoError(t, err)
 	defer stream.Close()

integration/proxy/main_test.go

@@ -0,0 +1,28 @@
+/*
+Copyright 2022 Gravitational, Inc.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package proxy
+
+import (
+	"testing"
+
+	"github.com/gravitational/teleport/integration/helpers"
+)
+
+// TestMain will re-execute Teleport to run a command if "exec" is passed to
+// it as an argument. Otherwise, it will run tests as normal.
+func TestMain(m *testing.M) {
+	helpers.TestMainImplementation(m)
+}

integration/proxy/proxy_helpers.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package integration
+package proxy
 
 import (
 	"bytes"
@@ -25,7 +25,6 @@ import (
 	"net"
 	"net/http"
 	"net/http/httptest"
-	"os/user"
 	"path/filepath"
 	"testing"
 	"time"
@@ -36,6 +35,7 @@ import (
 	apiutils "github.com/gravitational/teleport/api/utils"
 	"github.com/gravitational/teleport/integration/helpers"
 	"github.com/gravitational/teleport/lib/defaults"
+	"github.com/gravitational/teleport/lib/reversetunnel"
 	"github.com/gravitational/teleport/lib/service"
 	"github.com/gravitational/teleport/lib/services"
 	"github.com/gravitational/teleport/lib/srv/alpnproxy"
@@ -52,14 +52,14 @@ import (
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
 )
 
-type ProxySuite struct {
+type Suite struct {
 	root *helpers.TeleInstance
 	leaf *helpers.TeleInstance
 }
 
-type proxySuiteOptions struct {
-	rootConfigFunc func(suite *ProxySuite) *service.Config
-	leafConfigFunc func(suite *ProxySuite) *service.Config
+type suiteOptions struct {
+	rootConfigFunc func(suite *Suite) *service.Config
+	leafConfigFunc func(suite *Suite) *service.Config
 
 	rootConfigModFunc []func(config *service.Config)
 	leafConfigModFunc []func(config *service.Config)
@@ -70,22 +70,22 @@ type proxySuiteOptions struct {
 	rootClusterListeners helpers.InstanceListenerSetupFunc
 	leafClusterListeners helpers.InstanceListenerSetupFunc
 
-	rootTrustedSecretFunc func(suite *ProxySuite) []*helpers.InstanceSecrets
-	leafTrustedFunc       func(suite *ProxySuite) []*helpers.InstanceSecrets
+	rootTrustedSecretFunc func(suite *Suite) []*helpers.InstanceSecrets
+	leafTrustedFunc       func(suite *Suite) []*helpers.InstanceSecrets
 
 	rootClusterRoles      []types.Role
 	leafClusterRoles      []types.Role
-	updateRoleMappingFunc func(t *testing.T, suite *ProxySuite)
+	updateRoleMappingFunc func(t *testing.T, suite *Suite)
 
 	trustedCluster types.TrustedCluster
 }
 
-func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite {
-	options := proxySuiteOptions{
-		rootClusterNodeName:  Host,
-		leafClusterNodeName:  Host,
-		rootClusterListeners: helpers.SingleProxyPortSetupOn(Host),
-		leafClusterListeners: helpers.SingleProxyPortSetupOn(Host),
+func newSuite(t *testing.T, opts ...proxySuiteOptionsFunc) *Suite {
+	options := suiteOptions{
+		rootClusterNodeName:  helpers.Host,
+		leafClusterNodeName:  helpers.Host,
+		rootClusterListeners: helpers.SingleProxyPortSetupOn(helpers.Host),
+		leafClusterListeners: helpers.SingleProxyPortSetupOn(helpers.Host),
 	}
 	for _, opt := range opts {
 		opt(&options)
@@ -111,12 +111,12 @@ func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite {
 	}
 	lCfg.Listeners = options.leafClusterListeners(t, &lCfg.Fds)
 	lc := helpers.NewInstance(t, lCfg)
-	suite := &ProxySuite{
+	suite := &Suite{
 		root: rc,
 		leaf: lc,
 	}
 
-	user := mustGetCurrentUser(t)
+	user := helpers.MustGetCurrentUser(t)
 	for _, role := range options.rootClusterRoles {
 		rc.AddUserWithRole(user.Username, role)
 	}
@@ -162,14 +162,14 @@ func newProxySuite(t *testing.T, opts ...proxySuiteOptionsFunc) *ProxySuite {
 	}
 
 	if options.trustedCluster != nil {
-		tryCreateTrustedCluster(t, suite.leaf.Process.GetAuthServer(), options.trustedCluster)
-		waitForTunnelConnections(t, suite.root.Process.GetAuthServer(), suite.leaf.Secrets.SiteName, 1)
+		helpers.TryCreateTrustedCluster(t, suite.leaf.Process.GetAuthServer(), options.trustedCluster)
+		helpers.WaitForTunnelConnections(t, suite.root.Process.GetAuthServer(), suite.leaf.Secrets.SiteName, 1)
 	}
 
 	return suite
 }
 
-func (p *ProxySuite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname string) {
+func (p *Suite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname string) {
 	nodeConfig := func() *service.Config {
 		tconf := service.MakeDefaultConfig()
 		tconf.Console = nil
@@ -192,17 +192,17 @@ func (p *ProxySuite) addNodeToLeafCluster(t *testing.T, tunnelNodeHostname strin
 	require.NoError(t, err)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(p.root.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(p.root.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(p.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(p.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	// Wait for both nodes to show up before attempting to dial to them.
-	err = waitForNodeCount(context.Background(), p.root, p.leaf.Secrets.SiteName, 2)
+	err = helpers.WaitForNodeCount(context.Background(), p.root, p.leaf.Secrets.SiteName, 2)
 	require.NoError(t, err)
 }
 
-func (p *ProxySuite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config helpers.ClientConfig) {
+func (p *Suite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config helpers.ClientConfig) {
 	const (
 		deadline         = time.Second * 5
 		nextIterWaitTime = time.Millisecond * 100
@@ -225,73 +225,72 @@ func (p *ProxySuite) mustConnectToClusterAndRunSSHCommand(t *testing.T, config h
 	require.Equal(t, "hello world\n", output.String())
 }
 
-type proxySuiteOptionsFunc func(*proxySuiteOptions)
+type proxySuiteOptionsFunc func(*suiteOptions)
 
 func withRootClusterRoles(roles ...types.Role) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		options.rootClusterRoles = roles
 	}
 }
 
 func withLeafClusterRoles(roles ...types.Role) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		options.leafClusterRoles = roles
 	}
 }
 
 func withRootAndLeafClusterRoles(roles ...types.Role) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		withRootClusterRoles(roles...)(options)
 		withLeafClusterRoles(roles...)(options)
-
 	}
 }
 
-func withLeafClusterConfig(fn func(suite *ProxySuite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+func withLeafClusterConfig(fn func(suite *Suite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc {
+	return func(options *suiteOptions) {
 		options.leafConfigFunc = fn
 		options.leafConfigModFunc = append(options.leafConfigModFunc, configModFunctions...)
 	}
 }
 
-func withRootClusterConfig(fn func(suite *ProxySuite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+func withRootClusterConfig(fn func(suite *Suite) *service.Config, configModFunctions ...func(config *service.Config)) proxySuiteOptionsFunc {
+	return func(options *suiteOptions) {
 		options.rootConfigFunc = fn
 		options.rootConfigModFunc = append(options.rootConfigModFunc, configModFunctions...)
 	}
 }
 
 func withRootAndLeafTrustedClusterReset() proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
-		options.rootTrustedSecretFunc = func(suite *ProxySuite) []*helpers.InstanceSecrets {
+	return func(options *suiteOptions) {
+		options.rootTrustedSecretFunc = func(suite *Suite) []*helpers.InstanceSecrets {
 			return nil
 		}
-		options.leafTrustedFunc = func(suite *ProxySuite) []*helpers.InstanceSecrets {
+		options.leafTrustedFunc = func(suite *Suite) []*helpers.InstanceSecrets {
 			return nil
 		}
 	}
 }
 
 func withRootClusterNodeName(nodeName string) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		options.rootClusterNodeName = nodeName
 	}
 }
 
 func withLeafClusterNodeName(nodeName string) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		options.leafClusterNodeName = nodeName
 	}
 }
 
 func withRootClusterListeners(fn helpers.InstanceListenerSetupFunc) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		options.rootClusterListeners = fn
 	}
 }
 
 func withLeafClusterListeners(fn helpers.InstanceListenerSetupFunc) proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
+	return func(options *suiteOptions) {
 		options.leafClusterListeners = fn
 	}
 }
@@ -306,8 +305,8 @@ func newRole(t *testing.T, roleName string, username string) types.Role {
 	return role
 }
 
-func rootClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Config {
-	return func(suite *ProxySuite) *service.Config {
+func rootClusterStandardConfig(t *testing.T) func(suite *Suite) *service.Config {
+	return func(suite *Suite) *service.Config {
 		rc := suite.root
 		config := service.MakeDefaultConfig()
 		config.DataDir = t.TempDir()
@@ -326,8 +325,8 @@ func rootClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Co
 	}
 }
 
-func leafClusterStandardConfig(t *testing.T) func(suite *ProxySuite) *service.Config {
-	return func(suite *ProxySuite) *service.Config {
+func leafClusterStandardConfig(t *testing.T) func(suite *Suite) *service.Config {
+	return func(suite *Suite) *service.Config {
 		lc := suite.leaf
 		config := service.MakeDefaultConfig()
 		config.DataDir = t.TempDir()
@@ -355,11 +354,11 @@ func createTestRole(username string) types.Role {
 }
 
 func withStandardRoleMapping() proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
-		options.updateRoleMappingFunc = func(t *testing.T, suite *ProxySuite) {
+	return func(options *suiteOptions) {
+		options.updateRoleMappingFunc = func(t *testing.T, suite *Suite) {
 			rc := suite.root
 			lc := suite.leaf
-			role := suite.root.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0]
+			role := suite.root.Secrets.Users[helpers.MustGetCurrentUser(t).Username].Roles[0]
 			ca, err := lc.Process.GetAuthServer().GetCertAuthority(context.Background(), types.CertAuthID{
 				Type:       types.UserCA,
 				DomainName: rc.Secrets.SiteName,
@@ -375,11 +374,11 @@ func withStandardRoleMapping() proxySuiteOptionsFunc {
 }
 
 func withTrustedCluster() proxySuiteOptionsFunc {
-	return func(options *proxySuiteOptions) {
-		options.updateRoleMappingFunc = func(t *testing.T, suite *ProxySuite) {
+	return func(options *suiteOptions) {
+		options.updateRoleMappingFunc = func(t *testing.T, suite *Suite) {
 			root := suite.root
-			rootRole := suite.root.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0]
-			secondRole := suite.leaf.Secrets.Users[mustGetCurrentUser(t).Username].Roles[0]
+			rootRole := suite.root.Secrets.Users[helpers.MustGetCurrentUser(t).Username].Roles[0]
+			secondRole := suite.leaf.Secrets.Users[helpers.MustGetCurrentUser(t).Username].Roles[0]
 
 			trustedClusterToken := "trustedclustertoken"
 			err := root.Process.GetAuthServer().UpsertToken(context.Background(),
@@ -397,12 +396,6 @@ func withTrustedCluster() proxySuiteOptionsFunc {
 	}
 }
 
-func mustGetCurrentUser(t *testing.T) *user.User {
-	user, err := user.Current()
-	require.NoError(t, err)
-	return user
-}
-
 func mustRunPostgresQuery(t *testing.T, client *pgconn.PgConn) {
 	result, err := client.Exec(context.Background(), "select 1").ReadAll()
 	require.NoError(t, err)
@@ -602,3 +595,14 @@ func mustStartMockALBProxy(t *testing.T, proxyAddr string) *mockAWSALBProxy {
 	go m.serve(ctx, t)
 	return m
 }
+
+// waitForActivePeerProxyConnections waits for remote cluster to report a minimum number of active proxy peer connections
+func waitForActivePeerProxyConnections(t *testing.T, tunnel reversetunnel.Server, expectedCount int) {
+	require.Eventually(t, func() bool {
+		return tunnel.GetProxyPeerClient().GetConnectionsCount() >= expectedCount
+	},
+		30*time.Second,
+		time.Second,
+		"Peer proxy connections did not reach %v in the expected time frame", expectedCount,
+	)
+}

integration/proxy/proxy_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package integration
+package proxy
 
 import (
 	"bytes"
@@ -36,7 +36,9 @@ import (
 	"github.com/gravitational/teleport/api/breaker"
 	"github.com/gravitational/teleport/api/client"
 	"github.com/gravitational/teleport/api/types"
+	"github.com/gravitational/teleport/integration/appaccess"
 	"github.com/gravitational/teleport/integration/helpers"
+	"github.com/gravitational/teleport/integration/kube"
 	"github.com/gravitational/teleport/lib"
 	"github.com/gravitational/teleport/lib/auth/testauthority"
 	"github.com/gravitational/teleport/lib/defaults"
@@ -96,9 +98,9 @@ func TestALPNSNIProxyMultiCluster(t *testing.T) {
 			lib.SetInsecureDevMode(true)
 			defer lib.SetInsecureDevMode(false)
 
-			username := mustGetCurrentUser(t).Username
+			username := helpers.MustGetCurrentUser(t).Username
 
-			suite := newProxySuite(t,
+			suite := newSuite(t,
 				withRootClusterConfig(rootClusterStandardConfig(t), func(config *service.Config) {
 					config.Proxy.DisableALPNSNIListener = tc.disableALPNListenerOnRoot
 				}),
@@ -114,14 +116,14 @@ func TestALPNSNIProxyMultiCluster(t *testing.T) {
 			suite.mustConnectToClusterAndRunSSHCommand(t, helpers.ClientConfig{
 				Login:   username,
 				Cluster: suite.root.Secrets.SiteName,
-				Host:    Loopback,
+				Host:    helpers.Loopback,
 				Port:    helpers.Port(t, suite.root.SSH),
 			})
 			// Run command in leaf.
 			suite.mustConnectToClusterAndRunSSHCommand(t, helpers.ClientConfig{
 				Login:   username,
 				Cluster: suite.leaf.Secrets.SiteName,
-				Host:    Loopback,
+				Host:    helpers.Loopback,
 				Port:    helpers.Port(t, suite.leaf.SSH),
 			})
 		})
@@ -170,9 +172,9 @@ func TestALPNSNIProxyTrustedClusterNode(t *testing.T) {
 			lib.SetInsecureDevMode(true)
 			defer lib.SetInsecureDevMode(false)
 
-			username := mustGetCurrentUser(t).Username
+			username := helpers.MustGetCurrentUser(t).Username
 
-			suite := newProxySuite(t,
+			suite := newSuite(t,
 				withRootClusterConfig(rootClusterStandardConfig(t)),
 				withLeafClusterConfig(leafClusterStandardConfig(t)),
 				withRootClusterListeners(tc.mainClusterListenerSetup),
@@ -191,7 +193,7 @@ func TestALPNSNIProxyTrustedClusterNode(t *testing.T) {
 			suite.mustConnectToClusterAndRunSSHCommand(t, helpers.ClientConfig{
 				Login:   username,
 				Cluster: suite.leaf.Secrets.SiteName,
-				Host:    Loopback,
+				Host:    helpers.Loopback,
 				Port:    helpers.Port(t, suite.leaf.SSH),
 			})
 
@@ -219,7 +221,7 @@ func TestALPNSNIHTTPSProxy(t *testing.T) {
 	require.NoError(t, err)
 	t.Setenv("http_proxy", u.Host)
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 
 	// We need to use the non-loopback address for our Teleport cluster, as the
 	// Go HTTP library will recognize requests to the loopback address and
@@ -227,7 +229,7 @@ func TestALPNSNIHTTPSProxy(t *testing.T) {
 	addr, err := helpers.GetLocalIP()
 	require.NoError(t, err)
 
-	suite := newProxySuite(t,
+	suite := newSuite(t,
 		withRootClusterConfig(rootClusterStandardConfig(t)),
 		withLeafClusterConfig(leafClusterStandardConfig(t)),
 		withRootClusterNodeName(addr),
@@ -239,9 +241,9 @@ func TestALPNSNIHTTPSProxy(t *testing.T) {
 	)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	require.Greater(t, ph.Count(), 0, "proxy did not intercept any connection")
@@ -260,7 +262,7 @@ func TestMultiPortHTTPSProxy(t *testing.T) {
 	require.NoError(t, err)
 	t.Setenv("http_proxy", u.Host)
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 
 	// We need to use the non-loopback address for our Teleport cluster, as the
 	// Go HTTP library will recognize requests to the loopback address and
@@ -268,7 +270,7 @@ func TestMultiPortHTTPSProxy(t *testing.T) {
 	addr, err := helpers.GetLocalIP()
 	require.NoError(t, err)
 
-	suite := newProxySuite(t,
+	suite := newSuite(t,
 		withRootClusterConfig(rootClusterStandardConfig(t)),
 		withLeafClusterConfig(leafClusterStandardConfig(t)),
 		withRootClusterNodeName(addr),
@@ -280,9 +282,9 @@ func TestMultiPortHTTPSProxy(t *testing.T) {
 	)
 
 	// Wait for both cluster to see each other via reverse tunnels.
-	require.Eventually(t, waitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(suite.root.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
-	require.Eventually(t, waitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
+	require.Eventually(t, helpers.WaitForClusters(suite.leaf.Tunnel, 1), 10*time.Second, 1*time.Second,
 		"Two clusters do not see each other: tunnels are not working.")
 
 	require.Greater(t, ph.Count(), 0, "proxy did not intercept any connection")
@@ -300,18 +302,18 @@ func TestALPNSNIProxyKube(t *testing.T) {
 	kubeAPIMockSvr := startKubeAPIMock(t)
 	kubeConfigPath := mustCreateKubeConfigFile(t, k8ClientConfig(kubeAPIMockSvr.URL, localK8SNI))
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	kubeRoleSpec := types.RoleSpecV5{
 		Allow: types.RoleConditions{
 			Logins:     []string{username},
-			KubeGroups: []string{testImpersonationGroup},
+			KubeGroups: []string{kube.TestImpersonationGroup},
 			KubeUsers:  []string{k8User},
 		},
 	}
 	kubeRole, err := types.NewRoleV3(k8RoleName, kubeRoleSpec)
 	require.NoError(t, err)
 
-	suite := newProxySuite(t,
+	suite := newSuite(t,
 		withRootClusterConfig(rootClusterStandardConfig(t), func(config *service.Config) {
 			config.Proxy.Kube.Enabled = true
 			config.Proxy.Kube.KubeconfigPath = kubeConfigPath
@@ -322,13 +324,13 @@ func TestALPNSNIProxyKube(t *testing.T) {
 		withStandardRoleMapping(),
 	)
 
-	k8Client, _, err := kubeProxyClient(kubeProxyConfig{
-		t:                   suite.root,
-		username:            kubeRoleSpec.Allow.Logins[0],
-		kubeUsers:           kubeRoleSpec.Allow.KubeGroups,
-		kubeGroups:          kubeRoleSpec.Allow.KubeUsers,
-		customTLSServerName: localK8SNI,
-		targetAddress:       suite.root.Config.Proxy.WebAddr,
+	k8Client, _, err := kube.ProxyClient(kube.ProxyConfig{
+		T:                   suite.root,
+		Username:            kubeRoleSpec.Allow.Logins[0],
+		KubeUsers:           kubeRoleSpec.Allow.KubeGroups,
+		KubeGroups:          kubeRoleSpec.Allow.KubeUsers,
+		CustomTLSServerName: localK8SNI,
+		TargetAddress:       suite.root.Config.Proxy.WebAddr,
 	})
 	require.NoError(t, err)
 
@@ -352,18 +354,18 @@ func TestALPNSNIProxyKubeV2Leaf(t *testing.T) {
 	kubeAPIMockSvr := startKubeAPIMock(t)
 	kubeConfigPath := mustCreateKubeConfigFile(t, k8ClientConfig(kubeAPIMockSvr.URL, localK8SNI))
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	kubeRoleSpec := types.RoleSpecV5{
 		Allow: types.RoleConditions{
 			Logins:     []string{username},
-			KubeGroups: []string{testImpersonationGroup},
+			KubeGroups: []string{kube.TestImpersonationGroup},
 			KubeUsers:  []string{k8User},
 		},
 	}
 	kubeRole, err := types.NewRoleV3(k8RoleName, kubeRoleSpec)
 	require.NoError(t, err)
 
-	suite := newProxySuite(t,
+	suite := newSuite(t,
 		withRootClusterConfig(rootClusterStandardConfig(t), func(config *service.Config) {
 			config.Proxy.Kube.Enabled = true
 			config.Version = defaults.TeleportConfigVersionV2
@@ -374,7 +376,8 @@ func TestALPNSNIProxyKubeV2Leaf(t *testing.T) {
 
 			config.Kube.Enabled = true
 			config.Kube.KubeconfigPath = kubeConfigPath
-			config.Kube.ListenAddr = utils.MustParseAddr(net.JoinHostPort(Loopback, helpers.NewPortStr()))
+			config.Kube.ListenAddr = utils.MustParseAddr(
+				helpers.NewListener(t, service.ListenerKube, &config.FileDescriptors))
 		}),
 		withRootClusterRoles(kubeRole),
 		withLeafClusterRoles(kubeRole),
@@ -382,14 +385,14 @@ func TestALPNSNIProxyKubeV2Leaf(t *testing.T) {
 		withTrustedCluster(),
 	)
 
-	k8Client, _, err := kubeProxyClient(kubeProxyConfig{
-		t:                   suite.root,
-		username:            kubeRoleSpec.Allow.Logins[0],
-		kubeUsers:           kubeRoleSpec.Allow.KubeGroups,
-		kubeGroups:          kubeRoleSpec.Allow.KubeUsers,
-		customTLSServerName: localK8SNI,
-		targetAddress:       suite.root.Config.Proxy.WebAddr,
-		routeToCluster:      suite.leaf.Secrets.SiteName,
+	k8Client, _, err := kube.ProxyClient(kube.ProxyConfig{
+		T:                   suite.root,
+		Username:            kubeRoleSpec.Allow.Logins[0],
+		KubeUsers:           kubeRoleSpec.Allow.KubeGroups,
+		KubeGroups:          kubeRoleSpec.Allow.KubeUsers,
+		CustomTLSServerName: localK8SNI,
+		TargetAddress:       suite.root.Config.Proxy.WebAddr,
+		RouteToCluster:      suite.leaf.Secrets.SiteName,
 	})
 	require.NoError(t, err)
 
@@ -755,24 +758,24 @@ func TestALPNSNIProxyDatabaseAccess(t *testing.T) {
 
 // TestALPNSNIProxyAppAccess tests application access via ALPN SNI proxy service.
 func TestALPNSNIProxyAppAccess(t *testing.T) {
-	pack := setupWithOptions(t, appTestOptions{
-		rootClusterListeners: helpers.SingleProxyPortSetup,
-		leafClusterListeners: helpers.SingleProxyPortSetup,
-		rootConfig: func(config *service.Config) {
+	pack := appaccess.SetupWithOptions(t, appaccess.AppTestOptions{
+		RootClusterListeners: helpers.SingleProxyPortSetup,
+		LeafClusterListeners: helpers.SingleProxyPortSetup,
+		RootConfig: func(config *service.Config) {
 			config.Auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex)
 		},
-		leafConfig: func(config *service.Config) {
+		LeafConfig: func(config *service.Config) {
 			config.Auth.NetworkingConfig.SetProxyListenerMode(types.ProxyListenerMode_Multiplex)
 		},
 	})
 
-	sess := pack.createAppSession(t, pack.rootAppPublicAddr, pack.rootAppClusterName)
-	status, _, err := pack.makeRequest(sess, http.MethodGet, "/")
+	sess := pack.CreateAppSession(t, pack.RootAppPublicAddr(), pack.RootAppClusterName())
+	status, _, err := pack.MakeRequest(sess, http.MethodGet, "/")
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, status)
 
-	sess = pack.createAppSession(t, pack.leafAppPublicAddr, pack.leafAppClusterName)
-	status, _, err = pack.makeRequest(sess, http.MethodGet, "/")
+	sess = pack.CreateAppSession(t, pack.LeafAppPublicAddr(), pack.LeafAppClusterName())
+	status, _, err = pack.MakeRequest(sess, http.MethodGet, "/")
 	require.NoError(t, err)
 	require.Equal(t, http.StatusOK, status)
 }
@@ -783,9 +786,9 @@ func TestALPNProxyRootLeafAuthDial(t *testing.T) {
 	lib.SetInsecureDevMode(true)
 	defer lib.SetInsecureDevMode(false)
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 
-	suite := newProxySuite(t,
+	suite := newSuite(t,
 		withRootClusterConfig(rootClusterStandardConfig(t)),
 		withLeafClusterConfig(leafClusterStandardConfig(t)),
 		withRootClusterListeners(helpers.SingleProxyPortSetup),
@@ -834,7 +837,7 @@ func TestALPNProxyAuthClientConnectWithUserIdentity(t *testing.T) {
 	cfg := helpers.InstanceConfig{
 		ClusterName: "root.example.com",
 		HostID:      uuid.New().String(),
-		NodeName:    Loopback,
+		NodeName:    helpers.Loopback,
 		Log:         utils.NewLoggerForTests(),
 	}
 	cfg.Listeners = helpers.SingleProxyPortSetup(t, &cfg.Fds)
@@ -851,7 +854,7 @@ func TestALPNProxyAuthClientConnectWithUserIdentity(t *testing.T) {
 	rcConf.Version = "v2"
 	rcConf.CircuitBreakerConfig = breaker.NoopBreakerConfig()
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	rc.AddUser(username, []string{username})
 
 	err := rc.CreateEx(t, nil, rcConf)
@@ -889,14 +892,14 @@ func TestALPNProxyDialProxySSHWithoutInsecureMode(t *testing.T) {
 	rootCfg := helpers.InstanceConfig{
 		ClusterName: "root.example.com",
 		HostID:      uuid.New().String(),
-		NodeName:    Loopback,
+		NodeName:    helpers.Loopback,
 		Priv:        privateKey,
 		Pub:         publicKey,
 		Log:         utils.NewLoggerForTests(),
 	}
 	rootCfg.Listeners = helpers.StandardListenerSetup(t, &rootCfg.Fds)
 	rc := helpers.NewInstance(t, rootCfg)
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	rc.AddUser(username, []string{username})
 
 	// Make root cluster config.
@@ -966,7 +969,7 @@ func TestALPNProxyHTTPProxyNoProxyDial(t *testing.T) {
 	}
 	instanceCfg.Listeners = helpers.SingleProxyPortSetupOn(addr)(t, &instanceCfg.Fds)
 	rc := helpers.NewInstance(t, instanceCfg)
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	rc.AddUser(username, []string{username})
 
 	rcConf := service.MakeDefaultConfig()
@@ -1007,7 +1010,7 @@ func TestALPNProxyHTTPProxyNoProxyDial(t *testing.T) {
 	ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30))
 	defer cancel()
 
-	err = waitForNodeCount(ctx, rc, "root.example.com", 1)
+	err = helpers.WaitForNodeCount(ctx, rc, "root.example.com", 1)
 	require.NoError(t, err)
 
 	require.Zero(t, ph.Count())
@@ -1017,7 +1020,7 @@ func TestALPNProxyHTTPProxyNoProxyDial(t *testing.T) {
 	require.NoError(t, os.Unsetenv("no_proxy"))
 	_, err = rc.StartNode(makeNodeConfig("second-root-node", rcProxyAddr))
 	require.NoError(t, err)
-	err = waitForNodeCount(ctx, rc, "root.example.com", 2)
+	err = helpers.WaitForNodeCount(ctx, rc, "root.example.com", 2)
 	require.NoError(t, err)
 
 	require.NotZero(t, ph.Count())
@@ -1048,7 +1051,7 @@ func TestALPNProxyHTTPProxyBasicAuthDial(t *testing.T) {
 	rc := helpers.NewInstance(t, cfg)
 	log.Info("Teleport root cluster instance created")
 
-	username := mustGetCurrentUser(t).Username
+	username := helpers.MustGetCurrentUser(t).Username
 	rc.AddUser(username, []string{username})
 
 	rcConf := service.MakeDefaultConfig()
@@ -1109,7 +1112,7 @@ func TestALPNProxyHTTPProxyBasicAuthDial(t *testing.T) {
 	t.Setenv("http_proxy", helpers.MakeProxyAddr(validUser, validPass, proxyURL.Host))
 	_, err = rc.StartNode(makeNodeConfig("third-root-node", rcProxyAddr))
 	require.NoError(t, err)
-	err = waitForNodeCount(ctx, rc, "root.example.com", 1)
+	err = helpers.WaitForNodeCount(ctx, rc, "root.example.com", 1)
 	require.NoError(t, err)
 	require.NoError(t, authorizer.LastError())
 	require.NotZero(t, ph.Count())

integration/proxy/proxy_tunnel_strategy_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package integration
+package proxy
 
 import (
 	"bytes"
@@ -65,7 +65,7 @@ type proxyTunnelStrategy struct {
 func newProxyTunnelStrategy(t *testing.T, cluster string, strategy *types.TunnelStrategyV1) *proxyTunnelStrategy {
 	p := &proxyTunnelStrategy{
 		cluster:  cluster,
-		username: mustGetCurrentUser(t).Username,
+		username: helpers.MustGetCurrentUser(t).Username,
 		strategy: strategy,
 		log:      utils.NewLoggerForTests(),
 	}
@@ -100,8 +100,8 @@ func testProxyTunnelStrategyAgentMesh(t *testing.T) {
 				p.makeNode(t)
 
 				// wait for the node to be connected to both proxies
-				waitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1)
-				waitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1)
+				helpers.WaitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1)
+				helpers.WaitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1)
 
 				// make sure we can connect to the node going through any proxy.
 				p.waitForNodeToBeReachable(t)
@@ -114,8 +114,8 @@ func testProxyTunnelStrategyAgentMesh(t *testing.T) {
 				p.makeDatabase(t)
 
 				// wait for the node to be connected to both proxies
-				waitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1)
-				waitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1)
+				helpers.WaitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 1)
+				helpers.WaitForActiveTunnelConnections(t, p.proxies[1].Tunnel, p.cluster, 1)
 
 				// make sure we can connect to the database going through any proxy.
 				p.waitForDatabaseToBeReachable(t)
@@ -184,7 +184,7 @@ func testProxyTunnelStrategyProxyPeering(t *testing.T) {
 	p.makeDatabase(t)
 
 	// wait for the node and db to open reverse tunnels to the first proxy.
-	waitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 2)
+	helpers.WaitForActiveTunnelConnections(t, p.proxies[0].Tunnel, p.cluster, 2)
 
 	// bootstrap the second proxy instance after the node and db have already
 	// established reverse tunnels to the first proxy.
@@ -268,7 +268,9 @@ func (p *proxyTunnelStrategy) makeLoadBalancer(t *testing.T) {
 	ctx, cancel := context.WithCancel(context.Background())
 	t.Cleanup(cancel)
 
-	lbAddr := utils.MustParseAddr(net.JoinHostPort(Loopback, helpers.NewPortStr()))
+	// TODO(tcsc): fix ports before merging
+	lbAddr := utils.MustParseAddr(net.JoinHostPort(helpers.Loopback, "0"))
+	//lbAddr := utils.MustParseAddr(net.JoinHostPort(helpers.Loopback, helpers.NewPortStr()))
 	lb, err := utils.NewLoadBalancer(ctx, *lbAddr)
 	require.NoError(t, err)
 
@@ -293,7 +295,7 @@ func (p *proxyTunnelStrategy) makeAuth(t *testing.T) {
 	auth := helpers.NewInstance(t, helpers.InstanceConfig{
 		ClusterName: p.cluster,
 		HostID:      uuid.New().String(),
-		NodeName:    Loopback,
+		NodeName:    helpers.Loopback,
 		Priv:        privateKey,
 		Pub:         publicKey,
 		Log:         utils.NewLoggerForTests(),
@@ -321,7 +323,7 @@ func (p *proxyTunnelStrategy) makeProxy(t *testing.T) {
 	proxy := helpers.NewInstance(t, helpers.InstanceConfig{
 		ClusterName: p.cluster,
 		HostID:      uuid.New().String(),
-		NodeName:    Loopback,
+		NodeName:    helpers.Loopback,
 		Log:         utils.NewLoggerForTests(),
 	})
 
@@ -335,12 +337,11 @@ func (p *proxyTunnelStrategy) makeProxy(t *testing.T) {
 	conf.Auth.Enabled = false
 	conf.SSH.Enabled = false
 
-	// TODO: Replace old-style NewPortStr() call with preconfigured listener
 	conf.Proxy.Enabled = true
 	conf.Proxy.ReverseTunnelListenAddr.Addr = proxy.ReverseTunnel
 	conf.Proxy.SSHAddr.Addr = proxy.SSHProxy
 	conf.Proxy.WebAddr.Addr = proxy.Web
-	conf.Proxy.PeerAddr.Addr = net.JoinHostPort(Loopback, helpers.NewPortStr())
+	conf.Proxy.PeerAddr.Addr = helpers.NewListenerOn(t, helpers.Loopback, service.ListenerProxyPeer, &proxy.Fds)
 	conf.Proxy.PeerPublicAddr = conf.Proxy.PeerAddr
 	conf.Proxy.PublicAddrs = append(conf.Proxy.PublicAddrs, utils.FromAddr(p.lb.Addr()))
 	conf.Proxy.DisableWebInterface = true
@@ -367,7 +368,7 @@ func (p *proxyTunnelStrategy) makeNode(t *testing.T) {
 	node := helpers.NewInstance(t, helpers.InstanceConfig{
 		ClusterName: p.cluster,
 		HostID:      uuid.New().String(),
-		NodeName:    Loopback,
+		NodeName:    helpers.Loopback,
 		Log:         utils.NewLoggerForTests(),
 	})
 
@@ -397,19 +398,19 @@ func (p *proxyTunnelStrategy) makeDatabase(t *testing.T) {
 		require.Fail(t, "database already initialized")
 	}
 
-	dbListener, err := net.Listen("tcp", net.JoinHostPort(Host, "0"))
+	dbListener, err := net.Listen("tcp", net.JoinHostPort(helpers.Host, "0"))
 	require.NoError(t, err)
 
 	_, portStr, err := net.SplitHostPort(dbListener.Addr().String())
 	require.NoError(t, err)
 
-	dbAddr := net.JoinHostPort(Host, portStr)
+	dbAddr := net.JoinHostPort(helpers.Host, portStr)
 
 	// setup database service
 	db := helpers.NewInstance(t, helpers.InstanceConfig{
 		ClusterName: p.cluster,
 		HostID:      uuid.New().String(),
-		NodeName:    Loopback,
+		NodeName:    helpers.Loopback,
 		Log:         utils.NewLoggerForTests(),
 	})