teleport/lib/kube/proxy/websocket_client_testing.go

/*
Copyright 2022 Gravitational, Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package proxy

import (
	"bytes"
	"context"
	"crypto/tls"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"

	gwebsocket "github.com/gorilla/websocket"
	"github.com/gravitational/trace"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/rest"
	clientremotecommand "k8s.io/client-go/tools/remotecommand"
	"k8s.io/client-go/transport"

	testingkubemock "github.com/gravitational/teleport/lib/kube/proxy/testing/kube_server"
)

var (
	supportedProtocols = []string{v4BinaryWebsocketProtocol, preV4BinaryWebsocketProtocol}
	separator          = "\r\n"
)

const (
	// These constants match:
	// - pkg/kubelet/cri/streaming/remotecommand/websocket.go
	// - staging/src/k8s.io/apiserver/pkg/util/wsstream/conn.go

	streamStdin = iota
	streamStdout
	streamStderr
	streamErr
	streamResize
)

// wsStreamClient handles transporting standard shell streams over a websocket connection.
type wsStreamClient struct {
	config    *rest.Config
	tlsConfig *tls.Config
	method    string
	url       string
	protocols []string
	cacheBuff *bytes.Buffer
	conn      *gwebsocket.Conn
	mu        *sync.Mutex
	localPort *int32
	readyChan chan struct{}
	listener  net.Listener
}

type websocketOption func(*wsStreamClient)

func withLocalPortforwarding(port int32, readyChan chan struct{}) websocketOption {
	return func(c *wsStreamClient) {
		c.localPort = &port
		c.readyChan = readyChan
	}
}

// newWebSocketClient allows running exec commands via Websocket protocol.
// The existing code exists for tests purpose where the final endpoint is a fictional Kubernetes API.
// The code in question should never be used outside testing.
func newWebSocketClient(config *rest.Config, method string, u *url.URL, opts ...websocketOption) (*wsStreamClient, error) {
	c := &wsStreamClient{
		config:    config,
		method:    method,
		url:       u.String(),
		protocols: supportedProtocols,
		cacheBuff: bytes.NewBuffer(nil),
		mu:        &sync.Mutex{},
	}

	for _, opt := range opts {
		opt(c)
	}
	return c, nil
}

// StreamWithContext copies the contents from stdin into the connection and respective stdout and stderr
// from the connection into the desired buffers.
// This method will block until the server closes the connection.
// Unfortunately, the K8S Websocket protocol does not support half-closed streams,
// i.e. indicating that nothing else will be sent via stdin. If the server
// reads the stdin stream until io.EOF is received, it will block on reading.
// This issue is being tracked by https://github.com/kubernetes/kubernetes/issues/89899
// To prevent this issue, and uniquely for testing, this client will send
// an exit keyword specified by testingkubemock.CloseStreamMessage. Our mock server is expecting that
// keyword and will return once it's received.

// The protocol docs are at https://pkg.go.dev/k8s.io/apiserver/pkg/util/wsstream#pkg-constants
// Below we have a copy of the implemented binary protocol specification.

// The Websocket subprotocol "channel.k8s.io" prepends each binary message with a byte indicating
// the channel number (zero indexed) the message was sent on. Messages in both directions should
// prefix their messages with this channel byte. When used for remote execution, the channel numbers
// are by convention defined to match the POSIX file-descriptors assigned to STDIN, STDOUT, and STDERR
// (0, 1, and 2). No other conversion is performed on the raw subprotocol - writes are sent as they
// are received by the server.
//
// Example client session:
//
//	CONNECT http://server.com with subprotocol "channel.k8s.io"
//	WRITE []byte{0, 102, 111, 111, 10} # send "foo\n" on channel 0 (STDIN)
//	READ  []byte{1, 10}                # receive "\n" on channel 1 (STDOUT)
//	CLOSE
func (e *wsStreamClient) StreamWithContext(_ context.Context, options clientremotecommand.StreamOptions) error {
	return trace.Wrap(e.Stream(options))
}

// Stream copies the contents from stdin into the connection and respective stdout and stderr
// from the connection into the desired buffers.
// This method will block until the server closes the connection.
// Unfortunately, the K8S Websocket protocol does not support half-closed streams,
// i.e. indicating that nothing else will be sent via stdin. If the server
// reads the stdin stream until io.EOF is received, it will block on reading.
// This issue is being tracked by https://github.com/kubernetes/kubernetes/issues/89899
// To prevent this issue, and uniquely for testing, this client will send
// an exit keyword specified by testingkubemock.CloseStreamMessage. Our mock server is expecting that
// keyword and will return once it's received.

// The protocol docs are at https://pkg.go.dev/k8s.io/apiserver/pkg/util/wsstream#pkg-constants
// Below we have a copy of the implemented binary protocol specification.

// The Websocket subprotocol "channel.k8s.io" prepends each binary message with a byte indicating
// the channel number (zero indexed) the message was sent on. Messages in both directions should
// prefix their messages with this channel byte. When used for remote execution, the channel numbers
// are by convention defined to match the POSIX file-descriptors assigned to STDIN, STDOUT, and STDERR
// (0, 1, and 2). No other conversion is performed on the raw subprotocol - writes are sent as they
// are received by the server.
//
// Example client session:
//
//	CONNECT http://server.com with subprotocol "channel.k8s.io"
//	WRITE []byte{0, 102, 111, 111, 10} # send "foo\n" on channel 0 (STDIN)
//	READ  []byte{1, 10}                # receive "\n" on channel 1 (STDOUT)
//	CLOSE
func (e *wsStreamClient) Stream(options clientremotecommand.StreamOptions) error {
	if options.TerminalSizeQueue != nil || options.Tty {
		return fmt.Errorf("client does not support resizes or Tty shells")
	}
	err := e.connectViaWebsocket()
	if err != nil {
		return err
	}
	conn := e.conn
	// stream will block until execution is finished.
	defer conn.Close()
	streamingProto := conn.Subprotocol()

	found := false
	for _, p := range supportedProtocols {
		if p == streamingProto {
			found = true
			break
		}
	}
	if !found {
		return fmt.Errorf("unsupported streaming protocol: %q", streamingProto)
	}

	// hang until server closes streams.
	return e.stream(conn, options)
}

// ForwardPorts opens a listener at the specified port and waits for new connections.
// once a connection is received, it waits for the upstream data and writes it
// into the original connection.
// Due to portforward websocket limitations, the listener do not accept
// concurrent connections.
func (e *wsStreamClient) ForwardPorts() error {
	err := e.connectViaWebsocket()
	if err != nil {
		return err
	}
	conn := e.conn
	// stream will block until execution is finished.
	defer conn.Close()
	streamingProto := conn.Subprotocol()

	found := false
	for _, p := range supportedProtocols {
		if p == streamingProto {
			found = true
			break
		}
	}
	if !found {
		return fmt.Errorf("unsupported streaming protocol: %q", streamingProto)
	}

	// hang until server closes streams.
	return e.portforward(conn)
}

func (e *wsStreamClient) Close() {
	if e.listener != nil {
		e.listener.Close()
	}
}

// stream copies the contents from stdin into the connection and respective stdout and stderr
// from the connection into the desired buffers.
// This method will block until the server closes the connection.
// Unfortunately, the K8S Websocket protocol does not support half-closed streams,
// i.e. indicating that nothing else will be sent via stdin. If the server
// reads the stdin stream until io.EOF is received, it will block on reading.
// This issue is being tracked by https://github.com/kubernetes/kubernetes/issues/89899
// To prevent this issue, and uniquely for testing, this client will send
// an exit keyword specified by testingkubemock.CloseStreamMessage. Our mock server is expecting that
// keyword and will return once it's received.

// The protocol docs are at https://pkg.go.dev/k8s.io/apiserver/pkg/util/wsstream#pkg-constants
// Below we have a copy of the implemented binary protocol specification.

// The Websocket subprotocol "channel.k8s.io" prepends each binary message with a byte indicating
// the channel number (zero indexed) the message was sent on. Messages in both directions should
// prefix their messages with this channel byte. When used for remote execution, the channel numbers
// are by convention defined to match the POSIX file-descriptors assigned to STDIN, STDOUT, and STDERR
// (0, 1, and 2). No other conversion is performed on the raw subprotocol - writes are sent as they
// are received by the server.
//
// Example client session:
//
//	CONNECT http://server.com with subprotocol "channel.k8s.io"
//	WRITE []byte{0, 102, 111, 111, 10} # send "foo\n" on channel 0 (STDIN)
//	READ  []byte{1, 10}                # receive "\n" on channel 1 (STDOUT)
//	CLOSE
func (e *wsStreamClient) stream(conn *gwebsocket.Conn, options clientremotecommand.StreamOptions) error {
	errChan := make(chan error, 3)
	wg := sync.WaitGroup{}
	if options.Stdin != nil {
		wg.Add(1)
		go func() {
			defer wg.Done()
			buf := make([]byte, 1025)
			for {
				n, err := options.Stdin.Read(buf)
				if errors.Is(err, io.EOF) && n == 0 {
					// send the close payload indicating that there is nothing else to read from stdin.
					if err := conn.WriteMessage(
						gwebsocket.BinaryMessage,
						append([]byte{streamStdin}, []byte(testingkubemock.CloseStreamMessage)...),
					); err != nil {
						errChan <- err
					}
					return
				} else if err != nil && n == 0 {
					errChan <- err
					return
				}
				if n == 0 {
					continue
				}
				e.mu.Lock()
				// we must cache the last sdtdin line because the server will resend it together
				// with stdout/stderr streams.
				e.cacheBuff.Write([]byte{streamStdin})
				e.cacheBuff.Write(buf[0:n])

				if err := conn.WriteMessage(gwebsocket.BinaryMessage, e.cacheBuff.Bytes()); err != nil {
					e.mu.Unlock()
					errChan <- err
					return
				}
				// append the separator that the server will send
				e.cacheBuff.Write([]byte(separator))
				e.mu.Unlock()
			}
		}()
	}

	// If the client is not using stdout or stderr, we must listen for streamErr.
	wg.Add(1)
	go func() {
		defer wg.Done()
		for {
			_, buf, err := conn.ReadMessage()

			if len(buf) > 1 {
				var w io.Writer
				// We let the server send the stream number and we choose the desired stream accordingly.
				// If the stream is nil, we ignore the payload and continue.
				switch buf[0] {
				case streamStdout:
					w = options.Stdout
				case streamStderr:
					w = options.Stderr
				case streamErr:
					_, err := parseError(buf[1:])
					errChan <- err
					// Once we receive an error from streamErr, we must stop processing.
					// The server also stops the execution and closes the connection.
					return
				case streamResize:
					errChan <- fmt.Errorf("stream resize is not supported")
					return
				}
				if w == nil {
					continue
				}
				e.mu.Lock()
				// the stdout and stderr streams receive the last stdin input and we must trim it.
				s := strings.Replace(string(buf[1:]), e.cacheBuff.String(), "", -1)
				e.mu.Unlock()
				_, err = w.Write([]byte(s))
				if err != nil {
					errChan <- err
					return
				}
			}

			if err != nil {
				// check the connection was properly closed by server, and if true ignore the error.
				var websocketErr *gwebsocket.CloseError
				if errors.As(err, &websocketErr) && websocketErr.Code == gwebsocket.CloseNormalClosure {
					err = nil
				}
				errChan <- err
				return
			}
			e.mu.Lock()
			e.cacheBuff.Reset()
			e.mu.Unlock()

		}
	}()

	wg.Wait()
	close(errChan)
	err := <-errChan
	return err
}

const (
	// portforwardDataChan is the websocket channel for transferring data.
	portforwardDataChan = iota
	// portforwardErrChan is the websocket channel for transferring errors.
	portforwardErrChan
)

// portforward opens a listener at the specified port and waits for new connections.
// once a connection is received, it waits for the upstream data and writes it
// into the original connection.
// Due to portforward websocket limitations, the listener do not accept
// concurrent connections.
func (e *wsStreamClient) portforward(remoteConn *gwebsocket.Conn) (err error) {
	if e.localPort == nil || e.readyChan == nil {
		return trace.BadParameter("cannot use portforward without proper initialization")
	}

	e.listener, err = net.Listen("tcp", fmt.Sprintf("localhost:%d", *e.localPort))
	if err != nil {
		return trace.Wrap(err)
	}
	close(e.readyChan)
	for {
		conn, err := e.listener.Accept()
		if err != nil {
			if errors.Is(err, net.ErrClosed) {
				err = nil
			}
			return trace.Wrap(err)
		}
		if err := e.handlePortForwardRequest(conn, remoteConn); err != nil {
			return trace.Wrap(err)
		}
	}
}

// handlePortForwardRequest copies data streams from local connection to upstream and
// vice versa.
func (e *wsStreamClient) handlePortForwardRequest(conn net.Conn, remoteConn *gwebsocket.Conn) error {
	wg := sync.WaitGroup{}
	// errChan will receive the errors returned from 2 goroutines.
	// It needs to be buffered because we rely on sync.WaitGroup to control
	// when the two goroutines are terminated and the channel has to have enough
	// size to not block.
	errChan := make(chan error, 2)

	wg.Add(2)

	go func() {
		defer wg.Done()
		p := make([]byte, 1024)
		for {
			n, err := conn.Read(p)
			if n == 0 && err != nil {
				if errors.Is(err, io.EOF) {
					err = nil
				}
				errChan <- trace.Wrap(err)
				return
			}
			if err := remoteConn.WriteMessage(
				gwebsocket.BinaryMessage,
				append([]byte{portforwardDataChan}, p[:n]...),
			); err != nil {
				errChan <- trace.Wrap(err)
				return
			}
		}
	}()

	go func() {
		defer wg.Done()
		for {
			_, buf, err := remoteConn.ReadMessage()
			if err != nil {
				// check the connection was properly closed by server, and if true ignore the error.
				var websocketErr *gwebsocket.CloseError
				if errors.As(err, &websocketErr) && websocketErr.Code == gwebsocket.CloseNormalClosure {
					err = nil
				}
				errChan <- trace.Wrap(err)
				return
			}
			// if len(buf)==3, it is the mandatory protocol message. This message
			// includes the port number from proxy and we can safelly ignore it.
			// [channel, uint16]
			if len(buf) > 1 && len(buf) != 3 {
				// We let the server send the stream number and we choose the desired stream accordingly.
				// If the stream is nil, we ignore the payload and continue.
				switch buf[0] {
				case portforwardDataChan:
					_, err := conn.Write(buf[1:])
					if err != nil {
						errChan <- trace.Wrap(err)
						return
					}
				case portforwardErrChan:
					err := trace.Errorf(string(buf[1:]))
					errChan <- trace.Wrap(err)
					// Once we receive an error from streamErr, we must stop processing.
					// The server also stops the execution and closes the connection.
					return
				}
			}
		}
	}()

	wg.Wait()
	close(errChan)
	return trace.NewAggregateFromChannel(errChan, context.Background())
}

func (e *wsStreamClient) connectViaWebsocket() error {
	transportCfg, err := e.config.TransportConfig()
	if err != nil {
		return err
	}
	e.tlsConfig, err = transport.TLSConfigFor(transportCfg)
	if err != nil {
		return err
	}

	wrapper, err := transport.HTTPWrappersForConfig(transportCfg, e)
	if err != nil {
		return err
	}

	err = dial(wrapper, e.method, e.url)
	if err != nil {
		return err
	}

	return nil
}

// dial connects to the server via websocket and sends the supported protocols.
func dial(rt http.RoundTripper, method string, url string) error {
	req, err := http.NewRequest(method, url, nil)
	if err != nil {
		return err
	}

	resp, err := rt.RoundTrip(req)
	if err != nil {
		if !errors.Is(err, gwebsocket.ErrBadHandshake) {
			return err
		}
		// resp.Body contains (a part of) the response when err == gwebsocket.ErrBadHandshake
		responseErrorBytes, bodyErr := io.ReadAll(resp.Body)
		if bodyErr != nil {
			return err
		}
		// drain response body

		_ = resp.Body.Close()
		isStatusErr, err := parseError(responseErrorBytes)
		if isStatusErr {
			return err
		}
		return fmt.Errorf("unable to upgrade connection: %w", err)
	}
	// if request is successful, ignore body payload and close.
	io.Copy(io.Discard, resp.Body)
	resp.Body.Close()

	return nil
}

// RoundTrip connects to the remote websocket using TLS configuration.
func (e *wsStreamClient) RoundTrip(request *http.Request) (retResp *http.Response, retErr error) {
	dialer := gwebsocket.Dialer{
		TLSClientConfig: e.tlsConfig,
		Subprotocols:    supportedProtocols,
	}
	switch request.URL.Scheme {
	case "https":
		request.URL.Scheme = "wss"
	case "http":
		request.URL.Scheme = "ws"
	}
	wsConn, resp, err := dialer.DialContext(request.Context(), request.URL.String(), request.Header)
	e.conn = wsConn

	return resp, err
}

// parseError parses the error received from Kube API and checks if the returned error is *metav1.Status
func parseError(errorBytes []byte) (bool, error) {
	if obj, _, err := statusCodecs.UniversalDecoder().Decode(errorBytes, nil, &metav1.Status{}); err == nil {
		if status, ok := obj.(*metav1.Status); ok && status.Status == metav1.StatusSuccess {
			return true, nil
		} else if ok {
			return true, &apierrors.StatusError{ErrStatus: *status}
		}
		return false, fmt.Errorf("unexpected error type: %T", obj)
	}
	return false, fmt.Errorf("%s", bytes.TrimSpace(errorBytes))
}