/* * Teleport * Copyright (C) 2023 Gravitational, Inc. * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU Affero General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU Affero General Public License for more details. * * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ // Package sshutils contains the implementations of the base SSH // server used throughout Teleport. package sshutils import ( "context" "errors" "fmt" "io" "log/slog" "net" "sync" "sync/atomic" "time" "github.com/gravitational/trace" "github.com/jonboulle/clockwork" "github.com/prometheus/client_golang/prometheus" semconv "go.opentelemetry.io/otel/semconv/v1.10.0" oteltrace "go.opentelemetry.io/otel/trace" "golang.org/x/crypto/ssh" "github.com/gravitational/teleport" "github.com/gravitational/teleport/api/observability/tracing" tracessh "github.com/gravitational/teleport/api/observability/tracing/ssh" "github.com/gravitational/teleport/api/utils/sshutils" "github.com/gravitational/teleport/lib/defaults" "github.com/gravitational/teleport/lib/limiter" "github.com/gravitational/teleport/lib/observability/metrics" "github.com/gravitational/teleport/lib/srv/ingress" "github.com/gravitational/teleport/lib/utils" ) var proxyConnectionLimitHitCount = prometheus.NewCounter( prometheus.CounterOpts{ Name: teleport.MetricProxyConnectionLimitHit, Help: "Number of times the proxy connection limit was exceeded", }, ) // Server is a generic implementation of an SSH server. All Teleport // services (auth, proxy, ssh) use this as a base to accept SSH connections. type Server struct { sync.RWMutex logger *slog.Logger // component is a name of the facility which uses this server, // used for logging/debugging. typically it's "proxy" or "auth api", etc component string // addr is the address this server binds to and listens on addr utils.NetAddr // listener is usually the listening TCP/IP socket listener net.Listener newChanHandler NewChanHandler reqHandler RequestHandler newConnHandler NewConnHandler getHostSigners GetHostSignersFunc cfg ssh.ServerConfig limiter *limiter.Limiter closeContext context.Context closeFunc context.CancelFunc // userConns tracks amount of current active connections with user certificates. userConns int32 // shutdownPollPeriod sets polling period for shutdown shutdownPollPeriod time.Duration // insecureSkipHostValidation does not validate the host signers to make sure // they are a valid certificate. Used in tests. insecureSkipHostValidation bool // fips means Teleport started in a FedRAMP/FIPS 140-2 compliant // configuration. fips bool // tracerProvider is used to create tracers capable // of starting spans. tracerProvider oteltrace.TracerProvider // clock is used to control time. clock clockwork.Clock clusterName string // ingressReporter reports new and active connections. ingressReporter *ingress.Reporter // ingressService the service name passed to the ingress reporter. ingressService string } const ( // SSHVersionPrefix is the prefix of "server version" string which begins // every SSH handshake. It MUST start with "SSH-2.0" according to // https://tools.ietf.org/html/rfc4253#page-4 SSHVersionPrefix = "SSH-2.0-Teleport" // MaxVersionStringBytes is the maximum number of bytes allowed for a // SSH version string // https://tools.ietf.org/html/rfc4253 MaxVersionStringBytes = 255 ) // ServerOption is a functional argument for server type ServerOption func(cfg *Server) error // SetIngressReporter sets the reporter for reporting new and active connections. func SetIngressReporter(service string, r *ingress.Reporter) ServerOption { return func(s *Server) error { s.ingressReporter = r s.ingressService = service return nil } } // SetLogger sets the logger for the server func SetLogger(logger *slog.Logger) ServerOption { return func(s *Server) error { s.logger = logger.With(teleport.ComponentKey, teleport.Component("ssh", s.component)) return nil } } func SetLimiter(limiter *limiter.Limiter) ServerOption { return func(s *Server) error { s.limiter = limiter return nil } } // SetShutdownPollPeriod sets a polling period for graceful shutdowns of SSH servers func SetShutdownPollPeriod(period time.Duration) ServerOption { return func(s *Server) error { s.shutdownPollPeriod = period return nil } } // SetInsecureSkipHostValidation does not validate the host signers to make sure // they are a valid certificate. Used in tests. func SetInsecureSkipHostValidation() ServerOption { return func(s *Server) error { s.insecureSkipHostValidation = true return nil } } // SetTracerProvider sets the tracer provider for the server. func SetTracerProvider(provider oteltrace.TracerProvider) ServerOption { return func(s *Server) error { s.tracerProvider = provider return nil } } // SetClock sets the server's clock. func SetClock(clock clockwork.Clock) ServerOption { return func(s *Server) error { s.clock = clock return nil } } func SetClusterName(clusterName string) ServerOption { return func(s *Server) error { s.clusterName = clusterName return nil } } func NewServer( component string, a utils.NetAddr, h NewChanHandler, getHostSigners GetHostSignersFunc, ah AuthMethods, opts ...ServerOption, ) (*Server, error) { err := metrics.RegisterPrometheusCollectors(proxyConnectionLimitHitCount) if err != nil { return nil, trace.Wrap(err) } closeContext, cancel := context.WithCancel(context.TODO()) s := &Server{ logger: slog.With(teleport.ComponentKey, teleport.Component("ssh", component)), addr: a, newChanHandler: h, getHostSigners: getHostSigners, component: component, closeContext: closeContext, closeFunc: cancel, } s.limiter, err = limiter.NewLimiter(limiter.Config{}) if err != nil { return nil, trace.Wrap(err) } for _, o := range opts { if err := o(s); err != nil { return nil, err } } if s.shutdownPollPeriod == 0 { s.shutdownPollPeriod = defaults.ShutdownPollPeriod } if s.tracerProvider == nil { s.tracerProvider = tracing.DefaultProvider() } err = s.checkArguments(a, h, getHostSigners, ah) if err != nil { return nil, err } s.cfg.PublicKeyCallback = ah.PublicKey s.cfg.PasswordCallback = ah.Password s.cfg.NoClientAuth = ah.NoClient if s.fips { s.cfg.PublicKeyAuthAlgorithms = defaults.FIPSPubKeyAuthAlgorithms } // Teleport servers need to identify as such to allow passing of the client // IP from the client to the proxy to the destination node. s.cfg.ServerVersion = SSHVersionPrefix return s, nil } func SetSSHConfig(cfg ssh.ServerConfig) ServerOption { return func(s *Server) error { s.cfg = cfg return nil } } func SetRequestHandler(req RequestHandler) ServerOption { return func(s *Server) error { s.reqHandler = req return nil } } func SetNewConnHandler(handler NewConnHandler) ServerOption { return func(s *Server) error { s.newConnHandler = handler return nil } } func SetCiphers(ciphers []string) ServerOption { return func(s *Server) error { s.logger.DebugContext(context.Background(), "Supported ciphers updated", "ciphers", ciphers) if ciphers != nil { s.cfg.Ciphers = ciphers } return nil } } func SetKEXAlgorithms(kexAlgorithms []string) ServerOption { return func(s *Server) error { s.logger.DebugContext(context.Background(), "Supported KEX algorithms updated", "kex_algorithms", kexAlgorithms) if kexAlgorithms != nil { s.cfg.KeyExchanges = kexAlgorithms } return nil } } func SetMACAlgorithms(macAlgorithms []string) ServerOption { return func(s *Server) error { s.logger.DebugContext(context.Background(), "Supported MAC algorithms updated", "mac_algorithms", macAlgorithms) if macAlgorithms != nil { s.cfg.MACs = macAlgorithms } return nil } } func SetFIPS(fips bool) ServerOption { return func(s *Server) error { s.fips = fips return nil } } func (s *Server) Addr() string { s.RLock() defer s.RUnlock() if s.listener == nil { return "" } return s.listener.Addr().String() } func (s *Server) Serve(listener net.Listener) error { if err := s.SetListener(listener); err != nil { return trace.Wrap(err) } s.acceptConnections() return nil } func (s *Server) Start() error { if s.listener == nil { listener, err := net.Listen(s.addr.AddrNetwork, s.addr.Addr) if err != nil { return trace.ConvertSystemError(err) } listener, err = s.limiter.WrapListener(listener) if err != nil { return trace.Wrap(err) } if err := s.SetListener(listener); err != nil { return trace.Wrap(err) } } s.logger.DebugContext(s.closeContext, "Starting server", "addr", s.listener.Addr().String()) go s.acceptConnections() return nil } func (s *Server) SetListener(l net.Listener) error { s.Lock() defer s.Unlock() if s.listener != nil { return trace.BadParameter("listener is already set to %v", s.listener.Addr()) } s.listener = l return nil } // Wait waits until server stops serving new connections // on the listener socket func (s *Server) Wait(ctx context.Context) { select { case <-s.closeContext.Done(): case <-ctx.Done(): } } // Shutdown initiates graceful shutdown - waiting until all active // connections will get closed func (s *Server) Shutdown(ctx context.Context) error { // close listener to stop receiving new connections err := s.Close() s.Wait(ctx) activeConnections := s.trackUserConnections(0) if activeConnections == 0 { return err } minReportInterval := 10 * s.shutdownPollPeriod maxReportInterval := 600 * s.shutdownPollPeriod s.logger.InfoContext(ctx, "Shutdown: waiting for active connections to finish", "active_connections", activeConnections) reportedConnections := activeConnections lastReport := time.Now() reportInterval := minReportInterval ticker := time.NewTicker(s.shutdownPollPeriod) defer ticker.Stop() for { select { case now := <-ticker.C: activeConnections = s.trackUserConnections(0) if activeConnections == 0 { return err } if activeConnections != reportedConnections || now.Sub(lastReport) > reportInterval { s.logger.InfoContext(ctx, "Shutdown: waiting for active connections to finish", "active_connections", activeConnections) lastReport = now if activeConnections == reportedConnections { reportInterval = min(reportInterval*2, maxReportInterval) } else { reportInterval = minReportInterval reportedConnections = activeConnections } } case <-ctx.Done(): s.logger.InfoContext(ctx, "Context canceled wait, returning") return trace.ConnectionProblem(err, "context canceled") } } } // Close closes listening socket and stops accepting connections func (s *Server) Close() error { s.Lock() defer s.Unlock() defer s.closeFunc() if s.listener != nil { err := s.listener.Close() if utils.IsUseOfClosedNetworkError(err) { return nil } return trace.Wrap(err) } return nil } func (s *Server) acceptConnections() { defer s.closeFunc() logger := s.logger.With("listen_addr", s.Addr()) logger.DebugContext(s.closeContext, "Listening for connections") for { conn, err := s.listener.Accept() if err != nil { if trace.IsLimitExceeded(err) { proxyConnectionLimitHitCount.Inc() logger.ErrorContext(s.closeContext, "connection limit exceeded", "error", err) continue } if utils.IsUseOfClosedNetworkError(err) { logger.DebugContext(s.closeContext, "Server has closed") return } select { case <-s.closeContext.Done(): logger.DebugContext(s.closeContext, "Server has closed") return case <-time.After(5 * time.Second): logger.DebugContext(s.closeContext, "Applying backoff in response to network error", "error", err) } } else { go s.HandleConnection(conn) } } } func (s *Server) trackUserConnections(delta int32) int32 { return atomic.AddInt32(&s.userConns, delta) } // TrackUserConnection tracks a user connection that should prevent // the server from being terminated if active. The returned function // should be called when the connection is terminated. func (s *Server) TrackUserConnection() (release func()) { s.trackUserConnections(1) return sync.OnceFunc(func() { s.trackUserConnections(-1) }) } // ActiveConnections returns the number of connections that are // being served. func (s *Server) ActiveConnections() int32 { return atomic.LoadInt32(&s.userConns) } // HandleConnection is called every time an SSH server accepts a new // connection from a client. // // this is the foundation of all SSH connections in Teleport (between clients // and proxies, proxies and servers, servers and auth, etc), except for forwarding // SSH proxy that used when "recording on proxy" is enabled. func (s *Server) HandleConnection(conn net.Conn) { if s.ingressReporter != nil { s.ingressReporter.ConnectionAccepted(s.ingressService, conn) defer s.ingressReporter.ConnectionClosed(s.ingressService, conn) } hostSigners := s.getHostSigners() if err := s.validateHostSigners(hostSigners); err != nil { s.logger.ErrorContext(s.closeContext, "Error during server setup for a new SSH connection (this is a bug)", "error", err, "remote_addr", conn.RemoteAddr(), ) conn.Close() return } cfg := s.cfg for _, signer := range hostSigners { cfg.AddHostKey(signer) } if v := serverVersionOverrideFromConn(conn); v != "" && v != cfg.ServerVersion { cfg.ServerVersion = v } // apply idle read/write timeout to this connection. conn = utils.ObeyIdleTimeout(conn, defaults.DefaultIdleConnectionDuration) // Wrap connection with a tracker used to monitor how much data was // transmitted and received over the connection. conn = utils.NewTrackingConn(conn) sconn, chans, reqs, err := ssh.NewServerConn(conn, &cfg) if err != nil { // Ignore EOF as these are triggered by loadbalancer health checks if !errors.Is(err, io.EOF) { s.logger.WarnContext(s.closeContext, "Error occurred in handshake for new SSH conn", "error", err, "remote_addr", conn.RemoteAddr(), ) } conn.Close() return } if s.ingressReporter != nil { s.ingressReporter.ConnectionAuthenticated(s.ingressService, conn) defer s.ingressReporter.AuthenticatedConnectionClosed(s.ingressService, conn) } certType := "unknown" if sconn.Permissions != nil { certType = sconn.Permissions.Extensions[utils.ExtIntCertType] } if certType == utils.ExtIntCertTypeUser { s.trackUserConnections(1) defer s.trackUserConnections(-1) } user := sconn.User() if err := s.limiter.RegisterRequest(user); err != nil { s.logger.ErrorContext(s.closeContext, "user connection rate limit exceeded", "user", user, "error", err) sconn.Close() conn.Close() return } // Connection successfully initiated s.logger.DebugContext(s.closeContext, "handling incoming connection", "remote_addr", sconn.RemoteAddr(), "local_addr", sconn.LocalAddr(), "version", string(sconn.ClientVersion()), "cert_type", certType, ) // will be called when the connection is closed connClosed := func() { s.logger.DebugContext(s.closeContext, "Closed connection", "remote_addr", sconn.RemoteAddr()) } // The keepalive ticket will ensure that SSH keepalive requests are being sent // to the client at an interval much shorter than idle connection kill switch keepAliveTick := time.NewTicker(defaults.DefaultIdleConnectionDuration / 3) defer keepAliveTick.Stop() keepAlivePayload := [8]byte{0} // NOTE: we deliberately don't use s.closeContext here because the server's // closeContext field is used to trigger starvation on cancellation by halting // the acceptance of new connections; it is not intended to halt in-progress // connection handling, and is therefore orthogonal to the role of ConnectionContext. ctx, ccx := NewConnectionContext(context.Background(), conn, sconn, SetConnectionContextClock(s.clock)) defer ccx.Close() if s.newConnHandler != nil { // if newConnHandler was set, then we have additional setup work // to do before we can begin serving normally. Errors returned // from a NewConnHandler are rejections. ctx, err = s.newConnHandler.HandleNewConn(ctx, ccx) if err != nil { s.logger.WarnContext(ctx, "Dropping inbound ssh connection due to error", "error", err) // Immediately dropping the ssh connection results in an // EOF error for the client. We therefore wait briefly // to see if the client opens a channel or sends any global // requests, which will give us the opportunity to respond // with a human-readable error. waitCtx, waitCancel := context.WithTimeout(s.closeContext, time.Second) defer waitCancel() for { select { case req := <-reqs: if req == nil { connClosed() break } // wait for a request that wants a reply to send the error if !req.WantReply { continue } if err := req.Reply(false, []byte(err.Error())); err != nil { s.logger.WarnContext(ctx, "failed to reply to request", "request_type", req.Type, "error", err) } case firstChan := <-chans: // channel was closed, terminate the connection if firstChan == nil { break } if err := firstChan.Reject(ssh.Prohibited, err.Error()); err != nil { s.logger.WarnContext(ctx, "failed to reject channel", "channel_type", firstChan.ChannelType(), "error", err) } case <-waitCtx.Done(): } break } if err := sconn.Close(); err != nil && !utils.IsOKNetworkError(err) { s.logger.WarnContext(ctx, "failed to close ssh server connection", "error", err) } if err := conn.Close(); err != nil && !utils.IsOKNetworkError(err) { s.logger.WarnContext(ctx, "failed to close ssh client connection", "error", err) } return } } for { select { // handle out of band ssh requests case req := <-reqs: if req == nil { connClosed() return } s.logger.DebugContext(ctx, "Received out-of-band request", "request_type", req.Type) reqCtx := tracessh.ContextFromRequest(req) ctx, span := s.tracerProvider.Tracer("ssh").Start( oteltrace.ContextWithRemoteSpanContext(ctx, oteltrace.SpanContextFromContext(reqCtx)), fmt.Sprintf("ssh.GlobalRequest/%s", req.Type), oteltrace.WithSpanKind(oteltrace.SpanKindServer), oteltrace.WithAttributes( semconv.RPCServiceKey.String("ssh.Server"), semconv.RPCMethodKey.String("GlobalRequest"), semconv.RPCSystemKey.String("ssh"), ), ) if s.reqHandler != nil { go func(span oteltrace.Span) { defer span.End() s.reqHandler.HandleRequest(ctx, ccx, req) }(span) } else { span.End() } // handle channels: case nch := <-chans: if nch == nil { connClosed() return } chanCtx, nch := tracessh.ContextFromNewChannel(nch) ctx, span := s.tracerProvider.Tracer("ssh").Start( oteltrace.ContextWithRemoteSpanContext(ctx, oteltrace.SpanContextFromContext(chanCtx)), fmt.Sprintf("ssh.OpenChannel/%s", nch.ChannelType()), oteltrace.WithSpanKind(oteltrace.SpanKindServer), oteltrace.WithAttributes( semconv.RPCServiceKey.String("ssh.Server"), semconv.RPCMethodKey.String("OpenChannel"), semconv.RPCSystemKey.String("ssh"), ), ) go func(span oteltrace.Span) { defer span.End() s.newChanHandler.HandleNewChan(ctx, ccx, nch) }(span) // send keepalive pings to the clients case <-keepAliveTick.C: const wantReply = true _, _, err = sconn.SendRequest(teleport.KeepAliveReqType, wantReply, keepAlivePayload[:]) if err != nil { s.logger.ErrorContext(ctx, "Failed sending keepalive request", "error", err) } case <-ctx.Done(): s.logger.DebugContext(ctx, "Connection context canceled", "remote_addr", conn.RemoteAddr(), "local_addr", conn.LocalAddr()) return } } } type RequestHandler interface { HandleRequest(ctx context.Context, ccx *ConnectionContext, r *ssh.Request) } type NewChanHandler interface { HandleNewChan(context.Context, *ConnectionContext, ssh.NewChannel) } type NewChanHandlerFunc func(context.Context, *ConnectionContext, ssh.NewChannel) func (f NewChanHandlerFunc) HandleNewChan(ctx context.Context, ccx *ConnectionContext, ch ssh.NewChannel) { f(ctx, ccx, ch) } // NewConnHandler is called once per incoming connection. // Errors terminate the incoming connection. The returned context // must be the same as, or a child of, the passed in context. type NewConnHandler interface { HandleNewConn(ctx context.Context, ccx *ConnectionContext) (context.Context, error) } // NewConnHandlerFunc wraps a function to satisfy NewConnHandler interface. type NewConnHandlerFunc func(ctx context.Context, ccx *ConnectionContext) (context.Context, error) func (f NewConnHandlerFunc) HandleNewConn(ctx context.Context, ccx *ConnectionContext) (context.Context, error) { return f(ctx, ccx) } type AuthMethods struct { PublicKey PublicKeyFunc Password PasswordFunc NoClient bool } // GetHostSignersFunc is an infallible function that returns host signers for // use with a new SSH connection. It should not block, as it's called while the // SSH client is already connected and waiting for the SSH handshake. type GetHostSignersFunc = func() []ssh.Signer // StaticHostSigners returns a [GetHostSignersFunc] that always returns the // given host signers. func StaticHostSigners(hostSigners ...ssh.Signer) GetHostSignersFunc { return func() []ssh.Signer { return hostSigners } } func (s *Server) checkArguments(a utils.NetAddr, h NewChanHandler, getHostSigners GetHostSignersFunc, ah AuthMethods) error { // If the server is not in tunnel mode, an address must be specified. if s.listener != nil { if a.Addr == "" || a.AddrNetwork == "" { return trace.BadParameter("addr: specify network and the address for listening socket") } } if h == nil { return trace.BadParameter("missing NewChanHandler") } if getHostSigners == nil { return trace.BadParameter("missing GetHostSignersFunc") } if err := s.validateHostSigners(getHostSigners()); err != nil { return trace.Wrap(err) } if ah.PublicKey == nil && ah.Password == nil && !ah.NoClient { return trace.BadParameter("need at least one auth method") } return nil } func (s *Server) validateHostSigners(hostSigners []ssh.Signer) error { for _, signer := range hostSigners { if signer == nil { return trace.BadParameter("host signer can not be nil") } if s.insecureSkipHostValidation { continue } if err := validateHostSigner(s.fips, signer); err != nil { return trace.Wrap(err) } } return nil } // validateHostSigner make sure the signer is a valid certificate. func validateHostSigner(fips bool, signer ssh.Signer) error { cert, ok := signer.PublicKey().(*ssh.Certificate) if !ok { return trace.BadParameter("only host certificates supported") } if len(cert.ValidPrincipals) == 0 { return trace.BadParameter("at least one valid principal is required in host certificate") } certChecker := sshutils.CertChecker{ FIPS: fips, } err := certChecker.CheckCert(cert.ValidPrincipals[0], cert) if err != nil { return trace.Wrap(err) } return nil } type ( PublicKeyFunc func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) PasswordFunc func(conn ssh.ConnMetadata, password []byte) (*ssh.Permissions, error) ) // ClusterDetails specifies information about a cluster type ClusterDetails struct { RecordingProxy bool FIPSEnabled bool } // SSHServerVersionOverrider returns a SSH server version string that should be // used instead of the one from a static configuration (typically because the // version was already sent and can't be un-sent). If SSHServerVersionOverride // returns a blank string (which is an invalid version string, as version // strings should start with "SSH-2.0-") then no override is specified. The // string is intended to be passed as the [ssh.ServerConfig.ServerVersion], so // it should not include a trailing CRLF pair ("\r\n"). type SSHServerVersionOverrider interface { SSHServerVersionOverride() string } func serverVersionOverrideFromConn(nc net.Conn) string { for nc != nil { if overrider, ok := nc.(SSHServerVersionOverrider); ok { if v := overrider.SSHServerVersionOverride(); v != "" { return v } } netConner, ok := nc.(interface{ NetConn() net.Conn }) if !ok { break } nc = netConner.NetConn() } return "" }