From 41ad71fdbae1afff2773e8e33ab8ba8d6d503f2c Mon Sep 17 00:00:00 2001 From: zijiren <84728412+zijiren233@users.noreply.github.com> Date: Sun, 4 Jan 2026 17:03:38 +0800 Subject: [PATCH] fix(sshgate): helath check error log move to debug level (#6476) * fix(sshgate): helath check error log move to debug level * fix: ci lint --- service/sshgate/gateway/gateway.go | 59 ++++++++-- service/sshgate/gateway/gateway_test.go | 140 ++++++++++++++++++++++++ 2 files changed, 192 insertions(+), 7 deletions(-) diff --git a/service/sshgate/gateway/gateway.go b/service/sshgate/gateway/gateway.go index 67f8a7d8b..438687316 100644 --- a/service/sshgate/gateway/gateway.go +++ b/service/sshgate/gateway/gateway.go @@ -3,8 +3,11 @@ package gateway import ( + "errors" "fmt" + "io" "net" + "syscall" "time" "github.com/labring/sealos/service/sshgate/registry" @@ -148,20 +151,38 @@ func New(hostKey ssh.Signer, reg *registry.Registry, opts ...Option) *Gateway { return gw } -func (g *Gateway) HandleConnection(nConn net.Conn) { +// NewServerConn performs SSH handshake with deadline on the given connection. +// Returns the SSH connection, channels, requests, and any error that occurred. +func (g *Gateway) NewServerConn( + nConn net.Conn, +) (*ssh.ServerConn, <-chan ssh.NewChannel, <-chan *ssh.Request, error) { _ = nConn.SetDeadline(time.Now().Add(g.options.SSHHandshakeTimeout)) - conn, chans, reqs, err := ssh.NewServerConn(nConn, g.sshConfig) if err != nil { - g.logger.WithFields(log.Fields{ - "remote_addr": nConn.RemoteAddr().String(), - }).WithError(err).Warn("SSH handshake failed") + return nil, nil, nil, err + } + _ = nConn.SetDeadline(time.Time{}) + return conn, chans, reqs, nil +} + +func (g *Gateway) HandleConnection(nConn net.Conn) { + conn, chans, reqs, err := g.NewServerConn(nConn) + if err != nil { + // Check if this is likely a health check probe (LB, k8s, etc.) + // These probes typically just connect and disconnect without completing SSH handshake + if IsHealthCheckProbeError(err) { + g.logger.WithFields(log.Fields{ + "remote_addr": nConn.RemoteAddr().String(), + }).WithError(err).Debug("SSH handshake failed (likely health check probe)") + } else { + g.logger.WithFields(log.Fields{ + "remote_addr": nConn.RemoteAddr().String(), + }).WithError(err).Warn("SSH handshake failed") + } return } defer conn.Close() - _ = nConn.SetDeadline(time.Time{}) - info, err := g.getDevboxInfoFromPermissions(conn.Permissions) if err != nil { g.logger.WithFields(log.Fields{ @@ -245,3 +266,27 @@ func (g *Gateway) getLoggerFromPermissions(perms *ssh.Permissions) *log.Entry { return logger } + +// IsHealthCheckProbeError checks if the error is likely from a health check probe. +// Health check probes (LB, k8s, etc.) typically just establish TCP connection +// and close it without sending valid SSH protocol data, causing: +// - ECONNRESET: connection reset by peer (RST packet) +// - EPIPE: broken pipe (write to closed connection) +// - EOF: connection closed gracefully without data +func IsHealthCheckProbeError(err error) bool { + // Check for EOF (graceful close without SSH data) + if errors.Is(err, io.EOF) { + return true + } + + // Check for network errors + var opErr *net.OpError + if errors.As(err, &opErr) { + // ECONNRESET: connection reset by peer + // EPIPE: broken pipe + if errors.Is(opErr.Err, syscall.ECONNRESET) || errors.Is(opErr.Err, syscall.EPIPE) { + return true + } + } + return false +} diff --git a/service/sshgate/gateway/gateway_test.go b/service/sshgate/gateway/gateway_test.go index 87d8e224d..b3ebc3ed4 100644 --- a/service/sshgate/gateway/gateway_test.go +++ b/service/sshgate/gateway/gateway_test.go @@ -4,8 +4,10 @@ import ( "crypto/ed25519" "crypto/rand" "encoding/pem" + "io" "net" "os" + "syscall" "testing" "time" @@ -485,6 +487,144 @@ func TestGetUsernameFromPermissions(t *testing.T) { }) } +func TestIsHealthCheckProbeError(t *testing.T) { + tests := []struct { + name string + err error + expected bool + }{ + { + name: "nil error", + err: nil, + expected: false, + }, + { + name: "ECONNRESET wrapped in OpError", + err: &net.OpError{ + Op: "write", + Net: "tcp", + Err: syscall.ECONNRESET, + }, + expected: true, + }, + { + name: "EPIPE wrapped in OpError", + err: &net.OpError{ + Op: "write", + Net: "tcp", + Err: syscall.EPIPE, + }, + expected: true, + }, + { + name: "EOF", + err: io.EOF, + expected: true, + }, + { + name: "other syscall error (ETIMEDOUT)", + err: &net.OpError{ + Op: "read", + Net: "tcp", + Err: syscall.ETIMEDOUT, + }, + expected: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := gateway.IsHealthCheckProbeError(tt.err) + if result != tt.expected { + t.Errorf("IsHealthCheckProbeError(%v) = %v, want %v", tt.err, result, tt.expected) + } + }) + } +} + +// TestNewServerConn_HealthCheckProbe tests that health check probes produce errors +// that are correctly identified by IsHealthCheckProbeError. +// +// Real-world LB health check flow: +// 1. LB establishes TCP connection to sshgate +// 2. LB immediately closes connection (sends FIN or RST) +// 3. sshgate's ssh.NewServerConn tries to WRITE SSH version string +// 4. sshgate receives RST when writing to closed connection -> "write: connection reset by peer" +// +// Note: Local loopback may produce EPIPE instead of ECONNRESET due to timing differences, +// but both are handled by IsHealthCheckProbeError. +func TestNewServerConn_HealthCheckProbe(t *testing.T) { + reg := registry.New() + hostKey, _, _, _ := generateTestKeys(t) + gw := gateway.New(hostKey, reg, gateway.WithSSHHandshakeTimeout(2*time.Second)) + + // Start a test server + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("Failed to start listener: %v", err) + } + defer listener.Close() + + t.Run("TCP RST (SetLinger 0)", func(t *testing.T) { + // Simulate LB health check: connect and immediately close with RST + clientConn, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatalf("Failed to connect: %v", err) + } + + serverConn, err := listener.Accept() + if err != nil { + t.Fatalf("Failed to accept: %v", err) + } + + // Set SO_LINGER to 0 and close to send RST + if tcpConn, ok := clientConn.(*net.TCPConn); ok { + _ = tcpConn.SetLinger(0) + } + clientConn.Close() + + // Now try SSH handshake - should fail with health check error + _, _, _, sshErr := gw.NewServerConn(serverConn) + if sshErr == nil { + t.Fatal("Expected error, got nil") + } + + t.Logf("Got error: %v", sshErr) + + if !gateway.IsHealthCheckProbeError(sshErr) { + t.Errorf("Expected health check probe error, got: %v", sshErr) + } + }) + + t.Run("Immediate close (FIN)", func(t *testing.T) { + // Simulate: connect and gracefully close -> produces EOF + clientConn, err := net.Dial("tcp", listener.Addr().String()) + if err != nil { + t.Fatalf("Failed to connect: %v", err) + } + + serverConn, err := listener.Accept() + if err != nil { + t.Fatalf("Failed to accept: %v", err) + } + + // Gracefully close + clientConn.Close() + + // Now try SSH handshake - should fail with EOF + _, _, _, sshErr := gw.NewServerConn(serverConn) + if sshErr == nil { + t.Fatal("Expected error, got nil") + } + + t.Logf("Got error: %v", sshErr) + + if !gateway.IsHealthCheckProbeError(sshErr) { + t.Errorf("Expected health check probe error, got: %v", sshErr) + } + }) +} + func TestPublicKeyCallback_WithPodIP(t *testing.T) { reg := registry.New()