feat: add ServerTailnet logs and metrics for unreachable agents (#30133)

Add default-level metrics and warn-level diagnostics to investigate
ServerTailnet reachability failures causing agent-specific 502s, without
restarting coderd or enabling debug logging.

- Enrich request logs and PTY/chatd warnings with agent ID, failure
  reason, wait duration, coordinator state, WireGuard handshake and
  byte counts, and DERP regions.
- Add metrics for unreachable attempts by reason, reachability latency,
  embedded DERP connections, and offline app/terminal requests.
- Distinguish reachability failures from subsequent TCP dial failures,
  cap reverse proxy dials at one minute, and exclude expected chatd
  readiness polls and superseded-agent dials from unreachable counts.
- Report missing handshakes as zero time instead of 1970, also fixing
  `coder ping`.

Part of PLAT-680, a sub-issue of PLAT-360.

coderd/coderd.go

@@ -752,6 +752,7 @@ func New(options *Options) *API {
 		options.AgentInactiveDisconnectTimeout,
 		options.WorkspaceAppAuditSessionTimeout,
 		options.AppSigningKeyCache,
+		options.PrometheusRegistry,
 	)
 
 	f := appearance.NewDefaultFetcher(options.Database, api.DeploymentValues.DocsURL.String())

coderd/mcp_http.go

@@ -2,6 +2,7 @@ package coderd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/http"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/coder/coder/v2/coderd/httpmw"
 	"github.com/coder/coder/v2/coderd/mcp"
 	"github.com/coder/coder/v2/coderd/rbac/policy"
+	"github.com/coder/coder/v2/coderd/workspaceapps"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/toolsdk"
 	"github.com/coder/coder/v2/codersdk/workspacesdk"
@@ -74,7 +76,13 @@ func (api *API) mcpHTTPHandler() http.Handler {
 			if !api.Authorize(r, policy.ActionSSH, workspace) {
 				return nil, nil, xerrors.New("unauthorized: you do not have SSH access to this workspace")
 			}
-			return api.agentProvider.AgentConn(ctx, agentID)
+			conn, release, err := api.agentProvider.AgentConn(ctx, agentID)
+			if unreachable, ok := errors.AsType[*workspaceapps.AgentUnreachableError](err); ok {
+				// Tool errors are sent with HTTP 200, so the request log
+				// line is written at debug level.
+				api.Logger.Warn(ctx, "agent is unreachable", append(unreachable.Fields, slog.Error(err))...)
+			}
+			return conn, release, err
 		})
 
 		toolset := MCPToolset(r.URL.Query().Get("toolset"))

coderd/tailnet.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"net"
 	"net/http"
+	"net/http/httptrace"
 	"net/http/httputil"
 	"net/netip"
 	"net/url"
@@ -18,12 +19,14 @@ import (
 
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
+	"github.com/tailscale/wireguard-go/device"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"
 	"tailscale.com/derp"
 	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog/v3"
+	"github.com/coder/coder/v2/coderd/httpmw/loggermw"
 	"github.com/coder/coder/v2/coderd/tracing"
 	"github.com/coder/coder/v2/coderd/workspaceapps"
 	"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
@@ -72,11 +75,18 @@ func NewServerTailnet(
 	}
 	serverCtx, cancel := context.WithCancel(ctx)
 
+	derpConnects := prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: "coder",
+		Subsystem: "servertailnet",
+		Name:      "derp_connects_total",
+		Help:      "Number of times the server tailnet connected to the embedded DERP relay. Stays 0 when the embedded relay is disabled.",
+	})
+
 	// This is set to allow local DERP traffic to be proxied through memory
 	// instead of needing to hit the external access URL. Don't use the ctx
 	// given in this callback, it's only valid while connecting.
 	if derpServer != nil {
-		conn.SetDERPRegionDialer(embeddedDERPRegionDialer(ctx, logger, derpServer))
+		conn.SetDERPRegionDialer(embeddedDERPRegionDialer(ctx, logger, derpServer, derpConnects))
 	}
 
 	tracer := traceProvider.Tracer(tracing.TracerName)
@@ -90,15 +100,18 @@ func NewServerTailnet(
 	// TODO: support controller.TelemetryCtrl
 
 	tn := &ServerTailnet{
-		ctx:         serverCtx,
-		cancel:      cancel,
-		logger:      logger,
-		tracer:      tracer,
-		conn:        conn,
-		coordinatee: conn,
-		controller:  controller,
-		coordCtrl:   coordCtrl,
-		transport:   tailnetTransport.Clone(),
+		ctx:                 serverCtx,
+		cancel:              cancel,
+		logger:              logger,
+		tracer:              tracer,
+		conn:                conn,
+		coordinatee:         conn,
+		controller:          controller,
+		coordCtrl:           coordCtrl,
+		transport:           tailnetTransport.Clone(),
+		clock:               quartz.NewReal(),
+		readPeerDiagnostics: conn.GetPeerDiagnostics,
+		peerDiagnosticsSlot: make(chan struct{}, 1),
 		connsPerAgent: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Namespace: "coder",
 			Subsystem: "servertailnet",
@@ -111,6 +124,20 @@ func NewServerTailnet(
 			Name:      "connections_total",
 			Help:      "Total number of TCP connections made to workspace agents.",
 		}, []string{"network"}),
+		derpConnects: derpConnects,
+		agentUnreachable: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Namespace: "coder",
+			Subsystem: "servertailnet",
+			Name:      "agent_unreachable_total",
+			Help:      "Number of connection attempts where the workspace agent did not answer in time, by reason: no_node (this server has no node for the agent), peer_lost (the agent's or this server's coordinator connection ended), no_handshake (no WireGuard handshake yet), handshake_stale (last handshake over 180s ago), handshake_ok (last handshake under 180s ago), diagnostics_timeout (peer state not readable within 1s), client_canceled (client left within 5s).",
+		}, []string{"reason"}),
+		awaitReachable: prometheus.NewHistogram(prometheus.HistogramOpts{
+			Namespace: "coder",
+			Subsystem: "servertailnet",
+			Name:      "await_reachable_seconds",
+			Help:      "Time until a workspace agent answered the first ping when a new connection was opened.",
+			Buckets:   []float64{0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60},
+		}),
 	}
 	tn.transport.DialContext = tn.dialContext
 	// These options are mostly just picked at random, and they can likely be
@@ -145,11 +172,17 @@ func (s *ServerTailnet) Conn() *tailnet.Conn {
 func (s *ServerTailnet) Describe(descs chan<- *prometheus.Desc) {
 	s.connsPerAgent.Describe(descs)
 	s.totalConns.Describe(descs)
+	s.derpConnects.Describe(descs)
+	s.agentUnreachable.Describe(descs)
+	s.awaitReachable.Describe(descs)
 }
 
 func (s *ServerTailnet) Collect(metrics chan<- prometheus.Metric) {
 	s.connsPerAgent.Collect(metrics)
 	s.totalConns.Collect(metrics)
+	s.derpConnects.Collect(metrics)
+	s.agentUnreachable.Collect(metrics)
+	s.awaitReachable.Collect(metrics)
 }
 
 type ServerTailnet struct {
@@ -168,9 +201,18 @@ type ServerTailnet struct {
 	coordCtrl  *MultiAgentController
 
 	transport *http.Transport
+	clock     quartz.Clock
 
-	connsPerAgent *prometheus.GaugeVec
-	totalConns    *prometheus.CounterVec
+	connsPerAgent    *prometheus.GaugeVec
+	totalConns       *prometheus.CounterVec
+	derpConnects     prometheus.Counter
+	agentUnreachable *prometheus.CounterVec
+	awaitReachable   prometheus.Histogram
+
+	// peerDiagnosticsSlot holds one token while a peer diagnostics read is
+	// in flight.
+	peerDiagnosticsSlot chan struct{}
+	readPeerDiagnostics func(agentID uuid.UUID) tailnet.PeerDiagnostics
 }
 
 func (s *ServerTailnet) ReverseProxy(targetURL, dashboardURL *url.URL, agentID uuid.UUID, app appurl.ApplicationURL, wildcardHostname string) *httputil.ReverseProxy {
@@ -184,6 +226,7 @@ func (s *ServerTailnet) ReverseProxy(targetURL, dashboardURL *url.URL, agentID u
 
 	proxy := httputil.NewSingleHostReverseProxy(&tgt)
 	proxy.ErrorHandler = func(w http.ResponseWriter, r *http.Request, theErr error) {
+		theErr = s.reportProxyDialFailure(r.Context(), agentID, theErr)
 		var (
 			desc           = "Failed to proxy request to application: " + theErr.Error()
 			additionalInfo = ""
@@ -239,37 +282,214 @@ func (s *ServerTailnet) ReverseProxy(targetURL, dashboardURL *url.URL, agentID u
 
 type agentIDKey struct{}
 
-// director makes sure agentIDKey is set on the context in the reverse proxy.
-// This allows the transport to correctly identify which agent to dial to.
+// director makes sure agentIDKey and a dialState are set on the context in the
+// reverse proxy. The transport uses the agent ID to pick which agent to dial,
+// and the dial and the error handler share the dialState.
 func (*ServerTailnet) director(agentID uuid.UUID, prev func(req *http.Request)) func(req *http.Request) {
 	return func(req *http.Request) {
+		ds := &dialState{}
 		ctx := context.WithValue(req.Context(), agentIDKey{}, agentID)
+		ctx = context.WithValue(ctx, dialStateKey{}, ds)
+		// A pooled connection skips dialContext, so the dial phases cannot show
+		// that the request has a connection. GotConn fires either way. GetConn
+		// runs on the request goroutine before each attempt, so a retry clears
+		// the connection the previous attempt used.
+		ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{
+			GetConn: func(string) { ds.setGotConn(false) },
+			GotConn: func(httptrace.GotConnInfo) { ds.setGotConn(true) },
+		})
 		*req = *req.WithContext(ctx)
 		prev(req)
 	}
 }
 
+// proxyDialTimeout caps a reverse proxy dial. The HTTP transport keeps dialing
+// after the request ends, so without a cap an unreachable agent is pinged
+// until the ServerTailnet closes.
+const proxyDialTimeout = time.Minute
+
+// dialPhase is the step of a reverse proxy dial that was in progress when the
+// request ended.
+type dialPhase int
+
+const (
+	dialPhaseNone dialPhase = iota
+	dialPhaseAwaitReachable
+	dialPhaseTCPDial
+	dialPhaseDone
+)
+
+func (p dialPhase) String() string {
+	switch p {
+	case dialPhaseAwaitReachable:
+		return "await_reachable"
+	case dialPhaseTCPDial:
+		return "tcp_dial"
+	case dialPhaseDone:
+		return "done"
+	default:
+		return "none"
+	}
+}
+
+// dialState is shared by the request goroutine and the transport's dial
+// goroutine through the request context. The error handler reads it to tell
+// a dial that never finished from a failure on an established connection.
+type dialState struct {
+	mu      sync.Mutex
+	phase   dialPhase
+	since   time.Time
+	gotConn bool
+}
+
+type dialStateKey struct{}
+
+func dialStateFromContext(ctx context.Context) *dialState {
+	ds, _ := ctx.Value(dialStateKey{}).(*dialState)
+	return ds
+}
+
+func (d *dialState) set(phase dialPhase) {
+	if d == nil {
+		return
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.phase = phase
+	d.since = time.Now()
+}
+
+// get returns the current phase and how long it has been in progress.
+func (d *dialState) get() (dialPhase, time.Duration) {
+	if d == nil {
+		return dialPhaseNone, 0
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.phase, time.Since(d.since)
+}
+
+func (d *dialState) setGotConn(gotConn bool) {
+	if d == nil {
+		return
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	d.gotConn = gotConn
+}
+
+// hadConn reports whether the transport gave the request a connection.
+func (d *dialState) hadConn() bool {
+	if d == nil {
+		return false
+	}
+	d.mu.Lock()
+	defer d.mu.Unlock()
+	return d.gotConn
+}
+
 func (s *ServerTailnet) dialContext(ctx context.Context, network, addr string) (net.Conn, error) {
 	agentID, ok := ctx.Value(agentIDKey{}).(uuid.UUID)
 	if !ok {
 		return nil, xerrors.Errorf("no agent id attached")
 	}
+	// Only the connect steps below use the context. The returned conn is not
+	// tied to it, so canceling after the dial is safe.
+	ctx, cancel := context.WithTimeout(ctx, proxyDialTimeout)
+	defer cancel()
 
-	nc, err := s.DialAgentNetConn(ctx, agentID, network, addr)
+	// The reverse proxy's error handler records unreachable agents on the
+	// request goroutine when the request ends, which can be before this dial
+	// returns, so this path uses acquireAgent, which neither logs nor counts.
+	ds := dialStateFromContext(ctx)
+	ds.set(dialPhaseAwaitReachable)
+	conn, release, err := s.acquireAgent(ctx, agentID)
 	if err != nil {
-		return nil, err
+		return nil, xerrors.Errorf("acquire agent conn: %w", err)
 	}
 
+	ds.set(dialPhaseTCPDial)
+	nc, err := conn.DialContext(ctx, network, addr)
+	if err != nil {
+		release()
+		return nil, xerrors.Errorf("dial context: %w", err)
+	}
+	ds.set(dialPhaseDone)
+
 	s.connsPerAgent.WithLabelValues("tcp").Inc()
 	s.totalConns.WithLabelValues("tcp").Inc()
 	return &instrumentedConn{
-		Conn:          nc,
+		Conn:          &netConnCloser{Conn: nc, close: release},
 		agentID:       agentID,
 		connsPerAgent: s.connsPerAgent,
 	}, nil
 }
 
+// reportProxyDialFailure records a reverse proxy request that ended while its
+// dial was still in progress. It runs on the request goroutine before the 502
+// is written, which is the last point where the request log line can still
+// take fields: the transport detaches the dial from the request deadline, so
+// the dial goroutine learns of the failure later, or not at all. Errors on an
+// established connection are returned unchanged.
+func (s *ServerTailnet) reportProxyDialFailure(ctx context.Context, agentID uuid.UUID, err error) error {
+	ds := dialStateFromContext(ctx)
+	// The transport may give the request a pooled connection while the dial it
+	// started is still waiting on the agent. An error after that is from the
+	// connection the request used, so the dial phase does not describe it.
+	if ds.hadConn() {
+		return err
+	}
+	phase, elapsed := ds.get()
+	switch phase {
+	case dialPhaseAwaitReachable:
+		// A request that is still waiting can also see an error from before
+		// the first ping, such as a failed coordinator send. AgentConn does
+		// not count those, so neither does this path.
+		if !errors.Is(err, errAgentUnreachable) && ctx.Err() == nil {
+			return err
+		}
+		unreachable := s.recordAgentUnreachable(ctx, agentID, elapsed, slog.F("dial_phase", phase.String()))
+		if errors.Is(err, errAgentUnreachable) {
+			return err
+		}
+		return xerrors.Errorf("%s: %w", unreachable.Error(), err)
+	case dialPhaseTCPDial:
+		// The agent answered a ping, so the tunnel is up and the failure is
+		// above it: the netstack dial or an app that is not accepting. Name
+		// the phase on the log line, but do not count it as unreachable.
+		if rl := loggermw.RequestLoggerFromContext(ctx); rl != nil {
+			rl.WithFields(
+				slog.F("agent_id", agentID),
+				slog.F("dial_phase", phase.String()),
+				slog.F("tcp_dial_after", elapsed),
+			)
+		}
+		return err
+	default:
+		return err
+	}
+}
+
+// errAgentUnreachable is returned by acquireAgent when the agent did not
+// answer a ping before the context ended.
+var errAgentUnreachable = xerrors.New("agent is unreachable")
+
 func (s *ServerTailnet) AgentConn(ctx context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
+	start := s.clock.Now()
+	conn, release, err := s.acquireAgent(ctx, agentID)
+	cause := context.Cause(ctx)
+	if errors.Is(err, errAgentUnreachable) &&
+		!errors.Is(cause, workspacesdk.ErrDialAbandoned) &&
+		!errors.Is(cause, workspacesdk.ErrReadinessProbeTimeout) {
+		return nil, nil, s.recordAgentUnreachable(ctx, agentID, s.clock.Since(start))
+	}
+	return conn, release, err
+}
+
+// acquireAgent adds the agent to the coordinator and waits for it to answer a
+// ping. On timeout it returns errAgentUnreachable and leaves logging and
+// metrics to the caller, which knows whether a request is still waiting.
+func (s *ServerTailnet) acquireAgent(ctx context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
 	var (
 		conn workspacesdk.AgentConn
 		ret  func()
@@ -291,33 +511,137 @@ func (s *ServerTailnet) AgentConn(ctx context.Context, agentID uuid.UUID) (works
 	// Since we now have an open conn, be careful to close it if we error
 	// without returning it to the user.
 
+	start := s.clock.Now()
 	reachable := conn.AwaitReachable(ctx)
 	if !reachable {
 		ret()
-		return nil, nil, xerrors.New("agent is unreachable")
+		return nil, nil, errAgentUnreachable
 	}
+	s.awaitReachable.Observe(s.clock.Since(start).Seconds())
 
 	return conn, ret, nil
 }
 
-func (s *ServerTailnet) DialAgentNetConn(ctx context.Context, agentID uuid.UUID, network, addr string) (net.Conn, error) {
-	conn, release, err := s.AgentConn(ctx, agentID)
-	if err != nil {
-		return nil, xerrors.Errorf("acquire agent conn: %w", err)
+// peerDiagnosticsTimeout caps how long a failed connection waits for peer
+// diagnostics, which need WireGuard engine locks that may be busy during an
+// incident.
+const peerDiagnosticsTimeout = time.Second
+
+// clientCanceledWait is the longest wait that still counts as the client
+// leaving when the context was canceled. A connected agent answers its first
+// ping well within this time.
+const clientCanceledWait = 5 * time.Second
+
+// recordAgentUnreachable counts a connection attempt that the agent did not
+// answer and returns an error carrying the peer state. The fields go on the
+// request log line, or on a warn line when the context has no request logger.
+// A client that left quickly gets a plain error and no warn line.
+func (s *ServerTailnet) recordAgentUnreachable(ctx context.Context, agentID uuid.UUID, after time.Duration, extra ...slog.Field) error {
+	fields := append([]slog.Field{
+		slog.F("agent_id", agentID),
+		slog.F("unreachable_after", after),
+	}, extra...)
+	reason := "diagnostics_timeout"
+
+	// A quick cancellation is the client leaving, not the agent failing.
+	// A long wait counts as unreachable however the context ended, since a
+	// load balancer giving up on a hung request also cancels it.
+	clientCanceled := errors.Is(ctx.Err(), context.Canceled) && after < clientCanceledWait
+	if clientCanceled {
+		reason = "client_canceled"
+	} else if d, ok := s.peerDiagnostics(agentID); ok {
+		fields = append(fields, unreachableFields(d)...)
+		reason = unreachableReason(d, s.clock.Now())
 	}
+	fields = append(fields, slog.F("reason", reason))
+	s.agentUnreachable.WithLabelValues(reason).Inc()
 
-	// Since we now have an open conn, be careful to close it if we error
-	// without returning it to the user.
-
-	nc, err := conn.DialContext(ctx, network, addr)
-	if err != nil {
-		release()
-		return nil, xerrors.Errorf("dial context: %w", err)
+	if rl := loggermw.RequestLoggerFromContext(ctx); rl != nil {
+		rl.WithFields(fields...)
+	} else if !clientCanceled {
+		// Callers such as chatd have no request log line, so without this the
+		// counter rises with no record of which agent failed.
+		s.logger.Warn(ctx, "agent is unreachable", fields...)
 	}
+	if clientCanceled {
+		return xerrors.New("agent is unreachable: client canceled")
+	}
+	return &workspaceapps.AgentUnreachableError{Fields: fields}
+}
 
-	return &netConnCloser{Conn: nc, close: func() {
-		release()
-	}}, err
+// peerDiagnostics reads the peer state within peerDiagnosticsTimeout. The
+// read takes WireGuard engine locks that may be held during an incident, so
+// one read runs at a time and other callers wait for the slot within the
+// same timeout instead of each parking a goroutine on the lock.
+func (s *ServerTailnet) peerDiagnostics(agentID uuid.UUID) (tailnet.PeerDiagnostics, bool) {
+	timeout := s.clock.NewTimer(peerDiagnosticsTimeout, "peerDiagnostics")
+	defer timeout.Stop()
+	select {
+	case s.peerDiagnosticsSlot <- struct{}{}:
+	case <-timeout.C:
+		return tailnet.PeerDiagnostics{}, false
+	}
+	diagCh := make(chan tailnet.PeerDiagnostics, 1)
+	go func() {
+		defer func() { <-s.peerDiagnosticsSlot }()
+		diagCh <- s.readPeerDiagnostics(agentID)
+	}()
+	select {
+	case d := <-diagCh:
+		return d, true
+	case <-timeout.C:
+		return tailnet.PeerDiagnostics{}, false
+	}
+}
+
+// unreachableFields describes the peer state for the request log line.
+// server_preferred_derp is this server's home relay. The peer_* fields
+// describe the agent.
+func unreachableFields(d tailnet.PeerDiagnostics) []slog.Field {
+	fields := []slog.Field{
+		slog.F("peer_node_received", d.ReceivedNode != nil),
+		slog.F("server_preferred_derp", d.PreferredDERP),
+	}
+	if !d.LastWireguardHandshake.IsZero() {
+		fields = append(fields, slog.F("peer_last_handshake", d.LastWireguardHandshake))
+	}
+	if d.ReceivedNode != nil {
+		// The node's DERP is "127.3.3.40:N" where N is the region ID.
+		region := d.ReceivedNode.DERP
+		if _, after, found := strings.Cut(region, ":"); found {
+			region = after
+		}
+		fields = append(fields,
+			slog.F("peer_lost", d.Lost),
+			slog.F("peer_preferred_derp", region),
+			slog.F("peer_tx_bytes", d.TxBytes),
+			slog.F("peer_rx_bytes", d.RxBytes),
+		)
+	}
+	return fields
+}
+
+// unreachableReason is the reason label for agent_unreachable_total. A lost
+// peer comes before the handshake checks because the tunnel can look healthy
+// while the control plane is down. A handshake older than WireGuard's session
+// limit means our handshakes went unanswered while we waited.
+func unreachableReason(d tailnet.PeerDiagnostics, now time.Time) string {
+	switch {
+	// The node was never received, or was removed after a disconnect or a
+	// lost timeout.
+	case d.ReceivedNode == nil:
+		return "no_node"
+	// The agent's coordinator connection dropped, or this server's connection
+	// to the coordinator ended.
+	case d.Lost:
+		return "peer_lost"
+	case d.LastWireguardHandshake.IsZero():
+		return "no_handshake"
+	case now.Sub(d.LastWireguardHandshake) > device.RejectAfterTime:
+		return "handshake_stale"
+	default:
+		return "handshake_ok"
+	}
 }
 
 func (s *ServerTailnet) ServeHTTPDebug(w http.ResponseWriter, r *http.Request) {
@@ -327,7 +651,7 @@ func (s *ServerTailnet) ServeHTTPDebug(w http.ResponseWriter, r *http.Request) {
 // embeddedDERPRegionDialer serves the embedded DERP region over an in-memory
 // pipe. A nil conn makes the DERP client dial the access URL, so after ctx is
 // done it returns a closed conn: the handshake fails and magicsock backs off.
-func embeddedDERPRegionDialer(ctx context.Context, logger slog.Logger, derpServer *derp.Server) func(context.Context, *tailcfg.DERPRegion) net.Conn {
+func embeddedDERPRegionDialer(ctx context.Context, logger slog.Logger, derpServer *derp.Server, derpConnects prometheus.Counter) func(context.Context, *tailcfg.DERPRegion) net.Conn {
 	return func(_ context.Context, region *tailcfg.DERPRegion) net.Conn {
 		if !region.EmbeddedRelay {
 			return nil
@@ -337,6 +661,7 @@ func embeddedDERPRegionDialer(ctx context.Context, logger slog.Logger, derpServe
 			_ = right.Close()
 			return left
 		}
+		derpConnects.Inc()
 		logger.Debug(ctx, "connecting to embedded DERP via in-memory pipe")
 		go func() {
 			defer left.Close()

coderd/tailnet_internal_test.go

@@ -1,14 +1,29 @@
 package coderd
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httptrace"
+	"sync/atomic"
 	"testing"
+	"time"
 
+	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/require"
+	"github.com/tailscale/wireguard-go/device"
+	"golang.org/x/xerrors"
 	"tailscale.com/derp"
 	"tailscale.com/tailcfg"
 	"tailscale.com/types/key"
 
+	"cdr.dev/slog/v3"
+	"cdr.dev/slog/v3/sloggers/slogjson"
+	"github.com/coder/coder/v2/coderd/workspaceapps"
+	"github.com/coder/coder/v2/tailnet"
 	"github.com/coder/coder/v2/testutil"
 	"github.com/coder/quartz"
 )
@@ -19,7 +34,7 @@ func TestEmbeddedDERPRegionDialer(t *testing.T) {
 	srv := derp.NewServer(key.NewNode(), func(format string, args ...any) { t.Logf(format, args...) })
 	t.Cleanup(func() { _ = srv.Close() })
 	ctx, cancel := context.WithCancel(testutil.Context(t, testutil.WaitShort))
-	dial := embeddedDERPRegionDialer(ctx, testutil.Logger(t), srv)
+	dial := embeddedDERPRegionDialer(ctx, testutil.Logger(t), srv, prometheus.NewCounter(prometheus.CounterOpts{Name: "derp_connects_total"}))
 
 	require.Nil(t, dial(ctx, &tailcfg.DERPRegion{EmbeddedRelay: false}))
 
@@ -51,3 +66,349 @@ func TestPollingDERPClient_FirstRecvDoesNotWaitForTick(t *testing.T) {
 	}()
 	require.Equal(t, derpMap, testutil.RequireReceive(ctx, t, got))
 }
+
+func TestUnreachableReason(t *testing.T) {
+	t.Parallel()
+
+	now := time.Date(2026, 1, 7, 12, 0, 0, 0, time.UTC)
+	node := &tailcfg.Node{}
+	for _, tc := range []struct {
+		name string
+		diag tailnet.PeerDiagnostics
+		want string
+	}{
+		{name: "NoNode", diag: tailnet.PeerDiagnostics{}, want: "no_node"},
+		{name: "NoHandshake", diag: tailnet.PeerDiagnostics{ReceivedNode: node}, want: "no_handshake"},
+		{
+			name: "Lost",
+			diag: tailnet.PeerDiagnostics{ReceivedNode: node, Lost: true, LastWireguardHandshake: now.Add(-time.Minute)},
+			want: "peer_lost",
+		},
+		{
+			name: "Stale",
+			diag: tailnet.PeerDiagnostics{ReceivedNode: node, LastWireguardHandshake: now.Add(-device.RejectAfterTime - time.Second)},
+			want: "handshake_stale",
+		},
+		{
+			name: "OK",
+			diag: tailnet.PeerDiagnostics{ReceivedNode: node, LastWireguardHandshake: now.Add(-time.Minute)},
+			want: "handshake_ok",
+		},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+			require.Equal(t, tc.want, unreachableReason(tc.diag, now))
+		})
+	}
+}
+
+func fieldMap(fields []slog.Field) map[string]any {
+	m := make(map[string]any, len(fields))
+	for _, f := range fields {
+		m[f.Name] = f.Value
+	}
+	return m
+}
+
+func TestUnreachableFields(t *testing.T) {
+	t.Parallel()
+
+	t.Run("NoNode", func(t *testing.T) {
+		t.Parallel()
+		got := fieldMap(unreachableFields(tailnet.PeerDiagnostics{PreferredDERP: 1}))
+		require.Equal(t, false, got["peer_node_received"])
+		require.Equal(t, 1, got["server_preferred_derp"])
+		require.NotContains(t, got, "peer_last_handshake")
+		require.NotContains(t, got, "peer_preferred_derp")
+		require.NotContains(t, got, "peer_tx_bytes")
+		require.NotContains(t, got, "peer_rx_bytes")
+	})
+
+	t.Run("NodeReceived", func(t *testing.T) {
+		t.Parallel()
+		handshake := time.Date(2026, 1, 7, 12, 0, 0, 0, time.UTC)
+		got := fieldMap(unreachableFields(tailnet.PeerDiagnostics{
+			PreferredDERP:          1,
+			ReceivedNode:           &tailcfg.Node{DERP: "127.3.3.40:2"},
+			LastWireguardHandshake: handshake,
+			TxBytes:                148,
+			RxBytes:                0,
+		}))
+		require.Equal(t, true, got["peer_node_received"])
+		require.Equal(t, false, got["peer_lost"])
+		require.Equal(t, handshake, got["peer_last_handshake"])
+		require.Equal(t, "2", got["peer_preferred_derp"])
+		require.EqualValues(t, 148, got["peer_tx_bytes"])
+		require.EqualValues(t, 0, got["peer_rx_bytes"])
+	})
+}
+
+// newTestServerTailnet returns a ServerTailnet with only the fields that
+// recordAgentUnreachable uses. read stands in for the tailnet connection.
+func newTestServerTailnet(t *testing.T, read func(uuid.UUID) tailnet.PeerDiagnostics) (*ServerTailnet, *prometheus.Registry) {
+	t.Helper()
+	counter := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "agent_unreachable_total"}, []string{"reason"})
+	registry := prometheus.NewRegistry()
+	require.NoError(t, registry.Register(counter))
+	return &ServerTailnet{
+		clock:               quartz.NewReal(),
+		agentUnreachable:    counter,
+		readPeerDiagnostics: read,
+		peerDiagnosticsSlot: make(chan struct{}, 1),
+	}, registry
+}
+
+func noPeerDiagnostics(uuid.UUID) tailnet.PeerDiagnostics {
+	return tailnet.PeerDiagnostics{}
+}
+
+func TestRecordAgentUnreachable_DiagnosticsWait(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	readStarted := make(chan struct{}, 2)
+	release := make(chan struct{})
+	s, registry := newTestServerTailnet(t, func(uuid.UUID) tailnet.PeerDiagnostics {
+		readStarted <- struct{}{}
+		<-release
+		return tailnet.PeerDiagnostics{ReceivedNode: &tailcfg.Node{}}
+	})
+	// The mock clock never fires the timeout, so the second failure can only
+	// finish by getting the slot.
+	clock := quartz.NewMock(t)
+	s.clock = clock
+	trap := clock.Trap().NewTimer("peerDiagnostics")
+	defer trap.Close()
+
+	errCh := make(chan error, 2)
+	go func() {
+		errCh <- s.recordAgentUnreachable(ctx, uuid.New(), time.Second)
+	}()
+	trap.MustWait(ctx).MustRelease(ctx)
+	testutil.RequireReceive(ctx, t, readStarted)
+
+	go func() {
+		errCh <- s.recordAgentUnreachable(ctx, uuid.New(), time.Second)
+	}()
+	trap.MustWait(ctx).MustRelease(ctx)
+	close(release)
+
+	for range 2 {
+		err := testutil.RequireReceive(ctx, t, errCh)
+		var unreachable *workspaceapps.AgentUnreachableError
+		require.ErrorAs(t, err, &unreachable)
+		require.Equal(t, "no_handshake", fieldMap(unreachable.Fields)["reason"])
+	}
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 2, "agent_unreachable_total", "no_handshake"))
+}
+
+func TestRecordAgentUnreachable_NoRequestLogger(t *testing.T) {
+	t.Parallel()
+
+	s, _ := newTestServerTailnet(t, noPeerDiagnostics)
+	var logs bytes.Buffer
+	s.logger = slog.Make(slogjson.Sink(&logs))
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	_ = s.recordAgentUnreachable(ctx, uuid.New(), 100*time.Millisecond)
+	require.Empty(t, logs.String())
+
+	agentID := uuid.New()
+	_ = s.recordAgentUnreachable(testutil.Context(t, testutil.WaitShort), agentID, time.Second)
+	var entry struct {
+		Level  string         `json:"level"`
+		Msg    string         `json:"msg"`
+		Fields map[string]any `json:"fields"`
+	}
+	require.NoError(t, json.Unmarshal(logs.Bytes(), &entry))
+	require.Equal(t, "WARN", entry.Level)
+	require.Equal(t, "agent is unreachable", entry.Msg)
+	require.Equal(t, agentID.String(), entry.Fields["agent_id"])
+	require.Equal(t, "no_node", entry.Fields["reason"])
+}
+
+func TestRecordAgentUnreachable_ClientCanceled(t *testing.T) {
+	t.Parallel()
+
+	s, registry := newTestServerTailnet(t, noPeerDiagnostics)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+
+	// A short wait ended by the client is counted but is not an unreachable
+	// agent, so the PTY handler logs it at debug like any other dial error.
+	err := s.recordAgentUnreachable(ctx, uuid.New(), 100*time.Millisecond)
+	require.Error(t, err)
+	var unreachable *workspaceapps.AgentUnreachableError
+	require.NotErrorAs(t, err, &unreachable)
+
+	// A long wait counts however the context ended.
+	err = s.recordAgentUnreachable(ctx, uuid.New(), clientCanceledWait)
+	require.ErrorAs(t, err, &unreachable)
+	require.Equal(t, "no_node", fieldMap(unreachable.Fields)["reason"])
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 1, "agent_unreachable_total", "client_canceled"))
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 1, "agent_unreachable_total", "no_node"))
+}
+
+func TestReportProxyDialFailure_PooledConn(t *testing.T) {
+	t.Parallel()
+
+	s, registry := newTestServerTailnet(t, noPeerDiagnostics)
+
+	req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
+	s.director(uuid.New(), func(*http.Request) {})(req)
+	ctx, cancel := context.WithCancel(req.Context())
+	ds := dialStateFromContext(ctx)
+	require.NotNil(t, ds)
+
+	// The dial started for this request is still waiting on the agent, but
+	// the transport gave the request a pooled connection and the request
+	// ended while it was on that connection. The dial goroutine can reach
+	// dialContext after GotConn.
+	trace := httptrace.ContextClientTrace(ctx)
+	trace.GetConn("127.0.0.1:80")
+	trace.GotConn(httptrace.GotConnInfo{Reused: true})
+	ds.set(dialPhaseAwaitReachable)
+	require.True(t, ds.hadConn())
+	cancel()
+	err := s.reportProxyDialFailure(ctx, uuid.New(), context.Canceled)
+	require.Same(t, context.Canceled, err)
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.Empty(t, metrics)
+}
+
+func TestReportProxyDialFailure_RetryDial(t *testing.T) {
+	t.Parallel()
+
+	s, registry := newTestServerTailnet(t, noPeerDiagnostics)
+
+	req := httptest.NewRequest(http.MethodGet, "http://127.0.0.1", nil)
+	s.director(uuid.New(), func(*http.Request) {})(req)
+	ctx, cancel := context.WithCancel(req.Context())
+	ds := dialStateFromContext(ctx)
+	require.NotNil(t, ds)
+
+	// The pooled connection failed and the transport retried on a new dial,
+	// which is still waiting on the agent when the request ends.
+	trace := httptrace.ContextClientTrace(ctx)
+	trace.GetConn("127.0.0.1:80")
+	trace.GotConn(httptrace.GotConnInfo{Reused: true})
+	trace.GetConn("127.0.0.1:80")
+	ds.set(dialPhaseAwaitReachable)
+	require.False(t, ds.hadConn())
+	cancel()
+	err := s.reportProxyDialFailure(ctx, uuid.New(), context.Canceled)
+	require.ErrorIs(t, err, context.Canceled)
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 1, "agent_unreachable_total", "client_canceled"))
+}
+
+func TestReportProxyDialFailure_DialTimedOut(t *testing.T) {
+	t.Parallel()
+
+	s, registry := newTestServerTailnet(t, noPeerDiagnostics)
+
+	// The dial hit proxyDialTimeout before the request ended, so the error
+	// already names the unreachable agent and must not be prefixed again.
+	ds := &dialState{}
+	ds.set(dialPhaseAwaitReachable)
+	ctx := context.WithValue(testutil.Context(t, testutil.WaitShort), dialStateKey{}, ds)
+	dialErr := xerrors.Errorf("acquire agent conn: %w", errAgentUnreachable)
+	err := s.reportProxyDialFailure(ctx, uuid.New(), dialErr)
+	require.Same(t, dialErr, err)
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 1, "agent_unreachable_total", "no_node"))
+}
+
+func TestReportProxyDialFailure_CoordinatorError(t *testing.T) {
+	t.Parallel()
+
+	s, registry := newTestServerTailnet(t, noPeerDiagnostics)
+
+	// The request is still waiting and the dial failed before the first
+	// ping, so the agent was never asked.
+	ds := &dialState{}
+	ds.set(dialPhaseAwaitReachable)
+	ctx := context.WithValue(testutil.Context(t, testutil.WaitShort), dialStateKey{}, ds)
+	dialErr := xerrors.New("acquire agent conn: ensure agent: send failed")
+	err := s.reportProxyDialFailure(ctx, uuid.New(), dialErr)
+	require.Same(t, dialErr, err)
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.Empty(t, metrics)
+}
+
+func TestRecordAgentUnreachable_DiagnosticsSlot(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	readStarted := make(chan struct{}, 1)
+	release := make(chan struct{})
+	var reads atomic.Int32
+	s, registry := newTestServerTailnet(t, func(uuid.UUID) tailnet.PeerDiagnostics {
+		reads.Add(1)
+		readStarted <- struct{}{}
+		<-release
+		return tailnet.PeerDiagnostics{ReceivedNode: &tailcfg.Node{}}
+	})
+	clock := quartz.NewMock(t)
+	s.clock = clock
+	trap := clock.Trap().NewTimer("peerDiagnostics")
+	defer trap.Close()
+	var unreachable *workspaceapps.AgentUnreachableError
+
+	errCh := make(chan error, 1)
+	record := func() {
+		go func() {
+			errCh <- s.recordAgentUnreachable(ctx, uuid.New(), time.Second)
+		}()
+		trap.MustWait(ctx).MustRelease(ctx)
+	}
+
+	// The first read waits out the timeout while the read stays in flight.
+	record()
+	testutil.RequireReceive(ctx, t, readStarted)
+	clock.Advance(peerDiagnosticsTimeout).MustWait(ctx)
+	err := testutil.RequireReceive(ctx, t, errCh)
+	require.ErrorAs(t, err, &unreachable)
+	require.Equal(t, "diagnostics_timeout", fieldMap(unreachable.Fields)["reason"])
+
+	// A second failure during that read waits for the slot, times out, and
+	// does not start another read.
+	record()
+	clock.Advance(peerDiagnosticsTimeout).MustWait(ctx)
+	err = testutil.RequireReceive(ctx, t, errCh)
+	require.ErrorAs(t, err, &unreachable)
+	require.Equal(t, "diagnostics_timeout", fieldMap(unreachable.Fields)["reason"])
+	require.EqualValues(t, 1, reads.Load())
+
+	// Once the read returns, the slot frees and later failures read again.
+	// The mock clock does not advance, so this call cannot time out.
+	close(release)
+	record()
+	testutil.RequireReceive(ctx, t, readStarted)
+	err = testutil.RequireReceive(ctx, t, errCh)
+	require.ErrorAs(t, err, &unreachable)
+	got := fieldMap(unreachable.Fields)
+	require.Equal(t, "no_handshake", got["reason"])
+	require.Equal(t, true, got["peer_node_received"])
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 2, "agent_unreachable_total", "diagnostics_timeout"))
+	require.True(t, testutil.PromCounterHasValue(t, metrics, 1, "agent_unreachable_total", "no_handshake"))
+}

coderd/tailnet_test.go

@@ -1,7 +1,9 @@
 package coderd_test
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"net"
@@ -22,10 +24,14 @@ import (
 	"golang.org/x/xerrors"
 	"tailscale.com/tailcfg"
 
+	"cdr.dev/slog/v3"
+	"cdr.dev/slog/v3/sloggers/slogjson"
 	"github.com/coder/coder/v2/agent"
 	"github.com/coder/coder/v2/agent/agenttest"
 	"github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/coderd"
+	"github.com/coder/coder/v2/coderd/httpmw/loggermw"
+	"github.com/coder/coder/v2/coderd/workspaceapps"
 	"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
 	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
@@ -45,11 +51,167 @@ func TestServerTailnet_AgentConn_OK(t *testing.T) {
 	agents, serverTailnet := setupServerTailnetAgent(t, 1)
 	a := agents[0]
 
+	registry := prometheus.NewRegistry()
+	require.NoError(t, registry.Register(serverTailnet))
+
 	conn, release, err := serverTailnet.AgentConn(ctx, a.id)
 	require.NoError(t, err)
 	defer release()
 
 	assert.True(t, conn.AwaitReachable(ctx))
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	assert.EqualValues(t, 1, testutil.PromHistogramSampleCount(t, metrics, "coder_servertailnet_await_reachable_seconds"))
+	// A reachable agent must not be counted under any reason.
+	for _, mf := range metrics {
+		assert.NotEqual(t, "coder_servertailnet_agent_unreachable_total", mf.GetName())
+	}
+}
+
+func TestServerTailnet_AgentConn_Unreachable(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+	defer cancel()
+
+	// No agent is registered with the coordinator, so the ServerTailnet never
+	// receives a node for this ID and the wait ends when the deadline passes.
+	_, serverTailnet := setupServerTailnetAgent(t, 0)
+	agentID := uuid.New()
+
+	registry := prometheus.NewRegistry()
+	require.NoError(t, registry.Register(serverTailnet))
+
+	var logs bytes.Buffer
+	logger := slog.Make(slogjson.Sink(&logs)).Leveled(slog.LevelDebug)
+	requestLogger := loggermw.NewRequestLogger(logger, http.MethodGet, time.Now())
+	ctx = loggermw.WithRequestLogger(ctx, requestLogger)
+
+	dialCtx, dialCancel := context.WithTimeout(ctx, testutil.IntervalSlow)
+	defer dialCancel()
+	_, _, err := serverTailnet.AgentConn(dialCtx, agentID)
+	var unreachable *workspaceapps.AgentUnreachableError
+	require.ErrorAs(t, err, &unreachable)
+
+	requestLogger.WriteLog(ctx, http.StatusBadGateway)
+	var entry struct {
+		Level  string         `json:"level"`
+		Fields map[string]any `json:"fields"`
+	}
+	require.NoError(t, json.Unmarshal(logs.Bytes(), &entry))
+	assert.Equal(t, "WARN", entry.Level)
+	assert.Equal(t, agentID.String(), entry.Fields["agent_id"])
+	assert.Equal(t, false, entry.Fields["peer_node_received"])
+	assert.Contains(t, entry.Fields, "unreachable_after")
+	assert.NotContains(t, entry.Fields, "peer_last_handshake")
+	assert.Contains(t, entry.Fields, "server_preferred_derp")
+	assert.NotContains(t, entry.Fields, "peer_tx_bytes")
+	assert.NotContains(t, entry.Fields, "peer_rx_bytes")
+
+	// A second failure reads diagnostics again, so the first read released
+	// the slot.
+	dialCtx, dialCancel = context.WithTimeout(ctx, testutil.IntervalSlow)
+	defer dialCancel()
+	_, _, err = serverTailnet.AgentConn(dialCtx, uuid.New())
+	require.ErrorAs(t, err, &unreachable)
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	assert.True(t, testutil.PromCounterHasValue(t, metrics, 2, "coder_servertailnet_agent_unreachable_total", "no_node"))
+	assert.EqualValues(t, 0, testutil.PromHistogramSampleCount(t, metrics, "coder_servertailnet_await_reachable_seconds"))
+}
+
+func TestServerTailnet_AgentConn_NotCounted(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name    string
+		dialCtx func(ctx context.Context) (context.Context, context.CancelFunc)
+	}{
+		{
+			// The caller gave up on the dial on purpose, as chatd does when
+			// the workspace has a newer agent.
+			name: "Abandoned",
+			dialCtx: func(ctx context.Context) (context.Context, context.CancelFunc) {
+				dialCtx, dialCancel := context.WithCancelCause(ctx)
+				dialCancel(workspacesdk.ErrDialAbandoned)
+				return dialCtx, func() {}
+			},
+		},
+		{
+			// The caller is polling an agent that may still be booting, as
+			// the chat tools do after they create or start a workspace.
+			name: "ReadinessProbeTimeout",
+			dialCtx: func(ctx context.Context) (context.Context, context.CancelFunc) {
+				return context.WithTimeoutCause(ctx, testutil.IntervalSlow, workspacesdk.ErrReadinessProbeTimeout)
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+			defer cancel()
+
+			_, serverTailnet := setupServerTailnetAgent(t, 0)
+			agentID := uuid.New()
+
+			registry := prometheus.NewRegistry()
+			require.NoError(t, registry.Register(serverTailnet))
+
+			var logs bytes.Buffer
+			logger := slog.Make(slogjson.Sink(&logs)).Leveled(slog.LevelDebug)
+			requestLogger := loggermw.NewRequestLogger(logger, http.MethodGet, time.Now())
+			ctx = loggermw.WithRequestLogger(ctx, requestLogger)
+
+			dialCtx, dialCancel := tt.dialCtx(ctx)
+			defer dialCancel()
+			_, _, err := serverTailnet.AgentConn(dialCtx, agentID)
+			require.Error(t, err)
+			var unreachable *workspaceapps.AgentUnreachableError
+			require.NotErrorAs(t, err, &unreachable)
+
+			requestLogger.WriteLog(ctx, http.StatusBadGateway)
+			var entry struct {
+				Fields map[string]any `json:"fields"`
+			}
+			require.NoError(t, json.Unmarshal(logs.Bytes(), &entry))
+			assert.NotContains(t, entry.Fields, "agent_id")
+			assert.NotContains(t, entry.Fields, "reason")
+
+			metrics, err := registry.Gather()
+			require.NoError(t, err)
+			for _, m := range metrics {
+				assert.NotEqual(t, "coder_servertailnet_agent_unreachable_total", m.GetName())
+			}
+		})
+	}
+}
+
+func TestServerTailnet_DERPConnects(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+	defer cancel()
+
+	agents, serverTailnet := setupServerTailnetAgent(t, 1, tailnettest.DisableSTUN, tailnettest.DERPIsEmbedded)
+	a := agents[0]
+
+	registry := prometheus.NewRegistry()
+	require.NoError(t, registry.Register(serverTailnet))
+
+	conn, release, err := serverTailnet.AgentConn(ctx, a.id)
+	require.NoError(t, err)
+	defer release()
+	require.True(t, conn.AwaitReachable(ctx))
+
+	// One connection to the embedded relay after startup.
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	assert.True(t, testutil.PromCounterHasValue(t, metrics, 1, "coder_servertailnet_derp_connects_total"))
 }
 
 func TestServerTailnet_AgentConn_NoSTUN(t *testing.T) {
@@ -99,6 +261,124 @@ func TestServerTailnet_ReverseProxy_ProxyEnv(t *testing.T) {
 	assert.Equal(t, http.StatusOK, res.StatusCode)
 }
 
+func TestServerTailnet_ReverseProxy_Unreachable(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+	defer cancel()
+
+	// No agent is registered with the coordinator, so the dial never gets a
+	// pong. The request ends first, as when a load balancer gives up on a
+	// hung request, and the transport keeps dialing without it.
+	_, serverTailnet := setupServerTailnetAgent(t, 0)
+	agentID := uuid.New()
+
+	registry := prometheus.NewRegistry()
+	require.NoError(t, registry.Register(serverTailnet))
+
+	var logs bytes.Buffer
+	logger := slog.Make(slogjson.Sink(&logs)).Leveled(slog.LevelDebug)
+	requestLogger := loggermw.NewRequestLogger(logger, http.MethodGet, time.Now())
+
+	u, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", workspacesdk.AgentHTTPAPIServerPort))
+	require.NoError(t, err)
+
+	rp := serverTailnet.ReverseProxy(u, u, agentID, appurl.ApplicationURL{}, "")
+
+	reqCtx, reqCancel := context.WithTimeout(loggermw.WithRequestLogger(ctx, requestLogger), testutil.IntervalSlow)
+	defer reqCancel()
+	rw := httptest.NewRecorder()
+	req := httptest.NewRequest(
+		http.MethodGet,
+		u.String(),
+		nil,
+	).WithContext(reqCtx)
+
+	rp.ServeHTTP(rw, req)
+	res := rw.Result()
+	defer res.Body.Close()
+
+	require.Equal(t, http.StatusBadGateway, res.StatusCode)
+	body, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+	assert.Contains(t, string(body), "agent is unreachable")
+
+	requestLogger.WriteLog(ctx, res.StatusCode)
+	var entry struct {
+		Level  string         `json:"level"`
+		Fields map[string]any `json:"fields"`
+	}
+	require.NoError(t, json.Unmarshal(logs.Bytes(), &entry))
+	assert.Equal(t, "WARN", entry.Level)
+	assert.Equal(t, agentID.String(), entry.Fields["agent_id"])
+	assert.Equal(t, "await_reachable", entry.Fields["dial_phase"])
+	assert.Equal(t, false, entry.Fields["peer_node_received"])
+	assert.Contains(t, entry.Fields, "unreachable_after")
+	assert.NotContains(t, entry.Fields, "peer_last_handshake")
+	assert.Contains(t, entry.Fields, "server_preferred_derp")
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	assert.True(t, testutil.PromCounterHasValue(t, metrics, 1, "coder_servertailnet_agent_unreachable_total", "no_node"))
+}
+
+func TestServerTailnet_ReverseProxy_TCPDialFailure(t *testing.T) {
+	t.Parallel()
+
+	ctx, cancel := context.WithTimeout(context.Background(), testutil.WaitMedium)
+	defer cancel()
+
+	// The agent answers pings, so the tunnel is up, but nothing listens on
+	// the target port. The failure is above WireGuard and must not count as
+	// an unreachable agent.
+	agents, serverTailnet := setupServerTailnetAgent(t, 1)
+	a := agents[0]
+
+	registry := prometheus.NewRegistry()
+	require.NoError(t, registry.Register(serverTailnet))
+
+	var logs bytes.Buffer
+	logger := slog.Make(slogjson.Sink(&logs)).Leveled(slog.LevelDebug)
+	requestLogger := loggermw.NewRequestLogger(logger, http.MethodGet, time.Now())
+
+	u, err := url.Parse(fmt.Sprintf("http://127.0.0.1:%d", testutil.RandomPort(t)))
+	require.NoError(t, err)
+
+	rp := serverTailnet.ReverseProxy(u, u, a.id, appurl.ApplicationURL{}, "")
+
+	rw := httptest.NewRecorder()
+	req := httptest.NewRequest(
+		http.MethodGet,
+		u.String(),
+		nil,
+	).WithContext(loggermw.WithRequestLogger(ctx, requestLogger))
+
+	rp.ServeHTTP(rw, req)
+	res := rw.Result()
+	defer res.Body.Close()
+
+	require.Equal(t, http.StatusBadGateway, res.StatusCode)
+	body, err := io.ReadAll(res.Body)
+	require.NoError(t, err)
+	assert.NotContains(t, string(body), "agent is unreachable")
+
+	requestLogger.WriteLog(ctx, res.StatusCode)
+	var entry struct {
+		Fields map[string]any `json:"fields"`
+	}
+	require.NoError(t, json.Unmarshal(logs.Bytes(), &entry))
+	assert.Equal(t, a.id.String(), entry.Fields["agent_id"])
+	assert.Equal(t, "tcp_dial", entry.Fields["dial_phase"])
+	assert.Contains(t, entry.Fields, "tcp_dial_after")
+	assert.NotContains(t, entry.Fields, "unreachable_after")
+
+	metrics, err := registry.Gather()
+	require.NoError(t, err)
+	for _, m := range metrics {
+		assert.NotEqual(t, "coder_servertailnet_agent_unreachable_total", m.GetName())
+	}
+}
+
 func TestServerTailnet_ReverseProxy(t *testing.T) {
 	t.Parallel()
 

coderd/workspaceapps/db.go

@@ -14,6 +14,7 @@ import (
 
 	"github.com/go-jose/go-jose/v4/jwt"
 	"github.com/google/uuid"
+	"github.com/prometheus/client_golang/prometheus"
 	"golang.org/x/xerrors"
 
 	"cdr.dev/slog/v3"
@@ -47,6 +48,13 @@ type DBTokenProvider struct {
 	WorkspaceAgentInactiveTimeout   time.Duration
 	WorkspaceAppAuditSessionTimeout time.Duration
 	Keycache                        cryptokeys.SigningKeycache
+
+	// agentOfflineTotal counts requests that got the agent offline page. Unlike
+	// coderd_agents_connections, which samples agent status once a minute, it
+	// records every failed request, so it can alert on the offline page now
+	// that the page is a 404 and not a 502. Open tabs on workspaces with a
+	// disconnected agent produce a steady baseline, so alert on rate changes.
+	agentOfflineTotal *prometheus.CounterVec
 }
 
 var _ SignedTokenProvider = &DBTokenProvider{}
@@ -62,6 +70,7 @@ func NewDBTokenProvider(ctx context.Context,
 	workspaceAgentInactiveTimeout time.Duration,
 	workspaceAppAuditSessionTimeout time.Duration,
 	signer cryptokeys.SigningKeycache,
+	reg prometheus.Registerer,
 ) SignedTokenProvider {
 	if workspaceAgentInactiveTimeout == 0 {
 		workspaceAgentInactiveTimeout = 1 * time.Minute
@@ -70,6 +79,14 @@ func NewDBTokenProvider(ctx context.Context,
 		workspaceAppAuditSessionTimeout = time.Hour
 	}
 
+	agentOfflineTotal := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Namespace: "coder",
+		Subsystem: "workspaceapps",
+		Name:      "agent_offline_total",
+		Help:      "Number of workspace app requests that got the offline page because the agent was not connected.",
+	}, []string{"status", "access_method"})
+	reg.MustRegister(agentOfflineTotal)
+
 	return &DBTokenProvider{
 		Logger:                          log,
 		ctx:                             ctx,
@@ -82,6 +99,7 @@ func NewDBTokenProvider(ctx context.Context,
 		WorkspaceAgentInactiveTimeout:   workspaceAgentInactiveTimeout,
 		WorkspaceAppAuditSessionTimeout: workspaceAppAuditSessionTimeout,
 		Keycache:                        signer,
+		agentOfflineTotal:               agentOfflineTotal,
 	}
 }
 
@@ -233,6 +251,7 @@ func (p *DBTokenProvider) Issue(ctx context.Context, rw http.ResponseWriter, r *
 	// Check that the agent is online.
 	agentStatus := dbReq.Agent.Status(dbtime.Now(), p.WorkspaceAgentInactiveTimeout)
 	if agentStatus.Status != database.WorkspaceAgentStatusConnected {
+		p.agentOfflineTotal.WithLabelValues(string(agentStatus.Status), string(appReq.AccessMethod)).Inc()
 		WriteWorkspaceAppOffline(p.Logger, p.DashboardURL, rw, r, &appReq, fmt.Sprintf("Agent state is %q, not %q", agentStatus.Status, database.WorkspaceAgentStatusConnected))
 		return nil, "", false
 	}

coderd/workspaceapps/db_test.go

@@ -1062,6 +1062,14 @@ func Test_ResolveRequest(t *testing.T) {
 		// It'll either be "connecting" or "disconnected". Both are OK for this
 		// test.
 		require.Contains(t, bodyStr, `Agent state is "`)
+
+		metrics, err := api.PrometheusRegistry.Gather()
+		require.NoError(t, err)
+		counted := func(status string) bool {
+			return testutil.PromCounterAssertion(t, metrics, func(v float64) bool { return v >= 1 },
+				"coder_workspaceapps_agent_offline_total", "path", status)
+		}
+		require.True(t, counted("connecting") || counted("disconnected"), "offline page was not counted")
 	})
 
 	// Initializing apps are now permitted to connect anyways. This wasn't

coderd/workspaceapps/proxy.go

@@ -2,6 +2,7 @@ package workspaceapps
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -81,6 +82,20 @@ type AgentProvider interface {
 	Close() error
 }
 
+// AgentUnreachableError is returned by AgentProvider.AgentConn when the agent
+// did not answer before the context ended. It is not returned when the client
+// canceled within 5s, or when the context was canceled with
+// workspacesdk.ErrDialAbandoned or workspacesdk.ErrReadinessProbeTimeout.
+// Fields holds the peer diagnostics collected at that time. Callers with a
+// request log line that log the failure should include them.
+type AgentUnreachableError struct {
+	Fields []slog.Field
+}
+
+func (*AgentUnreachableError) Error() string {
+	return "agent is unreachable"
+}
+
 type ServerOptions struct {
 	Logger slog.Logger
 
@@ -802,7 +817,13 @@ func (s *Server) workspaceAgentPTY(rw http.ResponseWriter, r *http.Request) {
 
 	agentConn, release, err := s.AgentProvider.AgentConn(ctx, appToken.AgentID)
 	if err != nil {
-		log.Debug(ctx, "dial workspace agent", slog.Error(err))
+		// A 101 response logs the request at debug, so log unreachable agents
+		// at warn here. Use s.Logger: Fields already has agent_id.
+		if unreachable, ok := errors.AsType[*AgentUnreachableError](err); ok {
+			s.Logger.Warn(ctx, "agent is unreachable", append(unreachable.Fields, slog.Error(err))...)
+		} else {
+			log.Debug(ctx, "dial workspace agent", slog.Error(err))
+		}
 		_ = conn.Close(websocket.StatusInternalError, httpapi.WebsocketCloseSprintf("dial workspace agent: %s", err))
 		return
 	}

coderd/workspaceapps/proxy_internal_test.go

@@ -1,11 +1,29 @@
 package workspaceapps
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/http/httputil"
 	"net/url"
 	"strings"
 	"testing"
 
+	"github.com/go-chi/chi/v5"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/require"
+
+	"cdr.dev/slog/v3"
+	"cdr.dev/slog/v3/sloggers/slogjson"
+	"github.com/coder/coder/v2/coderd/httpapi"
+	"github.com/coder/coder/v2/coderd/tracing"
+	"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
+	"github.com/coder/coder/v2/codersdk/workspacesdk"
+	"github.com/coder/coder/v2/testutil"
+	"github.com/coder/quartz"
+	"github.com/coder/websocket"
 )
 
 // Test_originLocalURL checks that originLocalURL produces a redirect target that
@@ -115,3 +133,100 @@ func Test_originLocalURL(t *testing.T) {
 		}
 	})
 }
+
+// fakeTokenProvider returns the same token for every request.
+type fakeTokenProvider struct {
+	token *SignedToken
+}
+
+func (p fakeTokenProvider) FromRequest(*http.Request) (*SignedToken, bool) {
+	return p.token, true
+}
+
+func (fakeTokenProvider) Issue(context.Context, http.ResponseWriter, *http.Request, IssueTokenRequest) (*SignedToken, string, bool) {
+	panic("unexpected Issue call")
+}
+
+// unreachableAgentProvider fails every AgentConn with AgentUnreachableError.
+type unreachableAgentProvider struct {
+	fields []slog.Field
+}
+
+func (unreachableAgentProvider) ReverseProxy(*url.URL, *url.URL, uuid.UUID, appurl.ApplicationURL, string) *httputil.ReverseProxy {
+	panic("unexpected ReverseProxy call")
+}
+
+func (p unreachableAgentProvider) AgentConn(context.Context, uuid.UUID) (workspacesdk.AgentConn, func(), error) {
+	return nil, nil, &AgentUnreachableError{Fields: p.fields}
+}
+
+func (unreachableAgentProvider) ServeHTTPDebug(http.ResponseWriter, *http.Request) {}
+
+func (unreachableAgentProvider) Close() error { return nil }
+
+func TestWorkspaceAgentPTY_AgentUnreachable(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+
+	logs := testutil.NewWaitBuffer()
+	logger := slog.Make(slogjson.Sink(logs)).Leveled(slog.LevelDebug)
+
+	agentID := uuid.New()
+	basePath := fmt.Sprintf("/api/v2/workspaceagents/%s/pty", agentID)
+	accessURL, err := url.Parse("http://coder.test")
+	require.NoError(t, err)
+
+	s := NewServer(ServerOptions{
+		Logger:       logger,
+		DashboardURL: accessURL,
+		AccessURL:    accessURL,
+		SignedTokenProvider: fakeTokenProvider{token: &SignedToken{
+			Request: Request{
+				AccessMethod:  AccessMethodTerminal,
+				BasePath:      basePath,
+				AgentNameOrID: agentID.String(),
+			},
+			AgentID: agentID,
+		}},
+		AgentProvider: unreachableAgentProvider{fields: []slog.Field{
+			slog.F("agent_id", agentID),
+			slog.F("reason", "no_node"),
+		}},
+		WSWatcher: httpapi.NewWSWatcher(quartz.NewReal(), nil),
+	})
+	r := chi.NewRouter()
+	r.Use(tracing.StatusWriterMiddleware, tracing.Middleware(nil, tracing.DefaultRoutePatterns, "coderd"))
+	r.Get("/api/v2/workspaceagents/{workspaceagent}/pty", s.workspaceAgentPTY)
+	srv := httptest.NewServer(r)
+	t.Cleanup(srv.Close)
+
+	wsURL := "ws" + strings.TrimPrefix(srv.URL, "http") + basePath + "?reconnect=" + uuid.NewString()
+	conn, _, err := websocket.Dial(ctx, wsURL, nil) //nolint:bodyclose // The response body is nil after a successful upgrade.
+	require.NoError(t, err)
+	defer conn.Close(websocket.StatusNormalClosure, "")
+
+	// The handler closes the socket once the dial fails.
+	_, _, err = conn.Read(ctx)
+	var closeErr websocket.CloseError
+	require.ErrorAs(t, err, &closeErr)
+	require.Equal(t, websocket.StatusInternalError, closeErr.Code)
+	require.Contains(t, closeErr.Reason, "agent is unreachable")
+
+	var found bool
+	for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") {
+		var entry struct {
+			Level  string         `json:"level"`
+			Msg    string         `json:"msg"`
+			Fields map[string]any `json:"fields"`
+		}
+		require.NoError(t, json.Unmarshal([]byte(line), &entry), line)
+		if entry.Msg != "agent is unreachable" {
+			continue
+		}
+		found = true
+		require.Equal(t, "WARN", entry.Level)
+		require.Equal(t, agentID.String(), entry.Fields["agent_id"])
+		require.Equal(t, "no_node", entry.Fields["reason"])
+	}
+	require.True(t, found, "no agent is unreachable log line")
+}

coderd/x/chatd/chattool/createworkspace.go

@@ -635,9 +635,17 @@ func waitForAgentReady(
 		ticker := time.NewTicker(agentRetryInterval)
 		defer ticker.Stop()
 
+		// Dials time out while a new agent boots, so they are not counted as
+		// unreachable. An agent that connected before should answer, so its
+		// failures are counted.
+		var attemptCause error
+		if !agent.FirstConnectedAt.Valid {
+			attemptCause = workspacesdk.ErrReadinessProbeTimeout
+		}
+
 		var lastErr error
 		for {
-			attemptCtx, attemptCancel := context.WithTimeout(agentCtx, agentAttemptTimeout)
+			attemptCtx, attemptCancel := context.WithTimeoutCause(agentCtx, agentAttemptTimeout, attemptCause)
 			_, release, err := agentConnFn(attemptCtx, agentID)
 			attemptCancel()
 			if err == nil {

coderd/x/chatd/chattool/createworkspace_internal_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"database/sql"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 	"sync"
@@ -173,6 +174,43 @@ func TestWaitForAgentReady(t *testing.T) {
 		require.NotContains(t, result, "agent_error")
 	})
 
+	t.Run("AttemptTimeoutCause", func(t *testing.T) {
+		t.Parallel()
+
+		for _, tc := range []struct {
+			name             string
+			firstConnectedAt sql.NullTime
+			wantProbeTimeout bool
+		}{
+			{name: "NewAgent", wantProbeTimeout: true},
+			{name: "ConnectedBefore", firstConnectedAt: sql.NullTime{Time: time.Now(), Valid: true}},
+		} {
+			t.Run(tc.name, func(t *testing.T) {
+				t.Parallel()
+				ctrl := gomock.NewController(t)
+				db := newCreateWorkspaceMockStore(ctrl)
+				agentID := uuid.New()
+
+				db.EXPECT().
+					GetWorkspaceAgentLifecycleStateByID(gomock.Any(), agentID).
+					Return(database.GetWorkspaceAgentLifecycleStateByIDRow{
+						LifecycleState: database.WorkspaceAgentLifecycleStateReady,
+					}, nil)
+
+				var cause error
+				connFn := func(ctx context.Context, _ uuid.UUID) (workspacesdk.AgentConn, func(), error) {
+					<-ctx.Done()
+					cause = context.Cause(ctx)
+					return nil, func() {}, nil
+				}
+
+				agent := database.WorkspaceAgent{ID: agentID, FirstConnectedAt: tc.firstConnectedAt}
+				waitForAgentReady(context.Background(), db, agent, connFn)
+				require.Equal(t, tc.wantProbeTimeout, errors.Is(cause, workspacesdk.ErrReadinessProbeTimeout), "cause: %v", cause)
+			})
+		}
+	})
+
 	t.Run("AgentConnectsButStartupFails", func(t *testing.T) {
 		t.Parallel()
 		ctrl := gomock.NewController(t)

coderd/x/chatd/dialvalidation.go

@@ -61,7 +61,10 @@ func dialWithLazyValidation(
 		return xerrors.Errorf("dial with lazy validation: %w", err)
 	}
 
-	dialCtx, dialCancel := context.WithCancel(ctx)
+	// A parent that ends first sets its own cause, so this cause only marks
+	// dials this function gave up on.
+	dialCtx, dialCancelCause := context.WithCancelCause(ctx)
+	dialCancel := func() { dialCancelCause(workspacesdk.ErrDialAbandoned) }
 	results := make(chan dialOut, 1)
 	go func() {
 		conn, release, err := dialFn(dialCtx, agentID)

coderd/x/chatd/dialvalidation_internal_test.go

@@ -121,6 +121,7 @@ func TestDialWithLazyValidation_SlowDialNoCurrentAgent(t *testing.T) {
 	workspaceID := uuid.New()
 	dialStarted := make(chan struct{})
 	resultCh := make(chan error, 1)
+	staleCause := make(chan error, 1)
 
 	var dialCalls atomic.Int32
 	var validateCalls atomic.Int32
@@ -138,6 +139,7 @@ func TestDialWithLazyValidation_SlowDialNoCurrentAgent(t *testing.T) {
 				dialCalls.Add(1)
 				close(dialStarted)
 				<-ctx.Done()
+				staleCause <- context.Cause(ctx)
 				return nil, nil, ctx.Err()
 			},
 			func(_ context.Context, id uuid.UUID) (uuid.UUID, error) {
@@ -159,6 +161,12 @@ func TestDialWithLazyValidation_SlowDialNoCurrentAgent(t *testing.T) {
 	case <-time.After(testutil.WaitShort):
 		t.Fatal("dialWithLazyValidation blocked after validation reported no current agent")
 	}
+	select {
+	case cause := <-staleCause:
+		require.ErrorIs(t, cause, workspacesdk.ErrDialAbandoned)
+	case <-time.After(testutil.WaitShort):
+		t.Fatal("stale dial was not canceled")
+	}
 
 	require.EqualValues(t, 1, dialCalls.Load())
 	require.EqualValues(t, 1, validateCalls.Load())
@@ -182,6 +190,7 @@ func TestDialWithLazyValidation_SlowDialStaleAgent(t *testing.T) {
 		var validateCalls atomic.Int32
 		var staleReleaseCalls atomic.Int32
 		var currentReleaseCalls atomic.Int32
+		staleCause := make(chan error, 1)
 
 		result, err := dialWithLazyValidation(
 			context.Background(),
@@ -193,6 +202,7 @@ func TestDialWithLazyValidation_SlowDialStaleAgent(t *testing.T) {
 				switch id {
 				case staleAgentID:
 					<-ctx.Done()
+					staleCause <- context.Cause(ctx)
 					return staleConn, func() {
 						staleReleaseCalls.Add(1)
 					}, nil
@@ -224,6 +234,7 @@ func TestDialWithLazyValidation_SlowDialStaleAgent(t *testing.T) {
 		require.Eventually(t, func() bool {
 			return staleReleaseCalls.Load() == 1
 		}, testutil.WaitShort, testutil.IntervalFast)
+		require.ErrorIs(t, <-staleCause, workspacesdk.ErrDialAbandoned)
 
 		if result.Release != nil {
 			result.Release()

codersdk/workspacesdk/workspacesdk.go

@@ -28,6 +28,17 @@ import (
 
 var ErrSkipClose = xerrors.New("skip tailnet close")
 
+// ErrDialAbandoned is a cancel cause for a dial whose result the caller no
+// longer wants, such as a dial to an agent that a rebuild replaced. A dial
+// canceled with it is not counted as an unreachable agent.
+var ErrDialAbandoned = xerrors.New("dial abandoned")
+
+// ErrReadinessProbeTimeout is a timeout cause for one attempt in a loop that
+// waits for a starting agent to come online. The agent is expected to be
+// missing until it boots, so a dial that times out with it is not counted as
+// an unreachable agent.
+var ErrReadinessProbeTimeout = xerrors.New("agent readiness probe timed out")
+
 const (
 	AgentSSHPort             = tailnet.WorkspaceAgentSSHPort
 	AgentStandardSSHPort     = tailnet.WorkspaceAgentStandardSSHPort

docs/admin/integrations/prometheus.md

@@ -188,8 +188,12 @@ The `coder_ai_gateway_cost_control_*` metrics are exported only by `coderd`.
 | `coder_pubsub_send_duration_seconds`                                     | histogram | The time taken to send a message into a pubsub event channel                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | `backend`                                                                                             |
 | `coder_pubsub_send_latency_seconds`                                      | gauge     | The most recent time taken to send a message into a pubsub event channel. New integrations should prefer the coder_pubsub_send_duration_seconds histogram.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                             | `backend`                                                                                             |
 | `coder_pubsub_subscribes_total`                                          | counter   | Total number of calls to Subscribe/SubscribeWithErr                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | `backend` `success`                                                                                   |
+| `coder_servertailnet_agent_unreachable_total`                            | counter   | Number of connection attempts where the workspace agent did not answer in time, by reason: no_node (this server has no node for the agent), peer_lost (the agent's or this server's coordinator connection ended), no_handshake (no WireGuard handshake yet), handshake_stale (last handshake over 180s ago), handshake_ok (last handshake under 180s ago), diagnostics_timeout (peer state not readable within 1s), client_canceled (client left within 5s).                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | `reason`                                                                                              |
+| `coder_servertailnet_await_reachable_seconds`                            | histogram | Time until a workspace agent answered the first ping when a new connection was opened.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                 |                                                                                                       |
 | `coder_servertailnet_connections_total`                                  | counter   | Total number of TCP connections made to workspace agents.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | `network`                                                                                             |
+| `coder_servertailnet_derp_connects_total`                                | counter   | Number of times the server tailnet connected to the embedded DERP relay. Stays 0 when the embedded relay is disabled.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  |                                                                                                       |
 | `coder_servertailnet_open_connections`                                   | gauge     | Total number of TCP connections currently open to workspace agents.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                    | `network`                                                                                             |
+| `coder_workspaceapps_agent_offline_total`                                | counter   | Number of workspace app requests that got the offline page because the agent was not connected.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | `access_method` `status`                                                                              |
 | `coderd_agentapi_metadata_batch_size`                                    | histogram | Total number of metadata entries in each batch, updated before flushes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                |                                                                                                       |
 | `coderd_agentapi_metadata_batch_utilization`                             | histogram | Number of metadata keys per agent in each batch, updated before flushes.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               |                                                                                                       |
 | `coderd_agentapi_metadata_batches_total`                                 | counter   | Total number of metadata batches flushed.                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              | `reason`                                                                                              |

scripts/metricsdocgen/generated_metrics

@@ -211,12 +211,24 @@ coder_pubsub_send_latency_seconds{backend=""} 0
 # HELP coder_pubsub_subscribes_total Total number of calls to Subscribe/SubscribeWithErr
 # TYPE coder_pubsub_subscribes_total counter
 coder_pubsub_subscribes_total{backend="",success=""} 0
+# HELP coder_servertailnet_agent_unreachable_total Number of connection attempts where the workspace agent did not answer in time, by reason: no_node (this server has no node for the agent), peer_lost (the agent's or this server's coordinator connection ended), no_handshake (no WireGuard handshake yet), handshake_stale (last handshake over 180s ago), handshake_ok (last handshake under 180s ago), diagnostics_timeout (peer state not readable within 1s), client_canceled (client left within 5s).
+# TYPE coder_servertailnet_agent_unreachable_total counter
+coder_servertailnet_agent_unreachable_total{reason=""} 0
+# HELP coder_servertailnet_await_reachable_seconds Time until a workspace agent answered the first ping when a new connection was opened.
+# TYPE coder_servertailnet_await_reachable_seconds histogram
+coder_servertailnet_await_reachable_seconds 0
 # HELP coder_servertailnet_connections_total Total number of TCP connections made to workspace agents.
 # TYPE coder_servertailnet_connections_total counter
 coder_servertailnet_connections_total{network=""} 0
+# HELP coder_servertailnet_derp_connects_total Number of times the server tailnet connected to the embedded DERP relay. Stays 0 when the embedded relay is disabled.
+# TYPE coder_servertailnet_derp_connects_total counter
+coder_servertailnet_derp_connects_total 0
 # HELP coder_servertailnet_open_connections Total number of TCP connections currently open to workspace agents.
 # TYPE coder_servertailnet_open_connections gauge
 coder_servertailnet_open_connections{network=""} 0
+# HELP coder_workspaceapps_agent_offline_total Number of workspace app requests that got the offline page because the agent was not connected.
+# TYPE coder_workspaceapps_agent_offline_total counter
+coder_workspaceapps_agent_offline_total{status="",access_method=""} 0
 # HELP coderd_agentapi_metadata_batch_size Total number of metadata entries in each batch, updated before flushes.
 # TYPE coderd_agentapi_metadata_batch_size histogram
 coderd_agentapi_metadata_batch_size 0

tailnet/configmaps.go

@@ -656,11 +656,18 @@ func (c *configMaps) fillPeerDiagnostics(d *PeerDiagnostics, peerID uuid.UUID) {
 	}
 
 	d.ReceivedNode = lc.node
+	d.Lost = lc.lost
 	ps, ok := status.Peer[lc.node.Key]
 	if !ok {
 		return
 	}
-	d.LastWireguardHandshake = ps.LastHandshake
+	// WireGuard reports a peer with no handshake as the Unix epoch. Use the
+	// zero time so callers can check IsZero.
+	if ps.LastHandshake.Unix() > 0 {
+		d.LastWireguardHandshake = ps.LastHandshake
+	}
+	d.TxBytes = ps.TxBytes
+	d.RxBytes = ps.RxBytes
 }
 
 func (c *configMaps) knownPeerIDs() []uuid.UUID {

tailnet/configmaps_internal_test.go

@@ -1072,6 +1072,7 @@ func TestConfigMaps_fillPeerDiagnostics(t *testing.T) {
 		peerID:        p1ID,
 		node:          p1tcn,
 		lastHandshake: hst,
+		lost:          true,
 	}
 	uut.L.Unlock()
 
@@ -1085,7 +1086,55 @@ func TestConfigMaps_fillPeerDiagnostics(t *testing.T) {
 	// Then:
 	require.Equal(t, map[int]string{1: "AUH", 1001: "DXB"}, d.DERPRegionNames)
 	require.Equal(t, p1tcn, d.ReceivedNode)
+	require.True(t, d.Lost)
 	require.Equal(t, hst, d.LastWireguardHandshake)
+	require.EqualValues(t, 1234, d.TxBytes)
+	require.EqualValues(t, 5678, d.RxBytes)
+
+	done := make(chan struct{})
+	go func() {
+		defer close(done)
+		uut.close()
+	}()
+	_ = testutil.TryReceive(ctx, t, done)
+}
+
+func TestConfigMaps_fillPeerDiagnostics_NoHandshake(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logger := testutil.Logger(t)
+	fEng := newFakeEngineConfigurable()
+	nodePrivateKey := key.NewNode()
+	nodeID := tailcfg.NodeID(5)
+	discoKey := key.NewDisco()
+	uut := newConfigMaps(logger, fEng, nodeID, nodePrivateKey, discoKey.Public(), CoderDNSSuffixFQDN)
+	defer uut.close()
+
+	p1ID := uuid.UUID{1}
+	p1Node := newTestNode(1)
+	p1n, err := NodeToProto(p1Node)
+	require.NoError(t, err)
+	p1tcn, err := uut.protoNodeToTailcfg(p1n)
+	require.NoError(t, err)
+
+	uut.L.Lock()
+	uut.peers[p1ID] = &peerLifecycle{
+		peerID: p1ID,
+		node:   p1tcn,
+	}
+	uut.L.Unlock()
+
+	// WireGuard reports a peer that never completed a handshake as the epoch.
+	s0 := expectStatusWithHandshake(ctx, t, fEng, p1Node.Key, time.Unix(0, 0))
+
+	d := PeerDiagnostics{DERPRegionNames: make(map[int]string)}
+	uut.fillPeerDiagnostics(&d, p1ID)
+	testutil.TryReceive(ctx, t, s0)
+
+	require.Equal(t, p1tcn, d.ReceivedNode)
+	require.False(t, d.Lost)
+	require.True(t, d.LastWireguardHandshake.IsZero())
+	require.EqualValues(t, 1234, d.TxBytes)
 
 	done := make(chan struct{})
 	go func() {
@@ -1109,6 +1158,8 @@ func expectStatusWithHandshake(
 			b.AddPeer(k, &ipnstate.PeerStatus{
 				PublicKey:     k,
 				LastHandshake: lastHandshake,
+				TxBytes:       1234,
+				RxBytes:       5678,
 				Active:        true,
 			})
 			select {

tailnet/conn.go

@@ -964,8 +964,18 @@ type PeerDiagnostics struct {
 	SentNode bool
 	// ReceivedNode is the last Node we received for the peer, or nil if we haven't received the node.
 	ReceivedNode *tailcfg.Node
+	// Lost is true when the coordinator reported the peer lost or our own
+	// coordination stream ended. The node and WireGuard config stay until
+	// lostTimeout passes with no handshake, so the other fields may still
+	// describe a working tunnel.
+	Lost bool
 	// LastWireguardHandshake is the last time we completed a wireguard handshake
 	LastWireguardHandshake time.Time
+	// TxBytes and RxBytes are the WireGuard byte counters for the peer,
+	// cumulative since the peer was added to the device. Both are zero when
+	// the engine has no status for the peer.
+	TxBytes int64
+	RxBytes int64
 	// TODO: surface Discovery (disco) protocol problems
 }
 

vpn/tunnel.go

@@ -529,7 +529,12 @@ func (u *updater) convertAgentsLocked(agents []*agentWithPing) []*Agent {
 		}
 		if u.conn != nil {
 			diags := u.conn.GetPeerDiagnostics(agent.ID)
-			protoAgent.LastHandshake = timestamppb.New(diags.LastWireguardHandshake)
+			// Clients read an unset timestamp as no handshake yet. The zero
+			// time would encode as year 1, which the Windows client shows as
+			// connecting.
+			if !diags.LastWireguardHandshake.IsZero() {
+				protoAgent.LastHandshake = timestamppb.New(diags.LastWireguardHandshake)
+			}
 		}
 		out = append(out, protoAgent)
 	}

vpn/tunnel_internal_test.go

@@ -482,6 +482,27 @@ func TestUpdater_createPeerUpdate(t *testing.T) {
 	})
 }
 
+func TestUpdater_createPeerUpdate_NoHandshake(t *testing.T) {
+	t.Parallel()
+
+	ctx := testutil.Context(t, testutil.WaitShort)
+	updater := updater{
+		ctx:         ctx,
+		netLoopDone: make(chan struct{}),
+		agents:      map[uuid.UUID]agentWithPing{},
+		workspaces:  map[uuid.UUID]tailnet.Workspace{},
+		conn:        newFakeConn(tailnet.WorkspaceUpdate{}, time.Time{}),
+	}
+
+	update := updater.createPeerUpdateLocked(tailnet.WorkspaceUpdate{
+		UpsertedAgents: []*tailnet.Agent{
+			{ID: uuid.UUID{4}, Name: "w1a1", WorkspaceID: uuid.UUID{1}},
+		},
+	})
+	require.Len(t, update.UpsertedAgents, 1)
+	require.Nil(t, update.UpsertedAgents[0].LastHandshake)
+}
+
 func TestTunnel_sendAgentUpdate(t *testing.T) {
 	t.Parallel()