mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: structured disconnect attribution for agent logs (#25191)
Implements [PLAT-60](https://linear.app/codercom/issue/PLAT-60/enhance-disconnect-logs-with-structured-reason-attribution): adds structured disconnect attribution to disconnect logs throughout the agent and tailnet packages. Every disconnect log site now carries structured slog fields. All existing logs remain; existing messages are preserved with the fields added alongside. New fields on disconnect log lines: - `connect_type` — which layer disconnected: `server_to_agent`, `agent_to_client`, or `client_to_server` - `disconnect_reason` — categorical reason: `graceful`, `network_error`, `server_shutdown`, etc. - `disconnect_expected` — whether the disconnect is normal operation (`true`) or should be investigated (`false`) - `disconnect_initiator` — who started it: `client`, `agent`, `server`, or `network` (control-plane sites only) - `disconnect_detail` — free-form supplemental info (where useful) ## What's covered **Control plane (`server_to_agent`):** coordination RPC, DERP map subscriber, agent runLoop, agent Close, `BasicCoordination.Close`, `Controller.run`. **Data plane (`agent_to_client`):** SSH sessions, reconnecting PTY, JetBrains port-forwarding. <details> <summary>Control-plane sites</summary> | Site | Reason | Initiator | |---|---|---| | `agent/agent.go` `runLoop` EOF | `network_error` | `network` | | `agent/agent.go` `runCoordinator` deferred exit | `server_shutdown` / `graceful` / `network_error` | `agent` / `server` / `network` | | `agent/agent.go` `runDERPMapSubscriber` deferred exit | same (shared `classifyCoordinatorRPCExit`) | same | | `agent/agent.go` `Close` shutdown timeout | `server_shutdown` + detail | `agent` | | `agent/agent.go` `Close` clean coord disconnect | `server_shutdown` | `agent` | | `tailnet/controllers.go` `BasicCoordination.Close` | `graceful` or `network_error` | `c.initiator` | | `tailnet/controllers.go` `Controller.run` `net.ErrClosed` | `network_error` | `network` | </details> <details> <summary>Data-plane sites</summary> | Site | Reason | Notes | |---|---|---| | `agent/agentssh/agentssh.go` SSH session closed | free-form (`graceful`, `process exited with error status: N`, etc.) | Also sets `closeCause("normal exit")` for clean exits so coderd's `connection_log.DisconnectReason` is no longer empty | | `agent/reconnectingpty/server.go` PTY closed | `server_shutdown`, error string, or `graceful` | | | `agent/agentssh/jetbrainstrack.go` channel closed | `normal close` or error string | Previously passed empty reason | </details> <details> <summary>Bug fix</summary> The deferred `disconnected from coordination RPC` log no longer fires when the initial `Coordinate()` RPC call fails before any connection is established. </details> Refs PLAT-60. --- _This PR was prepared by Coder Agents on behalf of @Emyrk._ **Manually QA'd a lot of common disconnects** --------- Co-authored-by: Coder Agents <noreply@coder.com>
This commit is contained in:
co-authored by
Coder Agents
parent
c9c933a300
commit
1afc6d4fd0
+57
-5
@@ -154,6 +154,17 @@ type BasicCoordinationController struct {
|
||||
Logger slog.Logger
|
||||
Coordinatee Coordinatee
|
||||
SendAcks bool
|
||||
// Initiator labels which side of the coordination this controller
|
||||
// represents (e.g. "agent", "client", "server"). It is attached as
|
||||
// a slog field to disconnect-related logs emitted by this
|
||||
// coordination so operators can attribute connection-close events
|
||||
// without inspecting call sites. Leave unset on synthetic
|
||||
// controllers (tests, in-memory fakes) and the field will be
|
||||
// omitted from log output.
|
||||
Initiator codersdk.DisconnectInitiator
|
||||
// Direction labels the connection layer (server_to_agent,
|
||||
// agent_to_client, client_to_server) for disconnect logs.
|
||||
Direction codersdk.ConnectionDirection
|
||||
}
|
||||
|
||||
// New satisfies the method on the CoordinationController interface
|
||||
@@ -170,6 +181,8 @@ func (c *BasicCoordinationController) NewCoordination(client CoordinatorClient)
|
||||
client: client,
|
||||
respLoopDone: make(chan struct{}),
|
||||
sendAcks: c.SendAcks,
|
||||
initiator: c.Initiator,
|
||||
direction: c.Direction,
|
||||
}
|
||||
|
||||
c.Coordinatee.SetNodeCallback(func(node *Node) {
|
||||
@@ -211,6 +224,8 @@ type BasicCoordination struct {
|
||||
client CoordinatorClient
|
||||
respLoopDone chan struct{}
|
||||
sendAcks bool
|
||||
initiator codersdk.DisconnectInitiator
|
||||
direction codersdk.ConnectionDirection
|
||||
}
|
||||
|
||||
// CloseClient forcibly closes the underlying coordinator client connection
|
||||
@@ -243,11 +258,24 @@ func (c *BasicCoordination) Close(ctx context.Context) (retErr error) {
|
||||
err := c.client.Send(&proto.CoordinateRequest{Disconnect: &proto.CoordinateRequest_Disconnect{}})
|
||||
c.Unlock()
|
||||
if err != nil && !xerrors.Is(err, io.EOF) {
|
||||
reason := codersdk.DisconnectReasonNetworkError
|
||||
// Log but don't return early; we must still clean up below.
|
||||
c.logger.Warn(context.Background(), "failed to send disconnect", slog.Error(err))
|
||||
c.logger.Warn(context.Background(), "failed to send disconnect",
|
||||
c.direction.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
reason.SlogField(),
|
||||
c.initiator.SlogField(),
|
||||
slog.Error(err),
|
||||
)
|
||||
retErr = xerrors.Errorf("send disconnect: %w", err)
|
||||
} else {
|
||||
c.logger.Debug(context.Background(), "sent disconnect")
|
||||
reason := codersdk.DisconnectReasonGraceful
|
||||
c.logger.Debug(context.Background(), "sent disconnect",
|
||||
c.direction.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
reason.SlogField(),
|
||||
c.initiator.SlogField(),
|
||||
)
|
||||
}
|
||||
|
||||
// We shouldn't just close the protocol right away, because the way dRPC streams work is
|
||||
@@ -256,10 +284,23 @@ func (c *BasicCoordination) Close(ctx context.Context) (retErr error) {
|
||||
// Disconnect message, so we should wait around for that until the context expires.
|
||||
select {
|
||||
case <-c.respLoopDone:
|
||||
c.logger.Debug(ctx, "responses closed after disconnect")
|
||||
reason := codersdk.DisconnectReasonGraceful
|
||||
c.logger.Debug(ctx, "responses closed after disconnect",
|
||||
c.direction.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
reason.SlogField(),
|
||||
c.initiator.SlogField(),
|
||||
)
|
||||
return retErr
|
||||
case <-ctx.Done():
|
||||
c.logger.Warn(ctx, "context expired while waiting for coordinate responses to close")
|
||||
reason := codersdk.DisconnectReasonNetworkError
|
||||
c.logger.Warn(ctx, "context expired while waiting for coordinate responses to close",
|
||||
c.direction.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
reason.SlogField(),
|
||||
c.initiator.SlogField(),
|
||||
codersdk.SlogDisconnectDetail("context expired before coordinator hung up"),
|
||||
)
|
||||
}
|
||||
// forcefully close the stream
|
||||
protoErr := c.client.Close()
|
||||
@@ -369,6 +410,8 @@ func NewTunnelSrcCoordController(
|
||||
Logger: logger,
|
||||
Coordinatee: coordinatee,
|
||||
SendAcks: false,
|
||||
Initiator: codersdk.DisconnectInitiatorClient,
|
||||
Direction: codersdk.ConnectionDirectionClientToServer,
|
||||
},
|
||||
dests: make(map[uuid.UUID]struct{}),
|
||||
}
|
||||
@@ -508,6 +551,8 @@ func NewAgentCoordinationController(
|
||||
Logger: logger,
|
||||
Coordinatee: coordinatee,
|
||||
SendAcks: true,
|
||||
Initiator: codersdk.DisconnectInitiatorAgent,
|
||||
Direction: codersdk.ConnectionDirectionServerToAgent,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1505,7 +1550,14 @@ func (c *Controller) Run(ctx context.Context) {
|
||||
}
|
||||
|
||||
if errors.Is(err, net.ErrClosed) {
|
||||
c.logger.Warn(c.ctx, "control plane connection closed, retrying", slog.Error(err))
|
||||
reason := codersdk.DisconnectReasonNetworkError
|
||||
c.logger.Warn(c.ctx, "control plane connection closed, retrying",
|
||||
codersdk.ConnectionDirectionServerToAgent.SlogField(),
|
||||
reason.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
codersdk.DisconnectInitiatorNetwork.SlogField(),
|
||||
slog.Error(err),
|
||||
)
|
||||
continue
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user