mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: structured disconnect attribution for agent logs (#25191)
Implements [PLAT-60](https://linear.app/codercom/issue/PLAT-60/enhance-disconnect-logs-with-structured-reason-attribution): adds structured disconnect attribution to disconnect logs throughout the agent and tailnet packages. Every disconnect log site now carries structured slog fields. All existing logs remain; existing messages are preserved with the fields added alongside. New fields on disconnect log lines: - `connect_type` — which layer disconnected: `server_to_agent`, `agent_to_client`, or `client_to_server` - `disconnect_reason` — categorical reason: `graceful`, `network_error`, `server_shutdown`, etc. - `disconnect_expected` — whether the disconnect is normal operation (`true`) or should be investigated (`false`) - `disconnect_initiator` — who started it: `client`, `agent`, `server`, or `network` (control-plane sites only) - `disconnect_detail` — free-form supplemental info (where useful) ## What's covered **Control plane (`server_to_agent`):** coordination RPC, DERP map subscriber, agent runLoop, agent Close, `BasicCoordination.Close`, `Controller.run`. **Data plane (`agent_to_client`):** SSH sessions, reconnecting PTY, JetBrains port-forwarding. <details> <summary>Control-plane sites</summary> | Site | Reason | Initiator | |---|---|---| | `agent/agent.go` `runLoop` EOF | `network_error` | `network` | | `agent/agent.go` `runCoordinator` deferred exit | `server_shutdown` / `graceful` / `network_error` | `agent` / `server` / `network` | | `agent/agent.go` `runDERPMapSubscriber` deferred exit | same (shared `classifyCoordinatorRPCExit`) | same | | `agent/agent.go` `Close` shutdown timeout | `server_shutdown` + detail | `agent` | | `agent/agent.go` `Close` clean coord disconnect | `server_shutdown` | `agent` | | `tailnet/controllers.go` `BasicCoordination.Close` | `graceful` or `network_error` | `c.initiator` | | `tailnet/controllers.go` `Controller.run` `net.ErrClosed` | `network_error` | `network` | </details> <details> <summary>Data-plane sites</summary> | Site | Reason | Notes | |---|---|---| | `agent/agentssh/agentssh.go` SSH session closed | free-form (`graceful`, `process exited with error status: N`, etc.) | Also sets `closeCause("normal exit")` for clean exits so coderd's `connection_log.DisconnectReason` is no longer empty | | `agent/reconnectingpty/server.go` PTY closed | `server_shutdown`, error string, or `graceful` | | | `agent/agentssh/jetbrainstrack.go` channel closed | `normal close` or error string | Previously passed empty reason | </details> <details> <summary>Bug fix</summary> The deferred `disconnected from coordination RPC` log no longer fires when the initial `Coordinate()` RPC call fails before any connection is established. </details> Refs PLAT-60. --- _This PR was prepared by Coder Agents on behalf of @Emyrk._ **Manually QA'd a lot of common disconnects** --------- Co-authored-by: Coder Agents <noreply@coder.com>
This commit is contained in:
co-authored by
Coder Agents
parent
c9c933a300
commit
1afc6d4fd0
+58
-7
@@ -514,7 +514,12 @@ func (a *agent) runLoop() {
|
||||
return
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
a.logger.Info(ctx, "disconnected from coderd")
|
||||
a.logger.Info(ctx, "disconnected from coderd",
|
||||
codersdk.ConnectionDirectionServerToAgent.SlogField(),
|
||||
codersdk.DisconnectReasonNetworkError.SlogField(),
|
||||
codersdk.DisconnectReasonNetworkError.SlogExpectedField(),
|
||||
codersdk.DisconnectInitiatorNetwork.SlogField(),
|
||||
)
|
||||
continue
|
||||
}
|
||||
a.logger.Warn(ctx, "run exited with error", slog.Error(err))
|
||||
@@ -1878,16 +1883,43 @@ func (a *agent) createTailnet(
|
||||
return network, nil
|
||||
}
|
||||
|
||||
// classifyCoordinatorRPCExit determines the DisconnectReason and
|
||||
// DisconnectInitiator for a coordinator-style RPC (the coordination RPC
|
||||
// and the DERP map subscriber RPC) that has just returned. A canceled
|
||||
// local context means the agent itself is shutting down. A non-nil
|
||||
// return error without context cancellation means the stream broke
|
||||
// unexpectedly.
|
||||
func classifyCoordinatorRPCExit(ctx context.Context, retErr error) (codersdk.DisconnectReason, codersdk.DisconnectInitiator) {
|
||||
localShutdown := ctx.Err() != nil
|
||||
switch {
|
||||
case localShutdown:
|
||||
return codersdk.DisconnectReasonServerShutdown, codersdk.DisconnectInitiatorAgent
|
||||
case retErr == nil:
|
||||
return codersdk.DisconnectReasonGraceful, codersdk.DisconnectInitiatorServer
|
||||
default:
|
||||
return codersdk.DisconnectReasonNetworkError, codersdk.DisconnectInitiatorNetwork
|
||||
}
|
||||
}
|
||||
|
||||
// runCoordinator runs a coordinator and returns whether a reconnect
|
||||
// should occur.
|
||||
func (a *agent) runCoordinator(ctx context.Context, tClient tailnetproto.DRPCTailnetClient24, network *tailnet.Conn) error {
|
||||
defer a.logger.Debug(ctx, "disconnected from coordination RPC")
|
||||
func (a *agent) runCoordinator(ctx context.Context, tClient tailnetproto.DRPCTailnetClient24, network *tailnet.Conn) (retErr error) {
|
||||
// we run the RPC on the hardCtx so that we have a chance to send the disconnect message if we
|
||||
// gracefully shut down.
|
||||
coordinate, err := tClient.Coordinate(a.hardCtx)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("failed to connect to the coordinate endpoint: %w", err)
|
||||
}
|
||||
defer func() {
|
||||
reason, initiator := classifyCoordinatorRPCExit(ctx, retErr)
|
||||
a.logger.Debug(ctx, "disconnected from coordination RPC",
|
||||
codersdk.ConnectionDirectionServerToAgent.SlogField(),
|
||||
reason.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
initiator.SlogField(),
|
||||
slog.Error(retErr),
|
||||
)
|
||||
}()
|
||||
defer func() {
|
||||
cErr := coordinate.Close()
|
||||
if cErr != nil {
|
||||
@@ -1935,8 +1967,7 @@ func (a *agent) setCoordDisconnected() chan struct{} {
|
||||
}
|
||||
|
||||
// runDERPMapSubscriber runs a coordinator and returns if a reconnect should occur.
|
||||
func (a *agent) runDERPMapSubscriber(ctx context.Context, tClient tailnetproto.DRPCTailnetClient24, network *tailnet.Conn) error {
|
||||
defer a.logger.Debug(ctx, "disconnected from derp map RPC")
|
||||
func (a *agent) runDERPMapSubscriber(ctx context.Context, tClient tailnetproto.DRPCTailnetClient24, network *tailnet.Conn) (retErr error) {
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
defer cancel()
|
||||
stream, err := tClient.StreamDERPMaps(ctx, &tailnetproto.StreamDERPMapsRequest{})
|
||||
@@ -1948,6 +1979,15 @@ func (a *agent) runDERPMapSubscriber(ctx context.Context, tClient tailnetproto.D
|
||||
if cErr != nil {
|
||||
a.logger.Debug(ctx, "error closing DERPMap stream", slog.Error(err))
|
||||
}
|
||||
|
||||
reason, initiator := classifyCoordinatorRPCExit(ctx, retErr)
|
||||
a.logger.Debug(ctx, "disconnected from derp map RPC",
|
||||
codersdk.ConnectionDirectionServerToAgent.SlogField(),
|
||||
reason.SlogField(),
|
||||
reason.SlogExpectedField(),
|
||||
initiator.SlogField(),
|
||||
slog.Error(retErr),
|
||||
)
|
||||
}()
|
||||
a.logger.Info(ctx, "connected to derp map RPC")
|
||||
for {
|
||||
@@ -2260,9 +2300,20 @@ lifecycleWaitLoop:
|
||||
// Wait for graceful disconnect from the Coordinator RPC
|
||||
select {
|
||||
case <-a.hardCtx.Done():
|
||||
a.logger.Warn(context.Background(), "timed out waiting for Coordinator RPC disconnect")
|
||||
a.logger.Warn(context.Background(), "timed out waiting for Coordinator RPC disconnect",
|
||||
codersdk.ConnectionDirectionServerToAgent.SlogField(),
|
||||
codersdk.DisconnectReasonServerShutdown.SlogField(),
|
||||
codersdk.DisconnectReasonServerShutdown.SlogExpectedField(),
|
||||
codersdk.DisconnectInitiatorAgent.SlogField(),
|
||||
codersdk.SlogDisconnectDetail("timed out waiting for coordinator RPC to disconnect"),
|
||||
)
|
||||
case <-coordDisconnected:
|
||||
a.logger.Debug(context.Background(), "coordinator RPC disconnected")
|
||||
a.logger.Debug(context.Background(), "coordinator RPC disconnected",
|
||||
codersdk.ConnectionDirectionServerToAgent.SlogField(),
|
||||
codersdk.DisconnectReasonServerShutdown.SlogField(),
|
||||
codersdk.DisconnectReasonServerShutdown.SlogExpectedField(),
|
||||
codersdk.DisconnectInitiatorAgent.SlogField(),
|
||||
)
|
||||
}
|
||||
|
||||
// Wait for logs to be sent
|
||||
|
||||
Reference in New Issue
Block a user