opensource/coder
3
0
Fork 0
mirror of https://github.com/coder/coder.git synced 2026-09-24 15:04:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: agent disconnects from coordinator (#7430)

Browse Source 
* work around websocket deadline bug

Signed-off-by: Spike Curtis <spike@coder.com>

* Use test context to hold websocket open

Signed-off-by: Spike Curtis <spike@coder.com>

* Fix race creating test websocket

Signed-off-by: Spike Curtis <spike@coder.com>

* set write deadline to time.Time zero

Signed-off-by: Spike Curtis <spike@coder.com>

---------

Signed-off-by: Spike Curtis <spike@coder.com>
This commit is contained in:
Spike Curtis
2023-05-05 20:29:03 +04:00
committed by GitHub
parent 5ffa6dae50
commit dc3d39baf8
2 changed files with 67 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tailnet/coordinator.go
+18 -1
View File
@@ -206,6 +206,10 @@ func (t *TrackedConn) Close() error {
return t.conn.Close()
}
// WriteTimeout is the amount of time we wait to write a node update to a connection before we declare it hung.
// It is exported so that tests can use it.
const WriteTimeout = time.Second * 5
// SendUpdates reads node updates and writes them to the connection. Ends when writes hit an error or context is
// canceled.
func (t *TrackedConn) SendUpdates() {
@@ -223,7 +227,7 @@ func (t *TrackedConn) SendUpdates() {
// Set a deadline so that hung connections don't put back pressure on the system.
// Node updates are tiny, so even the dinkiest connection can handle them if it's not hung.
err = t.conn.SetWriteDeadline(time.Now().Add(5 * time.Second))
err = t.conn.SetWriteDeadline(time.Now().Add(WriteTimeout))
if err != nil {
// often, this is just because the connection is closed/broken, so only log at debug.
t.logger.Debug(t.ctx, "unable to set write deadline", slog.Error(err))
@@ -238,6 +242,19 @@ func (t *TrackedConn) SendUpdates() {
return
}
t.logger.Debug(t.ctx, "wrote nodes", slog.F("nodes", nodes))
// nhooyr.io/websocket has a bugged implementation of deadlines on a websocket net.Conn. What they are
// *supposed* to do is set a deadline for any subsequent writes to complete, otherwise the call to Write()
// fails. What nhooyr.io/websocket does is set a timer, after which it expires the websocket write context.
// If this timer fires, then the next write will fail *even if we set a new write deadline*. So, after
// our successful write, it is important that we reset the deadline before it fires.
err = t.conn.SetWriteDeadline(time.Time{})
if err != nil {
// often, this is just because the connection is closed/broken, so only log at debug.
t.logger.Debug(t.ctx, "unable to extend write deadline", slog.Error(err))
_ = t.Close()
return
}
}
}
}