mirror of
https://github.com/coder/coder.git
synced 2026-10-16 14:48:56 +08:00
Bobby Ho
dffa890d8f
Add telemetry to diagnose intermittent macOS `test-go-pg` stalls and preserve timeout diagnostics. - Log runner details at startup; sample load, swap, and `vm_stat` every second and `top` every 10 seconds to identify runner-wide pauses. - Add stall detection to `cli` and `agent` `TestMain`, logging heap, GC, and goroutine counts when ticks arrive more than 5 seconds late. - Run artifact uploads with `always() && !success()` so canceled test steps do not skip diagnostics. - Add a `TEST_TIMEOUT` override: use 20m on macOS to leave room for goroutine dumps within the 30m job, retaining the 25m default for the nightly gauntlet. Validated with race tests, binary compilation, `actionlint`, local macOS checks, and Makefile timeout checks.
58 lines
1.4 KiB
Go
58 lines
1.4 KiB
Go
package testutil
|
|
|
|
import (
|
|
"fmt"
|
|
"os"
|
|
"runtime"
|
|
"runtime/metrics"
|
|
"time"
|
|
)
|
|
|
|
// StartStallDetector reports when the test process stops running for
|
|
// longer than threshold. It checks a one second ticker and writes a line
|
|
// to stderr when a tick arrives late, so the report shows up in the test
|
|
// output next to the goroutine dump on a timeout. It stays quiet when the
|
|
// process runs normally. Call the returned function to stop it.
|
|
func StartStallDetector(threshold time.Duration) func() {
|
|
done := make(chan struct{})
|
|
stopped := make(chan struct{})
|
|
go func() {
|
|
defer close(stopped)
|
|
ticker := time.NewTicker(time.Second)
|
|
defer ticker.Stop()
|
|
last := time.Now()
|
|
for {
|
|
select {
|
|
case <-done:
|
|
return
|
|
case <-ticker.C:
|
|
}
|
|
now := time.Now()
|
|
gap := now.Sub(last)
|
|
last = now
|
|
if gap < threshold {
|
|
continue
|
|
}
|
|
samples := []metrics.Sample{
|
|
{Name: "/memory/classes/heap/objects:bytes"},
|
|
{Name: "/gc/cycles/total:gc-cycles"},
|
|
{Name: "/sched/goroutines:goroutines"},
|
|
}
|
|
metrics.Read(samples)
|
|
_, _ = fmt.Fprintf(os.Stderr,
|
|
"stall detector: %s passed between 1s ticks, ending at %s: heap=%dMiB gc_cycles=%d goroutines=%d gomaxprocs=%d\n",
|
|
gap.Round(time.Millisecond),
|
|
now.UTC().Format(time.RFC3339Nano),
|
|
samples[0].Value.Uint64()>>20,
|
|
samples[1].Value.Uint64(),
|
|
samples[2].Value.Uint64(),
|
|
runtime.GOMAXPROCS(0),
|
|
)
|
|
}
|
|
}()
|
|
return func() {
|
|
close(done)
|
|
<-stopped
|
|
}
|
|
}
|