test: add testutil.WaitBuffer and replace time.Sleep in tests (#22922)

WaitBuffer is a thread-safe io.Writer that supports blocking until
accumulated output matches a substring or custom predicate. It
replaces ad-hoc safeBuffer/syncWriter types and time.Sleep-based
poll loops in tests with signal-driven waits.

- WaitFor/WaitForNth/WaitForCond for blocking on output
- Replace custom buffer types in cli/sync_test.go and
  provisionersdk/agent_test.go
- Convert time.Sleep poll loops to require.Eventually/require.Never
  in cli/ssh_test.go, coderd/activitybump_test.go,
  coderd/workspaceagentsrpc_test.go, workspaceproxy_test.go, and
  scaletest tests

cli/ssh_test.go

@@ -180,15 +180,11 @@ func TestSSH(t *testing.T) {
 
 		// Delay until workspace is starting, otherwise the agent may be
 		// booted due to outdated build.
-		var err error
-		for {
+		require.Eventually(t, func() bool {
+			var err error
 			workspace, err = client.Workspace(ctx, workspace.ID)
-			require.NoError(t, err)
-			if workspace.LatestBuild.Transition == codersdk.WorkspaceTransitionStart {
-				break
-			}
-			time.Sleep(testutil.IntervalFast)
-		}
+			return err == nil && workspace.LatestBuild.Transition == codersdk.WorkspaceTransitionStart
+		}, testutil.WaitShort, testutil.IntervalFast)
 
 		// When the agent connects, the workspace was started, and we should
 		// have access to the shell.
@@ -763,15 +759,11 @@ func TestSSH(t *testing.T) {
 
 		// Delay until workspace is starting, otherwise the agent may be
 		// booted due to outdated build.
-		var err error
-		for {
+		require.Eventually(t, func() bool {
+			var err error
 			workspace, err = client.Workspace(ctx, workspace.ID)
-			require.NoError(t, err)
-			if workspace.LatestBuild.Transition == codersdk.WorkspaceTransitionStart {
-				break
-			}
-			time.Sleep(testutil.IntervalFast)
-		}
+			return err == nil && workspace.LatestBuild.Transition == codersdk.WorkspaceTransitionStart
+		}, testutil.WaitShort, testutil.IntervalFast)
 
 		// When the agent connects, the workspace was started, and we should
 		// have access to the shell.

cli/sync_test.go

@@ -6,8 +6,6 @@ import (
 	"os"
 	"path/filepath"
 	"runtime"
-	"strings"
-	"sync"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -104,20 +102,11 @@ func TestSyncCommands_Golden(t *testing.T) {
 		require.NoError(t, err)
 		client.Close()
 
-		// Use a writer that signals when the "Waiting" message has been
-		// written, so the goroutine can complete the dependency at the
-		// right time without relying on time.Sleep.
-		outBuf := newSyncWriter("Waiting")
-
-		// Start a goroutine to complete the dependency once the start
-		// command has printed its waiting message.
+		outBuf := testutil.NewWaitBuffer()
 		done := make(chan error, 1)
 		go func() {
-			// Block until the command prints the waiting message.
-			select {
-			case <-outBuf.matched:
-			case <-ctx.Done():
-				done <- ctx.Err()
+			if err := outBuf.WaitFor(ctx, "Waiting"); err != nil {
+				done <- err
 				return
 			}
 
@@ -339,36 +328,3 @@ func TestSyncCommands_Golden(t *testing.T) {
 		clitest.TestGoldenFile(t, "TestSyncCommands_Golden/status_json_format", outBuf.Bytes(), nil)
 	})
 }
-
-// syncWriter is a thread-safe io.Writer that wraps a bytes.Buffer and
-// closes a channel when the written content contains a signal string.
-type syncWriter struct {
-	mu        sync.Mutex
-	buf       bytes.Buffer
-	signal    string
-	matched   chan struct{}
-	closeOnce sync.Once
-}
-
-func newSyncWriter(signal string) *syncWriter {
-	return &syncWriter{
-		signal:  signal,
-		matched: make(chan struct{}),
-	}
-}
-
-func (w *syncWriter) Write(p []byte) (int, error) {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	n, err := w.buf.Write(p)
-	if w.signal != "" && strings.Contains(w.buf.String(), w.signal) {
-		w.closeOnce.Do(func() { close(w.matched) })
-	}
-	return n, err
-}
-
-func (w *syncWriter) Bytes() []byte {
-	w.mu.Lock()
-	defer w.mu.Unlock()
-	return w.buf.Bytes()
-}