Deflake TestMonitorStaleLocks (#26882)

This PR deflakes `TestMonitorStaleLocks` because it ends up waiting for
the retry time to pass and timeout.

This PR enforces the loop continue by adding the test `StaleC` channel
into the wait select.

Fixes #26866

lib/services/watcher.go

@@ -242,6 +242,11 @@ func (p *resourceWatcher) runWatchLoop() {
 		case <-p.ctx.Done():
 			p.Log.Debug("Closed, returning from watch loop.")
 			return
+		case <-p.StaleC:
+			// Used for testing that the watch routine is waiting for the
+			// next restart attempt. We don't want to wait for the full
+			// retry period in tests so we trigger the restart immediately.
+			p.Log.Debug("Stale view, continue watch loop.")
 		}
 		if err != nil {
 			p.Log.Warningf("Restart watch on error: %v.", err)

lib/srv/monitor_test.go

@@ -205,9 +205,6 @@ func TestMonitorStaleLocks(t *testing.T) {
 	case <-time.After(15 * time.Second):
 		t.Fatal("Timeout waiting for LockWatcher loop check.")
 	}
-	// StaleC is listened by multiple goroutines, so we need to close to ensure
-	// that all of them are unblocked.
-	close(asrv.LockWatcher.StaleC)
 
 	// ensure ResetC is drained
 	select {
@@ -222,6 +219,12 @@ func TestMonitorStaleLocks(t *testing.T) {
 	case <-time.After(15 * time.Second):
 		t.Fatal("Timeout waiting for LockWatcher reset.")
 	}
+	// StaleC is listened by multiple goroutines, so we need to close to ensure
+	// that all of them are unblocked and the stale state is detected.
+	close(asrv.LockWatcher.StaleC)
+	require.Eventually(t, func() bool {
+		return asrv.LockWatcher.IsStale()
+	}, 15*time.Second, 100*time.Millisecond, "Timeout waiting for LockWatcher to be stale.")
 	select {
 	case <-conn.closedC:
 	case <-time.After(15 * time.Second):