fix: stop holding Pubsub mutex while calling pq.Listener (#12518)

fixes #11950

https://github.com/coder/coder/issues/11950#issuecomment-1987756088 explains the bug

We were also calling into `Unlisten()` and `Close()` while holding the mutex.  I don't believe that `Close()` depends on the notification loop being unblocked, but it's hard to be sure, and the safest thing to do is assume it could block.

So, I added a unit test that fakes out `pq.Listener` and sends a bunch of notifies every time we call into it to hopefully prevent regression where we hold the mutex while calling into these functions.

It also removes the use of a `context.Context` to stop the PubSub -- it must be explicitly `Closed()`.  This simplifies a bunch of the logic, and is how we use the pubsub anyway.

coderd/database/pubsub/pubsub_internal_test.go

@@ -3,10 +3,15 @@ package pubsub
 import (
 	"context"
 	"fmt"
+	"sync"
 	"testing"
 
+	"github.com/lib/pq"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"cdr.dev/slog"
+	"cdr.dev/slog/sloggers/slogtest"
 	"github.com/coder/coder/v2/testutil"
 )
 
@@ -138,3 +143,115 @@ func Test_msgQueue_Full(t *testing.T) {
 	// for the error, so we read 2 less than we sent.
 	require.Equal(t, BufferSize, n)
 }
+
+func TestPubSub_DoesntBlockNotify(t *testing.T) {
+	t.Parallel()
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logger := slogtest.Make(t, nil).Leveled(slog.LevelDebug)
+
+	uut := newWithoutListener(logger, nil)
+	fListener := newFakePqListener()
+	uut.pgListener = fListener
+	go uut.listen()
+
+	cancels := make(chan func())
+	go func() {
+		subCancel, err := uut.Subscribe("bagels", func(ctx context.Context, message []byte) {
+			t.Logf("got message: %s", string(message))
+		})
+		assert.NoError(t, err)
+		cancels <- subCancel
+	}()
+	subCancel := testutil.RequireRecvCtx(ctx, t, cancels)
+	cancelDone := make(chan struct{})
+	go func() {
+		defer close(cancelDone)
+		subCancel()
+	}()
+	testutil.RequireRecvCtx(ctx, t, cancelDone)
+
+	closeErrs := make(chan error)
+	go func() {
+		closeErrs <- uut.Close()
+	}()
+	err := testutil.RequireRecvCtx(ctx, t, closeErrs)
+	require.NoError(t, err)
+}
+
+const (
+	numNotifications = 5
+	testMessage      = "birds of a feather"
+)
+
+// fakePqListener is a fake version of pq.Listener.  This test code tests for regressions of
+// https://github.com/coder/coder/issues/11950 where pq.Listener deadlocked because we blocked the
+// PGPubsub.listen() goroutine while calling other pq.Listener functions.  So, all function calls
+// into the fakePqListener will send 5 notifications before returning to ensure the listen()
+// goroutine is unblocked.
+type fakePqListener struct {
+	mu       sync.Mutex
+	channels map[string]struct{}
+	notify   chan *pq.Notification
+}
+
+func (f *fakePqListener) Close() error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	ch := f.getTestChanLocked()
+	for i := 0; i < numNotifications; i++ {
+		f.notify <- &pq.Notification{Channel: ch, Extra: testMessage}
+	}
+	// note that the realPqListener must only be closed once, so go ahead and
+	// close the notify unprotected here.  If it panics, we have a bug.
+	close(f.notify)
+	return nil
+}
+
+func (f *fakePqListener) Listen(s string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	ch := f.getTestChanLocked()
+	for i := 0; i < numNotifications; i++ {
+		f.notify <- &pq.Notification{Channel: ch, Extra: testMessage}
+	}
+	if _, ok := f.channels[s]; ok {
+		return pq.ErrChannelAlreadyOpen
+	}
+	f.channels[s] = struct{}{}
+	return nil
+}
+
+func (f *fakePqListener) Unlisten(s string) error {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	ch := f.getTestChanLocked()
+	for i := 0; i < numNotifications; i++ {
+		f.notify <- &pq.Notification{Channel: ch, Extra: testMessage}
+	}
+	if _, ok := f.channels[s]; ok {
+		delete(f.channels, s)
+		return nil
+	}
+	return pq.ErrChannelNotOpen
+}
+
+func (f *fakePqListener) NotifyChan() <-chan *pq.Notification {
+	return f.notify
+}
+
+// getTestChanLocked returns the name of a channel we are currently listening for, if there is one.
+// Otherwise, it just returns "test".  We prefer to send test notifications for channels that appear
+// in the tests, but if there are none, just return anything.
+func (f *fakePqListener) getTestChanLocked() string {
+	for c := range f.channels {
+		return c
+	}
+	return "test"
+}
+
+func newFakePqListener() *fakePqListener {
+	return &fakePqListener{
+		channels: make(map[string]struct{}),
+		notify:   make(chan *pq.Notification),
+	}
+}