fix: reduce excessive logging when database is unreachable (#17363)

Fixes #17045

---------

Signed-off-by: Danny Kopping <dannykopping@gmail.com>

coderd/coderd.go

@@ -675,10 +675,11 @@ func New(options *Options) *API {
 	api.Auditor.Store(&options.Auditor)
 	api.TailnetCoordinator.Store(&options.TailnetCoordinator)
 	dialer := &InmemTailnetDialer{
-		CoordPtr: &api.TailnetCoordinator,
-		DERPFn:   api.DERPMap,
-		Logger:   options.Logger,
-		ClientID: uuid.New(),
+		CoordPtr:            &api.TailnetCoordinator,
+		DERPFn:              api.DERPMap,
+		Logger:              options.Logger,
+		ClientID:            uuid.New(),
+		DatabaseHealthCheck: api.Database,
 	}
 	stn, err := NewServerTailnet(api.ctx,
 		options.Logger,

coderd/tailnet.go

@@ -24,9 +24,11 @@ import (
 	"tailscale.com/tailcfg"
 
 	"cdr.dev/slog"
+
 	"github.com/coder/coder/v2/coderd/tracing"
 	"github.com/coder/coder/v2/coderd/workspaceapps"
 	"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
+	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/workspacesdk"
 	"github.com/coder/coder/v2/site"
 	"github.com/coder/coder/v2/tailnet"
@@ -534,6 +536,10 @@ func NewMultiAgentController(ctx context.Context, logger slog.Logger, tracer tra
 	return m
 }
 
+type Pinger interface {
+	Ping(context.Context) (time.Duration, error)
+}
+
 // InmemTailnetDialer is a tailnet.ControlProtocolDialer that connects to a Coordinator and DERPMap
 // service running in the same memory space.
 type InmemTailnetDialer struct {
@@ -541,9 +547,17 @@ type InmemTailnetDialer struct {
 	DERPFn   func() *tailcfg.DERPMap
 	Logger   slog.Logger
 	ClientID uuid.UUID
+	// DatabaseHealthCheck is used to validate that the store is reachable.
+	DatabaseHealthCheck Pinger
 }
 
-func (a *InmemTailnetDialer) Dial(_ context.Context, _ tailnet.ResumeTokenController) (tailnet.ControlProtocolClients, error) {
+func (a *InmemTailnetDialer) Dial(ctx context.Context, _ tailnet.ResumeTokenController) (tailnet.ControlProtocolClients, error) {
+	if a.DatabaseHealthCheck != nil {
+		if _, err := a.DatabaseHealthCheck.Ping(ctx); err != nil {
+			return tailnet.ControlProtocolClients{}, xerrors.Errorf("%w: %v", codersdk.ErrDatabaseNotReachable, err)
+		}
+	}
+
 	coord := a.CoordPtr.Load()
 	if coord == nil {
 		return tailnet.ControlProtocolClients{}, xerrors.Errorf("tailnet coordinator not initialized")

coderd/tailnet_test.go

@@ -11,6 +11,7 @@ import (
 	"strconv"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/google/uuid"
 	"github.com/prometheus/client_golang/prometheus"
@@ -18,6 +19,7 @@ import (
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel/trace"
+	"golang.org/x/xerrors"
 	"tailscale.com/tailcfg"
 
 	"github.com/coder/coder/v2/agent"
@@ -25,6 +27,7 @@ import (
 	"github.com/coder/coder/v2/agent/proto"
 	"github.com/coder/coder/v2/coderd"
 	"github.com/coder/coder/v2/coderd/workspaceapps/appurl"
+	"github.com/coder/coder/v2/codersdk"
 	"github.com/coder/coder/v2/codersdk/agentsdk"
 	"github.com/coder/coder/v2/codersdk/workspacesdk"
 	"github.com/coder/coder/v2/tailnet"
@@ -365,6 +368,44 @@ func TestServerTailnet_ReverseProxy(t *testing.T) {
 	})
 }
 
+func TestDialFailure(t *testing.T) {
+	t.Parallel()
+
+	// Setup.
+	ctx := testutil.Context(t, testutil.WaitShort)
+	logger := testutil.Logger(t)
+
+	// Given: a tailnet coordinator.
+	coord := tailnet.NewCoordinator(logger)
+	t.Cleanup(func() {
+		_ = coord.Close()
+	})
+	coordPtr := atomic.Pointer[tailnet.Coordinator]{}
+	coordPtr.Store(&coord)
+
+	// Given: a fake DB healthchecker which will always fail.
+	fch := &failingHealthcheck{}
+
+	// When: dialing the in-memory coordinator.
+	dialer := &coderd.InmemTailnetDialer{
+		CoordPtr:            &coordPtr,
+		Logger:              logger,
+		ClientID:            uuid.UUID{5},
+		DatabaseHealthCheck: fch,
+	}
+	_, err := dialer.Dial(ctx, nil)
+
+	// Then: the error returned reflects the database has failed its healthcheck.
+	require.ErrorIs(t, err, codersdk.ErrDatabaseNotReachable)
+}
+
+type failingHealthcheck struct{}
+
+func (failingHealthcheck) Ping(context.Context) (time.Duration, error) {
+	// Simulate a database connection error.
+	return 0, xerrors.New("oops")
+}
+
 type wrappedListener struct {
 	net.Listener
 	dials int32

coderd/workspaceagents.go

@@ -997,6 +997,16 @@ func (api *API) derpMapUpdates(rw http.ResponseWriter, r *http.Request) {
 func (api *API) workspaceAgentClientCoordinate(rw http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
 
+	// Ensure the database is reachable before proceeding.
+	_, err := api.Database.Ping(ctx)
+	if err != nil {
+		httpapi.Write(ctx, rw, http.StatusInternalServerError, codersdk.Response{
+			Message: codersdk.DatabaseNotReachable,
+			Detail:  err.Error(),
+		})
+		return
+	}
+
 	// This route accepts user API key auth and workspace proxy auth. The moon actor has
 	// full permissions so should be able to pass this authz check.
 	workspace := httpmw.WorkspaceParam(r)