From 2fa3dd65bc0a800a8a199a5805c31463a2f5b32c Mon Sep 17 00:00:00 2001
From: Waleed <walif6@gmail.com>
Date: Fri, 26 Jun 2026 12:21:11 -0700
Subject: [PATCH] fix(db): retry the migration connection on transient slot
 exhaustion (#5226)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* ci(migrations): skip db:migrate on merges that change no migration files

Every push to main/staging ran db:migrate against the production/staging
database even when the merge changed no schema, so a no-op migration would dial
the DB and fail whenever it was at its connection limit (53300, slots reserved
for SUPERUSER) — red-X'ing UI-only merges.

Add a detect-migrations job (dorny/paths-filter on packages/db/migrations/**)
and pass the result into the reusable migrations workflow, which now skips the
apply step when no migration files changed. The migrate job still runs so
downstream build/deploy jobs that need it are never skipped, and the flag
defaults to 'true' so manual dispatch and any unknown value always apply
migrations — the gate only ever skips a provably-empty change.

* fix(db): retry the migration connection on transient slot exhaustion

The migration opens its session on the first query (the advisory-lock
acquire). When the deploy database briefly exhausts every non-superuser
connection slot at peak, that connect fails with 53300 ("remaining connection
slots are reserved for roles with the SUPERUSER attribute") and the whole
deploy's migrate step errors out — even when the spike clears within seconds.

Add a bounded connectWithRetry() before acquiring the lock that retries 53300,
the 08xxx connection_exception class, and the driver's transport errors with
backoff (10 attempts, ~90s ceiling). Non-transient errors (auth, bad config)
still fail fast. The migration is a single short-lived session, so waiting out
a transient spike is far safer than failing the deploy.

* ci: drop the migration paths-filter gate (out of scope)

Revert the detect-migrations gate carried over from the closed CI PR; we are
fixing the connection failure at its source (migrate.ts connection retry)
rather than gating db:migrate, which the reviewers correctly noted could leave
a previously-merged migration unapplied after a failed deploy.
---
 packages/db/scripts/migrate.ts | 57 ++++++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)

diff --git a/packages/db/scripts/migrate.ts b/packages/db/scripts/migrate.ts
index 6e9846ec02..08a14c22e2 100644
--- a/packages/db/scripts/migrate.ts
+++ b/packages/db/scripts/migrate.ts
@@ -71,10 +71,42 @@ const DDL_LOCK_TIMEOUT = '5s'
 const MAX_MIGRATE_ATTEMPTS = 8
 const MIGRATE_RETRY_BACKOFF = { baseMs: 2_000, maxMs: 30_000 } as const
 
+const CONNECT_MAX_ATTEMPTS = 10
+const CONNECT_RETRY_BACKOFF = { baseMs: 1_000, maxMs: 15_000 } as const
+
+/**
+ * Error codes that mean the database was momentarily unreachable rather than
+ * the migration being wrong: chiefly `53300` (too_many_connections — every
+ * non-superuser slot was taken, surfaced as "remaining connection slots are
+ * reserved for roles with the SUPERUSER attribute"), the `08xxx`
+ * connection_exception class, and the postgres-js driver's own transport
+ * codes. These are retried while opening the session; anything else is fatal.
+ */
+const TRANSIENT_CONNECT_CODES = new Set([
+  '53300',
+  '53400',
+  'CONNECT_TIMEOUT',
+  'CONNECTION_CLOSED',
+  'CONNECTION_DESTROYED',
+  'CONNECTION_ENDED',
+  'ECONNREFUSED',
+  'ECONNRESET',
+  'ETIMEDOUT',
+  'EHOSTUNREACH',
+  'ENOTFOUND',
+])
+
+function isTransientConnectError(error: unknown): boolean {
+  const code = getPostgresErrorCode(error)
+  if (!code) return false
+  return TRANSIENT_CONNECT_CODES.has(code) || code.startsWith('08')
+}
+
 /** Backend pid of the lock-holding session; a change means the lock was lost. */
 let lockSessionPid = 0
 
 try {
+  await connectWithRetry()
   await acquireMigrationLock()
   try {
     await runMigrationsWithRetry()
@@ -91,6 +123,31 @@ try {
   await client.end()
 }
 
+/**
+ * Open the migration session before taking the advisory lock, retrying
+ * transient connection failures with bounded backoff. The deploy database can
+ * briefly exhaust every non-superuser connection slot at peak (`53300`); the
+ * migration is a single short-lived session, so waiting out a spike that frees
+ * within seconds is far safer than failing the whole deploy. Non-transient
+ * errors (auth, unknown host config, etc.) still fail fast.
+ */
+async function connectWithRetry(): Promise<void> {
+  for (let attempt = 1; ; attempt++) {
+    try {
+      await client`SELECT 1`
+      return
+    } catch (error) {
+      if (!isTransientConnectError(error) || attempt >= CONNECT_MAX_ATTEMPTS) throw error
+      const delayMs = backoffWithJitter(attempt, null, CONNECT_RETRY_BACKOFF)
+      console.warn(
+        `WARN: database unavailable (${getPostgresErrorCode(error)}); ` +
+          `attempt ${attempt}/${CONNECT_MAX_ATTEMPTS}, retrying in ${Math.round(delayMs)}ms.`
+      )
+      await sleep(delayMs)
+    }
+  }
+}
+
 /**
  * Acquire the cross-process migration lock, failing loudly after the deadline
  * instead of blocking forever behind a wedged runner.