feat(secrets): ingest env secrets at container runtime instead of fanning into ECS taskdef (#5189)

* feat(secrets): ingest env secrets at container runtime instead of fanning into ECS taskdef

The app/socket ECS taskdefs were ~42KB, ~93% of which was the secrets[] array:
268 pointer entries each restating the full ~78-char secret ARN, marching toward
the 64KB taskdef limit and growing ~150 bytes per hosted key added. The secret
blob itself is only ~18KB/268 keys.

Move secret delivery to container boot: new @sim/runtime-secrets loadRuntimeSecrets()
reads SIM_ENV_SECRET_ID, fetches the combined secret once, and hydrates process.env
(no-clobber, no-op when unset, fail-fast). Bootstrap entrypoints for app + realtime
await it before importing the real server (env-flags reads env at module load). The
app bootstrap is bun-bundled in the Dockerfile builder stage since it runs outside
the Next standalone bundle; realtime keeps full node_modules and runs the TS entry.

Backward-compatible: with the current fan-out taskdef the loader no-ops and the app
reads the injected env vars unchanged. The matching infra change (empty secrets[] +
SIM_ENV_SECRET_ID) ships separately, after this image is live.

* fix(runtime-secrets): address review feedback

- Move the binary-secret guard outside the retry loop (sendWithRetry) so a
  missing SecretString throws immediately instead of burning 3 attempts + backoff.
- Bound each Secrets Manager request with AbortSignal.timeout(5s) so a stalled
  response can't hang boot indefinitely.
- Drop the redundant @aws-sdk/client-secrets-manager pin from apps/realtime; it
  resolves transitively via @sim/runtime-secrets.
- Add a test for the non-retriable binary-secret path.

apps/realtime/package.json

@@ -26,6 +26,7 @@
     "@sim/logger": "workspace:*",
     "@sim/platform-authz": "workspace:*",
     "@sim/realtime-protocol": "workspace:*",
+    "@sim/runtime-secrets": "workspace:*",
     "@sim/security": "workspace:*",
     "@sim/utils": "workspace:*",
     "@sim/workflow-persistence": "workspace:*",

apps/realtime/src/bootstrap.ts

@@ -0,0 +1,9 @@
+/**
+ * Container entrypoint. Hydrates `process.env` from the runtime secret before
+ * loading the Socket.IO server, whose modules (`@/env`, DB preflight) read env
+ * at import time. See `@sim/runtime-secrets`.
+ */
+import { loadRuntimeSecrets } from '@sim/runtime-secrets'
+
+await loadRuntimeSecrets()
+await import('@/index')

apps/sim/bootstrap.ts

@@ -0,0 +1,13 @@
+/**
+ * Container entrypoint. Hydrates `process.env` from the runtime secret before
+ * loading the Next.js standalone server, so application modules that read env at
+ * import time see the full configuration. See `@sim/runtime-secrets`.
+ */
+import { loadRuntimeSecrets } from '@sim/runtime-secrets'
+
+await loadRuntimeSecrets()
+// `server.js` is the Next standalone build artifact, a sibling of this file in
+// the image; it does not exist at type-check time, so the specifier is held in a
+// variable to keep it out of static module resolution.
+const standaloneServer = './server.js'
+await import(standaloneServer)

apps/sim/package.json

@@ -99,6 +99,7 @@
     "@sim/logger": "workspace:*",
     "@sim/platform-authz": "workspace:*",
     "@sim/realtime-protocol": "workspace:*",
+    "@sim/runtime-secrets": "workspace:*",
     "@sim/security": "workspace:*",
     "@sim/utils": "workspace:*",
     "@sim/workflow-persistence": "workspace:*",

bun.lock

@@ -1,5 +1,6 @@
 {
   "lockfileVersion": 1,
+  "configVersion": 0,
   "workspaces": {
     "": {
       "name": "simstudio",
@@ -67,6 +68,7 @@
         "@sim/logger": "workspace:*",
         "@sim/platform-authz": "workspace:*",
         "@sim/realtime-protocol": "workspace:*",
+        "@sim/runtime-secrets": "workspace:*",
         "@sim/security": "workspace:*",
         "@sim/utils": "workspace:*",
         "@sim/workflow-persistence": "workspace:*",
@@ -158,6 +160,7 @@
         "@sim/logger": "workspace:*",
         "@sim/platform-authz": "workspace:*",
         "@sim/realtime-protocol": "workspace:*",
+        "@sim/runtime-secrets": "workspace:*",
         "@sim/security": "workspace:*",
         "@sim/utils": "workspace:*",
         "@sim/workflow-persistence": "workspace:*",
@@ -400,6 +403,21 @@
         "typescript": "^5.7.3",
       },
     },
+    "packages/runtime-secrets": {
+      "name": "@sim/runtime-secrets",
+      "version": "0.1.0",
+      "dependencies": {
+        "@aws-sdk/client-secrets-manager": "3.1032.0",
+        "@sim/logger": "workspace:*",
+        "@sim/utils": "workspace:*",
+      },
+      "devDependencies": {
+        "@sim/tsconfig": "workspace:*",
+        "@types/node": "24.2.1",
+        "typescript": "^5.7.3",
+        "vitest": "^4.1.0",
+      },
+    },
     "packages/security": {
       "name": "@sim/security",
       "version": "0.1.0",
@@ -1469,6 +1487,8 @@
 
     "@sim/realtime-protocol": ["@sim/realtime-protocol@workspace:packages/realtime-protocol"],
 
+    "@sim/runtime-secrets": ["@sim/runtime-secrets@workspace:packages/runtime-secrets"],
+
     "@sim/security": ["@sim/security@workspace:packages/security"],
 
     "@sim/testing": ["@sim/testing@workspace:packages/testing"],
@@ -4237,6 +4257,8 @@
 
     "@sim/realtime/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="],
 
+    "@sim/runtime-secrets/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="],
+
     "@sim/security/@types/node": ["@types/node@24.2.1", "", { "dependencies": { "undici-types": "~7.10.0" } }, "sha512-DRh5K+ka5eJic8CjH7td8QpYEV6Zo10gfRkjHCO3weqZHWDtAaSTFtl4+VMqOJ4N5jcuhZ9/l+yy8rVgw7BQeQ=="],
 
     "@smithy/middleware-compression/fflate": ["fflate@0.8.1", "", {}, "sha512-/exOvEuc+/iaUm105QIiOt4LpBdMTWsXxqR0HDF35vx3fmaKzw7354gTilCh5rkzEt8WYyG//ku3h3nRmd7CHQ=="],
@@ -4749,6 +4771,8 @@
 
     "@sim/realtime/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
 
+    "@sim/runtime-secrets/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
+
     "@sim/security/@types/node/undici-types": ["undici-types@7.10.0", "", {}, "sha512-t5Fy/nfn+14LuOc2KNYg75vZqClpAiqscVvMygNnlsHBFpSXdJaYtXMcdNLpl/Qvc3P2cB3s6lOV51nqsFq4ag=="],
 
     "@trigger.dev/core/@opentelemetry/api-logs/@opentelemetry/api": ["@opentelemetry/api@1.9.1", "", {}, "sha512-gLyJlPHPZYdAk1JENA9LeHejZe1Ti77/pTeFm/nMXmQH/HFZlcS/O2XJB+L8fkbrNSqhdtlvjBVjxwUYanNH5Q=="],

docker/app.Dockerfile

@@ -81,6 +81,13 @@ RUN --mount=type=cache,id=next-cache-${TARGETPLATFORM},target=/app/apps/sim/.nex
     --mount=type=cache,id=turbo-cache-${TARGETPLATFORM},target=/app/.turbo \
     bun run build
 
+# Bundle the secrets-loading bootstrap into a self-contained entrypoint. It runs
+# before (and outside) the Next standalone server, so its dependencies
+# (@sim/runtime-secrets, AWS SDK) are inlined here rather than resolved from the
+# pruned standalone node_modules. The dynamic import of ./server.js stays a
+# runtime import.
+RUN bun build apps/sim/bootstrap.ts --target=bun --outfile=apps/sim/bootstrap.js
+
 # ========================================
 # Runner Stage: Run the actual app
 # ========================================
@@ -100,6 +107,10 @@ COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/public ./apps/sim/public
 COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/standalone ./
 COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/.next/static ./apps/sim/.next/static
 
+# Self-contained secrets-loading bootstrap (bundled in the builder stage). Runs
+# before the standalone server.js to hydrate process.env from the runtime secret.
+COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/bootstrap.js ./apps/sim/bootstrap.js
+
 # Copy blog/author content for runtime filesystem reads (not part of the JS bundle)
 COPY --from=builder --chown=nextjs:nodejs /app/apps/sim/content ./apps/sim/content
 
@@ -128,4 +139,4 @@ EXPOSE 3000
 ENV PORT=3000 \
     HOSTNAME="0.0.0.0"
 
-CMD ["bun", "apps/sim/server.js"]
+CMD ["bun", "apps/sim/bootstrap.js"]

docker/realtime.Dockerfile

@@ -49,4 +49,4 @@ USER nextjs
 
 EXPOSE 3002
 
-CMD ["bun", "apps/realtime/src/index.ts"]
+CMD ["bun", "apps/realtime/src/bootstrap.ts"]

packages/runtime-secrets/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@sim/runtime-secrets",
+  "version": "0.1.0",
+  "private": true,
+  "sideEffects": false,
+  "type": "module",
+  "license": "Apache-2.0",
+  "engines": {
+    "bun": ">=1.2.13",
+    "node": ">=20.0.0"
+  },
+  "exports": {
+    ".": {
+      "types": "./src/index.ts",
+      "default": "./src/index.ts"
+    }
+  },
+  "scripts": {
+    "type-check": "tsc --noEmit",
+    "lint": "biome check --write --unsafe .",
+    "lint:check": "biome check .",
+    "format": "biome format --write .",
+    "format:check": "biome format .",
+    "test": "vitest run",
+    "test:watch": "vitest"
+  },
+  "dependencies": {
+    "@aws-sdk/client-secrets-manager": "3.1032.0",
+    "@sim/logger": "workspace:*",
+    "@sim/utils": "workspace:*"
+  },
+  "devDependencies": {
+    "@sim/tsconfig": "workspace:*",
+    "@types/node": "24.2.1",
+    "typescript": "^5.7.3",
+    "vitest": "^4.1.0"
+  }
+}

packages/runtime-secrets/src/index.test.ts

@@ -0,0 +1,91 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'
+
+const { mockSend } = vi.hoisted(() => ({ mockSend: vi.fn() }))
+
+vi.mock('@aws-sdk/client-secrets-manager', () => ({
+  SecretsManagerClient: class SecretsManagerClient {
+    send = mockSend
+  },
+  GetSecretValueCommand: class GetSecretValueCommand {
+    constructor(public input: unknown) {}
+  },
+}))
+
+vi.mock('@sim/logger', () => ({
+  createLogger: () => ({ info: vi.fn(), warn: vi.fn(), error: vi.fn() }),
+}))
+
+vi.mock('@sim/utils/helpers', () => ({
+  sleep: vi.fn().mockResolvedValue(undefined),
+}))
+
+import { loadRuntimeSecrets } from './index'
+
+const TOUCHED = ['SIM_ENV_SECRET_ID', 'FOO', 'BAZ'] as const
+
+describe('loadRuntimeSecrets', () => {
+  beforeEach(() => {
+    vi.clearAllMocks()
+    for (const key of TOUCHED) delete process.env[key]
+  })
+
+  afterEach(() => {
+    for (const key of TOUCHED) delete process.env[key]
+  })
+
+  it('no-ops when SIM_ENV_SECRET_ID is unset', async () => {
+    await loadRuntimeSecrets()
+    expect(mockSend).not.toHaveBeenCalled()
+  })
+
+  it('hydrates process.env from the parsed secret JSON', async () => {
+    process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars'
+    mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'bar', BAZ: 'qux' }) })
+
+    await loadRuntimeSecrets()
+
+    expect(process.env.FOO).toBe('bar')
+    expect(process.env.BAZ).toBe('qux')
+  })
+
+  it('never overwrites an already-set env var', async () => {
+    process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars'
+    process.env.FOO = 'existing'
+    mockSend.mockResolvedValue({ SecretString: JSON.stringify({ FOO: 'new', BAZ: 'qux' }) })
+
+    await loadRuntimeSecrets()
+
+    expect(process.env.FOO).toBe('existing')
+    expect(process.env.BAZ).toBe('qux')
+  })
+
+  it('throws when the secret is not valid JSON', async () => {
+    process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars'
+    mockSend.mockResolvedValue({ SecretString: 'not json' })
+
+    await expect(loadRuntimeSecrets()).rejects.toThrow(/not valid JSON/)
+  })
+
+  it('throws when the secret JSON is not an object', async () => {
+    process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars'
+    mockSend.mockResolvedValue({ SecretString: JSON.stringify(['a', 'b']) })
+
+    await expect(loadRuntimeSecrets()).rejects.toThrow(/must be a JSON object/)
+  })
+
+  it('throws immediately on a binary secret (no SecretString), without retrying', async () => {
+    process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars'
+    mockSend.mockResolvedValue({})
+
+    await expect(loadRuntimeSecrets()).rejects.toThrow(/binary secrets/)
+    expect(mockSend).toHaveBeenCalledTimes(1)
+  })
+
+  it('retries then throws when the fetch keeps failing', async () => {
+    process.env.SIM_ENV_SECRET_ID = '/test/sim/env-vars'
+    mockSend.mockRejectedValue(new Error('boom'))
+
+    await expect(loadRuntimeSecrets()).rejects.toThrow(/Failed to fetch runtime secrets/)
+    expect(mockSend).toHaveBeenCalledTimes(3)
+  })
+})

packages/runtime-secrets/src/index.ts

@@ -0,0 +1,102 @@
+import type { GetSecretValueCommandOutput } from '@aws-sdk/client-secrets-manager'
+import { GetSecretValueCommand, SecretsManagerClient } from '@aws-sdk/client-secrets-manager'
+import { createLogger } from '@sim/logger'
+import { getErrorMessage } from '@sim/utils/errors'
+import { sleep } from '@sim/utils/helpers'
+import { backoffWithJitter } from '@sim/utils/retry'
+
+const logger = createLogger('RuntimeSecrets')
+
+/** Plaintext env var (set in the ECS task definition) naming the secret to ingest. */
+const SECRET_ID_ENV = 'SIM_ENV_SECRET_ID'
+
+const MAX_ATTEMPTS = 3
+
+/** Bounds each Secrets Manager request so a stalled response can't hang boot. */
+const REQUEST_TIMEOUT_MS = 5000
+
+/**
+ * Fetches the combined `/{env}/sim/env-vars` secret once at container boot and
+ * hydrates `process.env`, so secrets no longer have to be fanned out into the
+ * ECS task definition (which is approaching the 64 KB rendered-document limit).
+ *
+ * Must run before any application module that reads env at import time. No-ops
+ * when {@link SECRET_ID_ENV} is unset (local dev / self-hosted keep using their
+ * own env). Existing `process.env` keys are never overwritten, so explicit
+ * task-definition `environment` entries win. Throws on any fetch/parse failure
+ * so a misconfigured container crashes instead of booting without its config.
+ */
+export async function loadRuntimeSecrets(): Promise<void> {
+  const secretId = process.env[SECRET_ID_ENV]
+  if (!secretId) {
+    logger.info(`${SECRET_ID_ENV} not set; skipping runtime secret ingestion`)
+    return
+  }
+
+  const client = new SecretsManagerClient(
+    process.env.AWS_REGION ? { region: process.env.AWS_REGION } : {}
+  )
+
+  const secretString = await fetchSecretString(client, secretId)
+  const entries = parseSecretJson(secretString)
+
+  let loaded = 0
+  let skipped = 0
+  for (const [key, value] of Object.entries(entries)) {
+    if (key in process.env) {
+      skipped++
+      continue
+    }
+    process.env[key] = typeof value === 'string' ? value : JSON.stringify(value)
+    loaded++
+  }
+
+  logger.info('Runtime secrets ingested', { secretId, loaded, skipped })
+}
+
+async function fetchSecretString(client: SecretsManagerClient, secretId: string): Promise<string> {
+  const response = await sendWithRetry(client, secretId)
+  if (!response.SecretString) {
+    // Non-retriable: a binary secret will never become a string between attempts.
+    throw new Error('Secret has no SecretString (binary secrets are not supported)')
+  }
+  return response.SecretString
+}
+
+async function sendWithRetry(
+  client: SecretsManagerClient,
+  secretId: string
+): Promise<GetSecretValueCommandOutput> {
+  let lastError: unknown
+  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
+    try {
+      return await client.send(new GetSecretValueCommand({ SecretId: secretId }), {
+        abortSignal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
+      })
+    } catch (error) {
+      lastError = error
+      if (attempt < MAX_ATTEMPTS) {
+        const delay = backoffWithJitter(attempt, null, { baseMs: 200, maxMs: 2000 })
+        logger.warn(
+          `Failed to fetch runtime secrets (attempt ${attempt}/${MAX_ATTEMPTS}), retrying`,
+          { error: getErrorMessage(error) }
+        )
+        await sleep(delay)
+      }
+    }
+  }
+  throw new Error(`Failed to fetch runtime secrets from ${secretId}: ${getErrorMessage(lastError)}`)
+}
+
+function parseSecretJson(secretString: string): Record<string, unknown> {
+  let parsed: unknown
+  try {
+    parsed = JSON.parse(secretString)
+  } catch (error) {
+    throw new Error(`Runtime secret is not valid JSON: ${getErrorMessage(error)}`)
+  }
+  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {
+    throw new Error('Runtime secret must be a JSON object of key/value pairs')
+  }
+  return parsed as Record<string, unknown>
+}

packages/runtime-secrets/tsconfig.json

@@ -0,0 +1,5 @@
+{
+  "extends": "@sim/tsconfig/library.json",
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

packages/runtime-secrets/vitest.config.ts

@@ -0,0 +1,7 @@
+import { defineConfig } from 'vitest/config'
+
+export default defineConfig({
+  test: {
+    environment: 'node',
+  },
+})