Files
sim/scripts/setup/env-files.ts
T
Waleed 3de63c94e3 feat(self-host): align Docker Compose with Helm and overhaul self-hosting docs (#6225)
* feat(self-host): align Docker Compose with Helm and overhaul self-hosting docs

Docker Compose shipped no scheduler, so scheduled workflows, every polling
trigger, connector syncs, the outbox, and data drains silently never ran.
Adds a cron service running the same 18 jobs the Helm chart schedules as
CronJobs, and closes the remaining behavioral gaps between the two paths:
bundled Redis in the chart, no hosted plan caps in chart defaults, pinned
image tags, and fail-fast secrets. A CI check keeps the schedulers in sync.

Also rewrites the self-hosting docs: 14 new pages, 8 updated, reorganized
into Install / Configure / Operate.

* fix(self-host): drop bun install from chart CI, remove air-gapped and backup docs

The scheduler-parity check pulled a full dependency install into the
chart-validation job, which fails building isolated-vm on that runner.
Rewritten to use only node builtins so the job installs nothing.

Also removes the air-gapped and backup/restore pages, and stops pinning a
concrete release in the docs so the examples do not go stale each release.

* fix(helm): bundle Redis in secret-manager modes unless the URL is supplied

Suppressing Redis whenever a secret mode was active left those deployments
with no Redis at all — REDIS_URL is optional there and both shipped examples
omit it. The chart now steps aside only on a detectable signal: an explicit
app.env.REDIS_URL, an ESO remoteRefs.app.REDIS_URL mapping, or the new
redis.provideUrl=false opt-out for a pre-created Secret it cannot read.

* fix(compose): derive realtime BETTER_AUTH_URL from NEXT_PUBLIC_APP_URL

realtime read BETTER_AUTH_URL directly and fell back to localhost while
simstudio derived it from NEXT_PUBLIC_APP_URL, so setting only the public
origin left realtime authenticating against http://localhost:3000.

* fix(helm): deliver bundled REDIS_URL via ConfigMap so an operator value always wins

Injecting REDIS_URL as an inline container env made it beat every envFrom
source, so a REDIS_URL held in a pre-created Secret or synced by External
Secrets was silently shadowed and traffic moved to a fresh in-cluster Redis.

Kubernetes resolves duplicate envFrom keys by letting the last source win, so
the bundled URL now ships as a ConfigMap listed before the app Secret. Any
operator-supplied value overrides it without the chart needing to read it,
which also removes the redis.provideUrl flag the previous attempt required.

* docs(helm): spell out the egress rule external datastores need

The default NetworkPolicy allows 443 plus the bundled Postgres and Redis by
pod selector. Anything you run outside the chart on another port needs its own
rule, which is easiest to miss when REDIS_URL arrives via a Secret the chart
cannot inspect. Adds a copyable example to the production checklist and the
security guide.

* feat(helm): add networkPolicy.allowExternalEgress for managed datastores

The default policy allows 443 plus the bundled Postgres and Redis by pod
selector, so a managed datastore on another port needs a hand-written CIDR
rule — awkward when REDIS_URL arrives via a Secret the chart cannot inspect.

Adds an opt-in switch that drops the port restriction while still blocking the
cloud metadata endpoints. Defaults to false, keeping this chart stricter than
the common chart default of unrestricted egress.
2026-08-03 14:52:48 -07:00

179 lines
6.2 KiB
TypeScript

import { existsSync, readFileSync, renameSync, writeFileSync } from 'node:fs'
import path from 'node:path'
import { fileURLToPath } from 'node:url'
import { generateRandomHex } from '@sim/utils/random'
export const ROOT = path.resolve(fileURLToPath(new URL('.', import.meta.url)), '../..')
export type EnvTarget = 'sim' | 'realtime' | 'db' | 'root'
export const ENV_PATHS: Record<EnvTarget, string> = {
sim: path.join(ROOT, 'apps/sim/.env'),
realtime: path.join(ROOT, 'apps/realtime/.env'),
db: path.join(ROOT, 'packages/db/.env'),
root: path.join(ROOT, '.env'),
}
const EXAMPLE_PATHS: Partial<Record<EnvTarget, string>> = {
sim: path.join(ROOT, 'apps/sim/.env.example'),
realtime: path.join(ROOT, 'apps/realtime/.env.example'),
db: path.join(ROOT, 'packages/db/.env.example'),
}
/** Keys that must be byte-identical between apps/sim/.env and apps/realtime/.env. */
export const SHARED_KEYS = [
'DATABASE_URL',
'BETTER_AUTH_SECRET',
'INTERNAL_API_SECRET',
'BETTER_AUTH_URL',
'NEXT_PUBLIC_APP_URL',
] as const
export const SECRET_KEYS = [
'BETTER_AUTH_SECRET',
'ENCRYPTION_KEY',
'INTERNAL_API_SECRET',
'API_ENCRYPTION_KEY',
// Authenticates the scheduler against the background job endpoints. Without
// it the cron service exits and no scheduled work runs, so setup generates it.
'CRON_SECRET',
] as const
const PLACEHOLDER_VALUES = new Set([
'your_password',
'your_secret_key',
'your_encryption_key',
'your_internal_api_secret',
'your_api_encryption_key',
'your_better_auth_secret_min_32_chars',
'dev-secret-at-least-32-characters-long',
'dev-encryption-key-at-least-32-chars',
'dev-internal-api-secret-min-32-chars',
])
export interface EnvFile {
target: EnvTarget
path: string
exists: boolean
content: string
vars: Map<string, string>
}
const LINE_RE = /^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/
function parseValue(raw: string): string {
const trimmed = raw.trim()
if (trimmed.startsWith('"') || trimmed.startsWith("'")) {
const quote = trimmed[0]
const end = trimmed.indexOf(quote, 1)
return end === -1 ? trimmed.slice(1) : trimmed.slice(1, end)
}
return trimmed.replace(/\s+#.*$/, '').trim()
}
export function parseEnv(content: string): Map<string, string> {
const vars = new Map<string, string>()
for (const line of content.split('\n')) {
const match = LINE_RE.exec(line)
if (match && !vars.has(match[1])) vars.set(match[1], parseValue(match[2]))
}
return vars
}
export function readEnvFile(target: EnvTarget): EnvFile {
const filePath = ENV_PATHS[target]
const exists = existsSync(filePath)
const content = exists ? readFileSync(filePath, 'utf8') : ''
return { target, path: filePath, exists, content, vars: parseEnv(content) }
}
/**
* Sets a key in env-file content: replaces the active line, uncomments a
* commented-out line, or appends. Returns the new content.
*/
export function upsertEnv(content: string, key: string, value: string): string {
const lines = content.split('\n')
const activeRe = new RegExp(`^\\s*(?:export\\s+)?${key}\\s*=`)
const commentedRe = new RegExp(`^#\\s*${key}\\s*=`)
const activeIdx = lines.findIndex((l) => activeRe.test(l))
const idx = activeIdx !== -1 ? activeIdx : lines.findIndex((l) => commentedRe.test(l))
const newLine = `${key}=${value}`
if (idx === -1) {
const trailing = lines.length > 0 && lines[lines.length - 1] === ''
if (trailing) lines.splice(lines.length - 1, 0, newLine)
else lines.push(newLine)
} else {
lines[idx] = newLine
}
return lines.join('\n')
}
/** Writes values into an env file, seeding a missing file from its .env.example. */
export function writeEnvValues(target: EnvTarget, values: Record<string, string>): void {
const filePath = ENV_PATHS[target]
let content: string
if (existsSync(filePath)) {
content = readFileSync(filePath, 'utf8')
} else {
const example = EXAMPLE_PATHS[target]
content = example && existsSync(example) ? readFileSync(example, 'utf8') : ''
}
for (const [key, value] of Object.entries(values)) {
content = upsertEnv(content, key, value)
}
writeFileSync(filePath, content)
}
export function archiveEnvFile(target: EnvTarget): string | null {
const filePath = ENV_PATHS[target]
if (!existsSync(filePath)) return null
const backup = `${filePath}.bak-${new Date().toISOString().replace(/[:.]/g, '-')}`
renameSync(filePath, backup)
return backup
}
export function generateSecret(): string {
return generateRandomHex(64)
}
/**
* `ENCRYPTION_KEY` and `API_ENCRYPTION_KEY` are read as raw AES-256 material,
* so the app requires exactly 64 hex characters and throws on anything else
* (`lib/core/security/encryption.ts`, `lib/api-key/crypto.ts`). A merely-long
* passphrase passes a length check and then fails every encryption path at
* runtime, so those two are validated on format rather than length.
*
* Lives here so setup (which replaces an unusable secret) and doctor (which
* reports one) apply the same rule — they disagreed while it was duplicated.
*/
const HEX_KEY_PATTERN = /^[0-9a-f]{64}$/i
const HEX_SECRET_KEYS = new Set<string>(['ENCRYPTION_KEY', 'API_ENCRYPTION_KEY'])
export function isUsableSecret(key: string, value: string): boolean {
if (isPlaceholder(value)) return false
return HEX_SECRET_KEYS.has(key) ? HEX_KEY_PATTERN.test(value) : value.length >= 32
}
/** Human-readable reason a secret is unusable, for doctor's finding message. */
export function secretRequirement(key: string): string {
return HEX_SECRET_KEYS.has(key)
? 'must be exactly 64 hex characters (32-byte AES key)'
: 'must be at least 32 characters'
}
export function isPlaceholder(value: string): boolean {
return PLACEHOLDER_VALUES.has(value) || value.startsWith('your_')
}
/**
* Mirrors the app's `isTruthy` (apps/sim/lib/core/config/env.ts:633) exactly —
* `true` or `1` only. The app's separate `envBoolean` additionally accepts
* `yes`/`on`, but feature flags read through `isTruthy`, so accepting the wider
* set here made the wizard and doctor report a flag as on that the app treats
* as off.
*/
export function isTruthy(value: string | undefined): boolean {
if (value === undefined) return false
return value.toLowerCase() === 'true' || value === '1'
}