import { spawn, spawnSync } from 'node:child_process' import { getErrorMessage } from '@sim/utils/errors' import type { Detection } from '../detect.ts' import { ensureDocker } from '../docker.ts' import { generateSecret, ROOT } from '../env-files.ts' import { SetupError } from '../errors.ts' import { waitFor } from '../probes.ts' import * as p from '../prompter.ts' import { chatFlagValues, mothershipOverride, promptCopilotKey } from '../steps.ts' import { glyph, theme } from '../theme.ts' const APP_URL = 'http://localhost:3000' const RELEASE = 'sim-dev' const NAMESPACE = 'sim-dev' const LOCAL_CONTEXT_PREFIXES = ['kind-', 'docker-desktop', 'minikube', 'orbstack'] /** * `input` is piped on stdin rather than passed as arguments — argv is readable * by any process on the machine, so secrets must never travel that way. */ function run(command: string, args: string[], failMessage: string, input?: string): string { // `helm upgrade --install ./helm/sim` uses chart paths relative to the repo // root, so pin cwd regardless of where the wizard was invoked from. const result = spawnSync(command, args, { encoding: 'utf8', input, cwd: ROOT }) if (result.status !== 0) { throw new Error(`${failMessage}: ${result.stderr.trim() || result.stdout.trim()}`) } return result.stdout } function isLocalContext(context: string): boolean { return LOCAL_CONTEXT_PREFIXES.some((prefix) => context === prefix || context.startsWith(prefix)) } const LOCAL_SERVER_HOSTS = new Set([ '127.0.0.1', 'localhost', '0.0.0.0', '::1', 'kubernetes.docker.internal', 'host.docker.internal', ]) /** True when a context's API server is a loopback/host address — i.e. a local cluster. */ export function isLocalKubeContext(context: string): boolean { const server = contextServerHost(context) return server !== null && LOCAL_SERVER_HOSTS.has(server) } /** * Liveness probe — a kubeconfig entry can outlive a stopped or deleted cluster * (kind clusters are Docker containers that don't restart on their own), so a * context looking local is no guarantee its API server answers. */ function clusterReachable(context: string): boolean { return ( spawnSync('kubectl', ['cluster-info', '--context', context, '--request-timeout=5s'], { stdio: 'ignore', }).status === 0 ) } /** The API server host a context points at, or null if kubectl can't resolve it. */ function contextServerHost(context: string): string | null { const result = spawnSync( 'kubectl', [ 'config', 'view', '--minify', '--context', context, '-o', 'jsonpath={.clusters[0].cluster.server}', ], { encoding: 'utf8' } ) if (result.status !== 0) return null try { return new URL(result.stdout.trim()).hostname } catch { return null } } /** * POSIX-quote a value for a copyable shell hint. A kube-context passes only a * prefix check, so it can still contain whitespace or shell metacharacters that * would break the `--context` argument (or run embedded syntax) when copied. * Ordinary context names stay bare; only unsafe ones get single-quoted. */ function shq(value: string): string { if (/^[A-Za-z0-9._/-]+$/.test(value)) return value return `'${value.replace(/'/g, `'\\''`)}'` } async function ensureLocalContext(detection: Detection): Promise { if (!detection.binaries.helm || !detection.binaries.kubectl) { throw new SetupError('kubernetes mode needs kubectl and helm on PATH.', [ `install them: ${theme.command('brew install kubectl helm')}`, ]) } const context = detection.kubeContext if (context && isLocalContext(context)) { // The name is only a hint — a remote cluster can be named like a local one // (e.g. "kind-prod"). Verify the API server is a loopback/host address before // defaulting to "yes", so generated secrets can't silently ship to a remote // cluster on a blind Enter. const server = contextServerHost(context) if (server && LOCAL_SERVER_HOSTS.has(server)) { if (!clusterReachable(context)) { // The context is local but its cluster isn't answering — stopped or // deleted. Don't offer it (helm would just fail); fall through to the // kind path, which starts a stopped "sim" cluster or creates one. p.log.warn( `Context "${context}" points at a local cluster that isn't responding — it looks stopped or deleted. The wizard will start or recreate a kind cluster instead.` ) } else { const useIt = await p.confirm({ message: `Use current kube context "${context}"?`, initialValue: true, }) if (useIt) return context } } else { p.log.warn( `Context "${context}" is named like a local cluster, but its API server${server ? ` (${server})` : ''} does not look local. Continuing would deploy the generated secrets there.` ) const useIt = await p.confirm({ message: `Deploy to "${context}" anyway?`, initialValue: false, }) if (useIt) return context } } else if (context) { p.log.warn( `Current context "${context}" does not look like a local cluster. Deploying to remote clusters is not supported by the wizard yet — switch to a kind/docker-desktop context, or drive helm directly (see helm/sim/examples/values-production.yaml).` ) } if (!detection.binaries.kind) { throw new SetupError('no local cluster available.', [ `install kind: ${theme.command('brew install kind')} — the wizard creates the cluster for you`, 'or enable Kubernetes in Docker Desktop settings, then re-run', ]) } await ensureDocker(true) const clusters = run('kind', ['get', 'clusters'], 'kind get clusters failed') .trim() .split('\n') .filter(Boolean) if (clusters.includes('sim')) { run('kind', ['export', 'kubeconfig', '--name', 'sim'], 'kind export kubeconfig failed') if (clusterReachable('kind-sim')) { p.log.step('Reusing existing kind cluster "sim"') } else { // The cluster exists in kind but isn't answering — its node containers are // stopped (a Docker/machine restart). Start them and wait for the API. const spin = p.spinner() spin.start('kind cluster "sim" is stopped — starting it…') const nodes = run('kind', ['get', 'nodes', '--name', 'sim'], 'kind get nodes failed') .trim() .split('\n') .filter(Boolean) for (const node of nodes) spawnSync('docker', ['start', node], { stdio: 'ignore' }) const up = await waitFor(() => Promise.resolve(clusterReachable('kind-sim')), 60_000, 2000) if (!up) { spin.stop(`${glyph.fail} kind cluster "sim" would not start`) throw new SetupError('the kind cluster "sim" exists but will not come up.', [ `inspect it: ${theme.command('docker ps -a --filter name=sim-control-plane')}`, `recreate it: ${theme.command('kind delete cluster --name sim')}, then re-run ${theme.command('bun run setup')}`, ]) } spin.stop('kind cluster "sim" started') } } else { const spin = p.spinner() spin.start('Creating kind cluster "sim"…') run('kind', ['create', 'cluster', '--name', 'sim'], 'kind create cluster failed') spin.stop('kind cluster "sim" ready') } return 'kind-sim' } interface PodProgress { ready: number total: number detail: string } /** * One-line summary of what the cluster is doing, for the install spinner. Only * long-running workloads count — the chart's CronJobs spawn short-lived pods * that finish as Completed, and counting those makes "ready" jitter downward * for reasons that have nothing to do with the install. */ function podProgress(context: string): PodProgress | null { const result = spawnSync( 'kubectl', [ 'get', 'pods', '--context', context, '-n', NAMESPACE, '-o', 'jsonpath={range .items[*]}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{range .status.containerStatuses[*]}{.ready},{.state.waiting.reason}{" "}{end}{"\\n"}{end}', ], { encoding: 'utf8' } ) if (result.status !== 0) return null const rows = result.stdout.split('\n').filter(Boolean) if (rows.length === 0) return null let ready = 0 let total = 0 let pulling = 0 let crashing = 0 for (const row of rows) { const [, ownerKind = '', containers = ''] = row.split('\t') if (ownerKind === 'Job') continue total++ if (containers.includes('true,')) ready++ if (containers.includes('ContainerCreating') || containers.includes('PodInitializing')) pulling++ if (containers.includes('CrashLoopBackOff') || containers.includes('ImagePullBackOff')) crashing++ } if (total === 0) return null const notes: string[] = [] if (pulling > 0) notes.push(`${pulling} starting`) // Restarts while Postgres comes up are normal on a cold cluster; say so rather // than let a silent spinner imply nothing is happening. if (crashing > 0) notes.push(`${crashing} restarting`) return { ready, total, detail: notes.join(' · ') } } /** * `helm --wait` blocks for minutes with no output, so a slow image pull is * indistinguishable from a wedged install — the reason the old spinner was * unsatisfying. Run helm asynchronously and poll the cluster so the spinner * reports what is actually happening. */ async function helmInstall( args: string[], input: string, context: string, spin: ReturnType ): Promise { const child = spawn('helm', args, { cwd: ROOT, stdio: ['pipe', 'pipe', 'pipe'] }) child.stdin.write(input) child.stdin.end() let stderr = '' let stdout = '' child.stdout.on('data', (chunk) => { stdout += chunk }) child.stderr.on('data', (chunk) => { stderr += chunk }) const ticker = setInterval(() => { const progress = podProgress(context) if (!progress) return const suffix = progress.detail ? ` · ${progress.detail}` : '' spin.message(`${progress.ready}/${progress.total} pods ready${suffix}`) }, 3000) const code = await new Promise((resolve) => { child.once('close', (status) => resolve(status ?? 1)) }) clearInterval(ticker) if (code !== 0) { throw new Error(`helm upgrade --install failed: ${stderr.trim() || stdout.trim()}`) } } interface ReleaseValues { app?: { env?: Record } postgresql?: { auth?: { password?: string } } } function existingReleaseValues(context: string): ReleaseValues | null { const scope = ['--kube-context', context, '-n', NAMESPACE] const status = spawnSync('helm', ['status', RELEASE, ...scope], { stdio: 'ignore' }) if (status.status !== 0) return null return JSON.parse( run('helm', ['get', 'values', RELEASE, ...scope, '-o', 'json'], 'helm get values failed') ) as ReleaseValues } /** * The previous release's secrets, or `null` when any are missing — a partial set * cannot be reused, since regenerating only some of them invalidates sessions * and stored credentials encrypted under the originals. */ function reusableSecrets(values: ReleaseValues | null): Record | null { const env = values?.app?.env ?? {} const password = values?.postgresql?.auth?.password if ( !env.BETTER_AUTH_SECRET || !env.ENCRYPTION_KEY || !env.INTERNAL_API_SECRET || !env.CRON_SECRET || !password ) { return null } return { BETTER_AUTH_SECRET: env.BETTER_AUTH_SECRET, ENCRYPTION_KEY: env.ENCRYPTION_KEY, INTERNAL_API_SECRET: env.INTERNAL_API_SECRET, CRON_SECRET: env.CRON_SECRET, POSTGRES_PASSWORD: password, } } /** * Values document piped to helm on stdin instead of `--set`. `JSON.stringify` * quotes and escapes each value — JSON is a subset of YAML, so a secret * containing `#`, `:`, or a leading `*` can neither break the document nor be * reinterpreted as YAML syntax. */ function secretValues(secrets: Record): string { const { POSTGRES_PASSWORD, ...appEnv } = secrets const env = Object.entries(appEnv) .map(([key, value]) => ` ${key}: ${JSON.stringify(value)}`) .join('\n') return `app:\n env:\n${env}\npostgresql:\n auth:\n password: ${JSON.stringify(POSTGRES_PASSWORD)}\n` } export async function runK8sMode(detection: Detection): Promise { // Pin every subsequent call to the context we validated: the ambient context // can change between detection and deploy, which would send generated // credentials to an unintended cluster. const context = await ensureLocalContext(detection) const releaseValues = existingReleaseValues(context) const reused = reusableSecrets(releaseValues) const secrets = reused ?? { BETTER_AUTH_SECRET: generateSecret(), ENCRYPTION_KEY: generateSecret(), INTERNAL_API_SECRET: generateSecret(), CRON_SECRET: generateSecret(), POSTGRES_PASSWORD: generateSecret().slice(0, 24), } if (reused) p.log.step('Reusing secrets from the existing release') // Before the key is minted: a half-set override mints against one environment // and validates against the other, and warning afterwards is too late — the // bad key is already deployed. const overrides = mothershipOverride() const copilotKey = await promptCopilotKey(releaseValues?.app?.env?.COPILOT_API_KEY) // `helm upgrade` without `--reuse-values` keeps only what this document // carries, so a key the user chose to keep has to be re-supplied here. const appEnv: Record = { ...secrets, ...overrides, ...(copilotKey ? { COPILOT_API_KEY: copilotKey } : {}), ...chatFlagValues(copilotKey), } const spin = p.spinner() spin.start('helm upgrade --install (first run pulls images — this can take several minutes)…') try { await helmInstall( [ 'upgrade', '--install', RELEASE, './helm/sim', '--kube-context', context, '--namespace', NAMESPACE, '--create-namespace', '--values', './helm/sim/examples/values-development.yaml', '--values', '-', '--wait', '--timeout', '15m', ], secretValues(appEnv), context, spin ) } catch (error) { spin.stop(`${glyph.fail} helm install failed`) throw new SetupError(getErrorMessage(error), [ `pod status: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} get pods`)}`, `stuck pods: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} describe pod | tail -20`)}`, 'ImagePullBackOff on ghcr.io/simstudioai/* usually means the chart appVersion tag was never published — check Chart.yaml against ghcr', ]) } spin.stop('Release deployed, all pods ready') const testSpin = p.spinner() testSpin.start('Running helm test…') const test = spawnSync('helm', ['test', RELEASE, '--kube-context', context, '-n', NAMESPACE], { encoding: 'utf8', cwd: ROOT, }) if (test.status !== 0) { testSpin.stop(`${glyph.fail} helm test failed`) throw new SetupError(`helm test failed:\n${test.stdout}${test.stderr}`, [ `pod status: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} get pods`)}`, `app logs: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} logs deploy/${RELEASE}-app --tail 50`)}`, ]) } testSpin.stop('helm test passed') p.note( [ `open ${APP_URL} (needs both forwards below)`, `pods: kubectl --context ${shq(context)} -n ${NAMESPACE} get pods`, `app logs: kubectl --context ${shq(context)} -n ${NAMESPACE} logs deploy/${RELEASE}-app --tail 50`, // Both, always: the app alone loads but the editor's socket has nothing to // reach, which is the reconnect failure this change set exists to fix. ...forwardCommands(context).map( (command, index) => `${index === 0 ? 'forward: ' : ' '} ${command}` ), `tear down: helm uninstall ${RELEASE} --kube-context ${shq(context)} -n ${NAMESPACE}`, ].join('\n'), 'Reach your cluster' ) await offerPortForward(context) } /** * Both forwards, in the order a user should run them. Kept in one place so the * printed instructions and what the wizard actually runs cannot drift — the app * alone leaves the editor's socket dead. */ export function forwardCommands(context: string): string[] { const scope = `kubectl --context ${shq(context)} -n ${NAMESPACE} port-forward` return [`${scope} svc/${RELEASE}-app 3000:3000`, `${scope} svc/${RELEASE}-realtime 3002:3002`] } /** * The services are ClusterIP, so a healthy release is still unreachable from the * host — "Sim is ready" with nothing on :3000 is the least satisfying way to end * a setup. Offer the forward the same way dev mode offers to start the server, * and run it in the foreground so Ctrl-C ends it. * * Realtime needs its own forward (one resource per invocation) or the editor's * socket fails; it runs as a child in this process group, so the terminal's * Ctrl-C reaches it too, and it is killed explicitly once the app forward exits. */ async function offerPortForward(context: string): Promise { const forward = await p.confirm({ message: `Port-forward now so you can open ${APP_URL}?`, initialValue: true, }) if (!forward) { p.log.info( theme.muted( `Skipped — run both forwards when you want to reach it (the second keeps the editor's socket alive):\n${forwardCommands( context ) .map((command) => ` ${command}`) .join('\n')}` ) ) return } const scope = ['--context', context, '-n', NAMESPACE] const realtime = spawn( 'kubectl', [...scope, 'port-forward', `svc/${RELEASE}-realtime`, '3002:3002'], // stderr is kept so a failure can be explained; a silently dead second // forward looks exactly like a working setup until the editor won't connect. { stdio: ['ignore', 'ignore', 'pipe'] } ) let realtimeError = '' realtime.stderr?.on('data', (chunk) => { realtimeError += chunk }) // Only an exit we did not ask for is a problem — the kill below also fires this. let stopping = false realtime.once('exit', (code) => { if (stopping || code === 0) return p.log.warn( `The realtime forward (:3002) stopped — the editor's socket will not connect. ${ realtimeError.trim() || `Check that :3002 is free and svc/${RELEASE}-realtime exists.` }` ) }) p.log.step(`Forwarding ${APP_URL} (app) and :3002 (realtime) — Ctrl-C to stop`) spawnSync('kubectl', [...scope, 'port-forward', `svc/${RELEASE}-app`, '3000:3000'], { stdio: 'inherit', }) stopping = true realtime.kill() }