Files
sim/scripts/setup/modes/k8s.ts
T

521 lines
19 KiB
TypeScript

import { spawn, spawnSync } from 'node:child_process'
import { getErrorMessage } from '@sim/utils/errors'
import { KNOWLEDGE_EMBEDDINGS_SETUP } from '../capability-config.ts'
import { getCapabilitySetupFields, stageCapabilitySetupTransition } from '../capability-setup.ts'
import type { Detection } from '../detect.ts'
import { ensureDocker } from '../docker.ts'
import { generateSecret, ROOT } from '../env-files.ts'
import { SetupError } from '../errors.ts'
import { waitFor } from '../probes.ts'
import * as p from '../prompter.ts'
import {
chatFlagValues,
mothershipOverride,
promptCopilotKey,
promptKnowledgeEmbeddings,
} from '../steps.ts'
import { glyph, theme } from '../theme.ts'
import { APP_SIGNUP_URL, APP_URL } from '../urls.ts'
const RELEASE = 'sim-dev'
const NAMESPACE = 'sim-dev'
const LOCAL_CONTEXT_PREFIXES = ['kind-', 'docker-desktop', 'minikube', 'orbstack']
/**
* `input` is piped on stdin rather than passed as arguments — argv is readable
* by any process on the machine, so secrets must never travel that way.
*/
function run(command: string, args: string[], failMessage: string, input?: string): string {
// `helm upgrade --install ./helm/sim` uses chart paths relative to the repo
// root, so pin cwd regardless of where the wizard was invoked from.
const result = spawnSync(command, args, { encoding: 'utf8', input, cwd: ROOT })
if (result.status !== 0) {
throw new Error(`${failMessage}: ${result.stderr.trim() || result.stdout.trim()}`)
}
return result.stdout
}
function isLocalContext(context: string): boolean {
return LOCAL_CONTEXT_PREFIXES.some((prefix) => context === prefix || context.startsWith(prefix))
}
const LOCAL_SERVER_HOSTS = new Set([
'127.0.0.1',
'localhost',
'0.0.0.0',
'::1',
'kubernetes.docker.internal',
'host.docker.internal',
])
/** True when a context's API server is a loopback/host address — i.e. a local cluster. */
export function isLocalKubeContext(context: string): boolean {
const server = contextServerHost(context)
return server !== null && LOCAL_SERVER_HOSTS.has(server)
}
/**
* Liveness probe — a kubeconfig entry can outlive a stopped or deleted cluster
* (kind clusters are Docker containers that don't restart on their own), so a
* context looking local is no guarantee its API server answers.
*/
function clusterReachable(context: string): boolean {
return (
spawnSync('kubectl', ['cluster-info', '--context', context, '--request-timeout=5s'], {
stdio: 'ignore',
}).status === 0
)
}
/** The API server host a context points at, or null if kubectl can't resolve it. */
function contextServerHost(context: string): string | null {
const result = spawnSync(
'kubectl',
[
'config',
'view',
'--minify',
'--context',
context,
'-o',
'jsonpath={.clusters[0].cluster.server}',
],
{ encoding: 'utf8' }
)
if (result.status !== 0) return null
try {
return new URL(result.stdout.trim()).hostname
} catch {
return null
}
}
/**
* POSIX-quote a value for a copyable shell hint. A kube-context passes only a
* prefix check, so it can still contain whitespace or shell metacharacters that
* would break the `--context` argument (or run embedded syntax) when copied.
* Ordinary context names stay bare; only unsafe ones get single-quoted.
*/
function shq(value: string): string {
if (/^[A-Za-z0-9._/-]+$/.test(value)) return value
return `'${value.replace(/'/g, `'\\''`)}'`
}
async function ensureLocalContext(detection: Detection): Promise<string> {
if (!detection.binaries.helm || !detection.binaries.kubectl) {
throw new SetupError('kubernetes mode needs kubectl and helm on PATH.', [
`install them: ${theme.command('brew install kubectl helm')}`,
])
}
const context = detection.kubeContext
if (context && isLocalContext(context)) {
// The name is only a hint — a remote cluster can be named like a local one
// (e.g. "kind-prod"). Verify the API server is a loopback/host address before
// defaulting to "yes", so generated secrets can't silently ship to a remote
// cluster on a blind Enter.
const server = contextServerHost(context)
if (server && LOCAL_SERVER_HOSTS.has(server)) {
if (!clusterReachable(context)) {
// The context is local but its cluster isn't answering — stopped or
// deleted. Don't offer it (helm would just fail); fall through to the
// kind path, which starts a stopped "sim" cluster or creates one.
p.log.warn(
`Context "${context}" points at a local cluster that isn't responding — it looks stopped or deleted. The wizard will start or recreate a kind cluster instead.`
)
} else {
const useIt = await p.confirm({
message: `Use current kube context "${context}"?`,
initialValue: true,
})
if (useIt) return context
}
} else {
p.log.warn(
`Context "${context}" is named like a local cluster, but its API server${server ? ` (${server})` : ''} does not look local. Continuing would deploy the generated secrets there.`
)
const useIt = await p.confirm({
message: `Deploy to "${context}" anyway?`,
initialValue: false,
})
if (useIt) return context
}
} else if (context) {
p.log.warn(
`Current context "${context}" does not look like a local cluster. Deploying to remote clusters is not supported by the wizard yet — switch to a kind/docker-desktop context, or drive helm directly (see helm/sim/examples/values-production.yaml).`
)
}
if (!detection.binaries.kind) {
throw new SetupError('no local cluster available.', [
`install kind: ${theme.command('brew install kind')} — the wizard creates the cluster for you`,
'or enable Kubernetes in Docker Desktop settings, then re-run',
])
}
await ensureDocker(true)
const clusters = run('kind', ['get', 'clusters'], 'kind get clusters failed')
.trim()
.split('\n')
.filter(Boolean)
if (clusters.includes('sim')) {
run('kind', ['export', 'kubeconfig', '--name', 'sim'], 'kind export kubeconfig failed')
if (clusterReachable('kind-sim')) {
p.log.step('Reusing existing kind cluster "sim"')
} else {
// The cluster exists in kind but isn't answering — its node containers are
// stopped (a Docker/machine restart). Start them and wait for the API.
const spin = p.spinner()
spin.start('kind cluster "sim" is stopped — starting it…')
const nodes = run('kind', ['get', 'nodes', '--name', 'sim'], 'kind get nodes failed')
.trim()
.split('\n')
.filter(Boolean)
for (const node of nodes) spawnSync('docker', ['start', node], { stdio: 'ignore' })
const up = await waitFor(() => Promise.resolve(clusterReachable('kind-sim')), 60_000, 2000)
if (!up) {
spin.stop(`${glyph.fail} kind cluster "sim" would not start`)
throw new SetupError('the kind cluster "sim" exists but will not come up.', [
`inspect it: ${theme.command('docker ps -a --filter name=sim-control-plane')}`,
`recreate it: ${theme.command('kind delete cluster --name sim')}, then re-run ${theme.command('bun run setup')}`,
])
}
spin.stop('kind cluster "sim" started')
}
} else {
const spin = p.spinner()
spin.start('Creating kind cluster "sim"…')
run('kind', ['create', 'cluster', '--name', 'sim'], 'kind create cluster failed')
spin.stop('kind cluster "sim" ready')
}
return 'kind-sim'
}
interface PodProgress {
ready: number
total: number
detail: string
}
/**
* One-line summary of what the cluster is doing, for the install spinner. Only
* long-running workloads count — the chart's CronJobs spawn short-lived pods
* that finish as Completed, and counting those makes "ready" jitter downward
* for reasons that have nothing to do with the install.
*/
function podProgress(context: string): PodProgress | null {
const result = spawnSync(
'kubectl',
[
'get',
'pods',
'--context',
context,
'-n',
NAMESPACE,
'-o',
'jsonpath={range .items[*]}{.status.phase}{"\\t"}{.metadata.ownerReferences[0].kind}{"\\t"}{range .status.containerStatuses[*]}{.ready},{.state.waiting.reason}{" "}{end}{"\\n"}{end}',
],
{ encoding: 'utf8' }
)
if (result.status !== 0) return null
const rows = result.stdout.split('\n').filter(Boolean)
if (rows.length === 0) return null
let ready = 0
let total = 0
let pulling = 0
let crashing = 0
for (const row of rows) {
const [, ownerKind = '', containers = ''] = row.split('\t')
if (ownerKind === 'Job') continue
total++
if (containers.includes('true,')) ready++
if (containers.includes('ContainerCreating') || containers.includes('PodInitializing'))
pulling++
if (containers.includes('CrashLoopBackOff') || containers.includes('ImagePullBackOff'))
crashing++
}
if (total === 0) return null
const notes: string[] = []
if (pulling > 0) notes.push(`${pulling} starting`)
// Restarts while Postgres comes up are normal on a cold cluster; say so rather
// than let a silent spinner imply nothing is happening.
if (crashing > 0) notes.push(`${crashing} restarting`)
return { ready, total, detail: notes.join(' · ') }
}
/**
* `helm --wait` blocks for minutes with no output, so a slow image pull is
* indistinguishable from a wedged install — the reason the old spinner was
* unsatisfying. Run helm asynchronously and poll the cluster so the spinner
* reports what is actually happening.
*/
async function helmInstall(
args: string[],
input: string,
context: string,
spin: ReturnType<typeof p.spinner>
): Promise<void> {
const child = spawn('helm', args, { cwd: ROOT, stdio: ['pipe', 'pipe', 'pipe'] })
child.stdin.write(input)
child.stdin.end()
let stderr = ''
let stdout = ''
child.stdout.on('data', (chunk) => {
stdout += chunk
})
child.stderr.on('data', (chunk) => {
stderr += chunk
})
const ticker = setInterval(() => {
const progress = podProgress(context)
if (!progress) return
const suffix = progress.detail ? ` · ${progress.detail}` : ''
spin.message(`${progress.ready}/${progress.total} pods ready${suffix}`)
}, 3000)
const code = await new Promise<number>((resolve) => {
child.once('close', (status) => resolve(status ?? 1))
})
clearInterval(ticker)
if (code !== 0) {
throw new Error(`helm upgrade --install failed: ${stderr.trim() || stdout.trim()}`)
}
}
interface ReleaseValues {
app?: { env?: Record<string, string> }
postgresql?: { auth?: { password?: string } }
}
function existingReleaseValues(context: string): ReleaseValues | null {
const scope = ['--kube-context', context, '-n', NAMESPACE]
const status = spawnSync('helm', ['status', RELEASE, ...scope], { stdio: 'ignore' })
if (status.status !== 0) return null
return JSON.parse(
run('helm', ['get', 'values', RELEASE, ...scope, '-o', 'json'], 'helm get values failed')
) as ReleaseValues
}
/**
* The previous release's secrets, or `null` when any are missing — a partial set
* cannot be reused, since regenerating only some of them invalidates sessions
* and stored credentials encrypted under the originals.
*/
function reusableSecrets(values: ReleaseValues | null): Record<string, string> | null {
const env = values?.app?.env ?? {}
const password = values?.postgresql?.auth?.password
if (
!env.BETTER_AUTH_SECRET ||
!env.ENCRYPTION_KEY ||
!env.INTERNAL_API_SECRET ||
!env.CRON_SECRET ||
!password
) {
return null
}
return {
BETTER_AUTH_SECRET: env.BETTER_AUTH_SECRET,
ENCRYPTION_KEY: env.ENCRYPTION_KEY,
INTERNAL_API_SECRET: env.INTERNAL_API_SECRET,
CRON_SECRET: env.CRON_SECRET,
POSTGRES_PASSWORD: password,
}
}
/**
* Values document piped to helm on stdin instead of `--set`. `JSON.stringify`
* quotes and escapes each value — JSON is a subset of YAML, so a secret
* containing `#`, `:`, or a leading `*` can neither break the document nor be
* reinterpreted as YAML syntax.
*/
function secretValues(secrets: Record<string, string>): string {
const { POSTGRES_PASSWORD, ...appEnv } = secrets
const env = Object.entries(appEnv)
.map(([key, value]) => ` ${key}: ${JSON.stringify(value)}`)
.join('\n')
return `app:\n env:\n${env}\npostgresql:\n auth:\n password: ${JSON.stringify(POSTGRES_PASSWORD)}\n`
}
export async function runK8sMode(detection: Detection): Promise<void> {
// Pin every subsequent call to the context we validated: the ambient context
// can change between detection and deploy, which would send generated
// credentials to an unintended cluster.
const context = await ensureLocalContext(detection)
const releaseValues = existingReleaseValues(context)
const reused = reusableSecrets(releaseValues)
const secrets = reused ?? {
BETTER_AUTH_SECRET: generateSecret(),
ENCRYPTION_KEY: generateSecret(),
INTERNAL_API_SECRET: generateSecret(),
CRON_SECRET: generateSecret(),
POSTGRES_PASSWORD: generateSecret().slice(0, 24),
}
if (reused) p.log.step('Reusing secrets from the existing release')
// Before the key is minted: a half-set override mints against one environment
// and validates against the other, and warning afterwards is too late — the
// bad key is already deployed.
const overrides = mothershipOverride()
const copilotKey = await promptCopilotKey(releaseValues?.app?.env?.COPILOT_API_KEY)
// `helm upgrade` without `--reuse-values` keeps only what this document
// carries, so a key the user chose to keep has to be re-supplied here.
const appEnv: Record<string, string> = {
...secrets,
...overrides,
...(copilotKey ? { COPILOT_API_KEY: copilotKey } : {}),
...chatFlagValues(copilotKey),
}
const stagedVars = new Map(Object.entries(releaseValues?.app?.env ?? {}))
for (const [key, value] of Object.entries(appEnv)) stagedVars.set(key, value)
for (const key of getCapabilitySetupFields(KNOWLEDGE_EMBEDDINGS_SETUP)) {
const existing = stagedVars.get(key)
if (existing) appEnv[key] = existing
}
const remove = new Set<string>()
const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: true })
if (embeddings) {
stageCapabilitySetupTransition(stagedVars, appEnv, remove, embeddings)
}
const spin = p.spinner()
spin.start('helm upgrade --install (first run pulls images — this can take several minutes)…')
try {
await helmInstall(
[
'upgrade',
'--install',
RELEASE,
'./helm/sim',
'--kube-context',
context,
'--namespace',
NAMESPACE,
'--create-namespace',
'--values',
'./helm/sim/examples/values-development.yaml',
'--values',
'-',
'--wait',
'--timeout',
'15m',
],
secretValues(appEnv),
context,
spin
)
} catch (error) {
spin.stop(`${glyph.fail} helm install failed`)
throw new SetupError(getErrorMessage(error), [
`pod status: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} get pods`)}`,
`stuck pods: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} describe pod <name> | tail -20`)}`,
'ImagePullBackOff on ghcr.io/simstudioai/* usually means the chart appVersion tag was never published — check Chart.yaml against ghcr',
])
}
spin.stop('Release deployed, all pods ready')
const testSpin = p.spinner()
testSpin.start('Running helm test…')
const test = spawnSync('helm', ['test', RELEASE, '--kube-context', context, '-n', NAMESPACE], {
encoding: 'utf8',
cwd: ROOT,
})
if (test.status !== 0) {
testSpin.stop(`${glyph.fail} helm test failed`)
throw new SetupError(`helm test failed:\n${test.stdout}${test.stderr}`, [
`pod status: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} get pods`)}`,
`app logs: ${theme.command(`kubectl --context ${shq(context)} -n ${NAMESPACE} logs deploy/${RELEASE}-app --tail 50`)}`,
])
}
testSpin.stop('helm test passed')
p.note(
[
`open ${APP_SIGNUP_URL} (needs both forwards below)`,
`pods: kubectl --context ${shq(context)} -n ${NAMESPACE} get pods`,
`app logs: kubectl --context ${shq(context)} -n ${NAMESPACE} logs deploy/${RELEASE}-app --tail 50`,
// Both, always: the app alone loads but the editor's socket has nothing to
// reach, which is the reconnect failure this change set exists to fix.
...forwardCommands(context).map(
(command, index) => `${index === 0 ? 'forward: ' : ' '} ${command}`
),
`tear down: helm uninstall ${RELEASE} --kube-context ${shq(context)} -n ${NAMESPACE}`,
].join('\n'),
'Reach your cluster'
)
await offerPortForward(context)
}
/**
* Both forwards, in the order a user should run them. Kept in one place so the
* printed instructions and what the wizard actually runs cannot drift — the app
* alone leaves the editor's socket dead.
*/
export function forwardCommands(context: string): string[] {
const scope = `kubectl --context ${shq(context)} -n ${NAMESPACE} port-forward`
return [`${scope} svc/${RELEASE}-app 3000:3000`, `${scope} svc/${RELEASE}-realtime 3002:3002`]
}
/**
* The services are ClusterIP, so a healthy release is still unreachable from the
* host — "Sim is ready" with nothing on :3000 is the least satisfying way to end
* a setup. Offer the forward the same way dev mode offers to start the server,
* and run it in the foreground so Ctrl-C ends it.
*
* Realtime needs its own forward (one resource per invocation) or the editor's
* socket fails; it runs as a child in this process group, so the terminal's
* Ctrl-C reaches it too, and it is killed explicitly once the app forward exits.
*/
async function offerPortForward(context: string): Promise<void> {
const forward = await p.confirm({
message: `Port-forward now so you can open ${APP_SIGNUP_URL}?`,
initialValue: true,
})
if (!forward) {
p.log.info(
theme.muted(
`Skipped — run both forwards when you want to reach it (the second keeps the editor's socket alive):\n${forwardCommands(
context
)
.map((command) => ` ${command}`)
.join('\n')}`
)
)
return
}
const scope = ['--context', context, '-n', NAMESPACE]
const realtime = spawn(
'kubectl',
[...scope, 'port-forward', `svc/${RELEASE}-realtime`, '3002:3002'],
// stderr is kept so a failure can be explained; a silently dead second
// forward looks exactly like a working setup until the editor won't connect.
{ stdio: ['ignore', 'ignore', 'pipe'] }
)
let realtimeError = ''
realtime.stderr?.on('data', (chunk) => {
realtimeError += chunk
})
// Only an exit we did not ask for is a problem — the kill below also fires this.
let stopping = false
realtime.once('exit', (code) => {
if (stopping || code === 0) return
p.log.warn(
`The realtime forward (:3002) stopped — the editor's socket will not connect. ${
realtimeError.trim() || `Check that :3002 is free and svc/${RELEASE}-realtime exists.`
}`
)
})
p.log.step(`Forwarding ${APP_URL} (app) and :3002 (realtime) — Ctrl-C to stop`)
spawnSync('kubectl', [...scope, 'port-forward', `svc/${RELEASE}-app`, '3000:3000'], {
stdio: 'inherit',
})
stopping = true
realtime.kill()
}