improvement(setup): complete knowledge and update flows (#6521)

scripts/setup/capability-config.test.ts

@@ -10,6 +10,7 @@ import {
   defineCapabilitySetup,
   EMAIL_SETUP,
   getOAuthClientSetupFields,
+  KNOWLEDGE_EMBEDDINGS_SETUP,
   STORAGE_SETUP,
 } from './capability-config.ts'
 import { getCapabilitySetupOptions } from './capability-setup.ts'
@@ -76,6 +77,12 @@ describe('capability setup configuration', () => {
     )
   })
 
+  it('offers OpenAI first for fresh knowledge embedding setup', () => {
+    expect(
+      getCapabilitySetupOptions(KNOWLEDGE_EMBEDDINGS_SETUP).map((option) => option.id)
+    ).toEqual(['openai', 'azure-openai', 'openrouter'])
+  })
+
   it('maps every OAuth runtime field to a CLI input mode in runtime order', () => {
     for (const id of Object.keys(OAUTH_CLIENT_CAPABILITIES) as Array<
       keyof typeof OAUTH_CLIENT_CAPABILITIES

scripts/setup/capability-config.ts

@@ -823,7 +823,7 @@ export const KNOWLEDGE_EMBEDDINGS_SETUP = defineCapabilitySetup(KNOWLEDGE_EMBEDD
       ],
     },
   },
-  optionOrder: ['azure-openai', 'openai', 'openrouter'],
+  optionOrder: ['openai', 'azure-openai', 'openrouter'],
 })
 
 export const CAPABILITY_SETUPS = [

scripts/setup/capability-setup.ts

@@ -49,6 +49,8 @@ interface PromptState {
   values: Record<string, string>
 }
 
+const SKIP_OPTION_ID = '__skip-capability-setup__'
+
 /** Stages a capability transition into a larger setup run without losing prompt context. */
 export function stageCapabilitySetupTransition(
   currentValues: Map<string, string>,
@@ -176,11 +178,11 @@ export function getCapabilitySetupOptions(
   })
 }
 
-/** Resolves the setup option representing the effective current configuration. */
+/** Resolves the setup option representing the effective current configuration, if one exists. */
 export function resolveCurrentCapabilitySetupOptionId(
   setup: CapabilitySetupDefinition,
   values: EnvCapabilityValues
-): string {
+): string | undefined {
   const options = getCapabilitySetupOptions(setup)
   const explicitAction = options.find(
     (option) =>
@@ -214,9 +216,10 @@ export function resolveCurrentCapabilitySetupOptionId(
 
   const firstAction = options.find((option) => option.kind === 'action')
   if (firstAction) return firstAction.id
-  throw new Error(
-    `Capability ${setup.definition.id} has no setup option for its current configuration`
-  )
+  if (options.length === 0) {
+    throw new Error(`Capability ${setup.definition.id} has no setup options`)
+  }
+  return undefined
 }
 
 /** Applies selector and activation inference to the CLI-entered values. */
@@ -473,6 +476,21 @@ async function renderPrompts(prompts: readonly SetupPrompt[], state: PromptState
   }
 }
 
+async function promptSelectedCapabilitySetup(
+  setup: CapabilitySetupDefinition,
+  selected: ResolvedSetupOption,
+  currentValues: ReadonlyMap<string, string>
+): Promise<EnvCapabilitySetupTransition> {
+  const state: PromptState = {
+    setup,
+    optionId: selected.id,
+    currentValues,
+    values: {},
+  }
+  await renderPrompts(selected.prompts, state)
+  return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues)
+}
+
 /** Renders a CLI-owned capability setup and returns its validated environment transition. */
 export async function promptCapabilitySetup(
   setup: CapabilitySetupDefinition,
@@ -497,12 +515,44 @@ export async function promptCapabilitySetup(
     )
   }
 
-  const state: PromptState = {
-    setup,
-    optionId: selected.id,
-    currentValues,
-    values: {},
+  return promptSelectedCapabilitySetup(setup, selected, currentValues)
+}
+
+/** Offers capability providers while allowing the user to leave configuration unchanged. */
+export async function promptOptionalCapabilitySetup(
+  setup: CapabilitySetupDefinition,
+  currentValues: ReadonlyMap<string, string>,
+  context: CapabilitySetupContext,
+  skipHint: string
+): Promise<EnvCapabilitySetupTransition | null> {
+  const options = getCapabilitySetupOptions(setup)
+  if (options.some((option) => option.id === SKIP_OPTION_ID)) {
+    throw new Error(`Capability ${setup.definition.id} uses reserved option ${SKIP_OPTION_ID}`)
   }
-  await renderPrompts(selected.prompts, state)
-  return buildCapabilitySetupTransition(setup, selected.id, state.values, currentValues)
+  const currentOptionId = resolveCurrentCapabilitySetupOptionId(setup, currentValues)
+  const selectedOptionId = await p.select({
+    message: setup.message,
+    options: [
+      ...options.map((option) => ({
+        value: option.id,
+        label: option.label,
+        hint: markCurrentlyUsed(resolveHint(option.hint, context), option.id === currentOptionId),
+      })),
+      {
+        value: SKIP_OPTION_ID,
+        label: 'Not now',
+        hint: currentOptionId ? 'leave the current configuration unchanged' : skipHint,
+      },
+    ],
+    initialValue: currentOptionId ?? options[0]?.id,
+  })
+  if (selectedOptionId === SKIP_OPTION_ID) return null
+
+  const selected = options.find((option) => option.id === selectedOptionId)
+  if (!selected) {
+    throw new Error(
+      `Capability ${setup.definition.id} returned unknown setup option ${selectedOptionId}`
+    )
+  }
+  return promptSelectedCapabilitySetup(setup, selected, currentValues)
 }

scripts/setup/index.ts

@@ -18,6 +18,7 @@ const USAGE = `Usage:
   bun run sim setup <feature>                  configure one feature
   bun run sim doctor [--fix] [--json]          check your setup
   bun run sim start | stop | restart           bring your install up / down / cycle
+  bun run sim update                           pull/rebuild and apply Compose images
   bun run sim status                           what's installed and healthy
   bun run sim logs                             follow logs
   bun run sim down                             remove containers (data kept)

scripts/setup/lifecycle.test.ts

@@ -0,0 +1,14 @@
+import { describe, expect, it } from 'bun:test'
+import { getComposeUpdateMode, isLifecycleCommand } from './lifecycle.ts'
+
+describe('setup lifecycle', () => {
+  it('recognizes update as a lifecycle command', () => {
+    expect(isLifecycleCommand('update')).toBe(true)
+  })
+
+  it('pulls published installs and rebuilds source installs', () => {
+    expect(getComposeUpdateMode('/repo/docker-compose.prod.yml')).toBe('pull')
+    expect(getComposeUpdateMode('/repo/docker-compose.local.yml')).toBe('build')
+    expect(() => getComposeUpdateMode('/repo/compose.yml')).toThrow(/Unsupported Sim Compose file/)
+  })
+})

scripts/setup/lifecycle.ts

@@ -20,6 +20,7 @@ export const LIFECYCLE_COMMANDS = [
   'start',
   'stop',
   'restart',
+  'update',
   'status',
   'logs',
   'down',
@@ -325,6 +326,50 @@ function restart(install: Install): void {
   p.note(k8sReachHints(install.context), 'Kubernetes is managed with kubectl')
 }
 
+export type ComposeUpdateMode = 'pull' | 'build'
+
+/** Resolves how a setup-managed Compose install obtains its next image. */
+export function getComposeUpdateMode(file: string): ComposeUpdateMode {
+  const name = path.basename(file)
+  if (name === 'docker-compose.prod.yml') return 'pull'
+  if (name === 'docker-compose.local.yml') return 'build'
+  throw new Error(`Unsupported Sim Compose file: ${file}`)
+}
+
+function update(install: Install): void {
+  if (install.kind === 'dev') {
+    throw new SetupError('sim update is only available for Docker Compose installs.', [
+      'update the source checkout with git, run bun install, then restart bun run dev:full',
+    ])
+  }
+  if (install.kind === 'k8s') {
+    throw new SetupError('sim update does not upgrade Kubernetes releases.', [
+      'upgrade the release with helm after reviewing the chart and release notes',
+    ])
+  }
+
+  const mode = getComposeUpdateMode(install.file)
+  const spin = p.spinner()
+  if (mode === 'pull') {
+    spin.start('Pulling configured Sim images…')
+    dockerRun(composeArgs(install, 'pull'), 'docker compose pull failed', install.dir)
+  } else {
+    spin.start('Rebuilding Sim images with current base images…')
+    dockerRun(composeArgs(install, 'build', '--pull'), 'docker compose build failed', install.dir)
+  }
+  spin.message('Applying updated images and running migrations…')
+  dockerRun(composeArgs(install, 'up', '-d'), 'docker compose up failed', install.dir)
+  spin.stop('Sim updated (data volumes kept)')
+  p.note(
+    [
+      `version: ${theme.command(`SIM_VERSION in ${path.join(install.dir, '.env')}`)} (latest when unset)`,
+      `check:   ${theme.command('bun run sim status')}`,
+      `logs:    ${theme.command('bun run sim logs')}`,
+    ].join('\n'),
+    'Update complete'
+  )
+}
+
 function showLogs(install: Install): void {
   if (install.kind === 'compose') {
     dockerInherit(composeArgs(install, 'logs', '-f', '--tail', '100'), install.dir)
@@ -488,6 +533,8 @@ export async function runLifecycle(command: LifecycleCommand): Promise<void> {
       return stop(install)
     case 'restart':
       return restart(install)
+    case 'update':
+      return update(install)
     case 'logs':
       return showLogs(install)
     case 'down':

scripts/setup/modes/compose.ts

@@ -13,6 +13,7 @@ import {
   collectSecrets,
   mothershipOverride,
   promptCopilotKey,
+  promptKnowledgeEmbeddings,
   promptLlmKeys,
   promptSecurity,
   promptSignInProviders,
@@ -117,9 +118,13 @@ export async function runComposeMode(detection: Detection, quick: boolean): Prom
   if (copilotKey) values.COPILOT_API_KEY = copilotKey
   Object.assign(values, chatFlagValues(copilotKey))
   Object.assign(values, await promptLlmKeys(detection, !quick))
+  const stagedVars = new Map(root.vars)
+  for (const [key, value] of Object.entries(values)) stagedVars.set(key, value)
+  const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: true })
+  if (embeddings) {
+    stageCapabilitySetupTransition(stagedVars, values, remove, embeddings)
+  }
   if (!quick) {
-    const stagedVars = new Map(root.vars)
-    for (const [key, value] of Object.entries(values)) stagedVars.set(key, value)
     const storage = await promptCapabilitySetup(STORAGE_SETUP, stagedVars, {
       containerized: true,
     })

scripts/setup/modes/dev.ts

@@ -15,6 +15,7 @@ import {
   collectSecrets,
   mothershipOverride,
   promptCopilotKey,
+  promptKnowledgeEmbeddings,
   promptLlmKeys,
   promptSecurity,
   promptSignInProviders,
@@ -120,9 +121,14 @@ export async function runDevMode(
     writeEnvValues('realtime', { REDIS_URL: redisUrl })
   }
 
+  const stagedVars = new Map(simAfter.vars)
+  for (const [key, value] of Object.entries(values)) stagedVars.set(key, value)
+  const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: false })
+  if (embeddings) {
+    stageCapabilitySetupTransition(stagedVars, values, remove, embeddings)
+  }
+
   if (!quick) {
-    const stagedVars = new Map(simAfter.vars)
-    for (const [key, value] of Object.entries(values)) stagedVars.set(key, value)
     for (const setup of [JOBS_SETUP, STORAGE_SETUP, EMAIL_SETUP] as const) {
       const transition = await promptCapabilitySetup(setup, stagedVars, {
         containerized: false,

scripts/setup/modes/k8s.ts

@@ -1,12 +1,19 @@
 import { spawn, spawnSync } from 'node:child_process'
 import { getErrorMessage } from '@sim/utils/errors'
+import { KNOWLEDGE_EMBEDDINGS_SETUP } from '../capability-config.ts'
+import { getCapabilitySetupFields, stageCapabilitySetupTransition } from '../capability-setup.ts'
 import type { Detection } from '../detect.ts'
 import { ensureDocker } from '../docker.ts'
 import { generateSecret, ROOT } from '../env-files.ts'
 import { SetupError } from '../errors.ts'
 import { waitFor } from '../probes.ts'
 import * as p from '../prompter.ts'
-import { chatFlagValues, mothershipOverride, promptCopilotKey } from '../steps.ts'
+import {
+  chatFlagValues,
+  mothershipOverride,
+  promptCopilotKey,
+  promptKnowledgeEmbeddings,
+} from '../steps.ts'
 import { glyph, theme } from '../theme.ts'
 import { APP_SIGNUP_URL, APP_URL } from '../urls.ts'
 
@@ -362,6 +369,17 @@ export async function runK8sMode(detection: Detection): Promise<void> {
     ...(copilotKey ? { COPILOT_API_KEY: copilotKey } : {}),
     ...chatFlagValues(copilotKey),
   }
+  const stagedVars = new Map(Object.entries(releaseValues?.app?.env ?? {}))
+  for (const [key, value] of Object.entries(appEnv)) stagedVars.set(key, value)
+  for (const key of getCapabilitySetupFields(KNOWLEDGE_EMBEDDINGS_SETUP)) {
+    const existing = stagedVars.get(key)
+    if (existing) appEnv[key] = existing
+  }
+  const remove = new Set<string>()
+  const embeddings = await promptKnowledgeEmbeddings(stagedVars, { containerized: true })
+  if (embeddings) {
+    stageCapabilitySetupTransition(stagedVars, appEnv, remove, embeddings)
+  }
 
   const spin = p.spinner()
   spin.start('helm upgrade --install (first run pulls images — this can take several minutes)…')

scripts/setup/steps.test.ts

@@ -2,11 +2,12 @@ import { describe, expect, it } from 'bun:test'
 import {
   EMAIL_CAPABILITY,
   inspectCapability,
+  KNOWLEDGE_EMBEDDINGS_CAPABILITY,
   requireCapability,
   STORAGE_CAPABILITY,
   validateCapabilityFieldInput,
 } from '../../apps/sim/lib/core/config/env-capabilities.ts'
-import { EMAIL_SETUP, STORAGE_SETUP } from './capability-config.ts'
+import { EMAIL_SETUP, KNOWLEDGE_EMBEDDINGS_SETUP, STORAGE_SETUP } from './capability-config.ts'
 import {
   buildCapabilitySetupTransition,
   resolveCurrentCapabilitySetupOptionId,
@@ -41,6 +42,15 @@ describe('setup provider reconciliation', () => {
     ).toBe('smtp')
   })
 
+  it('allows setup when no knowledge embedding provider is configured', () => {
+    expect(resolveCurrentCapabilitySetupOptionId(KNOWLEDGE_EMBEDDINGS_SETUP, {})).toBeUndefined()
+    expect(
+      resolveCurrentCapabilitySetupOptionId(KNOWLEDGE_EMBEDDINGS_SETUP, {
+        OPENROUTER_API_KEY: 'openrouter-key',
+      })
+    ).toBe('openrouter')
+  })
+
   it('uses canonical storage selection for setup defaults', () => {
     expect(
       resolveCurrentCapabilitySetupOptionId(STORAGE_SETUP, new Map([['AWS_REGION', 'us-east-1']]))
@@ -79,6 +89,20 @@ describe('setup provider reconciliation', () => {
     expect(inspectCapability(EMAIL_CAPABILITY, reconciled).providerIds).toEqual(['resend', 'smtp'])
   })
 
+  it('configures a knowledge embedding provider from an empty state', () => {
+    const result = buildCapabilitySetupTransition(
+      KNOWLEDGE_EMBEDDINGS_SETUP,
+      'openrouter',
+      { OPENROUTER_API_KEY: 'openrouter-key' },
+      {}
+    )
+    const reconciled = applyResult({}, result)
+
+    expect(inspectCapability(KNOWLEDGE_EMBEDDINGS_CAPABILITY, reconciled).providerIds).toEqual([
+      'openrouter',
+    ])
+  })
+
   it('clears stale SMTP auth for an unauthenticated relay', () => {
     const result = buildCapabilitySetupTransition(
       EMAIL_SETUP,

scripts/setup/steps.ts

@@ -1,3 +1,9 @@
+import { KNOWLEDGE_EMBEDDINGS_SETUP } from './capability-config.ts'
+import {
+  type CapabilitySetupContext,
+  type EnvCapabilitySetupTransition,
+  promptOptionalCapabilitySetup,
+} from './capability-setup.ts'
 import { browserKeyFlow } from './cli-auth.ts'
 import type { Detection } from './detect.ts'
 import {
@@ -149,6 +155,19 @@ export async function promptLlmKeys(
   return values
 }
 
+/** Configures knowledge embeddings while allowing the user to explicitly defer them. */
+export function promptKnowledgeEmbeddings(
+  currentValues: ReadonlyMap<string, string>,
+  context: CapabilitySetupContext
+): Promise<EnvCapabilitySetupTransition | null> {
+  return promptOptionalCapabilitySetup(
+    KNOWLEDGE_EMBEDDINGS_SETUP,
+    currentValues,
+    context,
+    'knowledge-base indexing and semantic search will remain unavailable'
+  )
+}
+
 const PROVIDER_CONSOLES: Record<string, string> = {
   google: 'https://console.cloud.google.com/apis/credentials',
   github: 'https://github.com/settings/developers',

scripts/setup/wizard.ts

@@ -163,7 +163,7 @@ export async function runWizard(flags: WizardFlags): Promise<void> {
   p.note(
     [
       mode === 'k8s' ? `port-forward, then open ${APP_SIGNUP_URL}` : `open ${APP_SIGNUP_URL}`,
-      'manage it:  bun run sim start · stop · status · logs',
+      'manage it:  bun run sim start · stop · update · status · logs',
       'check your setup:  bun run sim doctor',
       mode === 'dev' && !startDevNow ? `start Sim:  bun run ${devScript}` : null,
       `prefer a bare "sim"? ${theme.command('bun link')} once (needs ~/.bun/bin on PATH)`,