perf(test): batch cli, tool, and misc dirs; weight shards by measured durations

Three probes proved cli/ (minus run/), tool/, and the small acp/lsp/mcp/
plugin/storage/... dirs isolation-safe in a shared process (1052 tests,
0 failures). Work items drop from 327 to 233.

Sharding now weights unbatched files by measured durations
(script/kilocode/test-durations.json, parsed from a full local run)
instead of byte size, so LPT packs shards evenly.

The batch builder now refuses to batch files with process-wide markers
(mock.module, AppRuntime.dispose, global fetch spies) so a future test
cannot silently poison a shared process.
This commit is contained in:
Yury Zialionka
2026-08-13 17:54:32 -06:00
parent 13a6b9d452
commit eed3a153e6
2 changed files with 399 additions and 4 deletions
@@ -0,0 +1,321 @@
{
"acp/config-option.test.ts": 1400,
"acp/content.test.ts": 1400,
"acp/directory.test.ts": 1400,
"acp/error.test.ts": 1200,
"acp/event.test.ts": 1500,
"acp/permission.test.ts": 1100,
"acp/service-session.test.ts": 1300,
"acp/session.test.ts": 1300,
"acp/tool.test.ts": 1200,
"acp/usage.test.ts": 1200,
"agent/agent.test.ts": 16000,
"agent/plan-mode-subagent-bypass.test.ts": 2900,
"agent/plugin-agent-regression.test.ts": 1800,
"auth/auth.test.ts": 1400,
"background/job.test.ts": 1700,
"bun/registry.test.ts": 1000,
"cli/account.test.ts": 1000,
"cli/acp/config-options.test.ts": 11500,
"cli/acp/initialize-auth.test.ts": 6100,
"cli/acp/lifecycle.test.ts": 15000,
"cli/acp/prompt-content.test.ts": 4800,
"cli/acp/skills.test.ts": 3400,
"cli/auto-mode.test.ts": 1100,
"cli/bin-kilo.test.ts": 1000,
"cli/cmd/tui/attention.test.ts": 1400,
"cli/effect-cmd-instance-als.test.ts": 3500,
"cli/error.test.ts": 1200,
"cli/github-action.test.ts": 1300,
"cli/github-remote.test.ts": 1300,
"cli/help/help-snapshots.test.ts": 8000,
"cli/import.test.ts": 1100,
"cli/mcp-add.test.ts": 5600,
"cli/plugin-auth-picker.test.ts": 1100,
"cli/pr.test.ts": 1000,
"cli/run/entry.body.test.ts": 1000,
"cli/run/footer.menu.test.ts": 1200,
"cli/run/footer.view.test.tsx": 2000,
"cli/run/footer.width.test.ts": 1000,
"cli/run/permission.shared.test.ts": 1100,
"cli/run/prompt.editor.test.ts": 1000,
"cli/run/prompt.shared.test.ts": 1200,
"cli/run/question.shared.test.ts": 1300,
"cli/run/run-process.test.ts": 169700,
"cli/run/runtime.boot.test.ts": 1400,
"cli/run/runtime.queue.test.ts": 1400,
"cli/run/runtime.stdin.test.ts": 1100,
"cli/run/runtime.test.ts": 1600,
"cli/run/scrollback.surface.test.ts": 2000,
"cli/run/session-data.test.ts": 1000,
"cli/run/session-replay.test.ts": 1200,
"cli/run/session.shared.test.ts": 1300,
"cli/run/stream.test.ts": 1000,
"cli/run/stream.transport.test.ts": 1800,
"cli/run/subagent-data.test.ts": 1000,
"cli/run/theme.test.ts": 1800,
"cli/run/variant.shared.test.ts": 1500,
"cli/serve/serve-process.test.ts": 4300,
"cli/smokes/read-only.test.ts": 12300,
"cli/tui/attach.test.ts": 1100,
"cli/tui/editor-context-zed.test.ts": 1300,
"cli/tui/editor-context.test.tsx": 1800,
"cli/tui/markdown.test.ts": 1100,
"cli/tui/plugin-add.test.ts": 2100,
"cli/tui/plugin-install.test.ts": 1900,
"cli/tui/plugin-lifecycle.test.ts": 2100,
"cli/tui/plugin-loader-entrypoint.test.ts": 2200,
"cli/tui/plugin-loader-pure.test.ts": 2100,
"cli/tui/plugin-loader.test.ts": 2200,
"cli/tui/plugin-toggle.test.ts": 2100,
"cli/tui/thread.test.ts": 5700,
"control-plane/adapters.test.ts": 1300,
"control-plane/workspace.test.ts": 22100,
"filesystem/filesystem.test.ts": 1400,
"fixture/fixture.test.ts": 1700,
"git/git.test.ts": 4900,
"ide/ide.test.ts": 1100,
"kilocode/acp/agent-interface.test.ts": 1200,
"kilocode/acp/permission.test.ts": 1100,
"kilocode/background-process-shutdown.test.ts": 1500,
"kilocode/background-process.test.ts": 46800,
"kilocode/cli-shutdown.test.ts": 400,
"kilocode/cli/cmd/console.test.ts": 1100,
"kilocode/cli/cmd/mcp.test.ts": 1000,
"kilocode/cli/cmd/remote.test.ts": 1200,
"kilocode/cli/cmd/run-message.test.ts": 1100,
"kilocode/cli/cmd/run-terminal.test.ts": 1100,
"kilocode/cli/cmd/run/interactive-terminal.test.ts": 1000,
"kilocode/cli/cmd/serve.test.ts": 3500,
"kilocode/cli/cmd/tui-worktree.test.ts": 2300,
"kilocode/cli/cmd/tui/attention.test.ts": 1400,
"kilocode/cli/cmd/tui/context/tui-config.test.ts": 1400,
"kilocode/cli/cmd/tui/cost-alert.test.ts": 1000,
"kilocode/cli/cmd/tui/memory-command.test.ts": 1600,
"kilocode/cli/cmd/tui/memory-status.test.tsx": 1400,
"kilocode/cli/cmd/tui/model-usage.test.ts": 1300,
"kilocode/cli/cmd/tui/prompt/vim.test.ts": 1000,
"kilocode/cli/cmd/tui/sidebar-usage-layout.test.tsx": 1200,
"kilocode/cli/dev-setup.test.ts": 1300,
"kilocode/cli/direct-mode-branding.test.ts": 1100,
"kilocode/cli/error.test.ts": 1100,
"kilocode/cli/install-artifact.test.ts": 2000,
"kilocode/cli/profile.test.ts": 1100,
"kilocode/cli/roll-call.test.ts": 2500,
"kilocode/cli/run/variant.test.ts": 1100,
"kilocode/cli/tui/remote-exit-bridge.test.ts": 1100,
"kilocode/cli/tui/remote-exit-worker.test.ts": 1000,
"kilocode/cli/tui/thread.test.ts": 7300,
"kilocode/cloud-session.test.ts": 1200,
"kilocode/command/override.test.ts": 1000,
"kilocode/commit-message/generate.test.ts": 1200,
"kilocode/commit-message/git-context.test.ts": 8500,
"kilocode/database/sqlite-error.test.ts": 1000,
"kilocode/event-service/client.test.ts": 1400,
"kilocode/indexing-startup.test.ts": 32400,
"kilocode/installation/upgrade.test.ts": 1300,
"kilocode/instance-vcs-watcher.test.ts": 3600,
"kilocode/kilo-sessions.test.ts": 36700,
"kilocode/lancedb-runtime.test.ts": 1100,
"kilocode/local-model.test.ts": 2000,
"kilocode/permission/agent-manager-prompt.test.ts": 1100,
"kilocode/permission/config-paths.test.ts": 1700,
"kilocode/permission/deny-provenance.test.ts": 1600,
"kilocode/permission/env-read.test.ts": 2300,
"kilocode/permission/external-directory-allow.test.ts": 8400,
"kilocode/permission/next.always-rules.test.ts": 15100,
"kilocode/permission/next.reply-http.test.ts": 4300,
"kilocode/permission/next.reply-routing.test.ts": 4200,
"kilocode/permission/permission-origins.test.ts": 1300,
"kilocode/permission/provenance.test.ts": 1000,
"kilocode/permission/skill-shell.test.ts": 4600,
"kilocode/run-auto.test.ts": 1300,
"kilocode/run-network.test.ts": 1700,
"kilocode/sandbox/config-network.test.ts": 1300,
"kilocode/sandbox/http-tools.test.ts": 1800,
"kilocode/sandbox/macos-confinement.test.ts": 11100,
"kilocode/sandbox/network.test.ts": 1200,
"kilocode/sandbox/policy.test.ts": 1700,
"kilocode/sandbox/sdk-config.test.ts": 1100,
"kilocode/sandbox/session-tools.test.ts": 4000,
"kilocode/sandbox/session.test.ts": 11400,
"kilocode/sandbox/shell-network.test.ts": 3900,
"kilocode/sandbox/state.test.ts": 5800,
"kilocode/sandbox/tui.test.ts": 1200,
"kilocode/server/agent-builder.test.ts": 4900,
"kilocode/server/cloud-session-import.test.ts": 7700,
"kilocode/server/commit-message-no-changes.test.ts": 2700,
"kilocode/server/config-model-state.test.ts": 2900,
"kilocode/server/config-overlay.test.ts": 60600,
"kilocode/server/config-rules.test.ts": 2600,
"kilocode/server/config-sources.test.ts": 3000,
"kilocode/server/fork-routing.test.ts": 1100,
"kilocode/server/httpapi-experimental.test.ts": 3200,
"kilocode/server/httpapi-global-sse.test.ts": 1500,
"kilocode/server/httpapi-kilo-edit.test.ts": 5800,
"kilocode/server/httpapi-memory.test.ts": 4500,
"kilocode/server/httpapi-public.test.ts": 2700,
"kilocode/server/kilo-gateway-statuses.test.ts": 1600,
"kilocode/server/listener-runtime.test.ts": 3300,
"kilocode/server/permission-allow-everything.test.ts": 4800,
"kilocode/server/prompt-training-model-filter.test.ts": 2900,
"kilocode/server/provider-auth-lifecycle.test.ts": 1100,
"kilocode/server/tui-config.test.ts": 5200,
"kilocode/server/worktree-list.test.ts": 3000,
"kilocode/session-compaction-chunks.test.ts": 3000,
"kilocode/session-fork-remap.test.ts": 9700,
"kilocode/session-processor-retry-limit.test.ts": 4900,
"kilocode/session-prompt-queue.test.ts": 31100,
"kilocode/session-prompt-steering.test.ts": 7200,
"kilocode/session-share.test.ts": 1500,
"kilocode/shell/shell.test.ts": 1200,
"kilocode/skill/discovery-validate.test.ts": 1100,
"kilocode/telemetry/feedback.test.ts": 1100,
"lsp/client.test.ts": 4100,
"lsp/index.test.ts": 3000,
"lsp/jdtls-root.test.ts": 1200,
"lsp/launch.test.ts": 1200,
"lsp/lifecycle.test.ts": 2700,
"mcp/auth.test.ts": 1200,
"mcp/catalog.test.ts": 1100,
"mcp/headers.test.ts": 2300,
"mcp/lifecycle.test.ts": 7500,
"mcp/oauth-auto-connect.test.ts": 3200,
"mcp/oauth-callback.test.ts": 1200,
"mcp/oauth-provider.test.ts": 1200,
"mcp/session-recovery.test.ts": 1300,
"mcp/transport.test.ts": 1000,
"permission-task.test.ts": 7200,
"permission/arity.test.ts": 1000,
"permission/next.test.ts": 12300,
"permission/next.toConfig.test.ts": 1300,
"plugin/auth-override.test.ts": 3000,
"plugin/cloudflare.test.ts": 1000,
"plugin/codex.test.ts": 1000,
"plugin/github-copilot-models.test.ts": 1400,
"plugin/install-concurrency.test.ts": 4600,
"plugin/install.test.ts": 1500,
"plugin/loader-shared.test.ts": 2600,
"plugin/meta.test.ts": 3400,
"plugin/modal-models.test.ts": 1100,
"plugin/openai-rollout.test.ts": 1000,
"plugin/openai-ws.test.ts": 3600,
"plugin/shared.test.ts": 1000,
"plugin/snowflake-cortex.test.ts": 1300,
"plugin/trigger.test.ts": 1600,
"plugin/workspace-adapter.test.ts": 1800,
"plugin/xai.test.ts": 1600,
"project/instance-bootstrap.test.ts": 6500,
"project/instance.test.ts": 9300,
"project/migrate-global.test.ts": 3600,
"project/project-directory.test.ts": 5600,
"project/project.test.ts": 20900,
"project/vcs.test.ts": 11800,
"project/worktree-remove.test.ts": 4100,
"project/worktree.test.ts": 25000,
"provider/amazon-bedrock.test.ts": 9000,
"provider/cf-ai-gateway-e2e.test.ts": 1100,
"provider/digitalocean.test.ts": 5000,
"provider/gitlab-duo.test.ts": 1000,
"provider/header-timeout.test.ts": 6600,
"provider/provider.test.ts": 52000,
"server/auth.test.ts": 1200,
"server/experimental-session-list.test.ts": 13700,
"server/global-session-list.test.ts": 3300,
"server/httpapi-authorization.test.ts": 1400,
"server/httpapi-compression.test.ts": 3300,
"server/httpapi-config.test.ts": 2300,
"server/httpapi-control-plane.test.ts": 1300,
"server/httpapi-cors-vary.test.ts": 2200,
"server/httpapi-cors.test.ts": 2900,
"server/httpapi-error-middleware.test.ts": 1300,
"server/httpapi-event.test.ts": 8500,
"server/httpapi-experimental.test.ts": 10300,
"server/httpapi-file.test.ts": 4600,
"server/httpapi-global.test.ts": 1200,
"server/httpapi-instance-context.test.ts": 12300,
"server/httpapi-instance-route-auth.test.ts": 4400,
"server/httpapi-instance.test.ts": 13900,
"server/httpapi-listen.test.ts": 22600,
"server/httpapi-mcp-oauth.test.ts": 1100,
"server/httpapi-mcp.test.ts": 3500,
"server/httpapi-mdns.test.ts": 2100,
"server/httpapi-promptasync-context.test.ts": 5200,
"server/httpapi-provider.test.ts": 4700,
"server/httpapi-pty.test.ts": 21700,
"server/httpapi-public-openapi.test.ts": 2000,
"server/httpapi-query-schema-drift.test.ts": 2500,
"server/httpapi-reference.test.ts": 3400,
"server/httpapi-schema-error-body.test.ts": 7800,
"server/httpapi-sdk.test.ts": 18600,
"server/httpapi-session.test.ts": 31800,
"server/httpapi-sync.test.ts": 4600,
"server/httpapi-ui.test.ts": 1500,
"server/httpapi-v2-location.test.ts": 3700,
"server/httpapi-v2-pty.test.ts": 12000,
"server/httpapi-workspace-routing.test.ts": 6100,
"server/httpapi-workspace.test.ts": 16100,
"server/negative-tokens-regression.test.ts": 2700,
"server/project-copy.test.ts": 4600,
"server/project-init-git.test.ts": 4400,
"server/proxy-util.test.ts": 1100,
"server/sdk-error-shape.test.ts": 2700,
"server/sdk-v1-smoke.test.ts": 5600,
"server/session-actions.test.ts": 5800,
"server/session-diff-missing-patch.test.ts": 4300,
"server/session-list.test.ts": 10000,
"server/session-messages.test.ts": 10500,
"server/session-select.test.ts": 5600,
"server/workspace-proxy.test.ts": 1500,
"server/workspace-routing.test.ts": 1200,
"server/worktree-endpoint-repro.test.ts": 22300,
"session/compaction.test.ts": 29400,
"session/instruction.test.ts": 1700,
"session/llm-native-recorded.test.ts": 3200,
"session/llm-native.test.ts": 1700,
"session/llm.test.ts": 6600,
"session/message-v2.test.ts": 1400,
"session/messages-pagination.test.ts": 2700,
"session/network.test.ts": 6300,
"session/processor-effect.test.ts": 22000,
"session/prompt.test.ts": 104600,
"session/retry.test.ts": 9400,
"session/revert-compact.test.ts": 19600,
"session/schema-decoding.test.ts": 1300,
"session/session-schema.test.ts": 1000,
"session/session.test.ts": 2600,
"session/shell-v2.test.ts": 1200,
"session/snapshot-tool-race.test.ts": 8600,
"session/structured-output-integration.test.ts": 1300,
"session/structured-output.test.ts": 1300,
"session/system.test.ts": 1100,
"skill/discovery.test.ts": 1400,
"skill/skill.test.ts": 18800,
"snapshot/snapshot.test.ts": 139700,
"storage/db.test.ts": 1100,
"storage/storage.test.ts": 2200,
"tool/apply_patch.test.ts": 8300,
"tool/code-mode-integration.test.ts": 1600,
"tool/code-mode.test.ts": 2100,
"tool/diagnostics-filter.test.ts": 1200,
"tool/edit.test.ts": 7800,
"tool/external-directory.test.ts": 1500,
"tool/glob.test.ts": 2000,
"tool/grep.test.ts": 2200,
"tool/lsp.test.ts": 9300,
"tool/parameters.test.ts": 1300,
"tool/question.test.ts": 2400,
"tool/read.test.ts": 12900,
"tool/recall.test.ts": 6400,
"tool/registry.test.ts": 5300,
"tool/shell.test.ts": 12700,
"tool/skill.test.ts": 5800,
"tool/suggest.test.ts": 1000,
"tool/task.test.ts": 12400,
"tool/tool-define.test.ts": 1400,
"tool/truncation.test.ts": 2500,
"tool/webfetch.test.ts": 2700,
"tool/websearch.test.ts": 1200,
"tool/write.test.ts": 6700,
"v2/session-message-updater.test.ts": 1100
}
+78 -4
View File
@@ -206,7 +206,16 @@ const DURATION_HINTS: Record<string, number> = {
"kilocode/daemon.test.ts": 65_000,
"tool/task.test.ts": 64_000,
}
const weight = (file: string) => DURATION_HINTS[file] ?? Bun.file(path.join(root, "test", file)).size
// Measured per-file durations (ms) from a full local run; regenerate by parsing the
// runner's PASS/FAIL lines. Relative order is what LPT needs, so a macOS measurement
// balances Windows shards fine. Hints above still win: they are Windows-observed maxima.
const measuredDurations: Record<string, number> = await Bun.file(
path.join(root, "script", "kilocode", "test-durations.json"),
)
.json()
.catch(() => ({}))
const weight = (file: string) =>
DURATION_HINTS[file] ?? measuredDurations[file] ?? Bun.file(path.join(root, "test", file)).size
// kilocode_change end
// kilocode_change start - fast tier: run isolation-safe test files in ONE shared `bun test`
@@ -217,9 +226,9 @@ const weight = (file: string) => DURATION_HINTS[file] ?? Bun.file(path.join(root
// The batch enters shard splitting as one pseudo-file with a duration-scale weight, so LPT
// places it like any other heavy file and exactly one shard executes it. Extend the list only
// with files proven to pass in a shared process (run: bun test <dirs> locally).
// Deliberately excludes directories whose runtime is real I/O work (git/, lsp/, storage/,
// auth/, ...): those parallelize well across the per-file worker pool, while a batch runs
// its members sequentially. The tier is for files where boot cost dominates execution.
// Deliberately excludes directories whose runtime is real I/O work (git/, server/, session/,
// project/, provider/, ...): those parallelize well across the per-file worker pool, while a
// batch runs its members sequentially. The tier is for files where boot cost dominates.
// Several independent batches (instead of one) keep each batch short enough to schedule
// like a normal heavy file, and let LPT spread them across shards and worker slots.
// Weights are observed single-process durations (ms), same unit as DURATION_HINTS.
@@ -268,6 +277,48 @@ const FAST_TIERS: Record<string, { weight: number; entries: string[] }> = {
weight: 50_000,
entries: ["kilocode/anaconda-desktop/", "kilocode/cloud/", "kilocode/tool/"],
},
// cli/run/ stays per-file: it spawns real non-interactive runs (SIGINT/daemon timing).
"fast-tier-cli": {
weight: 90_000,
entries: [
"cli/account.test.ts",
"cli/auto-mode.test.ts",
"cli/bin-kilo.test.ts",
"cli/effect-cmd-instance-als.test.ts",
"cli/error.test.ts",
"cli/github-action.test.ts",
"cli/github-remote.test.ts",
"cli/import.test.ts",
"cli/mcp-add.test.ts",
"cli/plugin-auth-picker.test.ts",
"cli/pr.test.ts",
"cli/acp/",
"cli/cmd/",
"cli/help/",
"cli/serve/",
"cli/smokes/",
"cli/tui/",
],
},
"fast-tier-misc": {
weight: 30_000,
entries: [
"acp/",
"auth/",
"bun/",
"filesystem/",
"ide/",
"lsp/",
"mcp/",
"plugin/",
"storage/",
"v2/",
],
},
"fast-tier-tool": {
weight: 55_000,
entries: ["tool/"],
},
}
// Top-level kilocode/*.test.ts files batch too, except files that mutate process-wide
// state and so can only run alone: bun's mock.module is process-wide and permanent,
@@ -325,6 +376,29 @@ if (patterns.length === 0 && !profile) {
batches.set(tier, members)
}
for (const [name, members] of batches) if (members.length < 2) batches.delete(name)
// A batch shares one process, so process-wide mutations poison every later file in it.
// Refuse to batch files with the known-unsafe markers instead of failing mysteriously
// later: bun's mock.module is process-wide and permanent, AppRuntime.dispose() kills the
// shared runtime, and global-fetch spies observe batch-mates' traffic.
const unsafe = [/\bmock\.module\s*\(/, /\bAppRuntime\.dispose\s*\(/, /\bspyOn\s*\(\s*globalThis\b/]
const violations: string[] = []
for (const members of batches.values()) {
for (const member of members) {
const source = await Bun.file(path.join(root, "test", member)).text()
if (unsafe.some((pattern) => pattern.test(source))) violations.push(member)
}
}
if (violations.length > 0) {
console.error(
[
"These test files use process-wide mocks/disposal/spies and cannot run in a shared batch process.",
"Exclude them from batching (KILOCODE_ROOT_EXCLUDES or the FAST_TIERS entries in this script):",
...violations.map((file) => `- ${file}`),
].join("\n"),
)
process.exit(2)
}
}
const inBatch = (file: string) => batches.size > 0 && tierOf(file) !== undefined && batches.has(tierOf(file)!)
const shardInput = [...batches.keys(), ...candidates.filter((file) => !inBatch(file))]