mirror of
https://github.com/Kilo-Org/kilocode.git
synced 2026-09-01 15:32:11 +08:00
perf(cli): seed Agent Manager snapshots from worktree index
This commit is contained in:
marius-kilocode
@@ -0,0 +1,5 @@
|
||||
---
|
||||
"@kilocode/cli": patch
|
||||
---
|
||||
|
||||
Speed up the first Agent Manager prompt in new worktrees by seeding snapshots from the checkout's Git index.
|
||||
@@ -0,0 +1,171 @@
|
||||
import { Effect, Stream } from "effect"
|
||||
import { ChildProcess } from "effect/unstable/process"
|
||||
import path from "path"
|
||||
import { AppFileSystem } from "@opencode-ai/core/filesystem"
|
||||
import * as Log from "@opencode-ai/core/util/log"
|
||||
|
||||
export namespace KiloSnapshotSeed {
|
||||
const log = Log.create({ service: "snapshot.seed" })
|
||||
|
||||
interface Result {
|
||||
readonly code: number
|
||||
readonly text: string
|
||||
readonly stderr: string
|
||||
}
|
||||
|
||||
type Git = (
|
||||
cmd: string[],
|
||||
opts?: { cwd?: string; env?: Record<string, string>; stdin?: ChildProcess.CommandInput },
|
||||
) => Effect.Effect<Result>
|
||||
|
||||
export interface Input {
|
||||
readonly dir: string
|
||||
readonly worktree: string
|
||||
readonly gitdir: string
|
||||
readonly limit: number
|
||||
readonly git: Git
|
||||
readonly fs: AppFileSystem.Interface
|
||||
}
|
||||
|
||||
export interface Output {
|
||||
readonly seeded: boolean
|
||||
readonly paths: number
|
||||
readonly dropped: number
|
||||
readonly reason?: string
|
||||
}
|
||||
|
||||
const list = (text: string) => text.split("\0").filter(Boolean)
|
||||
const feed = (items: string[]) => Stream.make(new TextEncoder().encode(items.join("\0") + "\0"))
|
||||
const snap = (input: Input, cmd: string[]) => ["--git-dir", input.gitdir, "--work-tree", input.worktree, ...cmd]
|
||||
// Match the existing snapshot add() stat fanout so seeding has the same filesystem pressure.
|
||||
const concurrency = 8
|
||||
|
||||
export const seed = Effect.fnUntraced(function* (input: Input) {
|
||||
const started = Date.now()
|
||||
const alt = path.join(input.gitdir, "objects", "info", "alternates")
|
||||
const changed = { value: false }
|
||||
const reset = Effect.fnUntraced(function* (reason: string, warn = false) {
|
||||
if (changed.value) {
|
||||
const cleared = yield* input.git(snap(input, ["read-tree", "--empty"]), { cwd: input.dir })
|
||||
if (cleared.code !== 0) {
|
||||
yield* input.fs.remove(path.join(input.gitdir, "index")).pipe(Effect.catch(() => Effect.void))
|
||||
}
|
||||
yield* input.fs.remove(path.join(input.gitdir, "index.lock")).pipe(Effect.catch(() => Effect.void))
|
||||
yield* input.fs.remove(alt).pipe(Effect.catch(() => Effect.void))
|
||||
}
|
||||
const fields = { reason, duration: Date.now() - started }
|
||||
if (warn) log.warn("snapshot seed failed; using cold initialization", fields)
|
||||
if (!warn) log.info("snapshot seed skipped", fields)
|
||||
return { seeded: false, paths: 0, dropped: 0, reason } satisfies Output
|
||||
})
|
||||
|
||||
const attempt = Effect.gen(function* () {
|
||||
if (path.resolve(input.dir) !== path.resolve(input.worktree)) return yield* reset("subdirectory")
|
||||
|
||||
const sparse = yield* input.git(["-C", input.worktree, "config", "--bool", "core.sparseCheckout"])
|
||||
if (sparse.code === 0 && sparse.text.trim() === "true") return yield* reset("sparse-checkout")
|
||||
|
||||
const unmerged = yield* input.git(["-C", input.worktree, "ls-files", "--unmerged", "-z"])
|
||||
if (unmerged.code !== 0) return yield* reset("unmerged-check-failed", true)
|
||||
if (unmerged.text) return yield* reset("unmerged-index")
|
||||
|
||||
const [src, root, idx, fmt, dst] = yield* Effect.all(
|
||||
[
|
||||
input.git(["-C", input.worktree, "rev-parse", "--path-format=absolute", "--git-dir"]),
|
||||
input.git(["-C", input.worktree, "rev-parse", "--path-format=absolute", "--git-common-dir"]),
|
||||
input.git(["-C", input.worktree, "rev-parse", "--path-format=absolute", "--git-path", "index"]),
|
||||
input.git(["-C", input.worktree, "rev-parse", "--show-object-format"]),
|
||||
input.git(["--git-dir", input.gitdir, "rev-parse", "--show-object-format"]),
|
||||
],
|
||||
{ concurrency: 5 },
|
||||
)
|
||||
if ([src, root, idx, fmt, dst].some((item) => item.code !== 0)) {
|
||||
return yield* reset("metadata", true)
|
||||
}
|
||||
if (fmt.text.trim() !== dst.text.trim()) return yield* reset("object-format")
|
||||
|
||||
const source = src.text.trim()
|
||||
const common = root.text.trim()
|
||||
const index = idx.text.trim()
|
||||
if (!source || !common || !index || !(yield* input.fs.exists(index))) {
|
||||
return yield* reset("source-index")
|
||||
}
|
||||
|
||||
const objects = path.join(common, "objects")
|
||||
if (!(yield* input.fs.exists(objects))) return yield* reset("source-objects")
|
||||
// Borrow committed objects to keep the first turn fast. Durable materialization can happen off this critical path.
|
||||
yield* input.fs.ensureDir(path.dirname(alt))
|
||||
changed.value = true
|
||||
yield* input.fs.writeFileString(alt, `${objects}\n`)
|
||||
|
||||
const tree = yield* input.git(["write-tree"], {
|
||||
cwd: input.dir,
|
||||
env: {
|
||||
GIT_DIR: source,
|
||||
GIT_WORK_TREE: input.worktree,
|
||||
GIT_INDEX_FILE: index,
|
||||
GIT_OBJECT_DIRECTORY: path.join(input.gitdir, "objects"),
|
||||
GIT_ALTERNATE_OBJECT_DIRECTORIES: objects,
|
||||
},
|
||||
})
|
||||
if (tree.code !== 0 || !tree.text.trim()) return yield* reset("write-tree", true)
|
||||
|
||||
const read = yield* input.git(snap(input, ["read-tree", tree.text.trim()]), { cwd: input.dir })
|
||||
if (read.code !== 0) return yield* reset("read-tree", true)
|
||||
|
||||
const tracked = yield* input.git(snap(input, ["ls-files", "-z", "--", "."]), { cwd: input.dir })
|
||||
if (tracked.code !== 0) return yield* reset("list", true)
|
||||
const files = list(tracked.text)
|
||||
if (!files.length) {
|
||||
log.info("snapshot seed complete", { paths: 0, dropped: 0, duration: Date.now() - started })
|
||||
return { seeded: true, paths: 0, dropped: 0 } satisfies Output
|
||||
}
|
||||
|
||||
const ignored = yield* input.git(["-C", input.worktree, "check-ignore", "--no-index", "--stdin", "-z"], {
|
||||
stdin: feed(files),
|
||||
})
|
||||
if (ignored.code !== 0 && ignored.code !== 1) return yield* reset("ignore", true)
|
||||
|
||||
const large = (yield* Effect.all(
|
||||
files.map((file) =>
|
||||
input.fs
|
||||
.stat(path.join(input.dir, file))
|
||||
.pipe(Effect.catch(() => Effect.void))
|
||||
.pipe(
|
||||
Effect.map((info) => {
|
||||
if (!info || info.type !== "File") return
|
||||
const size = typeof info.size === "bigint" ? Number(info.size) : info.size
|
||||
return size > input.limit ? file : undefined
|
||||
}),
|
||||
),
|
||||
),
|
||||
{ concurrency },
|
||||
)).filter((file): file is string => Boolean(file))
|
||||
|
||||
const dropped = Array.from(new Set([...list(ignored.text), ...large]))
|
||||
if (dropped.length) {
|
||||
const result = yield* input.git(
|
||||
snap(input, ["rm", "--cached", "-f", "--ignore-unmatch", "--pathspec-from-file=-", "--pathspec-file-nul"]),
|
||||
{ cwd: input.dir, stdin: feed(dropped) },
|
||||
)
|
||||
if (result.code !== 0) return yield* reset("drop", true)
|
||||
}
|
||||
|
||||
log.info("snapshot seed complete", {
|
||||
paths: files.length,
|
||||
dropped: dropped.length,
|
||||
ignored: list(ignored.text).length,
|
||||
large: large.length,
|
||||
duration: Date.now() - started,
|
||||
})
|
||||
return { seeded: true, paths: files.length, dropped: dropped.length } satisfies Output
|
||||
})
|
||||
|
||||
return yield* attempt.pipe(
|
||||
Effect.catch((err) => {
|
||||
log.warn("snapshot seed failed", { err })
|
||||
return reset("error", true)
|
||||
}),
|
||||
)
|
||||
})
|
||||
}
|
||||
@@ -12,6 +12,7 @@ import * as Log from "@opencode-ai/core/util/log"
|
||||
import { Flag } from "@opencode-ai/core/flag/flag" // kilocode_change
|
||||
import { DiffFull } from "../kilocode/snapshot/diff-full" // kilocode_change
|
||||
import { KiloSnapshotTrack } from "../kilocode/snapshot/track" // kilocode_change
|
||||
import { KiloSnapshotSeed } from "../kilocode/snapshot/seed" // kilocode_change
|
||||
import type { MessageID, SessionID } from "../session/schema" // kilocode_change
|
||||
import { withStatics } from "@opencode-ai/core/schema" // kilocode_change
|
||||
import { zod } from "@opencode-ai/core/effect-zod" // kilocode_change
|
||||
@@ -300,7 +301,7 @@ export const layer: Layer.Layer<Service, never, AppFileSystem.Service | AppProce
|
||||
)
|
||||
})
|
||||
|
||||
const track = Effect.fnUntraced(function* () {
|
||||
const track = Effect.fnUntraced(function* (opts?: Parameters<Interface["track"]>[0]) { // kilocode_change
|
||||
return yield* locked(
|
||||
Effect.gen(function* () {
|
||||
if (!(yield* enabled())) return
|
||||
@@ -314,6 +315,18 @@ export const layer: Layer.Layer<Service, never, AppFileSystem.Service | AppProce
|
||||
yield* git(["--git-dir", state.gitdir, "config", "core.longpaths", "true"])
|
||||
yield* git(["--git-dir", state.gitdir, "config", "core.symlinks", "true"])
|
||||
yield* git(["--git-dir", state.gitdir, "config", "core.fsmonitor", "false"])
|
||||
// kilocode_change start - seed new Agent Manager snapshots from the worktree index
|
||||
if (opts?.snapshotInitialization === "wait") {
|
||||
yield* KiloSnapshotSeed.seed({
|
||||
dir: state.directory,
|
||||
worktree: state.worktree,
|
||||
gitdir: state.gitdir,
|
||||
limit,
|
||||
git,
|
||||
fs,
|
||||
})
|
||||
}
|
||||
// kilocode_change end
|
||||
log.info("initialized")
|
||||
}
|
||||
yield* add()
|
||||
@@ -786,7 +799,7 @@ export const layer: Layer.Layer<Service, never, AppFileSystem.Service | AppProce
|
||||
// kilocode_change start - guard slow snapshots and surface progress to the active session
|
||||
track: Effect.fn("Snapshot.track")(function* (opts) {
|
||||
return yield* KiloSnapshotTrack.wrap({
|
||||
inner: InstanceState.useEffect(state, (s) => s.track()),
|
||||
inner: InstanceState.useEffect(state, (s) => s.track(opts)),
|
||||
state: trackState,
|
||||
snapshotInitialization: opts?.snapshotInitialization,
|
||||
sessionID: opts?.sessionID,
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
import { afterEach, expect, test } from "bun:test"
|
||||
import { $ } from "bun"
|
||||
import fs from "fs/promises"
|
||||
import path from "path"
|
||||
import { Effect } from "effect"
|
||||
import { Global } from "@opencode-ai/core/global"
|
||||
import { Hash } from "@opencode-ai/core/util/hash"
|
||||
import { Snapshot } from "../../src/snapshot"
|
||||
import { Instance } from "../../src/project/instance"
|
||||
import { Filesystem } from "../../src/util/filesystem"
|
||||
import { disposeAllInstances, provideInstance, tmpdir } from "../fixture/fixture"
|
||||
|
||||
const fwd = (...parts: string[]) => path.join(...parts).replaceAll("\\", "/")
|
||||
|
||||
function run<A>(dir: string, body: (snapshot: Snapshot.Interface) => Effect.Effect<A>) {
|
||||
return Effect.runPromise(
|
||||
Effect.gen(function* () {
|
||||
const snapshot = yield* Snapshot.Service
|
||||
const value = yield* body(snapshot)
|
||||
const gitdir = path.join(Global.Path.data, "snapshot", Instance.project.id, Hash.fast(Instance.worktree))
|
||||
return { value, gitdir }
|
||||
}).pipe(provideInstance(dir), Effect.provide(Snapshot.defaultLayer)),
|
||||
)
|
||||
}
|
||||
|
||||
async function setup(dir: string) {
|
||||
await $`git config filter.snapshot-test.clean "tr a-z A-Z"`.cwd(dir).quiet()
|
||||
await $`git config filter.snapshot-test.smudge cat`.cwd(dir).quiet()
|
||||
await $`git config filter.snapshot-test.required true`.cwd(dir).quiet()
|
||||
await Filesystem.write(path.join(dir, "dirty.txt"), "committed dirty\n")
|
||||
await Filesystem.write(path.join(dir, "staged.txt"), "committed staged\n")
|
||||
await Filesystem.write(path.join(dir, "deleted.txt"), "committed deleted\n")
|
||||
await Filesystem.write(path.join(dir, "tracked.log"), "tracked but ignored\n")
|
||||
await Filesystem.write(path.join(dir, "filtered.flt"), "committed filtered\n")
|
||||
await Filesystem.write(path.join(dir, "script.sh"), "#!/bin/sh\nexit 0\n")
|
||||
await Filesystem.write(path.join(dir, "huge.bin"), new Uint8Array(2 * 1024 * 1024 + 1))
|
||||
await Filesystem.write(path.join(dir, ".gitattributes"), "*.flt filter=snapshot-test\n")
|
||||
await $`git add .`.cwd(dir).quiet()
|
||||
await $`git commit -m baseline`.cwd(dir).quiet()
|
||||
await Filesystem.write(path.join(dir, ".gitignore"), "*.log\n")
|
||||
await $`git add .gitignore`.cwd(dir).quiet()
|
||||
await $`git commit -m ignore`.cwd(dir).quiet()
|
||||
}
|
||||
|
||||
async function dirty(dir: string) {
|
||||
await Filesystem.write(path.join(dir, "dirty.txt"), "user dirty\n")
|
||||
await Filesystem.write(path.join(dir, "staged.txt"), "user staged\n")
|
||||
await $`git add staged.txt`.cwd(dir).quiet()
|
||||
await fs.rm(path.join(dir, "deleted.txt"))
|
||||
await Filesystem.write(path.join(dir, "untracked.txt"), "user untracked\n")
|
||||
await Filesystem.write(path.join(dir, "filtered.flt"), "user filtered\n")
|
||||
await Filesystem.write(path.join(dir, "debug.log"), "ignored untracked\n")
|
||||
if (process.platform !== "win32") await fs.chmod(path.join(dir, "script.sh"), 0o755)
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await disposeAllInstances()
|
||||
})
|
||||
|
||||
test("Agent Manager cold seed matches full snapshot and preserves first-turn reset", async () => {
|
||||
await using source = await tmpdir({
|
||||
git: true,
|
||||
init: setup,
|
||||
})
|
||||
await using root = await tmpdir()
|
||||
const seeded = path.join(root.path, "seeded")
|
||||
await $`git worktree add --quiet -b snapshot-seed-test ${seeded} HEAD`.cwd(source.path)
|
||||
|
||||
await dirty(source.path)
|
||||
await dirty(seeded)
|
||||
|
||||
const cold = await run(source.path, (snapshot) => snapshot.track())
|
||||
const fast = await run(seeded, (snapshot) => snapshot.track({ snapshotInitialization: "wait" }))
|
||||
|
||||
expect(cold.value).toBeTruthy()
|
||||
expect(fast.value).toBe(cold.value)
|
||||
await expect(fs.access(path.join(cold.gitdir, "objects", "info", "alternates"))).rejects.toThrow()
|
||||
const common = (await $`git rev-parse --path-format=absolute --git-common-dir`.cwd(seeded).text()).trim()
|
||||
expect((await fs.readFile(path.join(fast.gitdir, "objects", "info", "alternates"), "utf8")).trim()).toBe(
|
||||
path.join(common, "objects"),
|
||||
)
|
||||
|
||||
expect((await run(seeded, (snapshot) => snapshot.patch(fast.value!))).value.files).toEqual([])
|
||||
|
||||
await Filesystem.write(path.join(seeded, "dirty.txt"), "assistant dirty\n")
|
||||
await Filesystem.write(path.join(seeded, "untracked.txt"), "assistant untracked\n")
|
||||
await Filesystem.write(path.join(seeded, "created.txt"), "assistant created\n")
|
||||
const patch = (await run(seeded, (snapshot) => snapshot.patch(fast.value!))).value
|
||||
expect(patch.files).toEqual(
|
||||
expect.arrayContaining([fwd(seeded, "dirty.txt"), fwd(seeded, "untracked.txt"), fwd(seeded, "created.txt")]),
|
||||
)
|
||||
|
||||
await run(seeded, (snapshot) => snapshot.revert([patch]))
|
||||
expect(await fs.readFile(path.join(seeded, "dirty.txt"), "utf8")).toBe("user dirty\n")
|
||||
expect(await fs.readFile(path.join(seeded, "untracked.txt"), "utf8")).toBe("user untracked\n")
|
||||
await expect(fs.access(path.join(seeded, "created.txt"))).rejects.toThrow()
|
||||
await expect(fs.access(path.join(seeded, "deleted.txt"))).rejects.toThrow()
|
||||
})
|
||||
|
||||
test("Agent Manager seed falls back for sparse checkouts", async () => {
|
||||
await using tmp = await tmpdir({
|
||||
git: true,
|
||||
init: async (dir) => {
|
||||
await Filesystem.write(path.join(dir, "tracked.txt"), "tracked\n")
|
||||
await $`git add tracked.txt`.cwd(dir).quiet()
|
||||
await $`git commit -m tracked`.cwd(dir).quiet()
|
||||
await $`git config core.sparseCheckout true`.cwd(dir).quiet()
|
||||
},
|
||||
})
|
||||
|
||||
const result = await run(tmp.path, (snapshot) => snapshot.track({ snapshotInitialization: "wait" }))
|
||||
expect(result.value).toBeTruthy()
|
||||
await expect(fs.access(path.join(result.gitdir, "objects", "info", "alternates"))).rejects.toThrow()
|
||||
})
|
||||
Reference in New Issue
Block a user