diff --git a/.changeset/grep-signal-controls.md b/.changeset/grep-signal-controls.md new file mode 100644 index 0000000000..917e6ed2f5 --- /dev/null +++ b/.changeset/grep-signal-controls.md @@ -0,0 +1,5 @@ +--- +"@kilocode/cli": patch +--- + +Add bounded, context-aware signal-to-noise controls to grep searches. diff --git a/packages/core/src/kilocode/ripgrep-grep.ts b/packages/core/src/kilocode/ripgrep-grep.ts new file mode 100644 index 0000000000..f92cc210f9 --- /dev/null +++ b/packages/core/src/kilocode/ripgrep-grep.ts @@ -0,0 +1,53 @@ +import type { Match } from "../filesystem/schema" + +export interface Options { + readonly context?: number + readonly literal?: boolean + readonly ignoreCase?: boolean +} + +export type GrepMatch = Match & { + readonly context: boolean + readonly textTruncated: boolean +} + +export const flags = (input: Options) => [ + ...(input.literal ? ["--fixed-strings"] : []), + ...(input.ignoreCase ? ["--ignore-case"] : []), + ...(input.context ? [`--context=${input.context}`] : []), +] + +export const stop = (limit: number) => { + let matches = 0 + return (row: { readonly context: boolean }) => !row.context && ++matches > limit +} + +export const select = < + A extends { + readonly context: boolean + readonly path: { readonly text: string } + readonly line_number: number + }, +>( + input: { readonly limit: number; readonly context?: number }, + items: readonly A[], +) => { + let count = 0 + const overflow = items.findIndex((row) => !row.context && ++count > input.limit) + const selected = items.slice(0, overflow === -1 ? items.length : overflow) + const matches = selected.filter((row) => !row.context) + return selected.filter( + (row) => + !row.context || + matches.some( + (match) => + match.path.text === row.path.text && Math.abs(match.line_number - row.line_number) <= (input.context ?? 0), + ), + ) +} + +export const decorate = (match: Match, context: boolean, textTruncated: boolean): GrepMatch => ({ + ...match, + context, + textTruncated, +}) diff --git a/packages/core/src/ripgrep.ts b/packages/core/src/ripgrep.ts index e154519560..5ee202a861 100644 --- a/packages/core/src/ripgrep.ts +++ b/packages/core/src/ripgrep.ts @@ -6,6 +6,7 @@ import path from "path" import { LayerNode } from "./effect/layer-node" import { Entry, Match } from "./filesystem/schema" import { FSUtil } from "./fs-util" +import * as KiloGrep from "./kilocode/ripgrep-grep" // kilocode_change import * as SpawnValidation from "./kilocode/spawn-validation" // kilocode_change import { AppProcess, collectStream, waitForAbort } from "./process" import { NonNegativeInt, PositiveInt, RelativePath } from "./schema" @@ -23,7 +24,7 @@ const MAX_RECORD_BYTES = 64 * 1024 const MAX_SUBMATCHES = 100 const RawMatch = Schema.Struct({ - type: Schema.Literal("match"), + type: Schema.Literals(["match", "context"]), // kilocode_change - retain requested context records data: Schema.Struct({ path: Schema.Struct({ text: Schema.String }), lines: Schema.Struct({ text: Schema.String }), @@ -39,7 +40,7 @@ const RawMatch = Schema.Struct({ }), }) -type RawMatchData = (typeof RawMatch.Type)["data"] +type RawMatchData = (typeof RawMatch.Type)["data"] & { readonly context: boolean } // kilocode_change export class Error extends Schema.TaggedErrorClass()("Ripgrep.Error", { message: Schema.String, @@ -71,7 +72,8 @@ export interface GlobInput { readonly validate?: Effect.Effect // kilocode_change - bind approved searches at spawn } -export interface GrepInput { +export interface GrepInput extends KiloGrep.Options { + // kilocode_change readonly cwd: string readonly pattern: string readonly file?: string @@ -84,7 +86,7 @@ export interface GrepInput { export interface Interface { readonly find: (input: FindInput) => Effect.Effect readonly glob: (input: GlobInput) => Effect.Effect, Error> // kilocode_change - readonly grep: (input: GrepInput) => Effect.Effect, Error | InvalidPatternError> // kilocode_change + readonly grep: (input: GrepInput) => Effect.Effect, Error | InvalidPatternError> // kilocode_change } // kilocode_change start - retain truncation state through model-facing tools @@ -116,6 +118,7 @@ export const layer = Layer.effect( readonly parse: (line: string) => Effect.Effect readonly pattern?: string readonly onItem?: (item: A) => Effect.Effect + readonly stop?: (item: A) => boolean // kilocode_change - stop bounded searches at the overflow match readonly validate?: Effect.Effect // kilocode_change - spawn-bound target validation }) => { const program = Effect.scoped( @@ -136,6 +139,13 @@ export const layer = Layer.effect( Effect.forkScoped, ) let observed = 0 + let stopped = false // kilocode_change + const take = input.stop // kilocode_change start + ? Stream.takeUntil((row) => { + stopped = input.stop?.(row) ?? false + return stopped + }) + : Stream.take(input.limit + 1) // kilocode_change end const rows = yield* Stream.decodeText(handle.stdout).pipe( Stream.splitLines, Stream.filter((line) => line.length > 0), @@ -145,11 +155,12 @@ export const layer = Layer.effect( if (!input.onItem || observed++ >= input.limit) return Effect.void return input.onItem(row) }), - Stream.take(input.limit + 1), + take, // kilocode_change Stream.runCollect, Effect.map((chunk) => [...chunk]), ) - const truncated = rows.length > input.limit + if (stopped) return { items: rows, truncated: true, partial: false } // kilocode_change + const truncated = input.stop ? false : rows.length > input.limit // kilocode_change - custom stop predicates own truncation if (truncated) return { items: rows.slice(0, input.limit), truncated, partial: false } const code = yield* handle.exitCode @@ -249,11 +260,13 @@ export const layer = Layer.effect( grep: (input) => run({ ...input, + stop: KiloGrep.stop(input.limit), // kilocode_change args: [ "--no-config", "--json", "--hidden", "--no-messages", + ...KiloGrep.flags(input), // kilocode_change ...(input.include ? [`--glob=${input.include}`] : []), "--glob=!**/.git/**", "--", @@ -269,13 +282,19 @@ export const layer = Layer.effect( }) ).pipe( Effect.flatMap((json) => { - if (!json || typeof json !== "object" || !("type" in json) || json.type !== "match") + if ( + !json || + typeof json !== "object" || + !("type" in json) || + (json.type !== "match" && json.type !== "context") // kilocode_change + ) return Effect.succeed(undefined) return Schema.decodeUnknownEffect(RawMatch)(json).pipe( Effect.map((match) => ({ ...match.data, path: { text: match.data.path.text.replace(/^\.[\\/]/, "") }, submatches: match.data.submatches.slice(0, MAX_SUBMATCHES), + context: match.type === "context", // kilocode_change })), Effect.mapError((cause) => failure("Invalid ripgrep match output", cause)), ) @@ -285,13 +304,13 @@ export const layer = Layer.effect( // kilocode_change start - retain spawn metadata after mapping matches Effect.map((result) => ({ ...result, - items: result.items.map((match) => { + items: KiloGrep.select(input, result.items).map((match) => { const relative = match.path.text .replace(/^(?:\.[\\/])+/u, "") .replace(/^[\\/]+/u, "") .replaceAll("\\", "/") const absolute = path.resolve(input.cwd, relative) - return new Match({ + const item = new Match({ entry: new Entry({ path: RelativePath.make(relative), type: "file", @@ -306,6 +325,7 @@ export const layer = Layer.effect( end: submatch.end, })), }) + return KiloGrep.decorate(item, match.context, match.lines.text.length > 2_000) }), })), // kilocode_change end diff --git a/packages/opencode/src/kilocode/tool/grep-signal-controls.ts b/packages/opencode/src/kilocode/tool/grep-signal-controls.ts new file mode 100644 index 0000000000..49eb582c64 --- /dev/null +++ b/packages/opencode/src/kilocode/tool/grep-signal-controls.ts @@ -0,0 +1,64 @@ +import { NonNegativeInt, PositiveInt } from "@opencode-ai/core/schema" +import { Schema } from "effect" + +export const DEFAULT_LIMIT = 100 + +export const fields = { + context: Schema.optional(NonNegativeInt).annotate({ + description: "Number of context lines to show before and after each match (default 0)", + }), + limit: Schema.optional(PositiveInt).annotate({ + description: "Maximum matching lines to return (default 100)", + }), + literal: Schema.optional(Schema.Boolean).annotate({ + description: "Treat pattern as plain text instead of a regex (default false)", + }), + ignoreCase: Schema.optional(Schema.Boolean).annotate({ + description: "Match without regard to letter case (default false)", + }), +} + +type Input = { + readonly context?: number + readonly limit?: number + readonly literal?: boolean + readonly ignoreCase?: boolean +} + +export const metadata = (input: Input, limit: number, context: number) => ({ + context, + limit, + literal: input.literal, + ignoreCase: input.ignoreCase, +}) + +export const options = (input: Input, limit: number, context: number) => ({ + limit, + context, + literal: input.literal, + ignoreCase: input.ignoreCase, +}) + +export const describe = (description: string) => `${description} +- Searches file contents using regular expressions by default; use literal=true for plain-text patterns +- Use ignoreCase=true for case-insensitive matching, context=N for surrounding lines, and limit=N to bound matches (default 100) +- Context lines are explicitly labeled when requested` + +export const line = ( + row: { readonly line: number; readonly text: string; readonly context: boolean }, + context: number, +) => { + const label = context === 0 ? `Line ${row.line}` : `${row.context ? "[context]" : "[match]"} Line ${row.line}` + return ` ${label}: ${row.text}` +} + +export const limitNotice = (limit: number) => + `${limit} matches limit reached. Use limit=${Math.min(Number.MAX_SAFE_INTEGER, limit * 2)} for more, or refine pattern.` + +export const notices = (rows: readonly { readonly textTruncated: boolean }[]) => { + const output: string[] = [] + if (rows.some((row) => row.textTruncated)) { + output.push("", "Some matching or context lines were truncated. Use read for full lines.") + } + return output +} diff --git a/packages/opencode/src/tool/grep.ts b/packages/opencode/src/tool/grep.ts index 30a1779208..f427783168 100644 --- a/packages/opencode/src/tool/grep.ts +++ b/packages/opencode/src/tool/grep.ts @@ -3,18 +3,20 @@ import { Effect, Schema } from "effect" import { InstanceState } from "@/effect/instance-state" import { FSUtil } from "@opencode-ai/core/fs-util" import { Ripgrep } from "@opencode-ai/core/ripgrep" +import * as KiloGrep from "@/kilocode/tool/grep-signal-controls" // kilocode_change import { assertExternalDirectoryEffect } from "./external-directory" import DESCRIPTION from "./grep.txt" import * as Tool from "./tool" export const Parameters = Schema.Struct({ - pattern: Schema.String.annotate({ description: "The regex pattern to search for in file contents" }), + pattern: Schema.String.annotate({ description: "Pattern to search for in file contents (regex by default)" }), // kilocode_change path: Schema.optional(Schema.String).annotate({ description: "The directory to search in. Defaults to the current working directory.", }), include: Schema.optional(Schema.String).annotate({ description: 'File pattern to include in the search (e.g. "*.js", "*.{ts,tsx}")', }), + ...KiloGrep.fields, // kilocode_change }) export const GrepTool = Tool.define( @@ -23,10 +25,12 @@ export const GrepTool = Tool.define( const fs = yield* FSUtil.Service const ripgrep = yield* Ripgrep.Service return { - description: DESCRIPTION, + description: KiloGrep.describe(DESCRIPTION), // kilocode_change parameters: Parameters, - execute: (params: { pattern: string; path?: string; include?: string }, ctx: Tool.Context) => + execute: (params: Schema.Schema.Type, ctx: Tool.Context) => Effect.gen(function* () { + const limit = params.limit ?? KiloGrep.DEFAULT_LIMIT // kilocode_change + const context = params.context ?? 0 // kilocode_change const empty = { title: params.pattern, metadata: { matches: 0, truncated: false }, @@ -44,6 +48,7 @@ export const GrepTool = Tool.define( pattern: params.pattern, path: params.path, include: params.include, + ...KiloGrep.metadata(params, limit, context), // kilocode_change }, }) @@ -66,7 +71,7 @@ export const GrepTool = Tool.define( file: info?.type === "File" ? path.basename(search) : undefined, // kilocode_change - constrain exact-file searches pattern: params.pattern, include: params.include, - limit: 100, + ...KiloGrep.options(params, limit, context), // kilocode_change signal: ctx.abort, // kilocode_change - stop ripgrep when the tool call is cancelled }) // kilocode_change start @@ -74,18 +79,20 @@ export const GrepTool = Tool.define( if (matches.length === 0) return empty // kilocode_change end - const rows = matches.map((item) => ({ // kilocode_change + const rows = matches.map((item) => ({ + // kilocode_change path: path.resolve(cwd, item.entry.path), line: item.line, text: item.text, + context: item.context, // kilocode_change + textTruncated: item.textTruncated, // kilocode_change })) - const limit = 100 const truncated = result.truncated // kilocode_change const final = rows if (final.length === 0) return empty - const total = rows.length + const total = rows.filter((row) => !row.context).length // kilocode_change const hasMore = truncated // kilocode_change const output = [`Found ${total} matches${hasMore ? " (more matches available)" : ""}`] @@ -96,13 +103,14 @@ export const GrepTool = Tool.define( current = match.path output.push(`${match.path}:`) } - output.push(` Line ${match.line}: ${match.text}`) + output.push(KiloGrep.line(match, context)) // kilocode_change } if (truncated) { output.push("") - output.push("(Results truncated. Consider using a more specific path or pattern.)") + output.push(KiloGrep.limitNotice(limit)) // kilocode_change } + output.push(...KiloGrep.notices(rows)) // kilocode_change if (result.partial) output.push("", "(Some paths were inaccessible.)") // kilocode_change return { diff --git a/packages/opencode/test/kilocode/tool/grep-signal-controls.test.ts b/packages/opencode/test/kilocode/tool/grep-signal-controls.test.ts new file mode 100644 index 0000000000..e6447866ea --- /dev/null +++ b/packages/opencode/test/kilocode/tool/grep-signal-controls.test.ts @@ -0,0 +1,192 @@ +import { describe, expect } from "bun:test" +import { Effect, Exit, Layer, Schema } from "effect" +import path from "path" +import { CrossSpawnSpawner } from "@opencode-ai/core/cross-spawn-spawner" +import { FSUtil } from "@opencode-ai/core/fs-util" +import { Ripgrep } from "@opencode-ai/core/ripgrep" +import { Agent } from "../../../src/agent/agent" +import { Git } from "../../../src/git" +import { GrepTool, Parameters } from "../../../src/tool/grep" +import { Truncate } from "../../../src/tool/truncate" +import { MessageID, SessionID } from "../../../src/session/schema" +import { TestInstance } from "../../fixture/fixture" +import { testEffect } from "../../lib/effect" + +const it = testEffect( + Layer.mergeAll( + CrossSpawnSpawner.defaultLayer, + FSUtil.defaultLayer, + Ripgrep.defaultLayer, + Truncate.defaultLayer, + Agent.defaultLayer, + Git.defaultLayer, + ), +) + +const ctx = { + sessionID: SessionID.make("ses_grep_signal_controls"), + messageID: MessageID.make("msg_grep_signal_controls"), + callID: "", + agent: "code", + abort: AbortSignal.any([]), + messages: [], + metadata: () => Effect.void, + ask: () => Effect.void, +} + +const file = (test: { readonly directory: string }, name: string) => path.join(test.directory, name) + +const init = Effect.gen(function* () { + const info = yield* GrepTool + return yield* info.init() +}) + +describe("Kilo grep signal-to-noise controls", () => { + it.effect("validates signal controls", () => + Effect.sync(() => { + expect(Schema.decodeUnknownSync(Parameters)({ pattern: "needle", context: 0, limit: 1 })).toMatchObject({ + context: 0, + limit: 1, + }) + expect(() => Schema.decodeUnknownSync(Parameters)({ pattern: "needle", context: -1 })).toThrow() + expect(() => Schema.decodeUnknownSync(Parameters)({ pattern: "needle", limit: 0 })).toThrow() + expect(() => Schema.decodeUnknownSync(Parameters)({ pattern: "needle", limit: 1.5 })).toThrow() + }), + ) + + it.instance("preserves the default match output", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => Bun.write(file(test, "default.txt"), "before\nneedle\nafter\n")) + const grep = yield* init + const result = yield* grep.execute({ pattern: "needle", path: test.directory }, ctx) + + expect(result.metadata.matches).toBe(1) + expect(result.output).toContain("Line 2: needle") + expect(result.output).not.toContain("[match]") + }), + ) + + it.instance("stops after the custom match limit", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => + Bun.write(file(test, "many.txt"), `${Array.from({ length: 20 }, () => "needle").join("\n")}\n`), + ) + const grep = yield* init + const result = yield* grep.execute({ pattern: "needle", path: test.directory, limit: 2 }, ctx) + + expect(result.metadata.matches).toBe(2) + expect(result.metadata.truncated).toBe(true) + expect(result.output).toContain("2 matches limit reached. Use limit=4 for more, or refine pattern.") + expect(result.output).not.toContain("(Results truncated") + expect(result.output).not.toContain("Line 3: needle") + }), + ) + + it.instance("supports literal and case-insensitive matching", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => + Promise.all([ + Bun.write(file(test, "literal.txt"), "a.*b\n"), + Bun.write(file(test, "regex.txt"), "azb\n"), + Bun.write(file(test, "case.txt"), "Needle\n"), + ]), + ) + const grep = yield* init + const literal = yield* grep.execute({ pattern: "a.*b", path: test.directory, literal: true }, ctx) + const insensitive = yield* grep.execute({ pattern: "needle", path: test.directory, ignoreCase: true }, ctx) + + expect(literal.metadata.matches).toBe(1) + expect(literal.output).toContain("literal.txt") + expect(literal.output).not.toContain("regex.txt") + expect(insensitive.metadata.matches).toBe(1) + expect(insensitive.output).toContain("case.txt") + }), + ) + + it.instance("formats only bounded context around returned matches", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => + Bun.write( + file(test, "context.txt"), + ["before", "needle", "after", "far", "later needle", "later after"].join("\n") + "\n", + ), + ) + const grep = yield* init + const result = yield* grep.execute({ pattern: "needle", path: test.directory, context: 1, limit: 1 }, ctx) + + expect(result.metadata.matches).toBe(1) + expect(result.output).toContain("[context] Line 1: before") + expect(result.output).toContain("[match] Line 2: needle") + expect(result.output).toContain("[context] Line 3: after") + expect(result.output).not.toContain("Line 4: far") + expect(result.output).not.toContain("later needle") + }), + ) + + it.instance("does not count context lines toward the match limit", () => + Effect.gen(function* () { + const test = yield* TestInstance + const content = Array.from( + { length: 35 }, + (_, index) => `before-${index}\nneedle-${index}\nafter-${index}\ngap-${index}`, + ).join("\n") + yield* Effect.promise(() => Bun.write(file(test, "context-limit.txt"), `${content}\n`)) + const grep = yield* init + const result = yield* grep.execute({ pattern: "needle", path: test.directory, context: 1, limit: 100 }, ctx) + + expect(result.metadata.matches).toBe(35) + expect(result.metadata.truncated).toBe(false) + expect(result.output).toContain("needle-34") + expect(result.output).not.toContain("matches limit reached") + }), + ) + + it.instance("guides the model to read truncated lines", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => Bun.write(file(test, "long.txt"), `${"x".repeat(2_100)}needle\n`)) + const grep = yield* init + const result = yield* grep.execute({ pattern: "needle", path: test.directory }, ctx) + + expect(result.metadata.matches).toBe(1) + expect(result.output).toContain("Some matching or context lines were truncated. Use read for full lines.") + }), + ) + + it.instance("retains include filtering", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => + Promise.all([ + Bun.write(file(test, "included.ts"), "needle\n"), + Bun.write(file(test, "excluded.txt"), "needle\n"), + ]), + ) + const grep = yield* init + const result = yield* grep.execute({ pattern: "needle", path: test.directory, include: "*.ts" }, ctx) + + expect(result.metadata.matches).toBe(1) + expect(result.output).toContain("included.ts") + expect(result.output).not.toContain("excluded.txt") + }), + ) + + it.instance("honors cancellation", () => + Effect.gen(function* () { + const test = yield* TestInstance + yield* Effect.promise(() => Bun.write(file(test, "cancel.txt"), "needle\n".repeat(10_000))) + const grep = yield* init + const controller = new AbortController() + controller.abort() + const exit = yield* grep + .execute({ pattern: "needle", path: test.directory }, { ...ctx, abort: controller.signal }) + .pipe(Effect.exit) + + expect(Exit.isFailure(exit)).toBe(true) + }), + ) +}) diff --git a/packages/opencode/test/tool/__snapshots__/parameters.test.ts.snap b/packages/opencode/test/tool/__snapshots__/parameters.test.ts.snap index 368225175b..a31700b7c0 100644 --- a/packages/opencode/test/tool/__snapshots__/parameters.test.ts.snap +++ b/packages/opencode/test/tool/__snapshots__/parameters.test.ts.snap @@ -114,16 +114,37 @@ exports[`tool parameters JSON Schema (wire shape) grep 1`] = ` { "$schema": "https://json-schema.org/draft/2020-12/schema", "properties": { + "context": { + "description": "Number of context lines to show before and after each match (default 0)", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer", + }, + "ignoreCase": { + "description": "Match without regard to letter case (default false)", + "type": "boolean", + }, "include": { "description": "File pattern to include in the search (e.g. "*.js", "*.{ts,tsx}")", "type": "string", }, + "limit": { + "description": "Maximum matching lines to return (default 100)", + "exclusiveMinimum": 0, + "maximum": 9007199254740991, + "minimum": -9007199254740991, + "type": "integer", + }, + "literal": { + "description": "Treat pattern as plain text instead of a regex (default false)", + "type": "boolean", + }, "path": { "description": "The directory to search in. Defaults to the current working directory.", "type": "string", }, "pattern": { - "description": "The regex pattern to search for in file contents", + "description": "Pattern to search for in file contents (regex by default)", "type": "string", }, }, diff --git a/packages/opencode/test/tool/grep.test.ts b/packages/opencode/test/tool/grep.test.ts index 630c355df3..41e6a4004f 100644 --- a/packages/opencode/test/tool/grep.test.ts +++ b/packages/opencode/test/tool/grep.test.ts @@ -149,7 +149,7 @@ describe("tool.grep", () => { const grep = yield* info.init() const result = yield* grep.execute({ pattern: "needle", path: test.directory, include: "*.txt" }, ctx) - expect(result.output).toContain("(Results truncated. Consider using a more specific path or pattern.)") + expect(result.output).toContain("100 matches limit reached. Use limit=200 for more, or refine pattern.") // kilocode_change expect(result.output).not.toMatch(/showing \d+ of \d+ matches/) }), )