mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
feat: defer MCP tool schemas behind a find_tools search (#28225)
## Summary When the `mcp-tool-search` experiment is enabled, chatd stops inlining connected MCP tool schemas into every generation. It instead exposes a built-in `find_tools` tool whose description carries a compact catalog of the deferred tools, and only ships full JSON schemas for tools the model has activated by searching or by calling them directly. Closes [CODAGT-760](https://linear.app/coder/issue/CODAGT-760). ## Problem Tool-heavy agent configurations (GitHub, Linear, Notion, and dev-tooling MCP servers) inline over 100k tokens of tool schema definitions into every generation. Initial uncached requests reached ~216k tokens with time-to-first-token close to nine minutes, while the model typically invokes only a handful of tools per turn. ## How it works - `decideMCPToolSearch` defers external and workspace `.mcp.json` MCP tools whenever the experiment is enabled. Native, dynamic, provider, skill, and transport tools are never deferred. - `find_tools` embeds a server-grouped catalog in its tool description (degrading to names-only, then counts-only, then a constant-size summary past a context-scaled size cap) and scores keyword matches across tool names, descriptions, parameter schemas, and server metadata. Queries can scope to one server with a `server:` prefix, and exact `names` arguments always activate. - Activation state is ephemeral: it is re-derived each generation from surviving chat history (`find_tools` results and direct calls to deferred tools), so activations naturally lapse when compaction summarizes them away. Aggregate activated schema weight is capped at 10% of the context window, shedding the least recently activated schemas first; `find_tools` shares that budget across parallel calls in one step. No new persistence. - Deferred tools stay registered for execution, so the model can call a cataloged tool directly without searching first; the schema is activated for subsequent steps. - Fail-open: the experiment being disabled, an empty candidate set, or an MCP tool named `find_tools` all disable deferral, leaving today's behavior byte-identical on the wire. - Prometheus counters/histograms track `find_tools` calls, matches, activations, and deferred token weight. - The conversation timeline renders `find_tools` calls with a collapsed search summary and expandable match list, falling back to the generic renderer on malformed payloads. ## Validation - Unit tests for the catalog, matcher, experiment-gated decision, and activation derivation; end-to-end chatd generation tests covering search-then-call, direct-call activation, experiment-off wire parity, compaction lapse, and subagent tool gating. - Storybook interaction tests for the timeline rendering and malformed-payload fallback. - Remote dogfood UAT on dev.coder.com passed: deferral with a real MCP server and Anthropic model, direct calls without prior search, activation persistence across turns, experiment-off parity, and clean UI/console. > Disclosure: Mux (AI agent) authored this PR on Mike's behalf.
This commit is contained in:
Michael Suchacz
Generated
+2
@@ -4991,6 +4991,7 @@ export type Experiment =
|
||||
| "chat-virtual-desktop"
|
||||
| "example"
|
||||
| "mcp-server-http"
|
||||
| "mcp-tool-search"
|
||||
| "nats_pubsub"
|
||||
| "notifications"
|
||||
| "oauth2"
|
||||
@@ -5006,6 +5007,7 @@ export const Experiments: Experiment[] = [
|
||||
"chat-virtual-desktop",
|
||||
"example",
|
||||
"mcp-server-http",
|
||||
"mcp-tool-search",
|
||||
"nats_pubsub",
|
||||
"notifications",
|
||||
"oauth2",
|
||||
|
||||
+209
@@ -571,6 +571,215 @@ export const LifecycleHookNoticeAfterEditedMessage: Story = {
|
||||
},
|
||||
};
|
||||
|
||||
export const FindToolsSearchResult: Story = {
|
||||
args: {
|
||||
...defaultArgs,
|
||||
parsedMessages: buildMessages([
|
||||
{
|
||||
...baseMessage,
|
||||
id: 1,
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool-call",
|
||||
tool_call_id: "find-tools-1",
|
||||
tool_name: "find_tools",
|
||||
args: {
|
||||
queries: JSON.stringify(["github issues", "pull requests"]),
|
||||
names: JSON.stringify(["github__list_issues"]),
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
...baseMessage,
|
||||
id: 2,
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
tool_call_id: "find-tools-1",
|
||||
tool_name: "find_tools",
|
||||
result: {
|
||||
matches: JSON.stringify([
|
||||
{
|
||||
name: "github__list_issues",
|
||||
description: "List issues in a GitHub repository.",
|
||||
},
|
||||
{
|
||||
name: "github__list_pull_requests",
|
||||
description: "List pull requests in a GitHub repository.",
|
||||
},
|
||||
]),
|
||||
activated: JSON.stringify([
|
||||
"github__list_issues",
|
||||
"github__list_pull_requests",
|
||||
]),
|
||||
total_deferred: "24",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
const summary = canvas.getByRole("button", {
|
||||
name: "Searched tools: github issues, pull requests, name:github__list_issues -> 2 matched",
|
||||
});
|
||||
expect(summary).toBeVisible();
|
||||
expect(canvas.queryByText("github__list_issues")).not.toBeInTheDocument();
|
||||
await userEvent.click(summary);
|
||||
expect(canvas.getByText("github__list_issues")).toBeVisible();
|
||||
expect(
|
||||
canvas.getByText("List issues in a GitHub repository."),
|
||||
).toBeVisible();
|
||||
expect(canvas.getByText("github__list_pull_requests")).toBeVisible();
|
||||
expect(
|
||||
canvas.getByText("List pull requests in a GitHub repository."),
|
||||
).toBeVisible();
|
||||
},
|
||||
};
|
||||
|
||||
export const FindToolsEmptyResult: Story = {
|
||||
args: {
|
||||
...defaultArgs,
|
||||
parsedMessages: buildMessages([
|
||||
{
|
||||
...baseMessage,
|
||||
id: 1,
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool-call",
|
||||
tool_call_id: "find-tools-empty",
|
||||
tool_name: "find_tools",
|
||||
args: { queries: JSON.stringify(["nonexistent capability"]) },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
...baseMessage,
|
||||
id: 2,
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
tool_call_id: "find-tools-empty",
|
||||
tool_name: "find_tools",
|
||||
result: {
|
||||
matches: JSON.stringify([]),
|
||||
activated: JSON.stringify([]),
|
||||
total_deferred: "24",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
const summary = canvas.getByText(
|
||||
"Searched tools: nonexistent capability -> 0 matched",
|
||||
);
|
||||
expect(summary).toBeVisible();
|
||||
expect(
|
||||
canvas.queryByRole("button", {
|
||||
name: "Searched tools: nonexistent capability -> 0 matched",
|
||||
}),
|
||||
).not.toBeInTheDocument();
|
||||
expect(canvas.queryByRole("img")).not.toBeInTheDocument();
|
||||
},
|
||||
};
|
||||
|
||||
export const FindToolsErrorResult: Story = {
|
||||
args: {
|
||||
...defaultArgs,
|
||||
parsedMessages: buildMessages([
|
||||
{
|
||||
...baseMessage,
|
||||
id: 1,
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool-call",
|
||||
tool_call_id: "find-tools-error",
|
||||
tool_name: "find_tools",
|
||||
args: { queries: JSON.stringify(["github issues"]) },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
...baseMessage,
|
||||
id: 2,
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
tool_call_id: "find-tools-error",
|
||||
tool_name: "find_tools",
|
||||
is_error: true,
|
||||
result: {
|
||||
error:
|
||||
"The schema budget for this step is exhausted; call the tools already activated or retry next step.",
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
expect(
|
||||
canvas.getByText("Searched tools: github issues -> 0 matched"),
|
||||
).toBeVisible();
|
||||
expect(
|
||||
canvas.getByRole("img", {
|
||||
name: "The schema budget for this step is exhausted; call the tools already activated or retry next step.",
|
||||
}),
|
||||
).toBeVisible();
|
||||
},
|
||||
};
|
||||
|
||||
export const FindToolsMalformedResultUsesDefaultRenderer: Story = {
|
||||
args: {
|
||||
...defaultArgs,
|
||||
parsedMessages: buildMessages([
|
||||
{
|
||||
...baseMessage,
|
||||
id: 1,
|
||||
role: "assistant",
|
||||
content: [
|
||||
{
|
||||
type: "tool-call",
|
||||
tool_call_id: "find-tools-invalid",
|
||||
tool_name: "find_tools",
|
||||
args: { queries: JSON.stringify(["github"]) },
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
...baseMessage,
|
||||
id: 2,
|
||||
role: "tool",
|
||||
content: [
|
||||
{
|
||||
type: "tool-result",
|
||||
tool_call_id: "find-tools-invalid",
|
||||
tool_name: "find_tools",
|
||||
result: { matches: "not-json" },
|
||||
},
|
||||
],
|
||||
},
|
||||
]),
|
||||
},
|
||||
play: async ({ canvasElement }) => {
|
||||
const canvas = within(canvasElement);
|
||||
expect(canvas.queryByText(/Searched tools:/)).not.toBeInTheDocument();
|
||||
expect(canvas.getByRole("button", { name: "find_tools" })).toBeVisible();
|
||||
},
|
||||
};
|
||||
|
||||
export const DurableListTemplatesToolLifecycle: Story = {
|
||||
args: {
|
||||
...defaultArgs,
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import type { FC } from "react";
|
||||
import { ToolCall } from "./ToolCall";
|
||||
import type { ToolStatus } from "./utils";
|
||||
|
||||
export type FindToolsMatch = {
|
||||
name: string;
|
||||
description: string;
|
||||
};
|
||||
|
||||
type FindToolsToolProps = {
|
||||
queries: readonly string[];
|
||||
names: readonly string[];
|
||||
matches: readonly FindToolsMatch[];
|
||||
status: ToolStatus;
|
||||
isError: boolean;
|
||||
errorMessage?: string;
|
||||
};
|
||||
|
||||
export const FindToolsTool: FC<FindToolsToolProps> = ({
|
||||
queries,
|
||||
names,
|
||||
matches,
|
||||
status,
|
||||
isError,
|
||||
errorMessage,
|
||||
}) => {
|
||||
const queryLabel =
|
||||
[...queries, ...names.map((name) => `name:${name}`)].join(", ") || "tools";
|
||||
const label =
|
||||
status === "running"
|
||||
? `Searching tools: ${queryLabel}`
|
||||
: `Searched tools: ${queryLabel} -> ${matches.length} matched`;
|
||||
|
||||
return (
|
||||
<ToolCall.Root
|
||||
className="w-full"
|
||||
status={status}
|
||||
isError={isError}
|
||||
errorMessage={errorMessage || "Failed to search tools"}
|
||||
hasContent={matches.length > 0}
|
||||
>
|
||||
<ToolCall.Header iconName="find_tools" label={label} />
|
||||
<ToolCall.Content>
|
||||
<ul className="mt-1.5 space-y-2 pl-6 text-[13px] text-content-secondary">
|
||||
{matches.map((match) => (
|
||||
<li key={match.name}>
|
||||
<div className="font-medium text-content-primary">
|
||||
{match.name}
|
||||
</div>
|
||||
{match.description ? <div>{match.description}</div> : null}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</ToolCall.Content>
|
||||
</ToolCall.Root>
|
||||
);
|
||||
};
|
||||
@@ -264,6 +264,20 @@ const allToolShowcaseItems: ToolShowcaseItem[] = [
|
||||
build_id: "a1b2c3d4-e5f6-7890-abcd-ef1234567890",
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "find_tools",
|
||||
args: { queries: ["github issues"] },
|
||||
result: {
|
||||
matches: [
|
||||
{
|
||||
name: "github__list_issues",
|
||||
description: "List issues in a GitHub repository.",
|
||||
},
|
||||
],
|
||||
activated: ["github__list_issues"],
|
||||
total_deferred: 12,
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "unknown_tool",
|
||||
args: { example: true },
|
||||
|
||||
@@ -15,6 +15,7 @@ import { CreateWorkspaceTool } from "./CreateWorkspaceTool";
|
||||
import { DiffFileHeader } from "./DiffFileHeader";
|
||||
import { EditFilesTool } from "./EditFilesTool";
|
||||
import { ExecuteTool as ExecuteToolComponent } from "./ExecuteTool";
|
||||
import { type FindToolsMatch, FindToolsTool } from "./FindToolsTool";
|
||||
import { ListAgentsTool } from "./ListAgentsTool";
|
||||
import { ListSubagentModelsTool } from "./ListSubagentModelsTool";
|
||||
import { ListTemplatesTool } from "./ListTemplatesTool";
|
||||
@@ -985,6 +986,97 @@ const GenericToolRenderer: FC<ToolRendererProps> = ({
|
||||
);
|
||||
};
|
||||
|
||||
const parseArray = <T,>(
|
||||
value: unknown,
|
||||
parseItem: (item: unknown) => T | null,
|
||||
): T[] | null => {
|
||||
let array = value;
|
||||
if (typeof array === "string") {
|
||||
try {
|
||||
array = JSON.parse(array);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (!Array.isArray(array)) {
|
||||
return null;
|
||||
}
|
||||
const items: T[] = [];
|
||||
for (const item of array) {
|
||||
const parsed = parseItem(item);
|
||||
if (parsed === null) {
|
||||
return null;
|
||||
}
|
||||
items.push(parsed);
|
||||
}
|
||||
return items;
|
||||
};
|
||||
|
||||
const parseStringList = (value: unknown): string[] | null =>
|
||||
parseArray(value, (item) =>
|
||||
typeof item === "string" ? item.trim() : null,
|
||||
)?.filter(Boolean) ?? null;
|
||||
|
||||
const parseFindToolsMatches = (value: unknown): FindToolsMatch[] | null =>
|
||||
parseArray(value, (item) => {
|
||||
const record = asRecord(item);
|
||||
return record &&
|
||||
typeof record.name === "string" &&
|
||||
typeof record.description === "string"
|
||||
? { name: record.name, description: record.description }
|
||||
: null;
|
||||
});
|
||||
|
||||
const FindToolsRenderer: FC<ToolRendererProps> = (props) => {
|
||||
const parsedArgs = parseArgs(props.args);
|
||||
if (!parsedArgs) {
|
||||
return <GenericToolRenderer {...props} />;
|
||||
}
|
||||
const queries =
|
||||
parsedArgs.queries === undefined ? [] : parseStringList(parsedArgs.queries);
|
||||
const names =
|
||||
parsedArgs.names === undefined ? [] : parseStringList(parsedArgs.names);
|
||||
if (!queries || !names) {
|
||||
return <GenericToolRenderer {...props} />;
|
||||
}
|
||||
const parsedResult = parseArgs(props.result);
|
||||
if (props.isError) {
|
||||
// Error results carry plain text or an error record instead of
|
||||
// matches, so they render through the specialized error state
|
||||
// rather than the malformed-result fallback.
|
||||
const errorMessage = parsedResult
|
||||
? asString(parsedResult.error || parsedResult.message)
|
||||
: asString(props.result);
|
||||
return (
|
||||
<FindToolsTool
|
||||
queries={queries}
|
||||
names={names}
|
||||
matches={[]}
|
||||
status={props.status}
|
||||
isError
|
||||
errorMessage={errorMessage || undefined}
|
||||
/>
|
||||
);
|
||||
}
|
||||
let matches: FindToolsMatch[] | null = [];
|
||||
if (props.status !== "running" || props.result !== undefined) {
|
||||
matches = parsedResult ? parseFindToolsMatches(parsedResult.matches) : null;
|
||||
}
|
||||
if (!matches) {
|
||||
return <GenericToolRenderer {...props} />;
|
||||
}
|
||||
|
||||
return (
|
||||
<FindToolsTool
|
||||
queries={queries}
|
||||
names={names}
|
||||
matches={matches}
|
||||
status={props.status}
|
||||
isError={false}
|
||||
/>
|
||||
);
|
||||
};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// process_signal promotes soft failures (success=false
|
||||
// in the result body, isError=false at protocol level) so the generic
|
||||
@@ -1033,6 +1125,7 @@ const StartWorkspaceRenderer: FC<ToolRendererProps> = ({
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
export const toolRenderers: Record<string, FC<ToolRendererProps>> = {
|
||||
find_tools: FindToolsRenderer,
|
||||
execute: ExecuteRenderer,
|
||||
process_output: ProcessOutputRenderer,
|
||||
process_signal: ProcessSignalRenderer,
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
MonitorIcon,
|
||||
PowerIcon,
|
||||
RouteIcon,
|
||||
SearchIcon,
|
||||
ServerIcon,
|
||||
TerminalIcon,
|
||||
WrenchIcon,
|
||||
@@ -44,6 +45,7 @@ export const toolIcons: Partial<Record<string, LucideIcon>> = {
|
||||
ask_user_question: BadgeQuestionMarkIcon,
|
||||
advisor: CompassIcon,
|
||||
computer: MonitorIcon,
|
||||
find_tools: SearchIcon,
|
||||
};
|
||||
|
||||
export const ToolIcon: React.FC<{
|
||||
|
||||
Reference in New Issue
Block a user