mirror of
https://github.com/coder/coder.git
synced 2026-11-01 21:13:14 +08:00
## Summary When the `mcp-tool-search` experiment is enabled, chatd stops inlining connected MCP tool schemas into every generation. It instead exposes a built-in `find_tools` tool whose description carries a compact catalog of the deferred tools, and only ships full JSON schemas for tools the model has activated by searching or by calling them directly. Closes [CODAGT-760](https://linear.app/coder/issue/CODAGT-760). ## Problem Tool-heavy agent configurations (GitHub, Linear, Notion, and dev-tooling MCP servers) inline over 100k tokens of tool schema definitions into every generation. Initial uncached requests reached ~216k tokens with time-to-first-token close to nine minutes, while the model typically invokes only a handful of tools per turn. ## How it works - `decideMCPToolSearch` defers external and workspace `.mcp.json` MCP tools whenever the experiment is enabled. Native, dynamic, provider, skill, and transport tools are never deferred. - `find_tools` embeds a server-grouped catalog in its tool description (degrading to names-only, then counts-only, then a constant-size summary past a context-scaled size cap) and scores keyword matches across tool names, descriptions, parameter schemas, and server metadata. Queries can scope to one server with a `server:` prefix, and exact `names` arguments always activate. - Activation state is ephemeral: it is re-derived each generation from surviving chat history (`find_tools` results and direct calls to deferred tools), so activations naturally lapse when compaction summarizes them away. Aggregate activated schema weight is capped at 10% of the context window, shedding the least recently activated schemas first; `find_tools` shares that budget across parallel calls in one step. No new persistence. - Deferred tools stay registered for execution, so the model can call a cataloged tool directly without searching first; the schema is activated for subsequent steps. - Fail-open: the experiment being disabled, an empty candidate set, or an MCP tool named `find_tools` all disable deferral, leaving today's behavior byte-identical on the wire. - Prometheus counters/histograms track `find_tools` calls, matches, activations, and deferred token weight. - The conversation timeline renders `find_tools` calls with a collapsed search summary and expandable match list, falling back to the generic renderer on malformed payloads. ## Validation - Unit tests for the catalog, matcher, experiment-gated decision, and activation derivation; end-to-end chatd generation tests covering search-then-call, direct-call activation, experiment-off wire parity, compaction lapse, and subagent tool gating. - Storybook interaction tests for the timeline rendering and malformed-payload fallback. - Remote dogfood UAT on dev.coder.com passed: deferral with a real MCP server and Anthropic model, direct calls without prior search, activation persistence across turns, experiment-off parity, and clean UI/console. > Disclosure: Mux (AI agent) authored this PR on Mike's behalf.
489 lines
24 KiB
Go
489 lines
24 KiB
Go
package chatd
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"strings"
|
|
"testing"
|
|
|
|
"charm.land/fantasy"
|
|
"github.com/google/uuid"
|
|
"github.com/stretchr/testify/require"
|
|
|
|
"github.com/coder/coder/v2/coderd/database"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
|
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
|
"github.com/coder/coder/v2/codersdk"
|
|
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
|
)
|
|
|
|
type deferredTestAgentTool struct {
|
|
info fantasy.ToolInfo
|
|
}
|
|
|
|
func (t deferredTestAgentTool) Info() fantasy.ToolInfo { return t.info }
|
|
func (deferredTestAgentTool) ProviderOptions() fantasy.ProviderOptions { return nil }
|
|
func (deferredTestAgentTool) SetProviderOptions(fantasy.ProviderOptions) {}
|
|
func (deferredTestAgentTool) Run(context.Context, fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
|
return fantasy.NewTextResponse("ok"), nil
|
|
}
|
|
|
|
func testDeferredTool(name, description string, parameters map[string]any) deferredMCPTool {
|
|
return deferredMCPTool{tool: deferredTestAgentTool{info: fantasy.ToolInfo{
|
|
Name: name, Description: description, Parameters: parameters,
|
|
}}}
|
|
}
|
|
|
|
func TestDecideMCPToolSearch(t *testing.T) {
|
|
t.Parallel()
|
|
candidates := []deferredMCPTool{testDeferredTool("server__small", "small", map[string]any{"value": map[string]any{"type": "string"}})}
|
|
|
|
tests := []struct {
|
|
name string
|
|
experiment bool
|
|
candidates []deferredMCPTool
|
|
dynamicNames map[string]bool
|
|
want bool
|
|
}{
|
|
{name: "experiment on", experiment: true, candidates: candidates, want: true},
|
|
{name: "experiment off", candidates: candidates},
|
|
{name: "empty", experiment: true},
|
|
{name: "collision", experiment: true, candidates: []deferredMCPTool{testDeferredTool(chattool.FindToolsName, "collision", nil)}},
|
|
{name: "dynamic collision", experiment: true, candidates: candidates, dynamicNames: map[string]bool{chattool.FindToolsName: true}},
|
|
{name: "dynamic no collision", experiment: true, candidates: candidates, dynamicNames: map[string]bool{"other": true}, want: true},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
t.Parallel()
|
|
require.Equal(t, tt.want, decideMCPToolSearch(mcpToolSearchInput{
|
|
experimentEnabled: tt.experiment,
|
|
candidates: tt.candidates,
|
|
dynamicToolNames: tt.dynamicNames,
|
|
}))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDeriveDeferredMCPActivations(t *testing.T) {
|
|
t.Parallel()
|
|
candidates := []deferredMCPTool{
|
|
testDeferredTool("server__first", "first", nil),
|
|
testDeferredTool("server__second", "second", nil),
|
|
}
|
|
findResult, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolResult("call-1", chattool.FindToolsName, []byte(`{"activated":["server__second","disconnected"]}`), false, false),
|
|
})
|
|
require.NoError(t, err)
|
|
directCall, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolCall("call-2", "server__first", []byte(`{}`)),
|
|
})
|
|
require.NoError(t, err)
|
|
malformed, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolResult("call-3", chattool.FindToolsName, []byte(`"not-json"`), false, false),
|
|
})
|
|
require.NoError(t, err)
|
|
rows := []database.ChatMessage{
|
|
{Role: database.ChatMessageRoleTool, Content: findResult, ContentVersion: chatprompt.CurrentContentVersion},
|
|
{Role: database.ChatMessageRoleAssistant, Content: directCall, ContentVersion: chatprompt.CurrentContentVersion},
|
|
{Role: database.ChatMessageRoleTool, Content: malformed, ContentVersion: chatprompt.CurrentContentVersion},
|
|
}
|
|
require.Equal(t, []string{"server__first", "server__second"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
|
"newest activations first")
|
|
require.Equal(t, []string{"server__first"}, deriveDeferredMCPActivations(rows[1:], candidates, 0),
|
|
"activations before a compaction summary are absent from the surviving prompt window")
|
|
|
|
firstWeight := estimateDeferredMCPToolTokens(candidates[:1])
|
|
require.Equal(t, []string{"server__first"}, deriveDeferredMCPActivations(rows, candidates, firstWeight),
|
|
"a token budget sheds the least recent activations")
|
|
require.Equal(t, []string{"server__first"}, deriveDeferredMCPActivations(rows, candidates, 0.001),
|
|
"the newest activation survives a budget smaller than its own schema")
|
|
}
|
|
|
|
func TestDeriveDeferredMCPActivationsSameStepDirectCallPriority(t *testing.T) {
|
|
t.Parallel()
|
|
candidates := []deferredMCPTool{
|
|
testDeferredTool("server__direct", "direct", nil),
|
|
testDeferredTool("server__searched", "searched", nil),
|
|
}
|
|
assistantStep, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolCall("call-search", chattool.FindToolsName, []byte(`{"queries":["direct"]}`)),
|
|
codersdk.ChatMessageToolCall("call-direct", "server__direct", []byte(`{}`)),
|
|
})
|
|
require.NoError(t, err)
|
|
searchResult, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolResult("call-search", chattool.FindToolsName, []byte(`{"activated":["server__searched"]}`), false, false),
|
|
})
|
|
require.NoError(t, err)
|
|
rows := []database.ChatMessage{
|
|
{Role: database.ChatMessageRoleAssistant, Content: assistantStep, ContentVersion: chatprompt.CurrentContentVersion},
|
|
{Role: database.ChatMessageRoleTool, Content: searchResult, ContentVersion: chatprompt.CurrentContentVersion},
|
|
}
|
|
require.Equal(t, []string{"server__direct", "server__searched"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
|
"a step's direct calls outrank its own search activations")
|
|
directWeight := estimateDeferredMCPToolTokens(candidates[:1])
|
|
require.Equal(t, []string{"server__direct"}, deriveDeferredMCPActivations(rows, candidates, directWeight),
|
|
"same-step search activations cannot shed a directly invoked tool's schema")
|
|
}
|
|
|
|
func TestDeriveDeferredMCPActivationsErroredDirectCallsActivateLast(t *testing.T) {
|
|
t.Parallel()
|
|
candidates := []deferredMCPTool{
|
|
testDeferredTool("server__errored", "errored", nil),
|
|
testDeferredTool("server__searched", "searched", nil),
|
|
}
|
|
assistantStep, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolCall("call-search", chattool.FindToolsName, []byte(`{"queries":["x"]}`)),
|
|
codersdk.ChatMessageToolCall("call-errored", "server__errored", []byte(`{}`)),
|
|
})
|
|
require.NoError(t, err)
|
|
toolRow, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolResult("call-errored", "server__errored", []byte(`"remote tool error"`), true, false),
|
|
codersdk.ChatMessageToolResult("call-search", chattool.FindToolsName, []byte(`{"activated":["server__searched"]}`), false, false),
|
|
})
|
|
require.NoError(t, err)
|
|
rows := []database.ChatMessage{
|
|
{Role: database.ChatMessageRoleAssistant, Content: assistantStep, ContentVersion: chatprompt.CurrentContentVersion},
|
|
{Role: database.ChatMessageRoleTool, Content: toolRow, ContentVersion: chatprompt.CurrentContentVersion},
|
|
}
|
|
require.Equal(t, []string{"server__searched", "server__errored"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
|
"an errored direct call activates last so the model keeps the schema for a corrected retry")
|
|
searchedWeight := estimateDeferredMCPToolTokens(candidates[1:])
|
|
require.Equal(t, []string{"server__searched"}, deriveDeferredMCPActivations(rows, candidates, searchedWeight),
|
|
"an errored direct call cannot consume budget promised to the search's reported activations")
|
|
|
|
erroredCall, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolCall("call-errored", "server__errored", []byte(`{}`)),
|
|
})
|
|
require.NoError(t, err)
|
|
erroredResult, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolResult("call-errored", "server__errored", []byte(`"remote tool error"`), true, false),
|
|
})
|
|
require.NoError(t, err)
|
|
erroredOnly := []database.ChatMessage{
|
|
{Role: database.ChatMessageRoleAssistant, Content: erroredCall, ContentVersion: chatprompt.CurrentContentVersion},
|
|
{Role: database.ChatMessageRoleTool, Content: erroredResult, ContentVersion: chatprompt.CurrentContentVersion},
|
|
}
|
|
require.Equal(t, []string{"server__errored"}, deriveDeferredMCPActivations(erroredOnly, candidates, 0.001),
|
|
"the newest errored call keeps the first-activation allowance when nothing else activates")
|
|
}
|
|
|
|
func TestDeferredMCPToolEntriesEmptyServerLabel(t *testing.T) {
|
|
t.Parallel()
|
|
empty := deferredMCPTool{tool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "_orphan__echo"}}}
|
|
|
|
entries := deferredMCPToolEntries([]deferredMCPTool{empty})
|
|
require.Equal(t, "workspace", entries[0].Server,
|
|
"an empty server identity gets the workspace label at construction, so scopes and grouping agree")
|
|
result, _ := chattool.SearchTools(entries, chattool.FindToolsArgs{Queries: []string{"workspace:"}}, chattool.SearchBudget{})
|
|
require.Equal(t, []string{"_orphan__echo"}, result.Activated,
|
|
"the advertised workspace scope reaches the relabeled entries")
|
|
|
|
literal := deferredMCPTool{
|
|
tool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "workspace__run"}},
|
|
server: "workspace",
|
|
}
|
|
colliding := deferredMCPToolEntries([]deferredMCPTool{empty, literal})
|
|
require.Equal(t, "workspace-2", colliding[0].Server,
|
|
"a literal workspace server keeps its own identity; the fallback label steps aside")
|
|
require.Equal(t, "workspace", colliding[1].Server)
|
|
result, _ = chattool.SearchTools(colliding, chattool.FindToolsArgs{Queries: []string{"workspace:"}}, chattool.SearchBudget{})
|
|
require.Equal(t, []string{"workspace__run"}, result.Activated,
|
|
"the workspace scope matches only the literal server")
|
|
result, _ = chattool.SearchTools(colliding, chattool.FindToolsArgs{Queries: []string{"workspace-2:"}}, chattool.SearchBudget{})
|
|
require.Equal(t, []string{"_orphan__echo"}, result.Activated,
|
|
"the suffixed label scopes the empty-identity server")
|
|
}
|
|
|
|
func TestDeriveDeferredMCPActivationsReusedCallIDs(t *testing.T) {
|
|
t.Parallel()
|
|
candidates := []deferredMCPTool{
|
|
testDeferredTool("server__a", "a", nil),
|
|
testDeferredTool("server__b", "b", nil),
|
|
testDeferredTool("server__c", "c", nil),
|
|
}
|
|
row := func(t *testing.T, role database.ChatMessageRole, parts ...codersdk.ChatMessagePart) database.ChatMessage {
|
|
t.Helper()
|
|
content, err := chatprompt.MarshalParts(parts)
|
|
require.NoError(t, err)
|
|
return database.ChatMessage{Role: role, Content: content, ContentVersion: chatprompt.CurrentContentVersion}
|
|
}
|
|
// call-1 errors for server__a, is reused for a successful
|
|
// server__b call, then server__c errors under its own ID. Only
|
|
// per-call pairing keeps server__b a success: a history-wide
|
|
// errored-ID set would demote it behind the newer errored
|
|
// server__c.
|
|
rows := []database.ChatMessage{
|
|
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__a", []byte(`{}`))),
|
|
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", "server__a", []byte(`"boom"`), true, false)),
|
|
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__b", []byte(`{}`))),
|
|
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", "server__b", []byte(`"ok"`), false, false)),
|
|
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-2", "server__c", []byte(`{}`))),
|
|
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-2", "server__c", []byte(`"boom"`), true, false)),
|
|
}
|
|
require.Equal(t, []string{"server__b", "server__c", "server__a"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
|
"a reused tool-call ID pairs each call with its own result, so the later success outranks errored calls")
|
|
bWeight := estimateDeferredMCPToolTokens(candidates[1:2])
|
|
require.Equal(t, []string{"server__b"}, deriveDeferredMCPActivations(rows, candidates, bWeight),
|
|
"under budget the reused-ID success wins over newer errored calls")
|
|
|
|
// server__a's result is missing entirely; a later step reuses its
|
|
// ID for server__b whose result errors. The error belongs to the
|
|
// newer call, and the abandoned older call counts as successful.
|
|
missingResult := []database.ChatMessage{
|
|
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__a", []byte(`{}`))),
|
|
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__b", []byte(`{}`))),
|
|
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", "server__b", []byte(`"boom"`), true, false)),
|
|
}
|
|
require.Equal(t, []string{"server__a", "server__b"}, deriveDeferredMCPActivations(missingResult, candidates, 0),
|
|
"a reused ID must not assign the newer call's result to the older missing-result call")
|
|
aWeight := estimateDeferredMCPToolTokens(candidates[:1])
|
|
require.Equal(t, []string{"server__a"}, deriveDeferredMCPActivations(missingResult, candidates, aWeight),
|
|
"under budget the abandoned call keeps its successful position")
|
|
|
|
// Compaction removed the orphan find_tools result's call row, and a
|
|
// later step reuses its ID for a fresh find_tools call. The orphan
|
|
// must be admitted at its own row, not stashed for a call that was
|
|
// already visited.
|
|
orphanSearch := []database.ChatMessage{
|
|
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", chattool.FindToolsName, []byte(`{"activated":["server__a"]}`), false, false)),
|
|
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", chattool.FindToolsName, []byte(`{"queries":["b"]}`))),
|
|
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", chattool.FindToolsName, []byte(`{"activated":["server__b"]}`), false, false)),
|
|
}
|
|
require.Equal(t, []string{"server__b", "server__a"}, deriveDeferredMCPActivations(orphanSearch, candidates, 0),
|
|
"an orphan search result with a reused ID is admitted at its own row")
|
|
}
|
|
|
|
func TestFlattenMCPParameterText(t *testing.T) {
|
|
t.Parallel()
|
|
text := flattenMCPParameterText(map[string]any{
|
|
"repository": map[string]any{"type": "string", "description": "Repository name"},
|
|
})
|
|
require.Contains(t, text, "repository")
|
|
require.Contains(t, text, "Repository name")
|
|
}
|
|
|
|
type deferredExternalTestTool struct {
|
|
deferredTestAgentTool
|
|
configID uuid.UUID
|
|
}
|
|
|
|
func (t deferredExternalTestTool) MCPServerConfigID() uuid.UUID { return t.configID }
|
|
|
|
func TestCollectDeferredMCPCandidates(t *testing.T) {
|
|
t.Parallel()
|
|
approvedID := uuid.New()
|
|
unapprovedID := uuid.New()
|
|
external := deferredExternalTestTool{
|
|
deferredTestAgentTool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "github__create_issue"}},
|
|
configID: approvedID,
|
|
}
|
|
unapproved := deferredExternalTestTool{
|
|
deferredTestAgentTool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "linear__create_issue"}},
|
|
configID: unapprovedID,
|
|
}
|
|
workspace := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "everything__echo"}}
|
|
input := deferredMCPCandidateInput{
|
|
mcpTools: []fantasy.AgentTool{external, unapproved},
|
|
workspaceMCPTools: []fantasy.AgentTool{workspace},
|
|
mcpConfigByID: map[uuid.UUID]database.MCPServerConfig{approvedID: {Slug: "github", Description: "GitHub"}},
|
|
approvedMCPConfigIDs: map[uuid.UUID]struct{}{approvedID: {}},
|
|
includeWorkspaceTools: true,
|
|
}
|
|
|
|
names := func(candidates []deferredMCPTool) []string {
|
|
out := make([]string, 0, len(candidates))
|
|
for _, candidate := range candidates {
|
|
out = append(out, candidate.tool.Info().Name)
|
|
}
|
|
return out
|
|
}
|
|
|
|
all := collectDeferredMCPCandidates(input)
|
|
require.Equal(t, []string{"github__create_issue", "linear__create_issue", "everything__echo"}, names(all))
|
|
require.Equal(t, "github", all[0].server)
|
|
require.Equal(t, "everything", all[2].server)
|
|
|
|
planInput := input
|
|
planInput.planMode = database.NullChatPlanMode{Valid: true, ChatPlanMode: database.ChatPlanModePlan}
|
|
require.Equal(t, []string{"github__create_issue"}, names(collectDeferredMCPCandidates(planInput)),
|
|
"plan mode keeps only approved external tools, matching filterToolsForTurn")
|
|
|
|
noWorkspace := input
|
|
noWorkspace.includeWorkspaceTools = false
|
|
require.Equal(t, []string{"github__create_issue", "linear__create_issue"}, names(collectDeferredMCPCandidates(noWorkspace)))
|
|
|
|
longServer := strings.Repeat("s", 70)
|
|
truncated := chattool.NewWorkspaceMCPTool(workspacesdk.MCPToolInfo{Name: longServer + "__echo"}, nil, nil)
|
|
require.NotContains(t, truncated.Info().Name, "__",
|
|
"sanitization must drop the separator for this scenario to be meaningful")
|
|
truncatedInput := deferredMCPCandidateInput{
|
|
workspaceMCPTools: []fantasy.AgentTool{truncated},
|
|
includeWorkspaceTools: true,
|
|
}
|
|
require.Equal(t, longServer, collectDeferredMCPCandidates(truncatedInput)[0].server,
|
|
"the server comes from the unsanitized routing name, not the capped model name")
|
|
|
|
padded := chattool.NewWorkspaceMCPTool(workspacesdk.MCPToolInfo{Name: " everything __echo"}, nil, nil)
|
|
paddedInput := deferredMCPCandidateInput{
|
|
workspaceMCPTools: []fantasy.AgentTool{padded},
|
|
includeWorkspaceTools: true,
|
|
}
|
|
require.Equal(t, "everything", collectDeferredMCPCandidates(paddedInput)[0].server,
|
|
"surrounding whitespace is trimmed so scope matching and catalog display see the canonical name")
|
|
|
|
unpadded := chattool.NewWorkspaceMCPTool(workspacesdk.MCPToolInfo{Name: "everything__ping"}, nil, nil)
|
|
collidingInput := deferredMCPCandidateInput{
|
|
workspaceMCPTools: []fantasy.AgentTool{padded, unpadded},
|
|
includeWorkspaceTools: true,
|
|
}
|
|
colliding := collectDeferredMCPCandidates(collidingInput)
|
|
require.Equal(t, " everything ", colliding[0].server,
|
|
"trimming must not collapse distinct servers into one catalog identity")
|
|
require.Equal(t, "everything", colliding[1].server)
|
|
|
|
slugColliding := deferredMCPCandidateInput{
|
|
mcpTools: []fantasy.AgentTool{external},
|
|
mcpConfigByID: map[uuid.UUID]database.MCPServerConfig{approvedID: {Slug: "everything"}},
|
|
approvedMCPConfigIDs: map[uuid.UUID]struct{}{approvedID: {}},
|
|
workspaceMCPTools: []fantasy.AgentTool{padded},
|
|
includeWorkspaceTools: true,
|
|
}
|
|
slugCands := collectDeferredMCPCandidates(slugColliding)
|
|
require.Equal(t, "everything", slugCands[0].server)
|
|
require.Equal(t, " everything ", slugCands[1].server,
|
|
"a workspace server whose trimmed name collides with an external slug keeps its raw name")
|
|
}
|
|
|
|
func TestConfigureDeferredMCPToolSearchGenerationFlows(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hot := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "read_file", Description: "Read a file"}}
|
|
first := testDeferredTool("github__create_issue", "Create an issue", nil)
|
|
second := testDeferredTool("github__list_issues", "List issues", nil)
|
|
candidates := []deferredMCPTool{first, second}
|
|
findTools := chattool.FindTools(chattool.FindToolsOptions{Entries: deferredMCPToolEntries(candidates)})
|
|
allTools := []fantasy.AgentTool{hot, first.tool, second.tool}
|
|
allActive := []string{"read_file", first.tool.Info().Name, second.tool.Info().Name}
|
|
|
|
ordered, active, allowInactive := configureDeferredMCPToolSearch(allTools, allActive, candidates, findTools, nil)
|
|
require.Equal(t, []string{"read_file", chattool.FindToolsName}, captureWireToolNames(t, ordered, active))
|
|
require.Equal(t, map[string]bool{
|
|
first.tool.Info().Name: true,
|
|
second.tool.Info().Name: true,
|
|
}, allowInactive)
|
|
|
|
result, _ := chattool.SearchTools(deferredMCPToolEntries(candidates), chattool.FindToolsArgs{Names: []string{second.tool.Info().Name}}, chattool.SearchBudget{})
|
|
resultJSON, err := json.Marshal(result)
|
|
require.NoError(t, err)
|
|
resultContent, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolResult("find-1", chattool.FindToolsName, resultJSON, false, false),
|
|
})
|
|
require.NoError(t, err)
|
|
history := []database.ChatMessage{{
|
|
Role: database.ChatMessageRoleTool, Content: resultContent, ContentVersion: chatprompt.CurrentContentVersion,
|
|
}}
|
|
activations := deriveDeferredMCPActivations(history, candidates, 0)
|
|
require.Equal(t, []string{second.tool.Info().Name}, activations)
|
|
|
|
ordered, active, _ = configureDeferredMCPToolSearch(allTools, allActive, candidates, findTools, activations)
|
|
require.Equal(t,
|
|
[]string{"read_file", chattool.FindToolsName, second.tool.Info().Name},
|
|
captureWireToolNames(t, ordered, active),
|
|
)
|
|
// Re-preparing the following turn from the same surviving history produces
|
|
// the same activation set without separate persisted state.
|
|
require.Equal(t, activations, deriveDeferredMCPActivations(history, candidates, 0))
|
|
}
|
|
|
|
func TestConfigureDeferredMCPToolSearchDirectCallAndCompaction(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
candidate := testDeferredTool("github__create_issue", "Create an issue", nil)
|
|
candidates := []deferredMCPTool{candidate}
|
|
directCall, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
|
codersdk.ChatMessageToolCall("direct-1", candidate.tool.Info().Name, []byte(`{"title":"bug"}`)),
|
|
})
|
|
require.NoError(t, err)
|
|
preSummary := []database.ChatMessage{{
|
|
Role: database.ChatMessageRoleAssistant, Content: directCall, ContentVersion: chatprompt.CurrentContentVersion,
|
|
}}
|
|
activation := deriveDeferredMCPActivations(preSummary, candidates, 0)
|
|
require.Equal(t, []string{candidate.tool.Info().Name}, activation)
|
|
|
|
findTools := chattool.FindTools(chattool.FindToolsOptions{Entries: deferredMCPToolEntries(candidates)})
|
|
ordered, active, _ := configureDeferredMCPToolSearch(
|
|
[]fantasy.AgentTool{candidate.tool},
|
|
[]string{candidate.tool.Info().Name},
|
|
candidates,
|
|
findTools,
|
|
activation,
|
|
)
|
|
require.Equal(t,
|
|
[]string{chattool.FindToolsName, candidate.tool.Info().Name},
|
|
captureWireToolNames(t, ordered, active),
|
|
)
|
|
|
|
// Prompt preparation passes only the post-summary history window, so an
|
|
// activation before chat_summarized naturally lapses after compaction.
|
|
require.Empty(t, deriveDeferredMCPActivations(nil, candidates, 0))
|
|
}
|
|
|
|
func TestMCPToolSearchExperimentDisabledPreservesWireTools(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hot := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "read_file"}}
|
|
candidate := testDeferredTool("github__list_issues", "List issues", nil)
|
|
tools := []fantasy.AgentTool{hot, candidate.tool}
|
|
active := []string{"read_file", candidate.tool.Info().Name}
|
|
withoutExperiment := captureWireToolNames(t, tools, active)
|
|
|
|
require.False(t, decideMCPToolSearch(mcpToolSearchInput{
|
|
candidates: []deferredMCPTool{candidate},
|
|
}))
|
|
require.Equal(t, withoutExperiment, captureWireToolNames(t, tools, active))
|
|
}
|
|
|
|
func TestMCPToolSearchExploreAllowlist(t *testing.T) {
|
|
t.Parallel()
|
|
|
|
hot := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "read_file"}}
|
|
external := deferredExternalTestTool{
|
|
deferredTestAgentTool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "github__list_issues"}},
|
|
configID: uuid.New(),
|
|
}
|
|
tools := []fantasy.AgentTool{hot, external}
|
|
exploreActive := allowedExploreToolNames(tools)
|
|
require.Equal(t, []string{"read_file", external.Info().Name}, exploreActive)
|
|
|
|
candidate := deferredMCPTool{tool: external}
|
|
findTools := chattool.FindTools(chattool.FindToolsOptions{Entries: deferredMCPToolEntries([]deferredMCPTool{candidate})})
|
|
_, deferredActive, _ := configureDeferredMCPToolSearch(tools, exploreActive, []deferredMCPTool{candidate}, findTools, nil)
|
|
require.Equal(t, []string{"read_file", chattool.FindToolsName}, deferredActive)
|
|
}
|
|
|
|
func captureWireToolNames(t *testing.T, tools []fantasy.AgentTool, active []string) []string {
|
|
t.Helper()
|
|
var names []string
|
|
model := &chattest.FakeModel{
|
|
ProviderName: "test",
|
|
ModelName: "test",
|
|
StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {
|
|
for _, tool := range call.Tools {
|
|
names = append(names, tool.GetName())
|
|
}
|
|
return func(yield func(fantasy.StreamPart) bool) {
|
|
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
|
}, nil
|
|
},
|
|
}
|
|
_, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
|
|
Model: model,
|
|
Tools: tools,
|
|
ActiveTools: active,
|
|
})
|
|
require.NoError(t, err)
|
|
return names
|
|
}
|