mirror of
https://github.com/coder/coder.git
synced 2026-09-22 05:05:20 +08:00
feat: defer MCP tool schemas behind a find_tools search (#28225)
## Summary When the `mcp-tool-search` experiment is enabled, chatd stops inlining connected MCP tool schemas into every generation. It instead exposes a built-in `find_tools` tool whose description carries a compact catalog of the deferred tools, and only ships full JSON schemas for tools the model has activated by searching or by calling them directly. Closes [CODAGT-760](https://linear.app/coder/issue/CODAGT-760). ## Problem Tool-heavy agent configurations (GitHub, Linear, Notion, and dev-tooling MCP servers) inline over 100k tokens of tool schema definitions into every generation. Initial uncached requests reached ~216k tokens with time-to-first-token close to nine minutes, while the model typically invokes only a handful of tools per turn. ## How it works - `decideMCPToolSearch` defers external and workspace `.mcp.json` MCP tools whenever the experiment is enabled. Native, dynamic, provider, skill, and transport tools are never deferred. - `find_tools` embeds a server-grouped catalog in its tool description (degrading to names-only, then counts-only, then a constant-size summary past a context-scaled size cap) and scores keyword matches across tool names, descriptions, parameter schemas, and server metadata. Queries can scope to one server with a `server:` prefix, and exact `names` arguments always activate. - Activation state is ephemeral: it is re-derived each generation from surviving chat history (`find_tools` results and direct calls to deferred tools), so activations naturally lapse when compaction summarizes them away. Aggregate activated schema weight is capped at 10% of the context window, shedding the least recently activated schemas first; `find_tools` shares that budget across parallel calls in one step. No new persistence. - Deferred tools stay registered for execution, so the model can call a cataloged tool directly without searching first; the schema is activated for subsequent steps. - Fail-open: the experiment being disabled, an empty candidate set, or an MCP tool named `find_tools` all disable deferral, leaving today's behavior byte-identical on the wire. - Prometheus counters/histograms track `find_tools` calls, matches, activations, and deferred token weight. - The conversation timeline renders `find_tools` calls with a collapsed search summary and expandable match list, falling back to the generic renderer on malformed payloads. ## Validation - Unit tests for the catalog, matcher, experiment-gated decision, and activation derivation; end-to-end chatd generation tests covering search-then-call, direct-call activation, experiment-off wire parity, compaction lapse, and subagent tool gating. - Storybook interaction tests for the timeline rendering and malformed-payload fallback. - Remote dogfood UAT on dev.coder.com passed: deferral with a real MCP server and Anthropic model, direct calls without prior search, activation persistence across turns, experiment-off parity, and clean UI/console. > Disclosure: Mux (AI agent) authored this PR on Mike's behalf.
This commit is contained in:
Generated
+4
@@ -20554,6 +20554,7 @@ const docTemplate = `{
|
||||
"workspace-usage",
|
||||
"oauth2",
|
||||
"mcp-server-http",
|
||||
"mcp-tool-search",
|
||||
"workspace-build-updates",
|
||||
"nats_pubsub",
|
||||
"workspace-capable-licensing",
|
||||
@@ -20570,6 +20571,7 @@ const docTemplate = `{
|
||||
"ExperimentChatVirtualDesktop": "Enables virtual desktop and computer use provider for agents.",
|
||||
"ExperimentExample": "This isn't used for anything.",
|
||||
"ExperimentMCPServerHTTP": "Enables the MCP HTTP server functionality.",
|
||||
"ExperimentMCPToolSearch": "Defers MCP tool schemas behind a searchable catalog in agent chats.",
|
||||
"ExperimentNATSPubsub": "Enables embedded NATS pubsub.",
|
||||
"ExperimentNotifications": "Sends notifications via SMTP and webhooks following certain events.",
|
||||
"ExperimentOAuth2": "Enables OAuth2 provider functionality.",
|
||||
@@ -20584,6 +20586,7 @@ const docTemplate = `{
|
||||
"Enables the new workspace usage tracking.",
|
||||
"Enables OAuth2 provider functionality.",
|
||||
"Enables the MCP HTTP server functionality.",
|
||||
"Defers MCP tool schemas behind a searchable catalog in agent chats.",
|
||||
"Enables publishing workspace build updates to the all builds pubsub channel.",
|
||||
"Enables embedded NATS pubsub.",
|
||||
"Counts only users holding the workspace-create permission toward the license seat limit.",
|
||||
@@ -20599,6 +20602,7 @@ const docTemplate = `{
|
||||
"ExperimentWorkspaceUsage",
|
||||
"ExperimentOAuth2",
|
||||
"ExperimentMCPServerHTTP",
|
||||
"ExperimentMCPToolSearch",
|
||||
"ExperimentWorkspaceBuildUpdates",
|
||||
"ExperimentNATSPubsub",
|
||||
"ExperimentWorkspaceCapableLicensing",
|
||||
|
||||
Generated
+4
@@ -18676,6 +18676,7 @@
|
||||
"workspace-usage",
|
||||
"oauth2",
|
||||
"mcp-server-http",
|
||||
"mcp-tool-search",
|
||||
"workspace-build-updates",
|
||||
"nats_pubsub",
|
||||
"workspace-capable-licensing",
|
||||
@@ -18692,6 +18693,7 @@
|
||||
"ExperimentChatVirtualDesktop": "Enables virtual desktop and computer use provider for agents.",
|
||||
"ExperimentExample": "This isn't used for anything.",
|
||||
"ExperimentMCPServerHTTP": "Enables the MCP HTTP server functionality.",
|
||||
"ExperimentMCPToolSearch": "Defers MCP tool schemas behind a searchable catalog in agent chats.",
|
||||
"ExperimentNATSPubsub": "Enables embedded NATS pubsub.",
|
||||
"ExperimentNotifications": "Sends notifications via SMTP and webhooks following certain events.",
|
||||
"ExperimentOAuth2": "Enables OAuth2 provider functionality.",
|
||||
@@ -18706,6 +18708,7 @@
|
||||
"Enables the new workspace usage tracking.",
|
||||
"Enables OAuth2 provider functionality.",
|
||||
"Enables the MCP HTTP server functionality.",
|
||||
"Defers MCP tool schemas behind a searchable catalog in agent chats.",
|
||||
"Enables publishing workspace build updates to the all builds pubsub channel.",
|
||||
"Enables embedded NATS pubsub.",
|
||||
"Counts only users holding the workspace-create permission toward the license seat limit.",
|
||||
@@ -18721,6 +18724,7 @@
|
||||
"ExperimentWorkspaceUsage",
|
||||
"ExperimentOAuth2",
|
||||
"ExperimentMCPServerHTTP",
|
||||
"ExperimentMCPToolSearch",
|
||||
"ExperimentWorkspaceBuildUpdates",
|
||||
"ExperimentNATSPubsub",
|
||||
"ExperimentWorkspaceCapableLicensing",
|
||||
|
||||
@@ -3048,8 +3048,7 @@ type Config struct {
|
||||
Clock quartz.Clock
|
||||
AIBridgeTransportFactory *atomic.Pointer[aibridge.TransportFactory]
|
||||
Experiments codersdk.Experiments
|
||||
|
||||
PrometheusRegistry prometheus.Registerer
|
||||
PrometheusRegistry prometheus.Registerer
|
||||
|
||||
AgentCapacityUnlock AgentCapacityUnlock
|
||||
|
||||
|
||||
@@ -798,6 +798,7 @@ func TestAllowedExploreToolNames(t *testing.T) {
|
||||
newTestAgentTool("read_skill"),
|
||||
newTestAgentTool("read_skill_file"),
|
||||
newTestAgentTool("ask_user_question"),
|
||||
newTestAgentTool(chattool.FindToolsName),
|
||||
})
|
||||
|
||||
require.Equal(t, []string{
|
||||
@@ -812,6 +813,7 @@ func TestAllowedExploreToolNames(t *testing.T) {
|
||||
require.NotContains(t, got, "start_workspace")
|
||||
require.NotContains(t, got, "stop_workspace")
|
||||
require.NotContains(t, got, "ask_user_question")
|
||||
require.NotContains(t, got, chattool.FindToolsName)
|
||||
}
|
||||
|
||||
func TestAllowedBehaviorToolNames(t *testing.T) {
|
||||
|
||||
@@ -60,6 +60,7 @@ import (
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
||||
"github.com/coder/coder/v2/codersdk/workspacesdk/agentconnmock"
|
||||
"github.com/coder/coder/v2/codersdk/x/agenthooks"
|
||||
"github.com/coder/coder/v2/provisioner/echo"
|
||||
proto "github.com/coder/coder/v2/provisionersdk/proto"
|
||||
"github.com/coder/coder/v2/testutil"
|
||||
@@ -802,6 +803,7 @@ func TestExploreChatUsesPersistedMCPSnapshot(t *testing.T) {
|
||||
|
||||
factory := chattest.NewMockAIBridgeTransport(t, openAIURL)
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory)
|
||||
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
||||
require.Equal(t, dbAgent.ID, agentID)
|
||||
@@ -1075,6 +1077,7 @@ func TestExploreChatSendMessageCannotMutateMCPSnapshot(t *testing.T) {
|
||||
|
||||
factory := chattest.NewMockAIBridgeTransport(t, openAIURL)
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory)
|
||||
})
|
||||
|
||||
@@ -1237,6 +1240,7 @@ func TestPlanModeRootChatAllowsApprovedExternalMCPTools(t *testing.T) {
|
||||
|
||||
factory := chattest.NewMockAIBridgeTransport(t, openAIURL)
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(factory)
|
||||
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
||||
require.Equal(t, dbAgent.ID, agentID)
|
||||
@@ -8494,6 +8498,14 @@ func newDebugEnabledTestServer(
|
||||
return server
|
||||
}
|
||||
|
||||
// withoutMCPToolSearch disables the mcp-tool-search experiment so a
|
||||
// test exercises direct MCP tool advertisement instead of deferral.
|
||||
func withoutMCPToolSearch(cfg *chatd.Config) {
|
||||
cfg.Experiments = slices.DeleteFunc(slices.Clone(cfg.Experiments), func(experiment codersdk.Experiment) bool {
|
||||
return experiment == codersdk.ExperimentMCPToolSearch
|
||||
})
|
||||
}
|
||||
|
||||
// newActiveTestServer creates a chatd server that actively polls for
|
||||
// and processes pending chats. Use this instead of newTestServer when
|
||||
// the test needs the chat loop to actually run. Optional config
|
||||
@@ -10410,6 +10422,415 @@ func (d *panicOnInTxDB) InTx(f func(database.Store) error, opts *database.TxOpti
|
||||
return d.Store.InTx(f, opts)
|
||||
}
|
||||
|
||||
func TestMCPToolSearchGenerationFlows(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
t.Run("search activates tools within and across turns", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
mcpSrv := newTestMCPServer("search-mcp")
|
||||
addTestMCPTextTool(mcpSrv, "alpha", "Alpha deferred action", "alpha: ")
|
||||
addTestMCPTextTool(mcpSrv, "beta", "Beta deferred action", "beta: ")
|
||||
mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv))
|
||||
t.Cleanup(mcpTS.Close)
|
||||
|
||||
var (
|
||||
streamCount atomic.Int32
|
||||
requestsMu sync.Mutex
|
||||
requests []recordedOpenAIRequest
|
||||
)
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
requestsMu.Lock()
|
||||
requests = append(requests, recordOpenAIRequest(req))
|
||||
requestsMu.Unlock()
|
||||
switch streamCount.Add(1) {
|
||||
case 1:
|
||||
return chattest.OpenAIStreamingResponse(
|
||||
chattest.OpenAIToolCallChunk(chattool.FindToolsName, `{"names":["search-mcp__alpha","search-mcp__beta"]}`),
|
||||
)
|
||||
default:
|
||||
return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("done")...)
|
||||
}
|
||||
})
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
|
||||
DisplayName: "Search MCP",
|
||||
Slug: "search-mcp",
|
||||
Url: mcpTS.URL,
|
||||
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
})
|
||||
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
|
||||
OrganizationID: org.ID,
|
||||
OwnerID: user.ID,
|
||||
Title: "deferred search",
|
||||
ModelConfigID: model.ID,
|
||||
MCPServerIDs: []uuid.UUID{mcpConfig.ID},
|
||||
InitialUserContent: []codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageText("find deferred actions"),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
|
||||
_, err = server.SendMessage(ctx, chatd.SendMessageOptions{
|
||||
ChatID: chat.ID,
|
||||
CreatedBy: user.ID,
|
||||
ModelConfigID: model.ID,
|
||||
Content: []codersdk.ChatMessagePart{codersdk.ChatMessageText("continue")},
|
||||
BusyBehavior: chatd.SendMessageBusyBehaviorQueue,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
|
||||
requestsMu.Lock()
|
||||
recorded := append([]recordedOpenAIRequest(nil), requests...)
|
||||
requestsMu.Unlock()
|
||||
require.Len(t, recorded, 3)
|
||||
require.Contains(t, recorded[0].Tools, "read_file")
|
||||
require.Contains(t, recorded[0].Tools, chattool.FindToolsName)
|
||||
require.NotContains(t, recorded[0].Tools, "search-mcp__alpha")
|
||||
require.NotContains(t, recorded[0].Tools, "search-mcp__beta")
|
||||
for _, request := range recorded[1:] {
|
||||
require.Contains(t, request.Tools, chattool.FindToolsName)
|
||||
require.Contains(t, request.Tools, "search-mcp__alpha")
|
||||
require.Contains(t, request.Tools, "search-mcp__beta")
|
||||
require.Less(t,
|
||||
slices.Index(request.Tools, "search-mcp__alpha"),
|
||||
slices.Index(request.Tools, "search-mcp__beta"),
|
||||
)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("direct call activates schema on next step", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
mcpSrv := newTestMCPServer("direct-mcp")
|
||||
addTestMCPTextTool(mcpSrv, "echo", "Echo deferred input", "echo: ")
|
||||
mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv))
|
||||
t.Cleanup(mcpTS.Close)
|
||||
|
||||
var (
|
||||
streamCount atomic.Int32
|
||||
requestsMu sync.Mutex
|
||||
requests []recordedOpenAIRequest
|
||||
)
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
requestsMu.Lock()
|
||||
requests = append(requests, recordOpenAIRequest(req))
|
||||
requestsMu.Unlock()
|
||||
if streamCount.Add(1) == 1 {
|
||||
return chattest.OpenAIStreamingResponse(
|
||||
chattest.OpenAIToolCallChunk("direct-mcp__echo", `{"input":"hello"}`),
|
||||
)
|
||||
}
|
||||
return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("done")...)
|
||||
})
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
|
||||
DisplayName: "Direct MCP",
|
||||
Slug: "direct-mcp",
|
||||
Url: mcpTS.URL,
|
||||
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
})
|
||||
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
|
||||
OrganizationID: org.ID,
|
||||
OwnerID: user.ID,
|
||||
Title: "direct deferred call",
|
||||
ModelConfigID: model.ID,
|
||||
MCPServerIDs: []uuid.UUID{mcpConfig.ID},
|
||||
InitialUserContent: []codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageText("call the deferred tool directly"),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
|
||||
requestsMu.Lock()
|
||||
recorded := append([]recordedOpenAIRequest(nil), requests...)
|
||||
requestsMu.Unlock()
|
||||
require.Len(t, recorded, 2)
|
||||
require.NotContains(t, recorded[0].Tools, "direct-mcp__echo")
|
||||
require.Contains(t, recorded[1].Tools, "direct-mcp__echo")
|
||||
require.True(t, openAIMessagesContain(recorded[1].Messages, "echo: hello"))
|
||||
})
|
||||
|
||||
t.Run("partition-denied find_tools calls count toward call totals", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
mcpSrv := newTestMCPServer("count-mcp")
|
||||
addTestMCPTextTool(mcpSrv, "echo", "Echo input", "echo: ")
|
||||
mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv))
|
||||
t.Cleanup(mcpTS.Close)
|
||||
|
||||
var streamCount atomic.Int32
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
switch streamCount.Add(1) {
|
||||
case 1:
|
||||
// Malformed JSON input: partitioned into a synthetic
|
||||
// denial before ExecuteLocalTools, so the tool's own
|
||||
// handler and decode never see this call.
|
||||
return chattest.OpenAIStreamingResponse(
|
||||
chattest.OpenAIToolCallChunk(chattool.FindToolsName, `{"queries":["echo"`),
|
||||
)
|
||||
default:
|
||||
return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("done")...)
|
||||
}
|
||||
})
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
|
||||
DisplayName: "Count MCP",
|
||||
Slug: "count-mcp",
|
||||
Url: mcpTS.URL,
|
||||
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
reg := prometheus.NewRegistry()
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
cfg.PrometheusRegistry = reg
|
||||
})
|
||||
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
|
||||
OrganizationID: org.ID,
|
||||
OwnerID: user.ID,
|
||||
Title: "count denied find_tools",
|
||||
ModelConfigID: model.ID,
|
||||
MCPServerIDs: []uuid.UUID{mcpConfig.ID},
|
||||
InitialUserContent: []codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageText("search"),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
|
||||
requireChatdMetricCounter(t, reg, "coderd_chatd_find_tools_calls_total", 1, nil)
|
||||
})
|
||||
|
||||
t.Run("hook-denied find_tools calls count toward call totals", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
mcpSrv := newTestMCPServer("hooked-mcp")
|
||||
addTestMCPTextTool(mcpSrv, "echo", "Echo input", "echo: ")
|
||||
mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv))
|
||||
t.Cleanup(mcpTS.Close)
|
||||
|
||||
var streamCount atomic.Int32
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
switch streamCount.Add(1) {
|
||||
case 1:
|
||||
return chattest.OpenAIStreamingResponse(
|
||||
chattest.OpenAIToolCallChunk(chattool.FindToolsName, `{"queries":["echo"]}`),
|
||||
)
|
||||
default:
|
||||
return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("done")...)
|
||||
}
|
||||
})
|
||||
consumer := preToolUseConsumer(t, func(data agenthooks.PreToolUseData) string {
|
||||
require.Equal(t, chattool.FindToolsName, data.ToolName)
|
||||
return `{"permission":{"decision":"deny","reason":"blocked by policy"}}`
|
||||
})
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
|
||||
DisplayName: "Hooked MCP",
|
||||
Slug: "hooked-mcp",
|
||||
Url: mcpTS.URL,
|
||||
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
reg := prometheus.NewRegistry()
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
cfg.HookDispatcher = newHookDispatcher(t, db, consumer)
|
||||
cfg.PrometheusRegistry = reg
|
||||
})
|
||||
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
|
||||
OrganizationID: org.ID,
|
||||
OwnerID: user.ID,
|
||||
Title: "count hook-denied find_tools",
|
||||
ModelConfigID: model.ID,
|
||||
MCPServerIDs: []uuid.UUID{mcpConfig.ID},
|
||||
InitialUserContent: []codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageText("search"),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
|
||||
requireChatdMetricCounter(t, reg, "coderd_chatd_find_tools_calls_total", 1, nil)
|
||||
})
|
||||
|
||||
t.Run("admission-failed find_tools calls count toward call totals", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
mcpSrv := newTestMCPServer("failing-mcp")
|
||||
addTestMCPTextTool(mcpSrv, "echo", "Echo input", "echo: ")
|
||||
mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv))
|
||||
t.Cleanup(mcpTS.Close)
|
||||
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
return chattest.OpenAIStreamingResponse(
|
||||
chattest.OpenAIToolCallChunk(chattool.FindToolsName, `{"queries":["echo"]}`),
|
||||
)
|
||||
})
|
||||
// A pre_tool_use dispatch failure errors admission before the
|
||||
// step commits, so the call never reaches executeLocalTools.
|
||||
consumer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
var request agenthooks.Request
|
||||
require.NoError(t, json.NewDecoder(r.Body).Decode(&request))
|
||||
if request.Type != agenthooks.EventPreToolUse {
|
||||
_, err := w.Write([]byte(`{}`))
|
||||
require.NoError(t, err)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(http.StatusInternalServerError)
|
||||
}))
|
||||
t.Cleanup(consumer.Close)
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
|
||||
DisplayName: "Failing MCP",
|
||||
Slug: "failing-mcp",
|
||||
Url: mcpTS.URL,
|
||||
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
reg := prometheus.NewRegistry()
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
cfg.HookDispatcher = newHookDispatcher(t, db, consumer)
|
||||
cfg.PrometheusRegistry = reg
|
||||
})
|
||||
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
|
||||
OrganizationID: org.ID,
|
||||
OwnerID: user.ID,
|
||||
Title: "count admission-failed find_tools",
|
||||
ModelConfigID: model.ID,
|
||||
MCPServerIDs: []uuid.UUID{mcpConfig.ID},
|
||||
InitialUserContent: []codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageText("search"),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusError)
|
||||
|
||||
requireChatdMetricCounter(t, reg, "coderd_chatd_find_tools_calls_total", 1, nil)
|
||||
})
|
||||
|
||||
t.Run("experiment gates deferral regardless of catalog size", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
run := func(t *testing.T, experimentEnabled bool) []byte {
|
||||
t.Helper()
|
||||
db, ps := dbtestutil.NewDB(t)
|
||||
ctx := testutil.Context(t, testutil.WaitLong)
|
||||
mcpSrv := newTestMCPServer("small-mcp")
|
||||
addTestMCPTextTool(mcpSrv, "echo", "Echo input", "echo: ")
|
||||
mcpTS := httptest.NewServer(testMCPHTTPHandler(mcpSrv))
|
||||
t.Cleanup(mcpTS.Close)
|
||||
var toolsJSON []byte
|
||||
var toolsMu sync.Mutex
|
||||
openAIURL := chattest.NewOpenAI(t, func(req *chattest.OpenAIRequest) chattest.OpenAIResponse {
|
||||
if !req.Stream {
|
||||
return chattest.OpenAINonStreamingResponse("title")
|
||||
}
|
||||
encoded, err := json.Marshal(req.Tools)
|
||||
require.NoError(t, err)
|
||||
toolsMu.Lock()
|
||||
toolsJSON = encoded
|
||||
toolsMu.Unlock()
|
||||
return chattest.OpenAIStreamingResponse(chattest.OpenAITextChunks("done")...)
|
||||
})
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
model.ContextLimit = 100_000
|
||||
model = updateChatModelContextLimit(t, db, model)
|
||||
mcpConfig := dbgen.MCPServerConfig(t, db, database.MCPServerConfig{
|
||||
DisplayName: "Small MCP",
|
||||
Slug: "small-mcp",
|
||||
Url: mcpTS.URL,
|
||||
CreatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
UpdatedBy: uuid.NullUUID{UUID: user.ID, Valid: true},
|
||||
})
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
if !experimentEnabled {
|
||||
withoutMCPToolSearch(cfg)
|
||||
}
|
||||
})
|
||||
chat, err := server.CreateChat(ctx, chatd.CreateOptions{
|
||||
OrganizationID: org.ID,
|
||||
OwnerID: user.ID,
|
||||
Title: "small deferred catalog",
|
||||
ModelConfigID: model.ID,
|
||||
MCPServerIDs: []uuid.UUID{mcpConfig.ID},
|
||||
InitialUserContent: []codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageText("finish"),
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
waitForChatStatus(ctx, t, db, chat.ID, database.ChatStatusWaiting)
|
||||
toolsMu.Lock()
|
||||
defer toolsMu.Unlock()
|
||||
return append([]byte(nil), toolsJSON...)
|
||||
}
|
||||
|
||||
toolNames := func(t *testing.T, toolsJSON []byte) []string {
|
||||
t.Helper()
|
||||
var tools []struct {
|
||||
Function struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"function"`
|
||||
}
|
||||
require.NoError(t, json.Unmarshal(toolsJSON, &tools))
|
||||
names := make([]string, 0, len(tools))
|
||||
for _, tool := range tools {
|
||||
names = append(names, tool.Function.Name)
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
withoutExperiment := toolNames(t, run(t, false))
|
||||
require.Contains(t, withoutExperiment, "small-mcp__echo",
|
||||
"without the experiment the MCP schema is advertised directly")
|
||||
require.NotContains(t, withoutExperiment, chattool.FindToolsName)
|
||||
|
||||
withExperiment := toolNames(t, run(t, true))
|
||||
require.Contains(t, withExperiment, chattool.FindToolsName,
|
||||
"the experiment defers every MCP schema behind find_tools, even a small catalog")
|
||||
require.NotContains(t, withExperiment, "small-mcp__echo")
|
||||
})
|
||||
}
|
||||
|
||||
// TestMCPServerToolInvocation verifies that when a chat has
|
||||
// mcp_server_ids set, the chat loop connects to those MCP servers,
|
||||
// discovers their tools, and the LLM can invoke them.
|
||||
@@ -10502,6 +10923,7 @@ func TestMCPServerToolInvocation(t *testing.T) {
|
||||
Return(io.NopCloser(strings.NewReader("")), "", nil).AnyTimes()
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
||||
require.Equal(t, dbAgent.ID, agentID)
|
||||
@@ -10652,6 +11074,7 @@ func TestPlanModeRootChatApprovedExternalMCPToolInvocation(t *testing.T) {
|
||||
})
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
})
|
||||
|
||||
@@ -10766,6 +11189,7 @@ func TestPlanModeRootChatApprovedExternalMCPWorkflowCanReachProposePlan(t *testi
|
||||
}).AnyTimes()
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
||||
require.Equal(t, dbAgent.ID, agentID)
|
||||
@@ -10973,6 +11397,7 @@ func TestMCPServerOAuth2TokenRefresh(t *testing.T) {
|
||||
mockConn.EXPECT().ReadFile(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
|
||||
Return(io.NopCloser(strings.NewReader("")), "", nil).AnyTimes()
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
cfg.AgentConn = func(_ context.Context, agentID uuid.UUID) (workspacesdk.AgentConn, func(), error) {
|
||||
require.Equal(t, dbAgent.ID, agentID)
|
||||
|
||||
@@ -238,10 +238,16 @@ type AssistantOutcome struct {
|
||||
|
||||
// ExecuteLocalToolsOptions configures one local tool execution batch.
|
||||
type ExecuteLocalToolsOptions struct {
|
||||
Tools []fantasy.AgentTool
|
||||
ActiveTools []string
|
||||
ProviderTools []ProviderTool
|
||||
ToolCalls []fantasy.ToolCallContent
|
||||
Tools []fantasy.AgentTool
|
||||
ActiveTools []string
|
||||
AllowInactiveTools map[string]bool
|
||||
ProviderTools []ProviderTool
|
||||
ToolCalls []fantasy.ToolCallContent
|
||||
// ObservedToolCalls optionally carries the step's full assistant
|
||||
// tool-call batch, including calls denied before execution, so
|
||||
// step observers account for denied siblings that derivation will
|
||||
// still count. Defaults to ToolCalls.
|
||||
ObservedToolCalls []fantasy.ToolCallContent
|
||||
|
||||
ExclusiveToolNames map[string]bool
|
||||
BuiltinToolNames map[string]bool
|
||||
@@ -594,8 +600,10 @@ func ExecuteLocalTools(ctx context.Context, opts ExecuteLocalToolsOptions) (Tool
|
||||
opts.Clock,
|
||||
opts.Tools,
|
||||
opts.ActiveTools,
|
||||
opts.AllowInactiveTools,
|
||||
opts.ProviderTools,
|
||||
localCalls,
|
||||
opts.ObservedToolCalls,
|
||||
opts.Metrics,
|
||||
opts.Logger,
|
||||
provider,
|
||||
@@ -1057,8 +1065,10 @@ func executeTools(
|
||||
clock quartz.Clock,
|
||||
allTools []fantasy.AgentTool,
|
||||
activeTools []string,
|
||||
allowInactiveTools map[string]bool,
|
||||
providerTools []ProviderTool,
|
||||
toolCalls []fantasy.ToolCallContent,
|
||||
observedToolCalls []fantasy.ToolCallContent,
|
||||
metrics *Metrics,
|
||||
logger slog.Logger,
|
||||
provider, model string,
|
||||
@@ -1109,47 +1119,83 @@ func executeTools(
|
||||
}
|
||||
}
|
||||
|
||||
observed := observedToolCalls
|
||||
if observed == nil {
|
||||
observed = localToolCalls
|
||||
}
|
||||
notifyStepToolCallObservers(toolMap, toolNameAliases, observed)
|
||||
|
||||
results := make([]fantasy.ToolResultContent, len(localToolCalls))
|
||||
completedAt := make([]time.Time, len(localToolCalls))
|
||||
runCall := func(i int, tc fantasy.ToolCallContent) {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
results[i] = fantasy.ToolResultContent{
|
||||
ToolCallID: tc.ToolCallID,
|
||||
ToolName: tc.ToolName,
|
||||
Result: fantasy.ToolResultOutputContentError{
|
||||
Error: xerrors.Errorf("tool panicked: %v", r),
|
||||
},
|
||||
}
|
||||
}
|
||||
// Record when this tool completed (or panicked).
|
||||
// Captured per call so parallel tools get
|
||||
// accurate individual completion times.
|
||||
completedAt[i] = clockNow(clock)
|
||||
}()
|
||||
results[i] = executeSingleTool(
|
||||
ctx,
|
||||
toolMap,
|
||||
tc,
|
||||
metrics,
|
||||
logger,
|
||||
provider,
|
||||
model,
|
||||
builtinToolNames,
|
||||
activeTools,
|
||||
allowInactiveTools,
|
||||
providerRunnerNames,
|
||||
resultProviderMetadata,
|
||||
maxResultBytes,
|
||||
toolNameAliases,
|
||||
)
|
||||
}
|
||||
// Calls to tools that opt in via SerialToolCalls run in tool-call
|
||||
// order after every concurrent sibling has settled. The step waits
|
||||
// for all calls anyway, so sequencing them last costs nothing, and
|
||||
// order-sensitive shared state (for example the find_tools
|
||||
// activation budget) is claimed deterministically after sibling
|
||||
// outcomes are known. All other calls stay concurrent.
|
||||
var serialIndexes []int
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(len(localToolCalls))
|
||||
for i, tc := range localToolCalls {
|
||||
if isSerialToolCall(toolMap, toolNameAliases, tc.ToolName) {
|
||||
serialIndexes = append(serialIndexes, i)
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
results[i] = fantasy.ToolResultContent{
|
||||
ToolCallID: tc.ToolCallID,
|
||||
ToolName: tc.ToolName,
|
||||
Result: fantasy.ToolResultOutputContentError{
|
||||
Error: xerrors.Errorf("tool panicked: %v", r),
|
||||
},
|
||||
}
|
||||
}
|
||||
// Record when this tool completed (or panicked).
|
||||
// Captured per-goroutine so parallel tools get
|
||||
// accurate individual completion times.
|
||||
completedAt[i] = clockNow(clock)
|
||||
}()
|
||||
results[i] = executeSingleTool(
|
||||
ctx,
|
||||
toolMap,
|
||||
tc,
|
||||
metrics,
|
||||
logger,
|
||||
provider,
|
||||
model,
|
||||
builtinToolNames,
|
||||
activeTools,
|
||||
providerRunnerNames,
|
||||
resultProviderMetadata,
|
||||
maxResultBytes,
|
||||
toolNameAliases,
|
||||
)
|
||||
runCall(i, tc)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
// Reconcile settled sibling outcomes before serial tools run, so
|
||||
// for example find_tools refunds reservations of errored direct
|
||||
// calls before its searches admit activations.
|
||||
settled := make([]fantasy.ToolResultContent, 0, len(results))
|
||||
for i := range results {
|
||||
if !slices.Contains(serialIndexes, i) {
|
||||
settled = append(settled, results[i])
|
||||
}
|
||||
}
|
||||
notifyStepToolResultObservers(toolMap, toolNameAliases, localToolCalls, observed, settled)
|
||||
|
||||
for _, i := range serialIndexes {
|
||||
runCall(i, localToolCalls[i])
|
||||
}
|
||||
|
||||
// Publish results in the original tool-call order so SSE
|
||||
// subscribers see a deterministic event sequence.
|
||||
if onResult != nil {
|
||||
@@ -1262,6 +1308,7 @@ func executeSingleTool(
|
||||
provider, model string,
|
||||
builtinToolNames map[string]bool,
|
||||
activeTools []string,
|
||||
allowInactiveTools map[string]bool,
|
||||
providerRunnerNames map[string]struct{},
|
||||
resultProviderMetadata map[string]func(fantasy.ToolResponse) fantasy.ProviderMetadata,
|
||||
maxResultBytes int,
|
||||
@@ -1294,7 +1341,7 @@ func executeSingleTool(
|
||||
}
|
||||
|
||||
_, isProviderRunner := providerRunnerNames[resolvedName]
|
||||
if !isProviderRunner && !isToolActive(resolvedName, activeTools) {
|
||||
if !isProviderRunner && !isToolActive(resolvedName, activeTools) && !allowInactiveTools[resolvedName] {
|
||||
result.Result = fantasy.ToolResultOutputContentError{
|
||||
Error: xerrors.New("Tool not active in this turn: " + resolvedName),
|
||||
}
|
||||
@@ -1455,6 +1502,119 @@ func isToolActive(name string, activeTools []string) bool {
|
||||
return len(activeTools) == 0 || slices.Contains(activeTools, name)
|
||||
}
|
||||
|
||||
// serialToolCaller is implemented by tools whose calls within one step
|
||||
// must execute in tool-call order because they claim from shared state.
|
||||
type serialToolCaller interface{ SerialToolCalls() bool }
|
||||
|
||||
// stepToolCallObserver is implemented by tools that need to see every
|
||||
// tool-call name in the step before any call executes, for example so
|
||||
// find_tools can charge same-step direct calls against its budget.
|
||||
type stepToolCallObserver interface{ ObserveStepToolCalls(names []string) }
|
||||
|
||||
// notifyStepToolCallObservers passes the step's resolved tool-call
|
||||
// names to each distinct called tool that observes them.
|
||||
func notifyStepToolCallObservers(toolMap map[string]fantasy.AgentTool, toolNameAliases map[string]string, calls []fantasy.ToolCallContent) {
|
||||
names := make([]string, 0, len(calls))
|
||||
for _, tc := range calls {
|
||||
name := tc.ToolName
|
||||
if alias, ok := toolNameAliases[name]; ok {
|
||||
name = alias
|
||||
}
|
||||
names = append(names, name)
|
||||
}
|
||||
notified := make(map[string]struct{}, len(names))
|
||||
for _, name := range names {
|
||||
if _, dup := notified[name]; dup {
|
||||
continue
|
||||
}
|
||||
notified[name] = struct{}{}
|
||||
tool, ok := toolMap[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if observer, ok := tool.(stepToolCallObserver); ok {
|
||||
observer.ObserveStepToolCalls(names)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stepToolResultObserver is implemented by tools that need the step's
|
||||
// per-call execution outcomes, for example so find_tools can refund
|
||||
// budget it reserved for a direct call whose execution errored. names
|
||||
// and errored are parallel slices in the observed tool-call order;
|
||||
// outcomes are kept per call because one tool can be called several
|
||||
// times in a step with different results.
|
||||
type stepToolResultObserver interface {
|
||||
ObserveStepToolResults(names []string, errored []bool)
|
||||
}
|
||||
|
||||
// notifyStepToolResultObservers passes the settled sibling outcomes to
|
||||
// each distinct called tool that observes them, per call in observed
|
||||
// order. Observed calls missing from the executed batch were rejected
|
||||
// before execution (for example malformed JSON partitioned into
|
||||
// synthetic denials) and settle as errored, since their persisted
|
||||
// results always carry IsError. Serial calls have not run yet, so
|
||||
// their own outcomes are reported as not errored; observers only need
|
||||
// the concurrent siblings they share state with.
|
||||
func notifyStepToolResultObservers(toolMap map[string]fantasy.AgentTool, toolNameAliases map[string]string, calls, observed []fantasy.ToolCallContent, settled []fantasy.ToolResultContent) {
|
||||
resolve := func(name string) string {
|
||||
if alias, ok := toolNameAliases[name]; ok {
|
||||
return alias
|
||||
}
|
||||
return name
|
||||
}
|
||||
erroredByID := make(map[string]bool, len(settled))
|
||||
for _, tr := range settled {
|
||||
_, isErr := tr.Result.(fantasy.ToolResultOutputContentError)
|
||||
erroredByID[tr.ToolCallID] = isErr
|
||||
}
|
||||
executedIDs := make(map[string]struct{}, len(calls))
|
||||
for _, tc := range calls {
|
||||
executedIDs[tc.ToolCallID] = struct{}{}
|
||||
}
|
||||
names := make([]string, 0, len(observed))
|
||||
errored := make([]bool, 0, len(observed))
|
||||
for _, tc := range observed {
|
||||
names = append(names, resolve(tc.ToolName))
|
||||
if isErr, ok := erroredByID[tc.ToolCallID]; ok {
|
||||
errored = append(errored, isErr)
|
||||
continue
|
||||
}
|
||||
_, executed := executedIDs[tc.ToolCallID]
|
||||
errored = append(errored, !executed)
|
||||
}
|
||||
notified := make(map[string]struct{}, len(calls))
|
||||
for _, tc := range calls {
|
||||
name := tc.ToolName
|
||||
if alias, ok := toolNameAliases[name]; ok {
|
||||
name = alias
|
||||
}
|
||||
if _, dup := notified[name]; dup {
|
||||
continue
|
||||
}
|
||||
notified[name] = struct{}{}
|
||||
tool, ok := toolMap[name]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if observer, ok := tool.(stepToolResultObserver); ok {
|
||||
observer.ObserveStepToolResults(names, errored)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func isSerialToolCall(toolMap map[string]fantasy.AgentTool, toolNameAliases map[string]string, name string) bool {
|
||||
if alias, ok := toolNameAliases[name]; ok {
|
||||
name = alias
|
||||
}
|
||||
tool, ok := toolMap[name]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
serial, ok := tool.(serialToolCaller)
|
||||
return ok && serial.SerialToolCalls()
|
||||
}
|
||||
|
||||
// buildToolDefinitions converts AgentTool definitions into the
|
||||
// fantasy.Tool slice expected by fantasy.Call. When activeTools
|
||||
// is non-empty, only function tools whose name appears in the
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"iter"
|
||||
"runtime"
|
||||
"slices"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
@@ -872,6 +874,313 @@ func TestSanitizeAnthropicProviderToolContent(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
type serialMarkerTool struct{ fantasy.AgentTool }
|
||||
|
||||
func (serialMarkerTool) SerialToolCalls() bool { return true }
|
||||
|
||||
type observerMarkerTool struct {
|
||||
fantasy.AgentTool
|
||||
observed func(names []string)
|
||||
}
|
||||
|
||||
func (t observerMarkerTool) ObserveStepToolCalls(names []string) { t.observed(names) }
|
||||
|
||||
func TestExecuteToolsNotifiesStepToolCallObservers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var mu sync.Mutex
|
||||
var observedNames []string
|
||||
observedBeforeRun := false
|
||||
observer := observerMarkerTool{
|
||||
AgentTool: fantasy.NewAgentTool(
|
||||
"observer_tool",
|
||||
"records sibling calls",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
mu.Lock()
|
||||
observedBeforeRun = observedNames != nil
|
||||
mu.Unlock()
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
),
|
||||
observed: func(names []string) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
observedNames = append([]string{}, names...)
|
||||
},
|
||||
}
|
||||
var uncalledObserved atomic.Bool
|
||||
uncalledObserver := observerMarkerTool{
|
||||
AgentTool: fantasy.NewAgentTool(
|
||||
"uncalled_observer",
|
||||
"never called this step",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
),
|
||||
observed: func([]string) { uncalledObserved.Store(true) },
|
||||
}
|
||||
other := fantasy.NewAgentTool(
|
||||
"other_tool",
|
||||
"plain tool",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
)
|
||||
|
||||
executeTools(
|
||||
context.Background(),
|
||||
quartz.NewReal(),
|
||||
[]fantasy.AgentTool{observer, uncalledObserver, other},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
[]fantasy.ToolCallContent{
|
||||
{ToolCallID: "1", ToolName: "observer_alias", Input: "{}"},
|
||||
{ToolCallID: "2", ToolName: "other_tool", Input: "{}"},
|
||||
},
|
||||
[]fantasy.ToolCallContent{
|
||||
{ToolCallID: "1", ToolName: "observer_alias", Input: "{}"},
|
||||
{ToolCallID: "2", ToolName: "other_tool", Input: "{}"},
|
||||
{ToolCallID: "3", ToolName: "denied_tool", Input: "{}"},
|
||||
},
|
||||
NewMetrics(prometheus.NewRegistry()),
|
||||
slog.Make(),
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
defaultToolResultBytes,
|
||||
map[string]string{"observer_alias": "observer_tool"},
|
||||
nil,
|
||||
)
|
||||
|
||||
require.Equal(t, []string{"observer_tool", "other_tool", "denied_tool"}, observedNames,
|
||||
"a called observer sees every observed tool-call name, including calls denied before execution")
|
||||
require.True(t, observedBeforeRun, "observers are notified before any tool call executes")
|
||||
require.False(t, uncalledObserved.Load(), "tools not called this step are not notified")
|
||||
}
|
||||
|
||||
type resultObserverMarkerTool struct {
|
||||
fantasy.AgentTool
|
||||
observedResults func(names []string, errored []bool)
|
||||
}
|
||||
|
||||
func (t resultObserverMarkerTool) ObserveStepToolResults(names []string, errored []bool) {
|
||||
t.observedResults(names, errored)
|
||||
}
|
||||
|
||||
func TestExecuteToolsNotifiesStepToolResultObservers(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var mu sync.Mutex
|
||||
var gotNames []string
|
||||
var gotErrored []bool
|
||||
notifications := 0
|
||||
observer := resultObserverMarkerTool{
|
||||
AgentTool: fantasy.NewAgentTool(
|
||||
"observer_tool",
|
||||
"records sibling outcomes",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
),
|
||||
observedResults: func(names []string, errored []bool) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
notifications++
|
||||
gotNames = append([]string{}, names...)
|
||||
gotErrored = append([]bool{}, errored...)
|
||||
},
|
||||
}
|
||||
failing := fantasy.NewAgentTool(
|
||||
"failing_tool",
|
||||
"returns an error result",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextErrorResponse("remote error"), nil
|
||||
},
|
||||
)
|
||||
|
||||
executed := []fantasy.ToolCallContent{
|
||||
{ToolCallID: "1", ToolName: "observer_alias", Input: "{}"},
|
||||
{ToolCallID: "2", ToolName: "failing_tool", Input: "{}"},
|
||||
{ToolCallID: "3", ToolName: "missing_tool", Input: "{}"},
|
||||
}
|
||||
executeTools(
|
||||
context.Background(),
|
||||
quartz.NewReal(),
|
||||
[]fantasy.AgentTool{observer, failing},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
executed,
|
||||
append(slices.Clone(executed), fantasy.ToolCallContent{
|
||||
ToolCallID: "4", ToolName: "rejected_tool", Input: "{not json",
|
||||
}),
|
||||
NewMetrics(prometheus.NewRegistry()),
|
||||
slog.Make(),
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
defaultToolResultBytes,
|
||||
map[string]string{"observer_alias": "observer_tool"},
|
||||
nil,
|
||||
)
|
||||
|
||||
require.Equal(t, 1, notifications, "each called observer is notified once per step")
|
||||
require.Equal(t, []string{"observer_tool", "failing_tool", "missing_tool", "rejected_tool"}, gotNames,
|
||||
"outcomes are reported per call in observed order with aliases resolved")
|
||||
require.Equal(t, []bool{false, true, true, true}, gotErrored,
|
||||
"error results, unresolvable tools, and observed calls rejected before execution all settle as errored outcomes")
|
||||
}
|
||||
|
||||
type serialResultObserverTool struct {
|
||||
fantasy.AgentTool
|
||||
observedResults func(names []string, errored []bool)
|
||||
}
|
||||
|
||||
func (serialResultObserverTool) SerialToolCalls() bool { return true }
|
||||
|
||||
func (t serialResultObserverTool) ObserveStepToolResults(names []string, errored []bool) {
|
||||
t.observedResults(names, errored)
|
||||
}
|
||||
|
||||
func TestExecuteToolsReconcilesResultsBeforeSerialCalls(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var mu sync.Mutex
|
||||
var erroredAtNotify []string
|
||||
var erroredAtRun []string
|
||||
notified := false
|
||||
serial := serialResultObserverTool{
|
||||
AgentTool: fantasy.NewAgentTool(
|
||||
"serial_observer",
|
||||
"observes sibling outcomes before running",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
mu.Lock()
|
||||
erroredAtRun = append([]string{}, erroredAtNotify...)
|
||||
mu.Unlock()
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
),
|
||||
observedResults: func(names []string, errored []bool) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
notified = true
|
||||
erroredAtNotify = nil
|
||||
for i, name := range names {
|
||||
if errored[i] {
|
||||
erroredAtNotify = append(erroredAtNotify, name)
|
||||
}
|
||||
}
|
||||
},
|
||||
}
|
||||
failing := fantasy.NewAgentTool(
|
||||
"failing_tool",
|
||||
"returns an error result",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextErrorResponse("remote error"), nil
|
||||
},
|
||||
)
|
||||
|
||||
results := executeTools(
|
||||
context.Background(),
|
||||
quartz.NewReal(),
|
||||
[]fantasy.AgentTool{serial, failing},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
[]fantasy.ToolCallContent{
|
||||
{ToolCallID: "1", ToolName: "serial_observer", Input: "{}"},
|
||||
{ToolCallID: "2", ToolName: "failing_tool", Input: "{}"},
|
||||
},
|
||||
nil,
|
||||
NewMetrics(prometheus.NewRegistry()),
|
||||
slog.Make(),
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
defaultToolResultBytes,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
require.True(t, notified)
|
||||
require.Equal(t, []string{"failing_tool"}, erroredAtRun,
|
||||
"a serial tool must see settled sibling outcomes before it executes")
|
||||
require.Len(t, results, 2)
|
||||
require.Equal(t, "1", results[0].ToolCallID, "results keep original call order")
|
||||
}
|
||||
|
||||
func TestExecuteToolsSerialToolCallOrder(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var mu sync.Mutex
|
||||
var events []string
|
||||
inFlight := 0
|
||||
maxInFlight := 0
|
||||
record := func(event string, delta int) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
inFlight += delta
|
||||
if inFlight > maxInFlight {
|
||||
maxInFlight = inFlight
|
||||
}
|
||||
events = append(events, event)
|
||||
}
|
||||
serial := serialMarkerTool{AgentTool: fantasy.NewAgentTool(
|
||||
"serial_tool",
|
||||
"records call order",
|
||||
func(_ context.Context, _ struct{}, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
record(call.ID+":start", 1)
|
||||
runtime.Gosched()
|
||||
record(call.ID+":end", -1)
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
)}
|
||||
parallelRan := make(chan struct{})
|
||||
parallel := fantasy.NewAgentTool(
|
||||
"parallel_tool",
|
||||
"plain tool",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
close(parallelRan)
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
)
|
||||
|
||||
calls := []fantasy.ToolCallContent{
|
||||
{ToolCallID: "a", ToolName: "serial_tool", Input: "{}"},
|
||||
{ToolCallID: "p", ToolName: "parallel_tool", Input: "{}"},
|
||||
{ToolCallID: "b", ToolName: "serial_tool", Input: "{}"},
|
||||
{ToolCallID: "c", ToolName: "serial_tool", Input: "{}"},
|
||||
}
|
||||
results := executeTools(
|
||||
context.Background(),
|
||||
quartz.NewReal(),
|
||||
[]fantasy.AgentTool{serial, parallel},
|
||||
nil,
|
||||
nil,
|
||||
nil,
|
||||
calls,
|
||||
nil,
|
||||
NewMetrics(prometheus.NewRegistry()),
|
||||
slog.Make(),
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
defaultToolResultBytes,
|
||||
nil,
|
||||
nil,
|
||||
)
|
||||
|
||||
require.Equal(t, []string{"a:start", "a:end", "b:start", "b:end", "c:start", "c:end"}, events,
|
||||
"serial tool calls must run one at a time in tool-call order")
|
||||
require.Equal(t, 1, maxInFlight)
|
||||
select {
|
||||
case <-parallelRan:
|
||||
default:
|
||||
t.Fatal("parallel tool call did not run")
|
||||
}
|
||||
require.Len(t, results, len(calls))
|
||||
for i, tc := range calls {
|
||||
require.Equal(t, tc.ToolCallID, results[i].ToolCallID, "results keep original call order")
|
||||
}
|
||||
}
|
||||
|
||||
func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -912,6 +1221,7 @@ func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
[]string{"screenshot"},
|
||||
nil,
|
||||
map[string]struct{}{},
|
||||
nil,
|
||||
defaultToolResultBytes,
|
||||
@@ -961,6 +1271,7 @@ func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
[]string{"screenshot"},
|
||||
nil,
|
||||
map[string]struct{}{},
|
||||
nil,
|
||||
defaultToolResultBytes,
|
||||
@@ -1005,6 +1316,7 @@ func TestExecuteSingleTool_MediaBase64Encoding(t *testing.T) {
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
[]string{"echo"},
|
||||
nil,
|
||||
map[string]struct{}{},
|
||||
nil,
|
||||
defaultToolResultBytes,
|
||||
@@ -1053,6 +1365,7 @@ func TestExecuteSingleTool_ResolvesToolNameAlias(t *testing.T) {
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
[]string{"interrupt_agent"},
|
||||
nil,
|
||||
map[string]struct{}{},
|
||||
nil,
|
||||
defaultToolResultBytes,
|
||||
@@ -1093,6 +1406,7 @@ func TestExecuteSingleTool_UnknownAliasFallsThrough(t *testing.T) {
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
[]string{"interrupt_agent"},
|
||||
nil,
|
||||
map[string]struct{}{},
|
||||
nil,
|
||||
defaultToolResultBytes,
|
||||
@@ -1103,3 +1417,32 @@ func TestExecuteSingleTool_UnknownAliasFallsThrough(t *testing.T) {
|
||||
require.True(t, ok, "expected error output, got %T", result.Result)
|
||||
require.Contains(t, errOutput.Error.Error(), "close_agent")
|
||||
}
|
||||
|
||||
func TestExecuteSingleTool_AllowsDeferredDirectCall(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool := fantasy.NewAgentTool(
|
||||
"server__direct",
|
||||
"direct",
|
||||
func(_ context.Context, _ struct{}, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
},
|
||||
)
|
||||
result := executeSingleTool(
|
||||
context.Background(),
|
||||
map[string]fantasy.AgentTool{"server__direct": tool},
|
||||
fantasy.ToolCallContent{ToolCallID: "call-direct", ToolName: "server__direct", Input: "{}"},
|
||||
NewMetrics(prometheus.NewRegistry()),
|
||||
slog.Make(),
|
||||
"fake", "fake-model",
|
||||
map[string]bool{},
|
||||
[]string{"find_tools"},
|
||||
map[string]bool{"server__direct": true},
|
||||
map[string]struct{}{},
|
||||
nil,
|
||||
defaultToolResultBytes,
|
||||
nil,
|
||||
)
|
||||
text, ok := result.Result.(fantasy.ToolResultOutputContentText)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "ok", text.Text)
|
||||
}
|
||||
|
||||
@@ -27,17 +27,21 @@ const (
|
||||
|
||||
// Metrics holds Prometheus metrics for the chatd subsystem.
|
||||
type Metrics struct {
|
||||
Chats *prometheus.GaugeVec
|
||||
MessageCount *prometheus.HistogramVec
|
||||
PromptSizeBytes *prometheus.HistogramVec
|
||||
ToolResultSizeBytes *prometheus.HistogramVec
|
||||
ToolResultTruncatedTotal *prometheus.CounterVec
|
||||
ToolErrorsTotal *prometheus.CounterVec
|
||||
TTFTSeconds *prometheus.HistogramVec
|
||||
CompactionTotal *prometheus.CounterVec
|
||||
StepsTotal *prometheus.CounterVec
|
||||
StreamRetriesTotal *prometheus.CounterVec
|
||||
StreamBufferDroppedTotal prometheus.Counter
|
||||
Chats *prometheus.GaugeVec
|
||||
MessageCount *prometheus.HistogramVec
|
||||
PromptSizeBytes *prometheus.HistogramVec
|
||||
ToolResultSizeBytes *prometheus.HistogramVec
|
||||
ToolResultTruncatedTotal *prometheus.CounterVec
|
||||
ToolErrorsTotal *prometheus.CounterVec
|
||||
TTFTSeconds *prometheus.HistogramVec
|
||||
CompactionTotal *prometheus.CounterVec
|
||||
StepsTotal *prometheus.CounterVec
|
||||
StreamRetriesTotal *prometheus.CounterVec
|
||||
StreamBufferDroppedTotal prometheus.Counter
|
||||
FindToolsCallsTotal prometheus.Counter
|
||||
FindToolsEmptyTotal prometheus.Counter
|
||||
FindToolsMatchCount prometheus.Histogram
|
||||
FindToolsActivationsTotal prometheus.Counter
|
||||
}
|
||||
|
||||
// NewMetrics creates a new Metrics instance registered with the
|
||||
@@ -109,6 +113,31 @@ func NewMetrics(reg prometheus.Registerer) *Metrics {
|
||||
Name: "stream_retries_total",
|
||||
Help: "Total LLM stream retries.",
|
||||
}, []string{"provider", "model", "kind"}),
|
||||
FindToolsCallsTotal: factory.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Subsystem: metricsSubsystem,
|
||||
Name: "find_tools_calls_total",
|
||||
Help: "Total find_tools calls.",
|
||||
}),
|
||||
FindToolsEmptyTotal: factory.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Subsystem: metricsSubsystem,
|
||||
Name: "find_tools_empty_total",
|
||||
Help: "Total find_tools calls with no matches.",
|
||||
}),
|
||||
FindToolsMatchCount: factory.NewHistogram(prometheus.HistogramOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Subsystem: metricsSubsystem,
|
||||
Name: "find_tools_match_count",
|
||||
Help: "Number of matches returned by find_tools calls.",
|
||||
Buckets: prometheus.LinearBuckets(0, 2, 11),
|
||||
}),
|
||||
FindToolsActivationsTotal: factory.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Subsystem: metricsSubsystem,
|
||||
Name: "find_tools_activations_total",
|
||||
Help: "Total deferred tool activations returned by find_tools.",
|
||||
}),
|
||||
StreamBufferDroppedTotal: factory.NewCounter(prometheus.CounterOpts{
|
||||
Namespace: metricsNamespace,
|
||||
Subsystem: metricsSubsystem,
|
||||
|
||||
@@ -0,0 +1,737 @@
|
||||
package chattool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"charm.land/fantasy"
|
||||
)
|
||||
|
||||
const (
|
||||
FindToolsName = "find_tools"
|
||||
findToolsMaxMatches = 20
|
||||
findToolsCatalogTokens = 4000
|
||||
// findToolsMaxQueries and findToolsMaxQueryTokens bound scoring
|
||||
// work: queries are model output, so one call could otherwise
|
||||
// carry arbitrarily many tokens scored against every entry.
|
||||
findToolsMaxQueries = 10
|
||||
findToolsMaxQueryTokens = 16
|
||||
// findToolsMaxNames bounds exact-name lookups the same way. Twice
|
||||
// the match cap leaves room for unknown or duplicate names.
|
||||
findToolsMaxNames = 2 * findToolsMaxMatches
|
||||
// findToolsSpentBudgetFloor replaces a spent or over-reserved budget
|
||||
// for searches so zero-cost reserved names remain activatable while
|
||||
// any real schema weight still exceeds it.
|
||||
findToolsSpentBudgetFloor = 0.000001
|
||||
)
|
||||
|
||||
var findToolsTokenSeparator = regexp.MustCompile(`[^\p{L}\p{N}]+`)
|
||||
|
||||
const findToolsBudgetExhausted = "the schema activation budget for this conversation is exhausted; call a cataloged tool directly by name to activate it in place of the least recently used schema"
|
||||
|
||||
// FindToolCatalogEntry is the searchable metadata for one deferred tool.
|
||||
type FindToolCatalogEntry struct {
|
||||
Name string
|
||||
Description string
|
||||
Server string
|
||||
ServerDescription string
|
||||
ParameterText string
|
||||
// SchemaTokens is the estimated prompt weight of the tool's full
|
||||
// definition, used to cap how much one search may activate.
|
||||
SchemaTokens float64
|
||||
}
|
||||
|
||||
// FindToolsCall records one catalog search for logging and metrics.
|
||||
type FindToolsCall struct {
|
||||
Queries []string
|
||||
Names []string
|
||||
MatchCount int
|
||||
Activated []string
|
||||
TotalDeferred int
|
||||
// Rejection is empty for successful searches. Rejected calls carry
|
||||
// "budget" or "arguments" so callers can count them without
|
||||
// polluting match or activation statistics.
|
||||
Rejection string
|
||||
}
|
||||
|
||||
const (
|
||||
findToolsRejectionBudget = "budget"
|
||||
findToolsRejectionArguments = "arguments"
|
||||
)
|
||||
|
||||
type FindToolsOptions struct {
|
||||
Entries []FindToolCatalogEntry
|
||||
// SchemaTokenBudget caps the aggregate SchemaTokens all searches on
|
||||
// this tool instance may activate, so results never report
|
||||
// activations that the activation budget would immediately shed.
|
||||
// The budget is shared across calls because one step can execute
|
||||
// several searches concurrently. <= 0 means unbounded.
|
||||
SchemaTokenBudget float64
|
||||
// CatalogTokenBudget lowers the default catalog size cap so small
|
||||
// context windows are not consumed by the catalog itself. <= 0 or
|
||||
// values above the default keep the default.
|
||||
CatalogTokenBudget float64
|
||||
OnCall func(context.Context, FindToolsCall)
|
||||
}
|
||||
|
||||
type FindToolsArgs struct {
|
||||
Queries []string `json:"queries,omitempty"`
|
||||
Names []string `json:"names,omitempty"`
|
||||
}
|
||||
|
||||
type FindToolsMatch struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
}
|
||||
|
||||
// FindToolsResult is persisted as the tool result and re-read on later steps.
|
||||
type FindToolsResult struct {
|
||||
Matches []FindToolsMatch `json:"matches"`
|
||||
Activated []string `json:"activated"`
|
||||
TotalDeferred int `json:"total_deferred"`
|
||||
}
|
||||
|
||||
// findToolsTool opts find_tools into in-order execution when one step
|
||||
// contains several calls, so the shared schema budget is claimed in
|
||||
// tool-call order rather than scheduler order. It also observes the
|
||||
// step's sibling tool-call names so direct calls to deferred tools are
|
||||
// charged against the budget before any search admits activations.
|
||||
type findToolsTool struct {
|
||||
fantasy.AgentTool
|
||||
reserveStepCalls func(names []string)
|
||||
settleStepResults func(names []string, errored []bool)
|
||||
onDecodeRejected func(ctx context.Context)
|
||||
}
|
||||
|
||||
func (findToolsTool) SerialToolCalls() bool { return true }
|
||||
|
||||
func (t findToolsTool) ObserveStepToolCalls(names []string) { t.reserveStepCalls(names) }
|
||||
|
||||
func (t findToolsTool) ObserveStepToolResults(names []string, errored []bool) {
|
||||
t.settleStepResults(names, errored)
|
||||
}
|
||||
|
||||
// Run counts calls the typed wrapper rejects during argument decoding,
|
||||
// which never reach the handler and would otherwise be missing from
|
||||
// call metrics. The response itself still comes from the wrapper's own
|
||||
// decode so its wording stays canonical.
|
||||
func (t findToolsTool) Run(ctx context.Context, call fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
var args FindToolsArgs
|
||||
if err := json.Unmarshal([]byte(call.Input), &args); err != nil && t.onDecodeRejected != nil {
|
||||
t.onDecodeRejected(ctx)
|
||||
}
|
||||
return t.AgentTool.Run(ctx, call)
|
||||
}
|
||||
|
||||
// FindTools returns the built-in used to discover deferred MCP tool schemas.
|
||||
func FindTools(options FindToolsOptions) fantasy.AgentTool {
|
||||
entries := slices.Clone(options.Entries)
|
||||
schemaTokensByName := make(map[string]float64, len(entries))
|
||||
for _, entry := range entries {
|
||||
schemaTokensByName[entry.Name] = entry.SchemaTokens
|
||||
}
|
||||
var budgetMu sync.Mutex
|
||||
remainingBudget := options.SchemaTokenBudget
|
||||
// Direct calls to deferred tools in the same step are admitted by
|
||||
// derivation before any search activations, per call in call order,
|
||||
// while their cumulative weight fits the budget (the first always
|
||||
// fits, mirroring derivation's newest-keep rule). Only that
|
||||
// retained prefix is free to activate; a call past it is
|
||||
// unclaimable this step because derivation marks it seen at its
|
||||
// rejected position, so no same-step search can inline its schema
|
||||
// either. Errored calls (including calls rejected before execution)
|
||||
// are skipped per call when siblings settle, exactly as derivation
|
||||
// postpones them by call ID, so one tool called several times with
|
||||
// mixed outcomes admits at its first successful call's position.
|
||||
type stepToolCall struct {
|
||||
name string
|
||||
errored bool
|
||||
}
|
||||
var stepCalls []stepToolCall
|
||||
reserved := make(map[string]struct{})
|
||||
unclaimable := make(map[string]struct{})
|
||||
// Derivation deduplicates activations by name, so a name an earlier
|
||||
// search already claimed is free for later searches in the step.
|
||||
claimedBySearch := make(map[string]struct{})
|
||||
searchClaimed := 0.0
|
||||
recompute := func() {
|
||||
clear(reserved)
|
||||
clear(unclaimable)
|
||||
charged := 0.0
|
||||
seen := make(map[string]struct{}, len(stepCalls))
|
||||
for _, call := range stepCalls {
|
||||
if call.errored {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[call.name]; dup {
|
||||
continue
|
||||
}
|
||||
seen[call.name] = struct{}{}
|
||||
weight := schemaTokensByName[call.name]
|
||||
if len(reserved) > 0 && charged+weight > options.SchemaTokenBudget {
|
||||
unclaimable[call.name] = struct{}{}
|
||||
continue
|
||||
}
|
||||
reserved[call.name] = struct{}{}
|
||||
charged += weight
|
||||
}
|
||||
remainingBudget = options.SchemaTokenBudget - charged - searchClaimed
|
||||
}
|
||||
rebuild := func(names []string, errored []bool) {
|
||||
stepCalls = stepCalls[:0]
|
||||
for i, name := range names {
|
||||
if _, ok := schemaTokensByName[name]; !ok {
|
||||
continue
|
||||
}
|
||||
stepCalls = append(stepCalls, stepToolCall{name: name, errored: len(errored) > i && errored[i]})
|
||||
}
|
||||
recompute()
|
||||
}
|
||||
reserve := func(names []string) {
|
||||
if options.SchemaTokenBudget <= 0 {
|
||||
return
|
||||
}
|
||||
budgetMu.Lock()
|
||||
defer budgetMu.Unlock()
|
||||
// Outcomes are unknown before execution, so every call charges;
|
||||
// settle rebuilds with real per-call outcomes before searches
|
||||
// run.
|
||||
rebuild(names, nil)
|
||||
}
|
||||
settle := func(names []string, errored []bool) {
|
||||
if options.SchemaTokenBudget <= 0 {
|
||||
return
|
||||
}
|
||||
budgetMu.Lock()
|
||||
defer budgetMu.Unlock()
|
||||
rebuild(names, errored)
|
||||
}
|
||||
onDecodeRejected := func(ctx context.Context) {
|
||||
if options.OnCall != nil {
|
||||
options.OnCall(ctx, FindToolsCall{
|
||||
TotalDeferred: len(entries),
|
||||
Rejection: findToolsRejectionArguments,
|
||||
})
|
||||
}
|
||||
}
|
||||
return findToolsTool{reserveStepCalls: reserve, settleStepResults: settle, onDecodeRejected: onDecodeRejected, AgentTool: fantasy.NewAgentTool(
|
||||
FindToolsName,
|
||||
buildFindToolsDescription(entries, options.CatalogTokenBudget),
|
||||
func(ctx context.Context, args FindToolsArgs, _ fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
if len(args.Queries) == 0 && len(args.Names) == 0 {
|
||||
if options.OnCall != nil {
|
||||
options.OnCall(ctx, FindToolsCall{
|
||||
TotalDeferred: len(entries),
|
||||
Rejection: findToolsRejectionArguments,
|
||||
})
|
||||
}
|
||||
return fantasy.NewTextErrorResponse("at least one query or name is required"), nil
|
||||
}
|
||||
budgetMu.Lock()
|
||||
searchEntries := entries
|
||||
if len(reserved) > 0 || len(unclaimable) > 0 || len(claimedBySearch) > 0 {
|
||||
searchEntries = make([]FindToolCatalogEntry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if _, ok := unclaimable[entry.Name]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := reserved[entry.Name]; ok {
|
||||
entry.SchemaTokens = 0
|
||||
}
|
||||
if _, ok := claimedBySearch[entry.Name]; ok {
|
||||
entry.SchemaTokens = 0
|
||||
}
|
||||
searchEntries = append(searchEntries, entry)
|
||||
}
|
||||
}
|
||||
searchBudget := remainingBudget
|
||||
if options.SchemaTokenBudget > 0 && searchBudget <= 0 {
|
||||
// A spent budget still admits zero-cost reserved names,
|
||||
// so search with a floor instead of failing outright.
|
||||
searchBudget = findToolsSpentBudgetFloor
|
||||
}
|
||||
budgetTouched := options.SchemaTokenBudget > 0 && remainingBudget < options.SchemaTokenBudget
|
||||
result, budgetSkipped := SearchTools(searchEntries, args, SearchBudget{
|
||||
SchemaTokens: searchBudget,
|
||||
AllowFirstOverBudget: !budgetTouched,
|
||||
})
|
||||
if options.SchemaTokenBudget > 0 {
|
||||
if len(result.Activated) == 0 && budgetSkipped > 0 {
|
||||
budgetMu.Unlock()
|
||||
if options.OnCall != nil {
|
||||
options.OnCall(ctx, FindToolsCall{
|
||||
Queries: args.Queries,
|
||||
Names: args.Names,
|
||||
TotalDeferred: len(entries),
|
||||
Rejection: findToolsRejectionBudget,
|
||||
})
|
||||
}
|
||||
return fantasy.NewTextErrorResponse(findToolsBudgetExhausted), nil
|
||||
}
|
||||
admitted := 0.0
|
||||
for _, name := range result.Activated {
|
||||
if _, ok := reserved[name]; ok {
|
||||
continue
|
||||
}
|
||||
if _, ok := claimedBySearch[name]; ok {
|
||||
continue
|
||||
}
|
||||
admitted += schemaTokensByName[name]
|
||||
}
|
||||
// Defensive invariant: with allowFirstOverBudget off,
|
||||
// a touched budget can never admit an over-claim. If
|
||||
// bookkeeping ever drifts, fail loudly rather than
|
||||
// report activations derivation would shed.
|
||||
if admitted > 0 && admitted > remainingBudget && budgetTouched {
|
||||
budgetMu.Unlock()
|
||||
if options.OnCall != nil {
|
||||
options.OnCall(ctx, FindToolsCall{
|
||||
Queries: args.Queries,
|
||||
Names: args.Names,
|
||||
TotalDeferred: len(entries),
|
||||
Rejection: findToolsRejectionBudget,
|
||||
})
|
||||
}
|
||||
return fantasy.NewTextErrorResponse(findToolsBudgetExhausted), nil
|
||||
}
|
||||
searchClaimed += admitted
|
||||
remainingBudget -= admitted
|
||||
for _, name := range result.Activated {
|
||||
if _, ok := reserved[name]; !ok {
|
||||
claimedBySearch[name] = struct{}{}
|
||||
}
|
||||
}
|
||||
}
|
||||
budgetMu.Unlock()
|
||||
// Unclaimable entries stay deferred; report the full count.
|
||||
result.TotalDeferred = len(entries)
|
||||
if options.OnCall != nil {
|
||||
options.OnCall(ctx, FindToolsCall{
|
||||
Queries: args.Queries,
|
||||
Names: args.Names,
|
||||
MatchCount: len(result.Matches),
|
||||
Activated: result.Activated,
|
||||
TotalDeferred: result.TotalDeferred,
|
||||
})
|
||||
}
|
||||
return marshalToolResponse(result), nil
|
||||
},
|
||||
)}
|
||||
}
|
||||
|
||||
// SearchBudget bounds the schema weight one search may activate.
|
||||
type SearchBudget struct {
|
||||
// SchemaTokens is the remaining activation budget. <= 0 means
|
||||
// unbounded.
|
||||
SchemaTokens float64
|
||||
// AllowFirstOverBudget admits the first match even over budget.
|
||||
// Callers set it only while the shared budget is untouched, where
|
||||
// derivation's newest-keep rule retains a sole over-budget claim.
|
||||
AllowFirstOverBudget bool
|
||||
}
|
||||
|
||||
// SearchTools includes exact name activations first, then fills the
|
||||
// remaining match slots with the top-scored keyword matches. The shared
|
||||
// cap and summary-length descriptions keep the persisted result small
|
||||
// enough that generic tool-result truncation can never corrupt the
|
||||
// activation JSON that later steps re-derive activations from. A
|
||||
// positive budget additionally skips matches whose schema weight would
|
||||
// push the aggregate past it, admitting later matches that still fit.
|
||||
// The second result counts matches skipped for budget, so callers can
|
||||
// tell an exhausted budget from no matches.
|
||||
func SearchTools(entries []FindToolCatalogEntry, args FindToolsArgs, budget SearchBudget) (FindToolsResult, int) {
|
||||
byName := make(map[string]FindToolCatalogEntry, len(entries))
|
||||
for _, entry := range entries {
|
||||
byName[entry.Name] = entry
|
||||
}
|
||||
|
||||
queryArgs := args.Queries
|
||||
if len(queryArgs) > findToolsMaxQueries {
|
||||
queryArgs = queryArgs[:findToolsMaxQueries]
|
||||
}
|
||||
queries := parseFindToolsQueries(entries, queryArgs)
|
||||
|
||||
type scoredEntry struct {
|
||||
entry FindToolCatalogEntry
|
||||
score int
|
||||
}
|
||||
scored := make([]scoredEntry, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
tokens := tokenizeFindToolsEntry(entry)
|
||||
score := 0
|
||||
for _, query := range queries {
|
||||
if query.server != "" {
|
||||
if query.exact && entry.Server != query.server {
|
||||
continue
|
||||
}
|
||||
if !query.exact && !strings.EqualFold(entry.Server, query.server) {
|
||||
continue
|
||||
}
|
||||
}
|
||||
if query.server != "" && len(query.tokens) == 0 {
|
||||
score++
|
||||
continue
|
||||
}
|
||||
for _, token := range query.tokens {
|
||||
score += tokens.score(token)
|
||||
}
|
||||
}
|
||||
if score > 0 {
|
||||
scored = append(scored, scoredEntry{entry: entry, score: score})
|
||||
}
|
||||
}
|
||||
slices.SortFunc(scored, func(a, b scoredEntry) int {
|
||||
if a.score != b.score {
|
||||
return b.score - a.score
|
||||
}
|
||||
return strings.Compare(a.entry.Name, b.entry.Name)
|
||||
})
|
||||
matches := make([]FindToolsMatch, 0, findToolsMaxMatches)
|
||||
activatedSet := make(map[string]struct{}, findToolsMaxMatches)
|
||||
usedSchemaTokens := 0.0
|
||||
budgetSkipped := 0
|
||||
appendMatch := func(entry FindToolCatalogEntry) {
|
||||
if _, exists := activatedSet[entry.Name]; exists {
|
||||
return
|
||||
}
|
||||
if len(matches) >= findToolsMaxMatches {
|
||||
return
|
||||
}
|
||||
overBudget := budget.SchemaTokens > 0 && usedSchemaTokens+entry.SchemaTokens > budget.SchemaTokens
|
||||
if overBudget && (len(matches) > 0 || !budget.AllowFirstOverBudget) {
|
||||
budgetSkipped++
|
||||
return
|
||||
}
|
||||
usedSchemaTokens += entry.SchemaTokens
|
||||
matches = append(matches, FindToolsMatch{
|
||||
Name: entry.Name,
|
||||
Description: truncateFindToolsSummary(entry.Description, 80),
|
||||
})
|
||||
activatedSet[entry.Name] = struct{}{}
|
||||
}
|
||||
nameArgs := args.Names
|
||||
if len(nameArgs) > findToolsMaxNames {
|
||||
nameArgs = nameArgs[:findToolsMaxNames]
|
||||
}
|
||||
for _, name := range nameArgs {
|
||||
if entry, ok := byName[name]; ok {
|
||||
appendMatch(entry)
|
||||
}
|
||||
}
|
||||
for _, item := range scored {
|
||||
appendMatch(item.entry)
|
||||
}
|
||||
activated := make([]string, 0, len(activatedSet))
|
||||
for name := range activatedSet {
|
||||
activated = append(activated, name)
|
||||
}
|
||||
slices.Sort(activated)
|
||||
return FindToolsResult{Matches: matches, Activated: activated, TotalDeferred: len(entries)}, budgetSkipped
|
||||
}
|
||||
|
||||
type scopedFindToolsQuery struct {
|
||||
server string
|
||||
// exact scopes to the one server whose name matched byte-for-byte;
|
||||
// otherwise the scope folds case and may span case-colliding
|
||||
// servers.
|
||||
exact bool
|
||||
tokens []string
|
||||
}
|
||||
|
||||
// parseFindToolsQueries treats "server: terms" as a scope only when the
|
||||
// prefix names a cataloged server, so queries like "error: timeout"
|
||||
// still search normally. Prefixes are matched against full cataloged
|
||||
// server names, longest first, because workspace server names may
|
||||
// themselves contain ":". An exact-case prefix wins before the
|
||||
// case-insensitive fallback, so servers whose names differ only by
|
||||
// case each stay reachable by their advertised catalog name.
|
||||
func parseFindToolsQueries(entries []FindToolCatalogEntry, queries []string) []scopedFindToolsQuery {
|
||||
servers := make([]string, 0, len(entries))
|
||||
seen := make(map[string]struct{}, len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.Server == "" {
|
||||
continue
|
||||
}
|
||||
if _, dup := seen[entry.Server]; dup {
|
||||
continue
|
||||
}
|
||||
seen[entry.Server] = struct{}{}
|
||||
servers = append(servers, entry.Server)
|
||||
}
|
||||
// Longest first, so a server named "jira:prod" wins over "jira"
|
||||
// when both are cataloged.
|
||||
slices.SortFunc(servers, func(a, b string) int { return len(b) - len(a) })
|
||||
parsed := make([]scopedFindToolsQuery, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
scoped := false
|
||||
trimmed := strings.TrimSpace(query)
|
||||
// The raw query is matched before whitespace normalization so
|
||||
// a whitespace-padded server name retained by collision
|
||||
// handling stays selectable; then the trimmed exact and
|
||||
// case-insensitive passes run as fallbacks.
|
||||
passes := []struct {
|
||||
text string
|
||||
exact bool
|
||||
}{
|
||||
{text: query, exact: true},
|
||||
{text: trimmed, exact: true},
|
||||
{text: trimmed, exact: false},
|
||||
}
|
||||
for _, pass := range passes {
|
||||
if scoped {
|
||||
break
|
||||
}
|
||||
for _, server := range servers {
|
||||
var rest string
|
||||
if pass.exact {
|
||||
var ok bool
|
||||
rest, ok = strings.CutPrefix(pass.text, server)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
var ok bool
|
||||
rest, ok = cutPrefixFold(pass.text, server)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
rest, ok := strings.CutPrefix(strings.TrimLeft(rest, " "), ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
parsed = append(parsed, scopedFindToolsQuery{server: server, exact: pass.exact, tokens: tokenizeFindToolsQuery(rest)})
|
||||
scoped = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !scoped {
|
||||
parsed = append(parsed, scopedFindToolsQuery{tokens: tokenizeFindToolsQuery(query)})
|
||||
}
|
||||
}
|
||||
return parsed
|
||||
}
|
||||
|
||||
// cutPrefixFold is a case-insensitive strings.CutPrefix. It compares
|
||||
// rune by rune with the same simple folding as strings.EqualFold, so a
|
||||
// prefix whose folded form differs in UTF-8 byte length (like S and
|
||||
// the long s) still matches and the cut lands on a rune boundary,
|
||||
// which byte-length slicing cannot guarantee.
|
||||
func cutPrefixFold(s, prefix string) (string, bool) {
|
||||
rest := s
|
||||
for _, prefixRune := range prefix {
|
||||
restRune, size := utf8.DecodeRuneInString(rest)
|
||||
if size == 0 || !runesFoldEqual(restRune, prefixRune) {
|
||||
return "", false
|
||||
}
|
||||
rest = rest[size:]
|
||||
}
|
||||
return rest, true
|
||||
}
|
||||
|
||||
func runesFoldEqual(a, b rune) bool {
|
||||
if a == b {
|
||||
return true
|
||||
}
|
||||
for r := unicode.SimpleFold(a); r != a; r = unicode.SimpleFold(r) {
|
||||
if r == b {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func tokenizeFindTools(value string) []string {
|
||||
parts := findToolsTokenSeparator.Split(strings.ToLower(value), -1)
|
||||
return slices.DeleteFunc(parts, func(part string) bool { return part == "" })
|
||||
}
|
||||
|
||||
// tokenizeFindToolsQuery caps model-supplied query tokens; catalog
|
||||
// fields are tokenized uncapped so every term stays searchable.
|
||||
func tokenizeFindToolsQuery(value string) []string {
|
||||
tokens := tokenizeFindTools(value)
|
||||
if len(tokens) > findToolsMaxQueryTokens {
|
||||
tokens = tokens[:findToolsMaxQueryTokens]
|
||||
}
|
||||
return tokens
|
||||
}
|
||||
|
||||
// findToolsEntryTokens holds an entry's fields tokenized once per
|
||||
// search, so scoring a token is a set lookup instead of re-splitting
|
||||
// name, description, parameter, and server text for every query token.
|
||||
type findToolsEntryTokens struct {
|
||||
name string
|
||||
nameTokens map[string]struct{}
|
||||
description map[string]struct{}
|
||||
parameters map[string]struct{}
|
||||
server map[string]struct{}
|
||||
}
|
||||
|
||||
func tokenizeFindToolsEntry(entry FindToolCatalogEntry) findToolsEntryTokens {
|
||||
toSet := func(value string) map[string]struct{} {
|
||||
tokens := tokenizeFindTools(value)
|
||||
set := make(map[string]struct{}, len(tokens))
|
||||
for _, token := range tokens {
|
||||
set[token] = struct{}{}
|
||||
}
|
||||
return set
|
||||
}
|
||||
return findToolsEntryTokens{
|
||||
name: strings.ToLower(entry.Name),
|
||||
nameTokens: toSet(entry.Name),
|
||||
description: toSet(entry.Description),
|
||||
parameters: toSet(entry.ParameterText),
|
||||
// Server metadata is shown in catalog headers, so its terms
|
||||
// must be searchable too. It applies to every tool on the
|
||||
// server, so it scores below tool-specific matches.
|
||||
server: toSet(entry.Server + " " + entry.ServerDescription),
|
||||
}
|
||||
}
|
||||
|
||||
func (t findToolsEntryTokens) score(token string) int {
|
||||
score := 0
|
||||
if _, ok := t.nameTokens[token]; ok {
|
||||
score += 8
|
||||
} else if strings.Contains(t.name, token) {
|
||||
score += 5
|
||||
}
|
||||
if _, ok := t.description[token]; ok {
|
||||
score += 2
|
||||
}
|
||||
if _, ok := t.parameters[token]; ok {
|
||||
score++
|
||||
}
|
||||
if _, ok := t.server[token]; ok {
|
||||
score++
|
||||
}
|
||||
return score
|
||||
}
|
||||
|
||||
func buildFindToolsDescription(entries []FindToolCatalogEntry, catalogTokenBudget float64) string {
|
||||
const usage = "Search deferred MCP tools by keyword, activate exact tool names, or scope a query to one server with a \"server: terms\" prefix. Calling a cataloged tool directly by name is allowed and auto-loads its schema, but search first for unfamiliar tools. At most 20 tools are returned and activated per call; call again for more.\n\n"
|
||||
budget := float64(findToolsCatalogTokens)
|
||||
if catalogTokenBudget > 0 && catalogTokenBudget < budget {
|
||||
budget = catalogTokenBudget
|
||||
}
|
||||
groups := groupFindToolsEntries(entries)
|
||||
catalog := detailedFindToolsCatalog(groups)
|
||||
if estimatedFindToolsTokens(usage+catalog) > budget {
|
||||
catalog = namesOnlyFindToolsCatalog(groups)
|
||||
}
|
||||
if estimatedFindToolsTokens(usage+catalog) > budget {
|
||||
catalog = countsOnlyFindToolsCatalog(groups)
|
||||
}
|
||||
// Server count and slug length are unbounded, so even the per-server
|
||||
// counts catalog needs a final constant-size fallback.
|
||||
if estimatedFindToolsTokens(usage+catalog) > budget {
|
||||
catalog = fmt.Sprintf("%d deferred tools across %d servers.\n", len(entries), len(groups))
|
||||
}
|
||||
return usage + catalog
|
||||
}
|
||||
|
||||
func detailedFindToolsCatalog(groups []findToolsGroup) string {
|
||||
var b strings.Builder
|
||||
for _, group := range groups {
|
||||
writeFindToolsGroupHeader(&b, group)
|
||||
for _, entry := range group.entries {
|
||||
_, _ = b.WriteString("- ")
|
||||
_, _ = b.WriteString(entry.Name)
|
||||
_, _ = b.WriteString(" - ")
|
||||
_, _ = b.WriteString(truncateFindToolsSummary(entry.Description, 80))
|
||||
_ = b.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func namesOnlyFindToolsCatalog(groups []findToolsGroup) string {
|
||||
var b strings.Builder
|
||||
for _, group := range groups {
|
||||
writeFindToolsGroupHeader(&b, group)
|
||||
names := make([]string, 0, len(group.entries))
|
||||
for _, entry := range group.entries {
|
||||
names = append(names, entry.Name)
|
||||
}
|
||||
_, _ = b.WriteString(strings.Join(names, " "))
|
||||
_ = b.WriteByte('\n')
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func countsOnlyFindToolsCatalog(groups []findToolsGroup) string {
|
||||
var b strings.Builder
|
||||
for _, group := range groups {
|
||||
_, _ = b.WriteString("## ")
|
||||
_, _ = b.WriteString(group.server)
|
||||
_, _ = b.WriteString(" (")
|
||||
_, _ = b.WriteString(strconv.Itoa(len(group.entries)))
|
||||
_, _ = b.WriteString(" tools)\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func writeFindToolsGroupHeader(b *strings.Builder, group findToolsGroup) {
|
||||
_, _ = b.WriteString("## ")
|
||||
_, _ = b.WriteString(group.server)
|
||||
if summary := truncateFindToolsSummary(group.description, 60); summary != "" {
|
||||
_, _ = b.WriteString(" - ")
|
||||
_, _ = b.WriteString(summary)
|
||||
}
|
||||
_ = b.WriteByte('\n')
|
||||
}
|
||||
|
||||
type findToolsGroup struct {
|
||||
server string
|
||||
description string
|
||||
entries []FindToolCatalogEntry
|
||||
}
|
||||
|
||||
func groupFindToolsEntries(entries []FindToolCatalogEntry) []findToolsGroup {
|
||||
grouped := make(map[string]*findToolsGroup)
|
||||
for _, entry := range entries {
|
||||
// Callers assign every entry a non-empty Server so display,
|
||||
// scope matching, and scoring share one identity; an empty
|
||||
// value groups as-is rather than under a label scopes cannot
|
||||
// reach.
|
||||
server := entry.Server
|
||||
group := grouped[server]
|
||||
if group == nil {
|
||||
group = &findToolsGroup{server: server, description: entry.ServerDescription}
|
||||
grouped[server] = group
|
||||
}
|
||||
group.entries = append(group.entries, entry)
|
||||
}
|
||||
groups := make([]findToolsGroup, 0, len(grouped))
|
||||
for _, group := range grouped {
|
||||
slices.SortFunc(group.entries, func(a, b FindToolCatalogEntry) int { return strings.Compare(a.Name, b.Name) })
|
||||
groups = append(groups, *group)
|
||||
}
|
||||
slices.SortFunc(groups, func(a, b findToolsGroup) int { return strings.Compare(a.server, b.server) })
|
||||
return groups
|
||||
}
|
||||
|
||||
func truncateFindToolsSummary(value string, maxRunes int) string {
|
||||
line, _, _ := strings.Cut(value, "\n")
|
||||
sentence, _, _ := strings.Cut(line, ". ")
|
||||
value = strings.TrimSpace(sentence)
|
||||
if utf8.RuneCountInString(value) <= maxRunes {
|
||||
return value
|
||||
}
|
||||
runes := []rune(value)
|
||||
if maxRunes <= 3 {
|
||||
return string(runes[:maxRunes])
|
||||
}
|
||||
return strings.TrimSpace(string(runes[:maxRunes-3])) + "..."
|
||||
}
|
||||
|
||||
func estimatedFindToolsTokens(value string) float64 {
|
||||
return float64(len(value)) / 2.5
|
||||
}
|
||||
@@ -0,0 +1,625 @@
|
||||
package chattool
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"charm.land/fantasy"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSearchTools(t *testing.T) {
|
||||
t.Parallel()
|
||||
entries := []FindToolCatalogEntry{
|
||||
{Name: "github__create_issue", Description: "Create an issue", ParameterText: "repository title body"},
|
||||
{Name: "github__search_issues", Description: "Search issue descriptions", ParameterText: "repository query"},
|
||||
{Name: "slack__post_message", Description: "Post a message", ParameterText: "channel text"},
|
||||
}
|
||||
|
||||
t.Run("weights and tie break", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
result, _ := SearchTools(entries, FindToolsArgs{Queries: []string{"issue"}}, SearchBudget{})
|
||||
require.Len(t, result.Matches, 2)
|
||||
require.Equal(t, []string{"github__create_issue", "github__search_issues"}, []string{result.Matches[0].Name, result.Matches[1].Name})
|
||||
})
|
||||
t.Run("parameter text", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
result, _ := SearchTools(entries, FindToolsArgs{Queries: []string{"channel"}}, SearchBudget{})
|
||||
require.Equal(t, "slack__post_message", result.Matches[0].Name)
|
||||
})
|
||||
t.Run("exact names", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
result, _ := SearchTools(entries, FindToolsArgs{Names: []string{"slack__post_message", "missing"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"slack__post_message"}, result.Activated)
|
||||
require.Equal(t, "slack__post_message", result.Matches[0].Name)
|
||||
})
|
||||
t.Run("empty queries", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
result, _ := SearchTools(entries, FindToolsArgs{}, SearchBudget{})
|
||||
require.Empty(t, result.Matches)
|
||||
require.Empty(t, result.Activated)
|
||||
})
|
||||
t.Run("cap", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
many := make([]FindToolCatalogEntry, 25)
|
||||
for i := range many {
|
||||
many[i] = FindToolCatalogEntry{Name: fmt.Sprintf("server__tool_%02d", i), Description: "common"}
|
||||
}
|
||||
result, _ := SearchTools(many, FindToolsArgs{Queries: []string{"common"}}, SearchBudget{})
|
||||
require.Len(t, result.Matches, findToolsMaxMatches)
|
||||
require.Equal(t, "server__tool_00", result.Matches[0].Name)
|
||||
})
|
||||
t.Run("names capped and prioritized over queries", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
many := make([]FindToolCatalogEntry, 25)
|
||||
names := make([]string, 0, len(many))
|
||||
for i := range many {
|
||||
many[i] = FindToolCatalogEntry{Name: fmt.Sprintf("server__tool_%02d", i), Description: "common"}
|
||||
names = append(names, many[i].Name)
|
||||
}
|
||||
result, _ := SearchTools(many, FindToolsArgs{Queries: []string{"common"}, Names: []string{"server__tool_24"}}, SearchBudget{})
|
||||
require.Len(t, result.Matches, findToolsMaxMatches)
|
||||
require.Equal(t, "server__tool_24", result.Matches[0].Name)
|
||||
require.Contains(t, result.Activated, "server__tool_24")
|
||||
|
||||
capped, _ := SearchTools(many, FindToolsArgs{Names: names}, SearchBudget{})
|
||||
require.Len(t, capped.Matches, findToolsMaxMatches)
|
||||
require.Len(t, capped.Activated, findToolsMaxMatches)
|
||||
})
|
||||
t.Run("names list is bounded", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
entries := []FindToolCatalogEntry{
|
||||
{Name: "server__target", Description: "does things"},
|
||||
}
|
||||
unknown := make([]string, findToolsMaxNames)
|
||||
for i := range unknown {
|
||||
unknown[i] = fmt.Sprintf("missing_%02d", i)
|
||||
}
|
||||
result, _ := SearchTools(entries, FindToolsArgs{Names: append(slices.Clone(unknown), "server__target")}, SearchBudget{})
|
||||
require.Empty(t, result.Activated,
|
||||
"a name past the inspection cap is not looked up")
|
||||
|
||||
result, _ = SearchTools(entries, FindToolsArgs{Names: append(unknown[:findToolsMaxNames-1], "server__target")}, SearchBudget{})
|
||||
require.Equal(t, []string{"server__target"}, result.Activated,
|
||||
"a name within the inspection cap still activates")
|
||||
})
|
||||
t.Run("server metadata", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
serverEntries := []FindToolCatalogEntry{
|
||||
{Name: "tracker__create", Description: "Create an item", Server: "tracker", ServerDescription: "Project tracking"},
|
||||
{Name: "docs__create", Description: "Create a project document", Server: "docs", ServerDescription: "Documentation"},
|
||||
}
|
||||
result, _ := SearchTools(serverEntries, FindToolsArgs{Queries: []string{"tracking"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"tracker__create"}, result.Activated)
|
||||
|
||||
result, _ = SearchTools(serverEntries, FindToolsArgs{Queries: []string{"project"}}, SearchBudget{})
|
||||
require.Equal(t, "docs__create", result.Matches[0].Name,
|
||||
"tool description match outranks server metadata match")
|
||||
require.Len(t, result.Matches, 2)
|
||||
})
|
||||
t.Run("server prefix scope", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
scopedEntries := []FindToolCatalogEntry{
|
||||
{Name: "ci__status", Description: "Pipeline status", Server: "ci"},
|
||||
{Name: "github__get_commit", Description: "Get commit status", Server: "github"},
|
||||
}
|
||||
result, _ := SearchTools(scopedEntries, FindToolsArgs{Queries: []string{"github: status"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"github__get_commit"}, result.Activated,
|
||||
"a known server prefix restricts matches to that server")
|
||||
|
||||
result, _ = SearchTools(scopedEntries, FindToolsArgs{Queries: []string{"github:"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"github__get_commit"}, result.Activated,
|
||||
"a bare server prefix lists that server's tools")
|
||||
|
||||
result, _ = SearchTools(scopedEntries, FindToolsArgs{Queries: []string{"error: status"}}, SearchBudget{})
|
||||
require.Len(t, result.Matches, 2,
|
||||
"an unknown prefix is searched as plain keywords")
|
||||
|
||||
result, _ = SearchTools(scopedEntries, FindToolsArgs{Queries: []string{"GitHub: status"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"github__get_commit"}, result.Activated,
|
||||
"a case-variant prefix still scopes to its server when no exact-case name collides")
|
||||
})
|
||||
t.Run("case-colliding server names", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
caseEntries := []FindToolCatalogEntry{
|
||||
{Name: "GitHub__enterprise_status", Description: "Enterprise status", Server: "GitHub"},
|
||||
{Name: "github__get_commit", Description: "Get commit status", Server: "github"},
|
||||
}
|
||||
result, _ := SearchTools(caseEntries, FindToolsArgs{Queries: []string{"GitHub: status"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"GitHub__enterprise_status"}, result.Activated,
|
||||
"an exact-case prefix scopes only to its own server")
|
||||
|
||||
result, _ = SearchTools(caseEntries, FindToolsArgs{Queries: []string{"github: status"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"github__get_commit"}, result.Activated,
|
||||
"the case-colliding sibling stays reachable by its own exact name")
|
||||
|
||||
result, _ = SearchTools(caseEntries, FindToolsArgs{Queries: []string{"GITHUB: status"}}, SearchBudget{})
|
||||
require.Len(t, result.Activated, 2,
|
||||
"a prefix matching no exact-case name falls back to spanning the case-colliding servers")
|
||||
})
|
||||
t.Run("folded scopes with different byte lengths", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
// The long s folds with S and s but is two UTF-8 bytes, so a
|
||||
// byte-length prefix slice can never line the two forms up.
|
||||
// Scope-only queries keep the assertion sharp: an unscoped
|
||||
// fallback tokenizes to a term that matches nothing because
|
||||
// ToLower does not case-fold the long s.
|
||||
foldedEntries := []FindToolCatalogEntry{
|
||||
{Name: "ſerver__tool", Description: "does things", Server: "ſerver"},
|
||||
}
|
||||
result, _ := SearchTools(foldedEntries, FindToolsArgs{Queries: []string{"Server:"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"ſerver__tool"}, result.Activated,
|
||||
"a folded scope with fewer bytes than the server name still scopes")
|
||||
|
||||
asciiEntries := []FindToolCatalogEntry{
|
||||
{Name: "server__tool", Description: "does things", Server: "server"},
|
||||
}
|
||||
result, _ = SearchTools(asciiEntries, FindToolsArgs{Queries: []string{"ſerver:"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"server__tool"}, result.Activated,
|
||||
"a folded scope with more bytes than the server name still scopes")
|
||||
})
|
||||
t.Run("bounded query work", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
entries := []FindToolCatalogEntry{
|
||||
{Name: "server__match", Description: "Matches the last token", Server: "server"},
|
||||
}
|
||||
// The matching term is placed beyond both caps, so a match
|
||||
// proves the caps were not applied.
|
||||
overflowQuery := strings.Repeat("filler ", findToolsMaxQueryTokens) + "matches"
|
||||
result, _ := SearchTools(entries, FindToolsArgs{Queries: []string{overflowQuery}}, SearchBudget{})
|
||||
require.Empty(t, result.Activated,
|
||||
"tokens beyond the per-query cap are not scored")
|
||||
|
||||
queries := make([]string, findToolsMaxQueries+1)
|
||||
for i := range queries {
|
||||
queries[i] = "filler"
|
||||
}
|
||||
queries[len(queries)-1] = "matches"
|
||||
result, _ = SearchTools(entries, FindToolsArgs{Queries: queries}, SearchBudget{})
|
||||
require.Empty(t, result.Activated,
|
||||
"queries beyond the per-call cap are not scored")
|
||||
|
||||
result, _ = SearchTools(entries, FindToolsArgs{Queries: []string{"matches"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"server__match"}, result.Activated,
|
||||
"capped search still scores in-bound tokens")
|
||||
})
|
||||
t.Run("whitespace-colliding server names", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
paddedEntries := []FindToolCatalogEntry{
|
||||
{Name: "_everything___ping", Description: "Ping status", Server: " everything "},
|
||||
{Name: "everything__status", Description: "Get status", Server: "everything"},
|
||||
}
|
||||
result, _ := SearchTools(paddedEntries, FindToolsArgs{Queries: []string{"everything: status"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"everything__status"}, result.Activated,
|
||||
"the exact-form scope matches only its own server, not a whitespace-padded sibling")
|
||||
|
||||
result, _ = SearchTools(paddedEntries, FindToolsArgs{Queries: []string{" everything : ping"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"_everything___ping"}, result.Activated,
|
||||
"the raw query prefix is matched before trimming, so the padded server stays selectable")
|
||||
})
|
||||
t.Run("server names containing colons", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
colonEntries := []FindToolCatalogEntry{
|
||||
{Name: "jira_prod__list_issues", Description: "List issues", Server: "jira:prod"},
|
||||
{Name: "jira__list_issues", Description: "List issues", Server: "jira"},
|
||||
{Name: "ci__status", Description: "Issue pipeline status", Server: "ci"},
|
||||
}
|
||||
result, _ := SearchTools(colonEntries, FindToolsArgs{Queries: []string{"jira:prod: issues"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"jira_prod__list_issues"}, result.Activated,
|
||||
"the longest cataloged server name wins over its colon-split prefix")
|
||||
|
||||
result, _ = SearchTools(colonEntries, FindToolsArgs{Queries: []string{"jira: issues"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"jira__list_issues"}, result.Activated,
|
||||
"the shorter server still scopes its own queries")
|
||||
})
|
||||
t.Run("unicode terms", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
unicodeEntries := []FindToolCatalogEntry{
|
||||
{Name: "docs__検索", Description: "ドキュメント検索"},
|
||||
{Name: "docs__erstellen", Description: "Dokument ERSTELLEN"},
|
||||
}
|
||||
result, _ := SearchTools(unicodeEntries, FindToolsArgs{Queries: []string{"検索"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"docs__検索"}, result.Activated)
|
||||
|
||||
result, _ = SearchTools(unicodeEntries, FindToolsArgs{Queries: []string{"Erstellen"}}, SearchBudget{})
|
||||
require.Equal(t, []string{"docs__erstellen"}, result.Activated)
|
||||
})
|
||||
t.Run("schema token budget", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
weighted := []FindToolCatalogEntry{
|
||||
{Name: "server__big_a", Description: "big", SchemaTokens: 60},
|
||||
{Name: "server__big_b", Description: "big", SchemaTokens: 60},
|
||||
{Name: "server__huge", Description: "big", SchemaTokens: 500},
|
||||
}
|
||||
result, _ := SearchTools(weighted, FindToolsArgs{Queries: []string{"big"}}, SearchBudget{SchemaTokens: 100, AllowFirstOverBudget: true})
|
||||
require.Equal(t, []string{"server__big_a"}, result.Activated,
|
||||
"matches stop once the schema budget is spent")
|
||||
|
||||
result, _ = SearchTools(weighted, FindToolsArgs{Names: []string{"server__huge"}}, SearchBudget{SchemaTokens: 100, AllowFirstOverBudget: true})
|
||||
require.Equal(t, []string{"server__huge"}, result.Activated,
|
||||
"the first match is kept even when it alone exceeds the budget")
|
||||
})
|
||||
t.Run("result descriptions are summarized", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
long := []FindToolCatalogEntry{{
|
||||
Name: "server__verbose",
|
||||
Description: strings.Repeat("word ", 100),
|
||||
}}
|
||||
result, _ := SearchTools(long, FindToolsArgs{Names: []string{"server__verbose"}}, SearchBudget{})
|
||||
require.LessOrEqual(t, len([]rune(result.Matches[0].Description)), 80)
|
||||
})
|
||||
}
|
||||
|
||||
func TestFindTools(t *testing.T) {
|
||||
t.Parallel()
|
||||
var recorded FindToolsCall
|
||||
tool := FindTools(FindToolsOptions{
|
||||
Entries: []FindToolCatalogEntry{{Name: "github__create_issue", Description: "Create an issue"}},
|
||||
OnCall: func(_ context.Context, call FindToolsCall) { recorded = call },
|
||||
})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"queries":["issue"]}`})
|
||||
require.NoError(t, err)
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"github__create_issue"}, result.Activated)
|
||||
require.Equal(t, 1, recorded.MatchCount)
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError)
|
||||
}
|
||||
|
||||
func TestFindToolsSerialToolCalls(t *testing.T) {
|
||||
t.Parallel()
|
||||
serial, ok := FindTools(FindToolsOptions{}).(interface{ SerialToolCalls() bool })
|
||||
require.True(t, ok, "find_tools must opt into serial execution so shared-budget admission follows tool-call order")
|
||||
require.True(t, serial.SerialToolCalls())
|
||||
}
|
||||
|
||||
func TestFindToolsDirectCallReservation(t *testing.T) {
|
||||
t.Parallel()
|
||||
newTool := func(budget float64) (fantasy.AgentTool, interface{ ObserveStepToolCalls([]string) }) {
|
||||
tool := FindTools(FindToolsOptions{
|
||||
Entries: []FindToolCatalogEntry{
|
||||
{Name: "server__a", SchemaTokens: 60},
|
||||
{Name: "server__b", SchemaTokens: 50},
|
||||
{Name: "server__c", SchemaTokens: 30},
|
||||
},
|
||||
SchemaTokenBudget: budget,
|
||||
})
|
||||
observer, ok := tool.(interface{ ObserveStepToolCalls([]string) })
|
||||
require.True(t, ok, "find_tools must observe step tool calls to reserve direct-call schema weight")
|
||||
return tool, observer
|
||||
}
|
||||
|
||||
t.Run("direct calls charge the budget before searches", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(100)
|
||||
observer.ObserveStepToolCalls([]string{"server__a", "server__a", "unknown", FindToolsName})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "a search claim exceeding the budget left by direct calls must fail loudly")
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__c"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError, "duplicate direct-call names are charged once")
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__c"}, result.Activated)
|
||||
})
|
||||
|
||||
t.Run("rejected calls still reach OnCall", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
var calls []FindToolsCall
|
||||
tool := FindTools(FindToolsOptions{
|
||||
Entries: []FindToolCatalogEntry{
|
||||
{Name: "server__a", SchemaTokens: 60},
|
||||
{Name: "server__b", SchemaTokens: 50},
|
||||
},
|
||||
SchemaTokenBudget: 60,
|
||||
OnCall: func(_ context.Context, call FindToolsCall) { calls = append(calls, call) },
|
||||
})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError)
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError)
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"queries":"github"}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "a type mismatch is rejected by the argument decoder")
|
||||
require.Len(t, calls, 4, "rejected calls count toward call totals")
|
||||
require.Empty(t, calls[0].Rejection)
|
||||
require.Equal(t, "budget", calls[1].Rejection)
|
||||
require.Equal(t, []string{"server__b"}, calls[1].Names)
|
||||
require.Empty(t, calls[1].Activated, "a rejected call reports no activations")
|
||||
require.Equal(t, "arguments", calls[2].Rejection, "empty-argument calls are counted as rejected")
|
||||
require.Empty(t, calls[2].Activated)
|
||||
require.Equal(t, "arguments", calls[3].Rejection,
|
||||
"calls rejected during argument decoding are counted before the handler is reached")
|
||||
})
|
||||
|
||||
t.Run("a touched budget skips oversized matches and admits later fits", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool := FindTools(FindToolsOptions{
|
||||
Entries: []FindToolCatalogEntry{
|
||||
{Name: "server__a", SchemaTokens: 60},
|
||||
{Name: "server__b", SchemaTokens: 50},
|
||||
{Name: "server__c", SchemaTokens: 30},
|
||||
},
|
||||
SchemaTokenBudget: 100,
|
||||
})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b","server__c"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError, "an oversized top match must not fail the call when a later match fits")
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__c"}, result.Activated,
|
||||
"the oversized first match is skipped and the fitting later match admitted")
|
||||
})
|
||||
|
||||
t.Run("an errored direct call refunds its reservation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(100)
|
||||
settler, ok := tool.(interface {
|
||||
ObserveStepToolResults(names []string, errored []bool)
|
||||
})
|
||||
require.True(t, ok, "find_tools must observe step results to refund errored reservations")
|
||||
observer.ObserveStepToolCalls([]string{"server__a"})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "the pre-execution reservation holds while the outcome is unknown")
|
||||
|
||||
settler.ObserveStepToolResults([]string{"server__a", "unknown"}, []bool{true, true})
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError, "the refunded reservation admits later searches")
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__b"}, result.Activated)
|
||||
})
|
||||
|
||||
t.Run("a name that executed successfully keeps its reservation", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(100)
|
||||
settler, ok := tool.(interface {
|
||||
ObserveStepToolResults(names []string, errored []bool)
|
||||
})
|
||||
require.True(t, ok)
|
||||
observer.ObserveStepToolCalls([]string{"server__a", "server__a"})
|
||||
settler.ObserveStepToolResults([]string{"server__a", "server__a"}, []bool{false, true})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "a successful execution pins the reservation even when a later call errors")
|
||||
})
|
||||
|
||||
t.Run("mixed outcomes admit at the first successful call position", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(100)
|
||||
settler, ok := tool.(interface {
|
||||
ObserveStepToolResults(names []string, errored []bool)
|
||||
})
|
||||
require.True(t, ok)
|
||||
// A errors, B succeeds, then A succeeds: derivation postpones
|
||||
// the errored A by call ID, admits B, and budget-rejects the
|
||||
// later A (60 over the 50 already charged), so only B's schema
|
||||
// reaches the next request.
|
||||
observer.ObserveStepToolCalls([]string{"server__a", "server__b", "server__a"})
|
||||
settler.ObserveStepToolResults([]string{"server__a", "server__b", "server__a"}, []bool{true, false, false})
|
||||
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Empty(t, result.Activated,
|
||||
"a name derivation budget-rejects at its successful call position is unclaimable, not free")
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__b"}, result.Activated, "the admitted call stays free")
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__c"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__c"}, result.Activated,
|
||||
"only the admitted prefix is charged, so the leftover budget admits new claims")
|
||||
})
|
||||
|
||||
t.Run("aggregate overflow frees only the prefix derivation retains", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(100)
|
||||
settler, ok := tool.(interface {
|
||||
ObserveStepToolResults(names []string, errored []bool)
|
||||
})
|
||||
require.True(t, ok)
|
||||
observer.ObserveStepToolCalls([]string{"server__a", "server__b"})
|
||||
settler.ObserveStepToolResults([]string{"server__a", "server__b"}, []bool{false, false})
|
||||
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Empty(t, result.Activated,
|
||||
"a direct call past the retained prefix cannot be reported activated: derivation sheds it")
|
||||
require.Equal(t, 3, result.TotalDeferred, "unclaimable entries still count as deferred")
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__a"}, result.Activated, "the retained prefix stays free")
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__c"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__c"}, result.Activated,
|
||||
"the skipped call's weight is not charged, so later searches keep the leftover budget")
|
||||
})
|
||||
|
||||
t.Run("a name claimed by an earlier search is free for later searches", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, _ := newTool(60)
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError, "derivation deduplicates by name, so a repeated claim costs nothing")
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__a"}, result.Activated)
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__c"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "the repeated claim must not have refunded the spent budget")
|
||||
})
|
||||
|
||||
t.Run("an errored prefix call promotes the next observed name", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(100)
|
||||
settler, ok := tool.(interface {
|
||||
ObserveStepToolResults(names []string, errored []bool)
|
||||
})
|
||||
require.True(t, ok)
|
||||
observer.ObserveStepToolCalls([]string{"server__a", "server__b"})
|
||||
settler.ObserveStepToolResults([]string{"server__a", "server__b"}, []bool{true, false})
|
||||
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__b"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError)
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__b"}, result.Activated,
|
||||
"the errored call leaves the prefix, so the succeeding call becomes free")
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError,
|
||||
"the errored call is claimable at full weight, which exceeds the leftover budget")
|
||||
})
|
||||
|
||||
t.Run("reserved names stay activatable after the budget is spent", func(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool, observer := newTool(50)
|
||||
observer.ObserveStepToolCalls([]string{"server__a"})
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__a"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError, "derivation retains direct calls, so reporting them activated is free")
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
require.Equal(t, []string{"server__a"}, result.Activated)
|
||||
|
||||
resp, err = tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__c"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "an over-reserved budget admits no new schema weight")
|
||||
})
|
||||
}
|
||||
|
||||
func TestFindToolsSharedSchemaBudget(t *testing.T) {
|
||||
t.Parallel()
|
||||
tool := FindTools(FindToolsOptions{
|
||||
Entries: []FindToolCatalogEntry{
|
||||
{Name: "server__a", SchemaTokens: 60},
|
||||
{Name: "server__b", SchemaTokens: 60},
|
||||
{Name: "server__c", SchemaTokens: 60},
|
||||
{Name: "server__d", SchemaTokens: 60},
|
||||
},
|
||||
SchemaTokenBudget: 200,
|
||||
})
|
||||
activated := func(input string) []string {
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: input})
|
||||
require.NoError(t, err)
|
||||
var result FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &result))
|
||||
return result.Activated
|
||||
}
|
||||
|
||||
require.Equal(t, []string{"server__a", "server__b"}, activated(`{"names":["server__a","server__b"]}`))
|
||||
require.Equal(t, []string{"server__c"}, activated(`{"names":["server__c","server__d"]}`),
|
||||
"the second call spends the remaining shared budget, not a fresh one")
|
||||
|
||||
resp, err := tool.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__d"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError,
|
||||
"a call whose claims cannot fit the remaining budget errors instead of over-claiming")
|
||||
|
||||
huge := FindTools(FindToolsOptions{
|
||||
Entries: []FindToolCatalogEntry{
|
||||
{Name: "server__huge", SchemaTokens: 500},
|
||||
{Name: "server__other", SchemaTokens: 60},
|
||||
},
|
||||
SchemaTokenBudget: 200,
|
||||
})
|
||||
resp, err = huge.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__huge"]}`})
|
||||
require.NoError(t, err)
|
||||
var hugeResult FindToolsResult
|
||||
require.NoError(t, json.Unmarshal([]byte(resp.Content), &hugeResult))
|
||||
require.Equal(t, []string{"server__huge"}, hugeResult.Activated,
|
||||
"an untouched budget may over-claim once; derivation's newest-keep retains the sole claim")
|
||||
resp, err = huge.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__huge"]}`})
|
||||
require.NoError(t, err)
|
||||
require.False(t, resp.IsError, "repeating an already claimed name costs nothing")
|
||||
resp, err = huge.Run(context.Background(), fantasy.ToolCall{Input: `{"names":["server__other"]}`})
|
||||
require.NoError(t, err)
|
||||
require.True(t, resp.IsError, "the spent budget rejects further new activations")
|
||||
}
|
||||
|
||||
func TestBuildFindToolsDescription(t *testing.T) {
|
||||
t.Parallel()
|
||||
entries := []FindToolCatalogEntry{
|
||||
{Name: "zeta__last", Description: "Last tool. More detail", Server: "zeta", ServerDescription: strings.Repeat("z", 80)},
|
||||
{Name: "alpha__second", Description: strings.Repeat("x", 100), Server: "alpha", ServerDescription: "Alpha server"},
|
||||
{Name: "alpha__first", Description: "First tool\nmore detail", Server: "alpha", ServerDescription: "Alpha server"},
|
||||
}
|
||||
description := buildFindToolsDescription(entries, 0)
|
||||
require.Less(t, strings.Index(description, "## alpha"), strings.Index(description, "## zeta"))
|
||||
require.Less(t, strings.Index(description, "alpha__first"), strings.Index(description, "alpha__second"))
|
||||
require.Contains(t, description, "First tool")
|
||||
require.NotContains(t, description, "more detail")
|
||||
require.Contains(t, description, "...")
|
||||
|
||||
many := make([]FindToolCatalogEntry, 300)
|
||||
for i := range many {
|
||||
many[i] = FindToolCatalogEntry{
|
||||
Name: fmt.Sprintf("server__tool_%03d_%s", i, strings.Repeat("n", 40)),
|
||||
Description: strings.Repeat("description ", 20),
|
||||
Server: "server",
|
||||
}
|
||||
}
|
||||
degraded := buildFindToolsDescription(many, 0)
|
||||
require.Contains(t, degraded, "## server (300 tools)")
|
||||
require.NotContains(t, degraded, "server__tool_000")
|
||||
|
||||
manyServers := make([]FindToolCatalogEntry, 500)
|
||||
for i := range manyServers {
|
||||
manyServers[i] = FindToolCatalogEntry{
|
||||
Name: fmt.Sprintf("server_%03d_%s__tool", i, strings.Repeat("s", 40)),
|
||||
Server: fmt.Sprintf("server_%03d_%s", i, strings.Repeat("s", 40)),
|
||||
}
|
||||
}
|
||||
countsExceeded := buildFindToolsDescription(manyServers, 0)
|
||||
require.Contains(t, countsExceeded, "500 deferred tools across 500 servers.")
|
||||
require.NotContains(t, countsExceeded, "## server_000")
|
||||
require.LessOrEqual(t, estimatedFindToolsTokens(countsExceeded), float64(findToolsCatalogTokens))
|
||||
|
||||
smallWindow := buildFindToolsDescription(entries, 150)
|
||||
require.NotContains(t, smallWindow, "First tool",
|
||||
"a small context window budget forces catalog degradation below the 4000-token default")
|
||||
}
|
||||
@@ -120,6 +120,17 @@ func buildWorkspaceMCPTool(
|
||||
}
|
||||
}
|
||||
|
||||
// ServerName returns the originating MCP server name from the unsanitized
|
||||
// routing name. The model-facing info.Name can lose the "__" separator to
|
||||
// sanitization or length capping, so it cannot be parsed for the server.
|
||||
func (t *WorkspaceMCPTool) ServerName() string {
|
||||
server, _, ok := strings.Cut(t.routingName, "__")
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
// sanitizeModelToolName returns the provider-safe form of a workspace MCP tool
|
||||
// name: characters outside [a-zA-Z0-9_-] become "_" and the result is capped
|
||||
// at maxModelToolNameLen. The "__" server/tool separator survives because
|
||||
|
||||
@@ -90,6 +90,7 @@ func TestCreateChat_ForceOnMCPServerEnforced(t *testing.T) {
|
||||
})
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
})
|
||||
|
||||
@@ -159,6 +160,7 @@ func TestSendMessage_ForceOnMCPServerEnforced(t *testing.T) {
|
||||
})
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
})
|
||||
|
||||
@@ -231,6 +233,7 @@ func TestGeneration_ForceOnMCPServerEnforcedForExistingChats(t *testing.T) {
|
||||
user, org, model := seedChatDependenciesWithProvider(t, db, "openai-compat", openAIURL)
|
||||
|
||||
server := newActiveTestServer(t, db, ps, func(cfg *chatd.Config) {
|
||||
withoutMCPToolSearch(cfg)
|
||||
cfg.AIBridgeTransportFactory = chatAIGatewayTransportFactoryPointer(chattest.NewMockAIBridgeTransport(t, openAIURL))
|
||||
})
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ import (
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprovider"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatretry"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatstate"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/messagepartbuffer"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/codersdk/x/agenthooks"
|
||||
@@ -40,13 +41,14 @@ type generationPrepared struct {
|
||||
Chat database.Chat
|
||||
Messages []database.ChatMessage
|
||||
|
||||
Model chatprovider.Model
|
||||
Prompt []fantasy.Message
|
||||
Tools []fantasy.AgentTool
|
||||
ActiveTools []string
|
||||
ProviderTools []chatloop.ProviderTool
|
||||
ModelRoute aiGatewayModelRoute
|
||||
ModelBuildOptions modelBuildOptions
|
||||
Model chatprovider.Model
|
||||
Prompt []fantasy.Message
|
||||
Tools []fantasy.AgentTool
|
||||
ActiveTools []string
|
||||
AllowInactiveTools map[string]bool
|
||||
ProviderTools []chatloop.ProviderTool
|
||||
ModelRoute aiGatewayModelRoute
|
||||
ModelBuildOptions modelBuildOptions
|
||||
|
||||
// ResolvedProvider is the configured provider identity used to label
|
||||
// user-facing errors. See chatloop.GenerateAssistantOptions.ErrorProvider.
|
||||
@@ -781,20 +783,46 @@ func (s *taskStarter) admitStepToolCalls(
|
||||
if len(toolCalls) == 0 || exclusiveBatchRejected(toolCalls, prepared.ExclusiveToolNames) {
|
||||
return chathooks.PreToolUseExecutionResult{}, nil
|
||||
}
|
||||
// An admission error discards the whole batch before it can be
|
||||
// committed, so its find_tools calls would otherwise never reach
|
||||
// the executeLocalTools counter; count them at each error exit.
|
||||
countBatch := func() {
|
||||
if !prepared.BuiltinToolNames[chattool.FindToolsName] {
|
||||
return
|
||||
}
|
||||
for _, toolCall := range toolCalls {
|
||||
if toolCall.ToolName == chattool.FindToolsName {
|
||||
s.server.metrics.FindToolsCallsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
// Check the full batch first: a call removed below still occupies its ID
|
||||
// in the step, so filtering before this would hide the collision.
|
||||
if err := chathooks.RejectDuplicateToolUseIDs(toolCalls); err != nil {
|
||||
countBatch()
|
||||
return chathooks.PreToolUseExecutionResult{}, chathooks.GenerationDispatchError(agenthooks.EventPreToolUse, err)
|
||||
}
|
||||
unambiguous, ambiguous := partitionAmbiguousToolCalls(prepared, toolCalls)
|
||||
preflight, err := s.server.hooks.PreflightPendingToolCalls(ctx, chathooks.ChatFor(prepared.Chat, input.hookTurnID()), unambiguous)
|
||||
if err != nil {
|
||||
countBatch()
|
||||
return chathooks.PreToolUseExecutionResult{}, chathooks.GenerationDispatchError(agenthooks.EventPreToolUse, err)
|
||||
}
|
||||
if err := validateOverriddenToolInputs(prepared, preflight); err != nil {
|
||||
countBatch()
|
||||
return chathooks.PreToolUseExecutionResult{}, chathooks.GenerationDispatchError(agenthooks.EventPreToolUse, err)
|
||||
}
|
||||
preflight.Denied = append(preflight.Denied, ambiguous...)
|
||||
// Calls denied at admission persist synthetic results with the
|
||||
// assistant step, so they never surface as unresolved calls where
|
||||
// executeLocalTools counts find_tools invocations; count them here.
|
||||
if prepared.BuiltinToolNames[chattool.FindToolsName] {
|
||||
for _, result := range preflight.Denied {
|
||||
if result.ToolName == chattool.FindToolsName {
|
||||
s.server.metrics.FindToolsCallsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
return preflight, nil
|
||||
}
|
||||
|
||||
@@ -810,6 +838,17 @@ func (s *taskStarter) executeLocalTools(
|
||||
if !exclusiveBatchRejected(decision.localToolCalls, prepared.ExclusiveToolNames) {
|
||||
allowed, denied = partitionAmbiguousToolCalls(prepared, decision.localToolCalls)
|
||||
}
|
||||
// find_tools calls are counted here, at the single point every
|
||||
// model-emitted call passes through, because rejections upstream of
|
||||
// the tool (partition denials, hook denials, exclusive-policy
|
||||
// batches) never reach its handler or OnCall.
|
||||
if prepared.BuiltinToolNames[chattool.FindToolsName] {
|
||||
for _, toolCall := range decision.localToolCalls {
|
||||
if toolCall.ToolName == chattool.FindToolsName {
|
||||
s.server.metrics.FindToolsCallsTotal.Inc()
|
||||
}
|
||||
}
|
||||
}
|
||||
attempt, err := s.beginGenerationAttempt(ctx, machine, input)
|
||||
if err != nil {
|
||||
return xerrors.Errorf("beginGenerationAttempt: %w", err)
|
||||
@@ -827,8 +866,10 @@ func (s *taskStarter) executeLocalTools(
|
||||
outcome, err = chatloop.ExecuteLocalTools(ctx, chatloop.ExecuteLocalToolsOptions{
|
||||
Tools: prepared.Tools,
|
||||
ActiveTools: prepared.ActiveTools,
|
||||
AllowInactiveTools: prepared.AllowInactiveTools,
|
||||
ProviderTools: prepared.ProviderTools,
|
||||
ToolCalls: allowed,
|
||||
ObservedToolCalls: decision.localToolCalls,
|
||||
ExclusiveToolNames: prepared.ExclusiveToolNames,
|
||||
BuiltinToolNames: prepared.BuiltinToolNames,
|
||||
ModelProvider: provider,
|
||||
|
||||
@@ -529,6 +529,19 @@ func (server *Server) prepareGeneration(
|
||||
builtinToolNames[t.Info().Name] = true
|
||||
}
|
||||
|
||||
mcpConfigByID := make(map[uuid.UUID]database.MCPServerConfig, len(mcpConnectConfigs))
|
||||
for _, config := range mcpConnectConfigs {
|
||||
mcpConfigByID[config.ID] = config
|
||||
}
|
||||
deferredCandidates := collectDeferredMCPCandidates(deferredMCPCandidateInput{
|
||||
mcpTools: mcpTools,
|
||||
workspaceMCPTools: workspaceMCPTools,
|
||||
mcpConfigByID: mcpConfigByID,
|
||||
planMode: currentPlanMode,
|
||||
parentChatID: chat.ParentChatID,
|
||||
approvedMCPConfigIDs: approvedPlanMCPConfigIDs,
|
||||
includeWorkspaceTools: !isExploreSubagent,
|
||||
})
|
||||
tools = append(tools, mcpTools...)
|
||||
if !isExploreSubagent {
|
||||
tools = append(tools, workspaceMCPTools...)
|
||||
@@ -590,6 +603,51 @@ func (server *Server) prepareGeneration(
|
||||
if isExploreSubagent {
|
||||
activeToolNames = allowedExploreToolNames(tools)
|
||||
}
|
||||
var allowInactiveTools map[string]bool
|
||||
if decideMCPToolSearch(mcpToolSearchInput{
|
||||
experimentEnabled: server.experiments.Enabled(codersdk.ExperimentMCPToolSearch),
|
||||
candidates: deferredCandidates,
|
||||
dynamicToolNames: dynamicToolNames,
|
||||
}) {
|
||||
activationTokenBudget := float64(modelConfig.ContextLimit) / mcpToolSearchBudgetDivisor
|
||||
findTools := chattool.FindTools(chattool.FindToolsOptions{
|
||||
Entries: deferredMCPToolEntries(deferredCandidates),
|
||||
SchemaTokenBudget: activationTokenBudget,
|
||||
CatalogTokenBudget: activationTokenBudget,
|
||||
// Calls total is counted in executeLocalTools, which also
|
||||
// sees calls rejected before the tool runs; OnCall covers
|
||||
// only calls that reach the handler or its decode.
|
||||
OnCall: func(callCtx context.Context, call chattool.FindToolsCall) {
|
||||
if call.Rejection == "" {
|
||||
server.metrics.FindToolsMatchCount.Observe(float64(call.MatchCount))
|
||||
server.metrics.FindToolsActivationsTotal.Add(float64(len(call.Activated)))
|
||||
if call.MatchCount == 0 {
|
||||
server.metrics.FindToolsEmptyTotal.Inc()
|
||||
}
|
||||
}
|
||||
// Queries and names are model output that can echo
|
||||
// prompt content, so standard logs carry only
|
||||
// aggregate fields; raw values are visible through
|
||||
// the opt-in chat debug logging path.
|
||||
logger.Info(callCtx, "deferred MCP tool search",
|
||||
slog.F("query_count", len(call.Queries)),
|
||||
slog.F("name_count", len(call.Names)),
|
||||
slog.F("match_count", call.MatchCount),
|
||||
slog.F("activated_count", len(call.Activated)),
|
||||
slog.F("total_deferred", call.TotalDeferred),
|
||||
slog.F("rejection", call.Rejection),
|
||||
)
|
||||
},
|
||||
})
|
||||
tools, activeToolNames, allowInactiveTools = configureDeferredMCPToolSearch(
|
||||
tools,
|
||||
activeToolNames,
|
||||
deferredCandidates,
|
||||
findTools,
|
||||
deriveDeferredMCPActivations(promptRows, deferredCandidates, activationTokenBudget),
|
||||
)
|
||||
builtinToolNames[chattool.FindToolsName] = true
|
||||
}
|
||||
|
||||
toolNameToConfigID := make(map[string]uuid.UUID)
|
||||
for _, t := range tools {
|
||||
@@ -675,6 +733,7 @@ func (server *Server) prepareGeneration(
|
||||
Prompt: prompt,
|
||||
Tools: tools,
|
||||
ActiveTools: activeToolNames,
|
||||
AllowInactiveTools: allowInactiveTools,
|
||||
ProviderTools: providerTools,
|
||||
ModelRoute: modelRoute,
|
||||
ModelBuildOptions: modelOpts,
|
||||
|
||||
@@ -0,0 +1,378 @@
|
||||
package chatd
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"charm.land/fantasy"
|
||||
"github.com/google/uuid"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/mcpclient"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
)
|
||||
|
||||
// mcpToolSearchBudgetDivisor scales the activation and catalog budgets
|
||||
// to the model context window: activated schemas may re-inline up to
|
||||
// ContextLimit / 10 estimated tokens per generation.
|
||||
const mcpToolSearchBudgetDivisor = 10
|
||||
|
||||
type deferredMCPTool struct {
|
||||
tool fantasy.AgentTool
|
||||
server string
|
||||
serverDescription string
|
||||
}
|
||||
|
||||
type deferredMCPCandidateInput struct {
|
||||
mcpTools []fantasy.AgentTool
|
||||
workspaceMCPTools []fantasy.AgentTool
|
||||
mcpConfigByID map[uuid.UUID]database.MCPServerConfig
|
||||
planMode database.NullChatPlanMode
|
||||
parentChatID uuid.NullUUID
|
||||
approvedMCPConfigIDs map[uuid.UUID]struct{}
|
||||
includeWorkspaceTools bool
|
||||
}
|
||||
|
||||
// collectDeferredMCPCandidates applies the same turn policy that
|
||||
// filterToolsForTurn later applies to the executable tool set, so the
|
||||
// find_tools catalog never advertises tools the turn cannot run.
|
||||
func collectDeferredMCPCandidates(input deferredMCPCandidateInput) []deferredMCPTool {
|
||||
candidates := make([]deferredMCPTool, 0, len(input.mcpTools)+len(input.workspaceMCPTools))
|
||||
for _, tool := range input.mcpTools {
|
||||
if !toolAllowedForTurn(tool, input.planMode, input.parentChatID, input.approvedMCPConfigIDs) {
|
||||
continue
|
||||
}
|
||||
candidate := deferredMCPTool{tool: tool}
|
||||
if identified, ok := tool.(mcpclient.MCPToolIdentifier); ok {
|
||||
if config, exists := input.mcpConfigByID[identified.MCPServerConfigID()]; exists {
|
||||
candidate.server = config.Slug
|
||||
candidate.serverDescription = config.Description
|
||||
}
|
||||
}
|
||||
candidates = append(candidates, candidate)
|
||||
}
|
||||
if !input.includeWorkspaceTools {
|
||||
return candidates
|
||||
}
|
||||
wsStart := len(candidates)
|
||||
for _, tool := range input.workspaceMCPTools {
|
||||
if !toolAllowedForTurn(tool, input.planMode, input.parentChatID, input.approvedMCPConfigIDs) {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, deferredMCPTool{tool: tool, server: workspaceMCPServerName(tool)})
|
||||
}
|
||||
trimWorkspaceServerNames(candidates, wsStart)
|
||||
return candidates
|
||||
}
|
||||
|
||||
// trimWorkspaceServerNames trims surrounding whitespace from workspace
|
||||
// server names, which config validation permits but find_tools strips
|
||||
// from queries, so a padded server stays reachable by scope. Trimming
|
||||
// is skipped when it would collapse distinct servers into one catalog
|
||||
// identity (a padded and an unpadded sibling, or a collision with an
|
||||
// external slug): those keep their raw names, so each server keeps its
|
||||
// own catalog group and an exact-form scope matches only its own
|
||||
// server.
|
||||
func trimWorkspaceServerNames(candidates []deferredMCPTool, wsStart int) {
|
||||
sources := make(map[string]map[string]struct{}, len(candidates))
|
||||
for i, candidate := range candidates {
|
||||
key := candidate.server
|
||||
if i >= wsStart {
|
||||
key = strings.TrimSpace(key)
|
||||
}
|
||||
if sources[key] == nil {
|
||||
sources[key] = make(map[string]struct{}, 1)
|
||||
}
|
||||
sources[key][candidate.server] = struct{}{}
|
||||
}
|
||||
for i := wsStart; i < len(candidates); i++ {
|
||||
trimmed := strings.TrimSpace(candidates[i].server)
|
||||
if len(sources[trimmed]) == 1 {
|
||||
candidates[i].server = trimmed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// workspaceMCPServerName prefers the wrapper's unsanitized routing name
|
||||
// because sanitization can truncate the model-facing name before the
|
||||
// "__" separator, which would otherwise catalog each such tool under a
|
||||
// fake single-tool server that prefix scoping cannot reach.
|
||||
func workspaceMCPServerName(tool fantasy.AgentTool) string {
|
||||
if namer, ok := tool.(interface{ ServerName() string }); ok {
|
||||
return namer.ServerName()
|
||||
}
|
||||
if server, _, ok := strings.Cut(tool.Info().Name, "__"); ok {
|
||||
return server
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type mcpToolSearchInput struct {
|
||||
experimentEnabled bool
|
||||
candidates []deferredMCPTool
|
||||
dynamicToolNames map[string]bool
|
||||
}
|
||||
|
||||
// decideMCPToolSearch reports whether MCP tool schemas are deferred
|
||||
// behind find_tools. With the experiment enabled, every generation with
|
||||
// deferrable candidates defers.
|
||||
func decideMCPToolSearch(input mcpToolSearchInput) bool {
|
||||
if !input.experimentEnabled || len(input.candidates) == 0 {
|
||||
return false
|
||||
}
|
||||
// A client-executed dynamic tool named find_tools would otherwise be
|
||||
// advertised alongside the built-in and capture its calls as
|
||||
// requires_action, so a collision on either surface fails open.
|
||||
if input.dynamicToolNames[chattool.FindToolsName] {
|
||||
return false
|
||||
}
|
||||
for _, candidate := range input.candidates {
|
||||
if candidate.tool.Info().Name == chattool.FindToolsName {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func configureDeferredMCPToolSearch(
|
||||
tools []fantasy.AgentTool,
|
||||
activeToolNames []string,
|
||||
candidates []deferredMCPTool,
|
||||
findTools fantasy.AgentTool,
|
||||
activations []string,
|
||||
) ([]fantasy.AgentTool, []string, map[string]bool) {
|
||||
candidateNames := deferredMCPToolNameSet(candidates)
|
||||
ordered := make([]fantasy.AgentTool, 0, len(tools)+1)
|
||||
for _, tool := range tools {
|
||||
if !candidateNames[tool.Info().Name] {
|
||||
ordered = append(ordered, tool)
|
||||
}
|
||||
}
|
||||
ordered = append(ordered, findTools)
|
||||
for _, tool := range tools {
|
||||
if candidateNames[tool.Info().Name] {
|
||||
ordered = append(ordered, tool)
|
||||
}
|
||||
}
|
||||
|
||||
activeToolNames = slices.DeleteFunc(activeToolNames, func(name string) bool { return candidateNames[name] })
|
||||
activeToolNames = append(activeToolNames, chattool.FindToolsName)
|
||||
activeToolNames = append(activeToolNames, activations...)
|
||||
return ordered, activeToolNames, candidateNames
|
||||
}
|
||||
|
||||
func estimateDeferredMCPToolTokens(candidates []deferredMCPTool) float64 {
|
||||
chars := 0
|
||||
for _, candidate := range candidates {
|
||||
info := candidate.tool.Info()
|
||||
schema := map[string]any{"type": "object", "properties": info.Parameters}
|
||||
if len(info.Required) > 0 {
|
||||
schema["required"] = info.Required
|
||||
}
|
||||
serialized, _ := json.Marshal(schema)
|
||||
chars += len(info.Name) + len(info.Description) + len(serialized)
|
||||
}
|
||||
return float64(chars) / 2.5
|
||||
}
|
||||
|
||||
func deferredMCPToolEntries(candidates []deferredMCPTool) []chattool.FindToolCatalogEntry {
|
||||
// Workspace config validation permits an empty server key, and
|
||||
// candidates whose config lookup failed also carry no server, so
|
||||
// empty identities get a real label here. The label is the entry's
|
||||
// Server for grouping, scope matching, and scoring alike, and it is
|
||||
// collision-safe: a literal server with the same name keeps its own
|
||||
// group and scope.
|
||||
fallback := "workspace"
|
||||
taken := make(map[string]struct{}, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
if candidate.server != "" {
|
||||
taken[candidate.server] = struct{}{}
|
||||
}
|
||||
}
|
||||
for suffix := 2; ; suffix++ {
|
||||
if _, collides := taken[fallback]; !collides {
|
||||
break
|
||||
}
|
||||
fallback = "workspace-" + strconv.Itoa(suffix)
|
||||
}
|
||||
entries := make([]chattool.FindToolCatalogEntry, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
info := candidate.tool.Info()
|
||||
server := candidate.server
|
||||
if server == "" {
|
||||
server = fallback
|
||||
}
|
||||
entries = append(entries, chattool.FindToolCatalogEntry{
|
||||
Name: info.Name,
|
||||
Description: info.Description,
|
||||
Server: server,
|
||||
ServerDescription: candidate.serverDescription,
|
||||
ParameterText: flattenMCPParameterText(info.Parameters),
|
||||
SchemaTokens: estimateDeferredMCPToolTokens([]deferredMCPTool{candidate}),
|
||||
})
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
func flattenMCPParameterText(value any) string {
|
||||
var values []string
|
||||
var walk func(any)
|
||||
walk = func(value any) {
|
||||
switch typed := value.(type) {
|
||||
case map[string]any:
|
||||
keys := make([]string, 0, len(typed))
|
||||
for key := range typed {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
slices.Sort(keys)
|
||||
for _, key := range keys {
|
||||
values = append(values, key)
|
||||
walk(typed[key])
|
||||
}
|
||||
case []any:
|
||||
for _, item := range typed {
|
||||
walk(item)
|
||||
}
|
||||
case string:
|
||||
values = append(values, typed)
|
||||
}
|
||||
}
|
||||
walk(value)
|
||||
return strings.Join(values, " ")
|
||||
}
|
||||
|
||||
// deriveDeferredMCPActivations walks the surviving history newest first
|
||||
// so that when the aggregate schema weight of activations exceeds
|
||||
// tokenBudget, the least recently activated schemas are shed. The newest
|
||||
// activation is always kept even when its schema alone exceeds the
|
||||
// budget, so the tool the model just requested stays usable. Shed tools
|
||||
// stay in the catalog and remain directly callable, which reactivates
|
||||
// them as most recent. A tokenBudget <= 0 means unbounded.
|
||||
//
|
||||
// find_tools results are admitted at their tool-call row, after that
|
||||
// row's direct tool calls, so a step's own search activations cannot
|
||||
// shed the schema of a tool the model invoked directly. Results whose
|
||||
// call row was compacted away are admitted at the result row.
|
||||
//
|
||||
// Direct calls whose tool result is an error are admitted last, newest
|
||||
// first. History cannot distinguish a pre-execution denial (hook
|
||||
// policy, input validation) from an executed call whose MCP server
|
||||
// returned an error, so errored calls activate only with budget left
|
||||
// after every other activation: the schema stays available for a
|
||||
// corrected retry without displacing schemas that find_tools results
|
||||
// or successful calls already claimed. Search-time reservations mirror
|
||||
// this order because sibling calls settle before a step's searches run
|
||||
// and the step result observer refunds errored reservations first, so
|
||||
// searches see the same leftover budget.
|
||||
func deriveDeferredMCPActivations(rows []database.ChatMessage, candidates []deferredMCPTool, tokenBudget float64) []string {
|
||||
candidateByName := make(map[string]deferredMCPTool, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
candidateByName[candidate.tool.Info().Name] = candidate
|
||||
}
|
||||
seen := make(map[string]struct{}, len(candidates))
|
||||
activated := make([]string, 0, len(candidates))
|
||||
usedTokens := 0.0
|
||||
appendName := func(name string) {
|
||||
candidate, ok := candidateByName[name]
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if _, dup := seen[name]; dup {
|
||||
return
|
||||
}
|
||||
seen[name] = struct{}{}
|
||||
weight := estimateDeferredMCPToolTokens([]deferredMCPTool{candidate})
|
||||
if len(activated) > 0 && tokenBudget > 0 && usedTokens+weight > tokenBudget {
|
||||
return
|
||||
}
|
||||
usedTokens += weight
|
||||
activated = append(activated, name)
|
||||
}
|
||||
parsedParts := make([][]codersdk.ChatMessagePart, len(rows))
|
||||
for i := range rows {
|
||||
parts, err := chatprompt.ParseContent(rows[i])
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
parsedParts[i] = parts
|
||||
}
|
||||
// Providers may reuse a tool-call ID in a later step, so a result
|
||||
// settles the newest unpaired call with its ID: a new call abandons
|
||||
// any older unpaired call, whose own result was lost or compacted
|
||||
// away, and abandoned calls count as successful rather than
|
||||
// adopting a later call's result. Results paired to a call
|
||||
// occurrence are recorded so orphan results, whose call row was
|
||||
// compacted away, are admitted at their own row even when a later
|
||||
// step reuses their ID.
|
||||
type partRef struct{ row, part int }
|
||||
callErrored := make(map[partRef]bool)
|
||||
resultPaired := make(map[partRef]struct{})
|
||||
pendingByID := make(map[string]partRef)
|
||||
for i := range rows {
|
||||
for j, part := range parsedParts[i] {
|
||||
if part.ToolCallID == "" {
|
||||
continue
|
||||
}
|
||||
switch part.Type {
|
||||
case codersdk.ChatMessagePartTypeToolCall:
|
||||
pendingByID[part.ToolCallID] = partRef{row: i, part: j}
|
||||
case codersdk.ChatMessagePartTypeToolResult:
|
||||
if ref, ok := pendingByID[part.ToolCallID]; ok {
|
||||
callErrored[ref] = part.IsError
|
||||
resultPaired[partRef{row: i, part: j}] = struct{}{}
|
||||
delete(pendingByID, part.ToolCallID)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
pendingSearch := make(map[string][]string)
|
||||
var erroredNames []string
|
||||
for i := len(rows) - 1; i >= 0; i-- {
|
||||
for j, part := range parsedParts[i] {
|
||||
if part.Type == codersdk.ChatMessagePartTypeToolCall && part.ToolName != chattool.FindToolsName {
|
||||
if callErrored[partRef{row: i, part: j}] {
|
||||
erroredNames = append(erroredNames, part.ToolName)
|
||||
continue
|
||||
}
|
||||
appendName(part.ToolName)
|
||||
}
|
||||
}
|
||||
for j, part := range parsedParts[i] {
|
||||
switch {
|
||||
case part.Type == codersdk.ChatMessagePartTypeToolResult && part.ToolName == chattool.FindToolsName:
|
||||
var result chattool.FindToolsResult
|
||||
if err := json.Unmarshal(part.Result, &result); err != nil {
|
||||
continue
|
||||
}
|
||||
if _, paired := resultPaired[partRef{row: i, part: j}]; paired {
|
||||
pendingSearch[part.ToolCallID] = result.Activated
|
||||
continue
|
||||
}
|
||||
for _, name := range result.Activated {
|
||||
appendName(name)
|
||||
}
|
||||
case part.Type == codersdk.ChatMessagePartTypeToolCall && part.ToolName == chattool.FindToolsName:
|
||||
for _, name := range pendingSearch[part.ToolCallID] {
|
||||
appendName(name)
|
||||
}
|
||||
delete(pendingSearch, part.ToolCallID)
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, name := range erroredNames {
|
||||
appendName(name)
|
||||
}
|
||||
return activated
|
||||
}
|
||||
|
||||
func deferredMCPToolNameSet(candidates []deferredMCPTool) map[string]bool {
|
||||
names := make(map[string]bool, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
names[candidate.tool.Info().Name] = true
|
||||
}
|
||||
return names
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
package chatd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"charm.land/fantasy"
|
||||
"github.com/google/uuid"
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/coder/coder/v2/coderd/database"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatloop"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatprompt"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chattool"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
"github.com/coder/coder/v2/codersdk/workspacesdk"
|
||||
)
|
||||
|
||||
type deferredTestAgentTool struct {
|
||||
info fantasy.ToolInfo
|
||||
}
|
||||
|
||||
func (t deferredTestAgentTool) Info() fantasy.ToolInfo { return t.info }
|
||||
func (deferredTestAgentTool) ProviderOptions() fantasy.ProviderOptions { return nil }
|
||||
func (deferredTestAgentTool) SetProviderOptions(fantasy.ProviderOptions) {}
|
||||
func (deferredTestAgentTool) Run(context.Context, fantasy.ToolCall) (fantasy.ToolResponse, error) {
|
||||
return fantasy.NewTextResponse("ok"), nil
|
||||
}
|
||||
|
||||
func testDeferredTool(name, description string, parameters map[string]any) deferredMCPTool {
|
||||
return deferredMCPTool{tool: deferredTestAgentTool{info: fantasy.ToolInfo{
|
||||
Name: name, Description: description, Parameters: parameters,
|
||||
}}}
|
||||
}
|
||||
|
||||
func TestDecideMCPToolSearch(t *testing.T) {
|
||||
t.Parallel()
|
||||
candidates := []deferredMCPTool{testDeferredTool("server__small", "small", map[string]any{"value": map[string]any{"type": "string"}})}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
experiment bool
|
||||
candidates []deferredMCPTool
|
||||
dynamicNames map[string]bool
|
||||
want bool
|
||||
}{
|
||||
{name: "experiment on", experiment: true, candidates: candidates, want: true},
|
||||
{name: "experiment off", candidates: candidates},
|
||||
{name: "empty", experiment: true},
|
||||
{name: "collision", experiment: true, candidates: []deferredMCPTool{testDeferredTool(chattool.FindToolsName, "collision", nil)}},
|
||||
{name: "dynamic collision", experiment: true, candidates: candidates, dynamicNames: map[string]bool{chattool.FindToolsName: true}},
|
||||
{name: "dynamic no collision", experiment: true, candidates: candidates, dynamicNames: map[string]bool{"other": true}, want: true},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
t.Parallel()
|
||||
require.Equal(t, tt.want, decideMCPToolSearch(mcpToolSearchInput{
|
||||
experimentEnabled: tt.experiment,
|
||||
candidates: tt.candidates,
|
||||
dynamicToolNames: tt.dynamicNames,
|
||||
}))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestDeriveDeferredMCPActivations(t *testing.T) {
|
||||
t.Parallel()
|
||||
candidates := []deferredMCPTool{
|
||||
testDeferredTool("server__first", "first", nil),
|
||||
testDeferredTool("server__second", "second", nil),
|
||||
}
|
||||
findResult, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolResult("call-1", chattool.FindToolsName, []byte(`{"activated":["server__second","disconnected"]}`), false, false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
directCall, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolCall("call-2", "server__first", []byte(`{}`)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
malformed, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolResult("call-3", chattool.FindToolsName, []byte(`"not-json"`), false, false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
rows := []database.ChatMessage{
|
||||
{Role: database.ChatMessageRoleTool, Content: findResult, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
{Role: database.ChatMessageRoleAssistant, Content: directCall, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
{Role: database.ChatMessageRoleTool, Content: malformed, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
}
|
||||
require.Equal(t, []string{"server__first", "server__second"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
||||
"newest activations first")
|
||||
require.Equal(t, []string{"server__first"}, deriveDeferredMCPActivations(rows[1:], candidates, 0),
|
||||
"activations before a compaction summary are absent from the surviving prompt window")
|
||||
|
||||
firstWeight := estimateDeferredMCPToolTokens(candidates[:1])
|
||||
require.Equal(t, []string{"server__first"}, deriveDeferredMCPActivations(rows, candidates, firstWeight),
|
||||
"a token budget sheds the least recent activations")
|
||||
require.Equal(t, []string{"server__first"}, deriveDeferredMCPActivations(rows, candidates, 0.001),
|
||||
"the newest activation survives a budget smaller than its own schema")
|
||||
}
|
||||
|
||||
func TestDeriveDeferredMCPActivationsSameStepDirectCallPriority(t *testing.T) {
|
||||
t.Parallel()
|
||||
candidates := []deferredMCPTool{
|
||||
testDeferredTool("server__direct", "direct", nil),
|
||||
testDeferredTool("server__searched", "searched", nil),
|
||||
}
|
||||
assistantStep, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolCall("call-search", chattool.FindToolsName, []byte(`{"queries":["direct"]}`)),
|
||||
codersdk.ChatMessageToolCall("call-direct", "server__direct", []byte(`{}`)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
searchResult, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolResult("call-search", chattool.FindToolsName, []byte(`{"activated":["server__searched"]}`), false, false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
rows := []database.ChatMessage{
|
||||
{Role: database.ChatMessageRoleAssistant, Content: assistantStep, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
{Role: database.ChatMessageRoleTool, Content: searchResult, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
}
|
||||
require.Equal(t, []string{"server__direct", "server__searched"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
||||
"a step's direct calls outrank its own search activations")
|
||||
directWeight := estimateDeferredMCPToolTokens(candidates[:1])
|
||||
require.Equal(t, []string{"server__direct"}, deriveDeferredMCPActivations(rows, candidates, directWeight),
|
||||
"same-step search activations cannot shed a directly invoked tool's schema")
|
||||
}
|
||||
|
||||
func TestDeriveDeferredMCPActivationsErroredDirectCallsActivateLast(t *testing.T) {
|
||||
t.Parallel()
|
||||
candidates := []deferredMCPTool{
|
||||
testDeferredTool("server__errored", "errored", nil),
|
||||
testDeferredTool("server__searched", "searched", nil),
|
||||
}
|
||||
assistantStep, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolCall("call-search", chattool.FindToolsName, []byte(`{"queries":["x"]}`)),
|
||||
codersdk.ChatMessageToolCall("call-errored", "server__errored", []byte(`{}`)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
toolRow, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolResult("call-errored", "server__errored", []byte(`"remote tool error"`), true, false),
|
||||
codersdk.ChatMessageToolResult("call-search", chattool.FindToolsName, []byte(`{"activated":["server__searched"]}`), false, false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
rows := []database.ChatMessage{
|
||||
{Role: database.ChatMessageRoleAssistant, Content: assistantStep, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
{Role: database.ChatMessageRoleTool, Content: toolRow, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
}
|
||||
require.Equal(t, []string{"server__searched", "server__errored"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
||||
"an errored direct call activates last so the model keeps the schema for a corrected retry")
|
||||
searchedWeight := estimateDeferredMCPToolTokens(candidates[1:])
|
||||
require.Equal(t, []string{"server__searched"}, deriveDeferredMCPActivations(rows, candidates, searchedWeight),
|
||||
"an errored direct call cannot consume budget promised to the search's reported activations")
|
||||
|
||||
erroredCall, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolCall("call-errored", "server__errored", []byte(`{}`)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
erroredResult, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolResult("call-errored", "server__errored", []byte(`"remote tool error"`), true, false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
erroredOnly := []database.ChatMessage{
|
||||
{Role: database.ChatMessageRoleAssistant, Content: erroredCall, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
{Role: database.ChatMessageRoleTool, Content: erroredResult, ContentVersion: chatprompt.CurrentContentVersion},
|
||||
}
|
||||
require.Equal(t, []string{"server__errored"}, deriveDeferredMCPActivations(erroredOnly, candidates, 0.001),
|
||||
"the newest errored call keeps the first-activation allowance when nothing else activates")
|
||||
}
|
||||
|
||||
func TestDeferredMCPToolEntriesEmptyServerLabel(t *testing.T) {
|
||||
t.Parallel()
|
||||
empty := deferredMCPTool{tool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "_orphan__echo"}}}
|
||||
|
||||
entries := deferredMCPToolEntries([]deferredMCPTool{empty})
|
||||
require.Equal(t, "workspace", entries[0].Server,
|
||||
"an empty server identity gets the workspace label at construction, so scopes and grouping agree")
|
||||
result, _ := chattool.SearchTools(entries, chattool.FindToolsArgs{Queries: []string{"workspace:"}}, chattool.SearchBudget{})
|
||||
require.Equal(t, []string{"_orphan__echo"}, result.Activated,
|
||||
"the advertised workspace scope reaches the relabeled entries")
|
||||
|
||||
literal := deferredMCPTool{
|
||||
tool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "workspace__run"}},
|
||||
server: "workspace",
|
||||
}
|
||||
colliding := deferredMCPToolEntries([]deferredMCPTool{empty, literal})
|
||||
require.Equal(t, "workspace-2", colliding[0].Server,
|
||||
"a literal workspace server keeps its own identity; the fallback label steps aside")
|
||||
require.Equal(t, "workspace", colliding[1].Server)
|
||||
result, _ = chattool.SearchTools(colliding, chattool.FindToolsArgs{Queries: []string{"workspace:"}}, chattool.SearchBudget{})
|
||||
require.Equal(t, []string{"workspace__run"}, result.Activated,
|
||||
"the workspace scope matches only the literal server")
|
||||
result, _ = chattool.SearchTools(colliding, chattool.FindToolsArgs{Queries: []string{"workspace-2:"}}, chattool.SearchBudget{})
|
||||
require.Equal(t, []string{"_orphan__echo"}, result.Activated,
|
||||
"the suffixed label scopes the empty-identity server")
|
||||
}
|
||||
|
||||
func TestDeriveDeferredMCPActivationsReusedCallIDs(t *testing.T) {
|
||||
t.Parallel()
|
||||
candidates := []deferredMCPTool{
|
||||
testDeferredTool("server__a", "a", nil),
|
||||
testDeferredTool("server__b", "b", nil),
|
||||
testDeferredTool("server__c", "c", nil),
|
||||
}
|
||||
row := func(t *testing.T, role database.ChatMessageRole, parts ...codersdk.ChatMessagePart) database.ChatMessage {
|
||||
t.Helper()
|
||||
content, err := chatprompt.MarshalParts(parts)
|
||||
require.NoError(t, err)
|
||||
return database.ChatMessage{Role: role, Content: content, ContentVersion: chatprompt.CurrentContentVersion}
|
||||
}
|
||||
// call-1 errors for server__a, is reused for a successful
|
||||
// server__b call, then server__c errors under its own ID. Only
|
||||
// per-call pairing keeps server__b a success: a history-wide
|
||||
// errored-ID set would demote it behind the newer errored
|
||||
// server__c.
|
||||
rows := []database.ChatMessage{
|
||||
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__a", []byte(`{}`))),
|
||||
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", "server__a", []byte(`"boom"`), true, false)),
|
||||
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__b", []byte(`{}`))),
|
||||
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", "server__b", []byte(`"ok"`), false, false)),
|
||||
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-2", "server__c", []byte(`{}`))),
|
||||
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-2", "server__c", []byte(`"boom"`), true, false)),
|
||||
}
|
||||
require.Equal(t, []string{"server__b", "server__c", "server__a"}, deriveDeferredMCPActivations(rows, candidates, 0),
|
||||
"a reused tool-call ID pairs each call with its own result, so the later success outranks errored calls")
|
||||
bWeight := estimateDeferredMCPToolTokens(candidates[1:2])
|
||||
require.Equal(t, []string{"server__b"}, deriveDeferredMCPActivations(rows, candidates, bWeight),
|
||||
"under budget the reused-ID success wins over newer errored calls")
|
||||
|
||||
// server__a's result is missing entirely; a later step reuses its
|
||||
// ID for server__b whose result errors. The error belongs to the
|
||||
// newer call, and the abandoned older call counts as successful.
|
||||
missingResult := []database.ChatMessage{
|
||||
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__a", []byte(`{}`))),
|
||||
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", "server__b", []byte(`{}`))),
|
||||
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", "server__b", []byte(`"boom"`), true, false)),
|
||||
}
|
||||
require.Equal(t, []string{"server__a", "server__b"}, deriveDeferredMCPActivations(missingResult, candidates, 0),
|
||||
"a reused ID must not assign the newer call's result to the older missing-result call")
|
||||
aWeight := estimateDeferredMCPToolTokens(candidates[:1])
|
||||
require.Equal(t, []string{"server__a"}, deriveDeferredMCPActivations(missingResult, candidates, aWeight),
|
||||
"under budget the abandoned call keeps its successful position")
|
||||
|
||||
// Compaction removed the orphan find_tools result's call row, and a
|
||||
// later step reuses its ID for a fresh find_tools call. The orphan
|
||||
// must be admitted at its own row, not stashed for a call that was
|
||||
// already visited.
|
||||
orphanSearch := []database.ChatMessage{
|
||||
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", chattool.FindToolsName, []byte(`{"activated":["server__a"]}`), false, false)),
|
||||
row(t, database.ChatMessageRoleAssistant, codersdk.ChatMessageToolCall("call-1", chattool.FindToolsName, []byte(`{"queries":["b"]}`))),
|
||||
row(t, database.ChatMessageRoleTool, codersdk.ChatMessageToolResult("call-1", chattool.FindToolsName, []byte(`{"activated":["server__b"]}`), false, false)),
|
||||
}
|
||||
require.Equal(t, []string{"server__b", "server__a"}, deriveDeferredMCPActivations(orphanSearch, candidates, 0),
|
||||
"an orphan search result with a reused ID is admitted at its own row")
|
||||
}
|
||||
|
||||
func TestFlattenMCPParameterText(t *testing.T) {
|
||||
t.Parallel()
|
||||
text := flattenMCPParameterText(map[string]any{
|
||||
"repository": map[string]any{"type": "string", "description": "Repository name"},
|
||||
})
|
||||
require.Contains(t, text, "repository")
|
||||
require.Contains(t, text, "Repository name")
|
||||
}
|
||||
|
||||
type deferredExternalTestTool struct {
|
||||
deferredTestAgentTool
|
||||
configID uuid.UUID
|
||||
}
|
||||
|
||||
func (t deferredExternalTestTool) MCPServerConfigID() uuid.UUID { return t.configID }
|
||||
|
||||
func TestCollectDeferredMCPCandidates(t *testing.T) {
|
||||
t.Parallel()
|
||||
approvedID := uuid.New()
|
||||
unapprovedID := uuid.New()
|
||||
external := deferredExternalTestTool{
|
||||
deferredTestAgentTool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "github__create_issue"}},
|
||||
configID: approvedID,
|
||||
}
|
||||
unapproved := deferredExternalTestTool{
|
||||
deferredTestAgentTool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "linear__create_issue"}},
|
||||
configID: unapprovedID,
|
||||
}
|
||||
workspace := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "everything__echo"}}
|
||||
input := deferredMCPCandidateInput{
|
||||
mcpTools: []fantasy.AgentTool{external, unapproved},
|
||||
workspaceMCPTools: []fantasy.AgentTool{workspace},
|
||||
mcpConfigByID: map[uuid.UUID]database.MCPServerConfig{approvedID: {Slug: "github", Description: "GitHub"}},
|
||||
approvedMCPConfigIDs: map[uuid.UUID]struct{}{approvedID: {}},
|
||||
includeWorkspaceTools: true,
|
||||
}
|
||||
|
||||
names := func(candidates []deferredMCPTool) []string {
|
||||
out := make([]string, 0, len(candidates))
|
||||
for _, candidate := range candidates {
|
||||
out = append(out, candidate.tool.Info().Name)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
all := collectDeferredMCPCandidates(input)
|
||||
require.Equal(t, []string{"github__create_issue", "linear__create_issue", "everything__echo"}, names(all))
|
||||
require.Equal(t, "github", all[0].server)
|
||||
require.Equal(t, "everything", all[2].server)
|
||||
|
||||
planInput := input
|
||||
planInput.planMode = database.NullChatPlanMode{Valid: true, ChatPlanMode: database.ChatPlanModePlan}
|
||||
require.Equal(t, []string{"github__create_issue"}, names(collectDeferredMCPCandidates(planInput)),
|
||||
"plan mode keeps only approved external tools, matching filterToolsForTurn")
|
||||
|
||||
noWorkspace := input
|
||||
noWorkspace.includeWorkspaceTools = false
|
||||
require.Equal(t, []string{"github__create_issue", "linear__create_issue"}, names(collectDeferredMCPCandidates(noWorkspace)))
|
||||
|
||||
longServer := strings.Repeat("s", 70)
|
||||
truncated := chattool.NewWorkspaceMCPTool(workspacesdk.MCPToolInfo{Name: longServer + "__echo"}, nil, nil)
|
||||
require.NotContains(t, truncated.Info().Name, "__",
|
||||
"sanitization must drop the separator for this scenario to be meaningful")
|
||||
truncatedInput := deferredMCPCandidateInput{
|
||||
workspaceMCPTools: []fantasy.AgentTool{truncated},
|
||||
includeWorkspaceTools: true,
|
||||
}
|
||||
require.Equal(t, longServer, collectDeferredMCPCandidates(truncatedInput)[0].server,
|
||||
"the server comes from the unsanitized routing name, not the capped model name")
|
||||
|
||||
padded := chattool.NewWorkspaceMCPTool(workspacesdk.MCPToolInfo{Name: " everything __echo"}, nil, nil)
|
||||
paddedInput := deferredMCPCandidateInput{
|
||||
workspaceMCPTools: []fantasy.AgentTool{padded},
|
||||
includeWorkspaceTools: true,
|
||||
}
|
||||
require.Equal(t, "everything", collectDeferredMCPCandidates(paddedInput)[0].server,
|
||||
"surrounding whitespace is trimmed so scope matching and catalog display see the canonical name")
|
||||
|
||||
unpadded := chattool.NewWorkspaceMCPTool(workspacesdk.MCPToolInfo{Name: "everything__ping"}, nil, nil)
|
||||
collidingInput := deferredMCPCandidateInput{
|
||||
workspaceMCPTools: []fantasy.AgentTool{padded, unpadded},
|
||||
includeWorkspaceTools: true,
|
||||
}
|
||||
colliding := collectDeferredMCPCandidates(collidingInput)
|
||||
require.Equal(t, " everything ", colliding[0].server,
|
||||
"trimming must not collapse distinct servers into one catalog identity")
|
||||
require.Equal(t, "everything", colliding[1].server)
|
||||
|
||||
slugColliding := deferredMCPCandidateInput{
|
||||
mcpTools: []fantasy.AgentTool{external},
|
||||
mcpConfigByID: map[uuid.UUID]database.MCPServerConfig{approvedID: {Slug: "everything"}},
|
||||
approvedMCPConfigIDs: map[uuid.UUID]struct{}{approvedID: {}},
|
||||
workspaceMCPTools: []fantasy.AgentTool{padded},
|
||||
includeWorkspaceTools: true,
|
||||
}
|
||||
slugCands := collectDeferredMCPCandidates(slugColliding)
|
||||
require.Equal(t, "everything", slugCands[0].server)
|
||||
require.Equal(t, " everything ", slugCands[1].server,
|
||||
"a workspace server whose trimmed name collides with an external slug keeps its raw name")
|
||||
}
|
||||
|
||||
func TestConfigureDeferredMCPToolSearchGenerationFlows(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hot := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "read_file", Description: "Read a file"}}
|
||||
first := testDeferredTool("github__create_issue", "Create an issue", nil)
|
||||
second := testDeferredTool("github__list_issues", "List issues", nil)
|
||||
candidates := []deferredMCPTool{first, second}
|
||||
findTools := chattool.FindTools(chattool.FindToolsOptions{Entries: deferredMCPToolEntries(candidates)})
|
||||
allTools := []fantasy.AgentTool{hot, first.tool, second.tool}
|
||||
allActive := []string{"read_file", first.tool.Info().Name, second.tool.Info().Name}
|
||||
|
||||
ordered, active, allowInactive := configureDeferredMCPToolSearch(allTools, allActive, candidates, findTools, nil)
|
||||
require.Equal(t, []string{"read_file", chattool.FindToolsName}, captureWireToolNames(t, ordered, active))
|
||||
require.Equal(t, map[string]bool{
|
||||
first.tool.Info().Name: true,
|
||||
second.tool.Info().Name: true,
|
||||
}, allowInactive)
|
||||
|
||||
result, _ := chattool.SearchTools(deferredMCPToolEntries(candidates), chattool.FindToolsArgs{Names: []string{second.tool.Info().Name}}, chattool.SearchBudget{})
|
||||
resultJSON, err := json.Marshal(result)
|
||||
require.NoError(t, err)
|
||||
resultContent, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolResult("find-1", chattool.FindToolsName, resultJSON, false, false),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
history := []database.ChatMessage{{
|
||||
Role: database.ChatMessageRoleTool, Content: resultContent, ContentVersion: chatprompt.CurrentContentVersion,
|
||||
}}
|
||||
activations := deriveDeferredMCPActivations(history, candidates, 0)
|
||||
require.Equal(t, []string{second.tool.Info().Name}, activations)
|
||||
|
||||
ordered, active, _ = configureDeferredMCPToolSearch(allTools, allActive, candidates, findTools, activations)
|
||||
require.Equal(t,
|
||||
[]string{"read_file", chattool.FindToolsName, second.tool.Info().Name},
|
||||
captureWireToolNames(t, ordered, active),
|
||||
)
|
||||
// Re-preparing the following turn from the same surviving history produces
|
||||
// the same activation set without separate persisted state.
|
||||
require.Equal(t, activations, deriveDeferredMCPActivations(history, candidates, 0))
|
||||
}
|
||||
|
||||
func TestConfigureDeferredMCPToolSearchDirectCallAndCompaction(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
candidate := testDeferredTool("github__create_issue", "Create an issue", nil)
|
||||
candidates := []deferredMCPTool{candidate}
|
||||
directCall, err := chatprompt.MarshalParts([]codersdk.ChatMessagePart{
|
||||
codersdk.ChatMessageToolCall("direct-1", candidate.tool.Info().Name, []byte(`{"title":"bug"}`)),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
preSummary := []database.ChatMessage{{
|
||||
Role: database.ChatMessageRoleAssistant, Content: directCall, ContentVersion: chatprompt.CurrentContentVersion,
|
||||
}}
|
||||
activation := deriveDeferredMCPActivations(preSummary, candidates, 0)
|
||||
require.Equal(t, []string{candidate.tool.Info().Name}, activation)
|
||||
|
||||
findTools := chattool.FindTools(chattool.FindToolsOptions{Entries: deferredMCPToolEntries(candidates)})
|
||||
ordered, active, _ := configureDeferredMCPToolSearch(
|
||||
[]fantasy.AgentTool{candidate.tool},
|
||||
[]string{candidate.tool.Info().Name},
|
||||
candidates,
|
||||
findTools,
|
||||
activation,
|
||||
)
|
||||
require.Equal(t,
|
||||
[]string{chattool.FindToolsName, candidate.tool.Info().Name},
|
||||
captureWireToolNames(t, ordered, active),
|
||||
)
|
||||
|
||||
// Prompt preparation passes only the post-summary history window, so an
|
||||
// activation before chat_summarized naturally lapses after compaction.
|
||||
require.Empty(t, deriveDeferredMCPActivations(nil, candidates, 0))
|
||||
}
|
||||
|
||||
func TestMCPToolSearchExperimentDisabledPreservesWireTools(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hot := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "read_file"}}
|
||||
candidate := testDeferredTool("github__list_issues", "List issues", nil)
|
||||
tools := []fantasy.AgentTool{hot, candidate.tool}
|
||||
active := []string{"read_file", candidate.tool.Info().Name}
|
||||
withoutExperiment := captureWireToolNames(t, tools, active)
|
||||
|
||||
require.False(t, decideMCPToolSearch(mcpToolSearchInput{
|
||||
candidates: []deferredMCPTool{candidate},
|
||||
}))
|
||||
require.Equal(t, withoutExperiment, captureWireToolNames(t, tools, active))
|
||||
}
|
||||
|
||||
func TestMCPToolSearchExploreAllowlist(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
hot := deferredTestAgentTool{info: fantasy.ToolInfo{Name: "read_file"}}
|
||||
external := deferredExternalTestTool{
|
||||
deferredTestAgentTool: deferredTestAgentTool{info: fantasy.ToolInfo{Name: "github__list_issues"}},
|
||||
configID: uuid.New(),
|
||||
}
|
||||
tools := []fantasy.AgentTool{hot, external}
|
||||
exploreActive := allowedExploreToolNames(tools)
|
||||
require.Equal(t, []string{"read_file", external.Info().Name}, exploreActive)
|
||||
|
||||
candidate := deferredMCPTool{tool: external}
|
||||
findTools := chattool.FindTools(chattool.FindToolsOptions{Entries: deferredMCPToolEntries([]deferredMCPTool{candidate})})
|
||||
_, deferredActive, _ := configureDeferredMCPToolSearch(tools, exploreActive, []deferredMCPTool{candidate}, findTools, nil)
|
||||
require.Equal(t, []string{"read_file", chattool.FindToolsName}, deferredActive)
|
||||
}
|
||||
|
||||
func captureWireToolNames(t *testing.T, tools []fantasy.AgentTool, active []string) []string {
|
||||
t.Helper()
|
||||
var names []string
|
||||
model := &chattest.FakeModel{
|
||||
ProviderName: "test",
|
||||
ModelName: "test",
|
||||
StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) {
|
||||
for _, tool := range call.Tools {
|
||||
names = append(names, tool.GetName())
|
||||
}
|
||||
return func(yield func(fantasy.StreamPart) bool) {
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
_, err := chatloop.GenerateAssistant(context.Background(), chatloop.GenerateAssistantOptions{
|
||||
Model: model,
|
||||
Tools: tools,
|
||||
ActiveTools: active,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return names
|
||||
}
|
||||
Reference in New Issue
Block a user