feat: record all tool call types (#26855)

## Summary

The Responses interceptor previously recorded only `function_call` and `custom_tool_call` output items, so interceptions that did real work via built-in tools (`web_search_call`, `computer_call`, `shell_call`, `mcp_call`, etc.) recorded no tool usage at all.

`recordNonInjectedToolUsage` now whitelists every tool-call output type and records it, with the tool name falling back to the item type when none is set.

`ToolUsageRecord` also gains an `ItemID` field so the two distinct Responses identifiers are captured without conflation (addresses review feedback on coder/aibridge#273):

- `ItemID`: the output item's unique `id` (always present).
- `ToolCallID`: the `call_id` correlation id (empty for hosted tools the provider runs server-side).

## Tests

- Extends `TestRecordToolUsage` with cases for the new hosted/agentic tool types.
- Adds blocking and streaming `web_search` fixtures (scrubbed of credentials and identifying metadata) plus `TestResponsesOutputMatchesUpstream` cases asserting a hosted tool records with an empty `ToolCallID` and a populated `ItemID`.

Linear: AIGOV-96

---
*This PR was produced by opencode (agent) using the `anthropic/claude-opus-4-8` model, under human direction and review.*
This commit is contained in:
Danny Kopping
2026-07-06 09:10:21 +02:00
committed by GitHub
parent 14d17abae6
commit 08a6359cac
7 changed files with 366 additions and 10 deletions
+6
View File
@@ -91,6 +91,9 @@ var (
//go:embed openai/responses/blocking/custom_tool.txtar
OaiResponsesBlockingCustomTool []byte
//go:embed openai/responses/blocking/web_search.txtar
OaiResponsesBlockingWebSearch []byte
//go:embed openai/responses/blocking/conversation.txtar
OaiResponsesBlockingConversation []byte
@@ -123,6 +126,9 @@ var (
//go:embed openai/responses/streaming/builtin_tool.txtar
OaiResponsesStreamingBuiltinTool []byte
//go:embed openai/responses/streaming/web_search.txtar
OaiResponsesStreamingWebSearch []byte
//go:embed openai/responses/streaming/multi_reasoning_builtin_tool.txtar
OaiResponsesStreamingMultiReasoningBuiltinTool []byte
@@ -0,0 +1,114 @@
-- request --
{
"input": [
{
"role": "user",
"content": "Search the web for the Example domain."
}
],
"model": "gpt-5.4",
"stream": false,
"tools": [
{
"type": "web_search"
}
]
}
-- non-streaming --
{
"id": "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b",
"object": "response",
"created_at": 1782819954,
"status": "completed",
"background": false,
"completed_at": 1782819963,
"error": null,
"incomplete_details": null,
"instructions": null,
"max_output_tokens": null,
"max_tool_calls": null,
"model": "gpt-5.4",
"output": [
{
"id": "msg_0b8f5f61bf0dee5f016a43ac78e6c0819c9d1852c768844565",
"type": "message",
"status": "completed",
"content": [
{
"type": "output_text",
"annotations": [],
"logprobs": [],
"text": "I'm invoking the web search tool now."
}
],
"role": "assistant"
},
{
"id": "ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9",
"type": "web_search_call",
"status": "completed",
"action": {
"type": "search",
"queries": [
"example domain"
],
"query": "example domain"
}
},
{
"id": "msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed",
"type": "message",
"status": "completed",
"content": [
{
"type": "output_text",
"annotations": [],
"logprobs": [],
"text": "The search ran successfully and returned normal web results."
}
],
"role": "assistant"
}
],
"parallel_tool_calls": true,
"previous_response_id": null,
"prompt_cache_key": null,
"prompt_cache_retention": null,
"reasoning": {
"effort": null,
"summary": null
},
"safety_identifier": null,
"service_tier": "default",
"store": true,
"temperature": 1.0,
"text": {
"format": {
"type": "text"
},
"verbosity": "medium"
},
"tool_choice": "auto",
"tools": [
{
"type": "web_search"
}
],
"top_logprobs": 0,
"top_p": 1.0,
"truncation": "disabled",
"usage": {
"input_tokens": 50,
"input_tokens_details": {
"cached_tokens": 0
},
"output_tokens": 30,
"output_tokens_details": {
"reasoning_tokens": 0
},
"total_tokens": 80
},
"user": null,
"metadata": {}
}
@@ -0,0 +1,60 @@
-- request --
{
"input": [
{
"role": "user",
"content": "Search the web for the Example domain."
}
],
"model": "gpt-5.4",
"stream": true,
"tools": [
{
"type": "web_search"
}
]
}
-- streaming --
event: response.created
data: {"type":"response.created","response":{"id":"resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b","object":"response","created_at":1782819954,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":0}
event: response.in_progress
data: {"type":"response.in_progress","response":{"id":"resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b","object":"response","created_at":1782819954,"status":"in_progress","background":false,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4","output":[],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"auto","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":null,"user":null,"metadata":{}},"sequence_number":1}
event: response.output_item.added
data: {"type":"response.output_item.added","item":{"id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","type":"web_search_call","status":"in_progress"},"output_index":0,"sequence_number":2}
event: response.web_search_call.in_progress
data: {"type":"response.web_search_call.in_progress","item_id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","output_index":0,"sequence_number":3}
event: response.web_search_call.searching
data: {"type":"response.web_search_call.searching","item_id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","output_index":0,"sequence_number":4}
event: response.web_search_call.completed
data: {"type":"response.web_search_call.completed","item_id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","output_index":0,"sequence_number":5}
event: response.output_item.done
data: {"type":"response.output_item.done","item":{"id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","type":"web_search_call","status":"completed","action":{"type":"search","queries":["example domain"],"query":"example domain"}},"output_index":0,"sequence_number":6}
event: response.output_item.added
data: {"type":"response.output_item.added","item":{"id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","type":"message","status":"in_progress","content":[],"role":"assistant"},"output_index":1,"sequence_number":7}
event: response.content_part.added
data: {"type":"response.content_part.added","content_index":0,"item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":""},"sequence_number":8}
event: response.output_text.delta
data: {"type":"response.output_text.delta","content_index":0,"delta":"The search ran successfully.","item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","logprobs":[],"output_index":1,"sequence_number":9}
event: response.output_text.done
data: {"type":"response.output_text.done","content_index":0,"item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","logprobs":[],"output_index":1,"sequence_number":10,"text":"The search ran successfully."}
event: response.content_part.done
data: {"type":"response.content_part.done","content_index":0,"item_id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","output_index":1,"part":{"type":"output_text","annotations":[],"logprobs":[],"text":"The search ran successfully."},"sequence_number":11}
event: response.output_item.done
data: {"type":"response.output_item.done","item":{"id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"The search ran successfully."}],"role":"assistant"},"output_index":1,"sequence_number":12}
event: response.completed
data: {"type":"response.completed","response":{"id":"resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b","object":"response","created_at":1782819954,"status":"completed","background":false,"completed_at":1782819963,"error":null,"incomplete_details":null,"instructions":null,"max_output_tokens":null,"max_tool_calls":null,"model":"gpt-5.4","output":[{"id":"ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9","type":"web_search_call","status":"completed","action":{"type":"search","queries":["example domain"],"query":"example domain"}},{"id":"msg_0b8f5f61bf0dee5f016a43ac7ae384819c8495d9f565ea2eed","type":"message","status":"completed","content":[{"type":"output_text","annotations":[],"logprobs":[],"text":"The search ran successfully."}],"role":"assistant"}],"parallel_tool_calls":true,"previous_response_id":null,"reasoning":{"effort":null,"summary":null},"service_tier":"default","store":true,"temperature":1.0,"text":{"format":{"type":"text"},"verbosity":"medium"},"tool_choice":"auto","tools":[{"type":"web_search"}],"top_logprobs":0,"top_p":1.0,"truncation":"disabled","usage":{"input_tokens":50,"input_tokens_details":{"cached_tokens":0},"output_tokens":30,"output_tokens_details":{"reasoning_tokens":0},"total_tokens":80},"user":null,"metadata":{}},"sequence_number":13}
+30 -6
View File
@@ -257,25 +257,49 @@ func (i *responsesInterceptionBase) recordNonInjectedToolUsage(ctx context.Conte
for _, item := range response.Output {
var args recorder.ToolArgs
// recording other function types to be considered: https://github.com/coder/aibridge/issues/121
// Whitelist the output item types that represent tool calls. Every
// other output type (message, reasoning, *_output, etc.) is skipped.
// Only function_call and custom_tool_call carry arguments we parse;
// the remaining built-in tool calls are recorded for visibility but
// have no uniform argument representation.
switch item.Type {
case string(constant.ValueOf[constant.FunctionCall]()):
args = i.parseFunctionCallJSONArgs(ctx, item.Arguments)
case string(constant.ValueOf[constant.CustomToolCall]()):
args = item.Input
case string(constant.ValueOf[constant.WebSearchCall]()),
// computer_call has no SDK constant; only computer_call_output does.
"computer_call",
string(constant.ValueOf[constant.LocalShellCall]()),
string(constant.ValueOf[constant.ShellCall]()),
string(constant.ValueOf[constant.ApplyPatchCall]()),
string(constant.ValueOf[constant.CodeInterpreterCall]()),
string(constant.ValueOf[constant.McpCall]()),
string(constant.ValueOf[constant.FileSearchCall]()),
string(constant.ValueOf[constant.ImageGenerationCall]()):
// Built-in tool calls carry no uniform argument payload.
default:
continue
}
// Built-in tools usually have no name, so fall back to the type.
toolName := item.Name
if toolName == "" {
toolName = item.Type
}
if err := i.recorder.RecordToolUsage(ctx, &recorder.ToolUsageRecord{
InterceptionID: i.ID().String(),
MsgID: response.ID,
ToolCallID: item.CallID,
Tool: item.Name,
Args: args,
Injected: false,
// ItemID is always present; ToolCallID (call_id) is empty for
// hosted tools that the provider executes internally.
ItemID: item.ID,
ToolCallID: item.CallID,
Tool: toolName,
Args: args,
Injected: false,
}); err != nil {
i.logger.Warn(ctx, "failed to record tool usage", slog.Error(err), slog.F("tool", item.Name))
i.logger.Warn(ctx, "failed to record tool usage", slog.Error(err), slog.F("tool", toolName))
}
}
}
@@ -200,6 +200,89 @@ func TestRecordToolUsage(t *testing.T) {
},
},
},
{
// Function/agentic tools expose both id and call_id; both are captured.
name: "function_call_captures_both_ids",
response: &oairesponses.Response{
ID: "resp_both",
Output: []oairesponses.ResponseOutputItemUnion{
{
Type: "function_call",
ID: "fc_item_1",
CallID: "call_both",
Name: "get_weather",
Arguments: `{"location": "NYC"}`,
},
},
},
expected: []*recorder.ToolUsageRecord{
{
InterceptionID: id.String(),
MsgID: "resp_both",
ItemID: "fc_item_1",
ToolCallID: "call_both",
Tool: "get_weather",
Args: map[string]any{"location": "NYC"},
Injected: false,
},
},
},
{
// Hosted tools only have id (no call_id) and usually no name, so
// the type is recorded as the tool name and ToolCallID is empty.
name: "hosted_tool_uses_item_id_no_call_id",
response: &oairesponses.Response{
ID: "resp_ws",
Output: []oairesponses.ResponseOutputItemUnion{
{
Type: "web_search_call",
ID: "ws_abc",
},
},
},
expected: []*recorder.ToolUsageRecord{
{
InterceptionID: id.String(),
MsgID: "resp_ws",
ItemID: "ws_abc",
ToolCallID: "",
Tool: "web_search_call",
Injected: false,
},
},
},
{
// Exercises every newly recorded tool type, the name-falls-back-to
// -type behavior, an explicit name override (mcp_call), and that
// non-tool output items (reasoning) are still skipped.
name: "all_additional_tool_types",
response: &oairesponses.Response{
ID: "resp_all",
Output: []oairesponses.ResponseOutputItemUnion{
{Type: "reasoning", ID: "rs_skip"},
{Type: "web_search_call", ID: "ws_1"},
{Type: "computer_call", ID: "cu_1", CallID: "call_cu"},
{Type: "local_shell_call", ID: "ls_1", CallID: "call_ls"},
{Type: "shell_call", ID: "sh_1", CallID: "call_sh"},
{Type: "apply_patch_call", ID: "ap_1", CallID: "call_ap"},
{Type: "code_interpreter_call", ID: "ci_1"},
{Type: "mcp_call", ID: "mcp_1", Name: "fetch"},
{Type: "file_search_call", ID: "fs_1"},
{Type: "image_generation_call", ID: "ig_1"},
},
},
expected: []*recorder.ToolUsageRecord{
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ws_1", Tool: "web_search_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "cu_1", ToolCallID: "call_cu", Tool: "computer_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ls_1", ToolCallID: "call_ls", Tool: "local_shell_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "sh_1", ToolCallID: "call_sh", Tool: "shell_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ap_1", ToolCallID: "call_ap", Tool: "apply_patch_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ci_1", Tool: "code_interpreter_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "mcp_1", Tool: "fetch"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "fs_1", Tool: "file_search_call"},
{InterceptionID: id.String(), MsgID: "resp_all", ItemID: "ig_1", Tool: "image_generation_call"},
},
},
}
for _, tc := range tests {
@@ -73,6 +73,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) {
MsgID: "resp_0da6045a8b68fa5200695fa23dcc2c81a19c849f627abf8a31",
Tool: "add",
ToolCallID: "call_CJSaa2u51JG996575oVljuNq",
ItemID: "fc_0da6045a8b68fa5200695fa23e198081a19bf68887d47ae93d",
Args: map[string]any{"a": float64(3), "b": float64(5)},
Injected: false,
},
@@ -113,6 +114,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) {
MsgID: "resp_09c614364030cdf000696942589da081a0af07f5859acb7308",
Tool: "code_exec",
ToolCallID: "call_haf8njtwrVZ1754Gm6fjAtuA",
ItemID: "ctc_09c614364030cdf0006969425bf33481a09cc0f9522af2d980",
Args: "print(\"hello world\")",
Injected: false,
},
@@ -127,6 +129,33 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) {
},
expectedClient: aibridge.ClientUnknown,
},
{
// web_search_call is a hosted tool executed server-side by the
// provider. It carries an item id but no call_id, so the recorded
// ToolCallID must be empty and the ItemID must be the output item's
// id.
name: "blocking_web_search",
fixture: fixtures.OaiResponsesBlockingWebSearch,
expectModel: "gpt-5.4",
expectPromptRecorded: "Search the web for the Example domain.",
expectToolRecorded: &recorder.ToolUsageRecord{
MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b",
Tool: "web_search_call",
ToolCallID: "",
ItemID: "ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9",
Injected: false,
},
expectTokenUsage: &recorder.TokenUsageRecord{
MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b",
Input: 50,
Output: 30,
ExtraTokenTypes: map[string]int64{
"output_reasoning": 0,
"total_tokens": 80,
},
},
expectedClient: aibridge.ClientUnknown,
},
{
name: "blocking_conversation",
fixture: fixtures.OaiResponsesBlockingConversation,
@@ -205,6 +234,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) {
MsgID: "resp_0c3fb28cfcf463a500695fa2f0239481a095ec6ce3dfe4d458",
Tool: "add",
ToolCallID: "call_7VaiUXZYuuuwWwviCrckxq6t",
ItemID: "fc_0c3fb28cfcf463a500695fa2f0b0a881a0890103ba88b0628e",
Args: map[string]any{"a": float64(3), "b": float64(5)},
Injected: false,
},
@@ -247,6 +277,7 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) {
MsgID: "resp_0c26996bc41c2a0500696942e83634819fb71b2b8ff8a4a76c",
Tool: "code_exec",
ToolCallID: "call_2gSnF58IEhXLwlbnqbm5XKMd",
ItemID: "ctc_0c26996bc41c2a0500696942ee6db8819fa6e841317eecbfb2",
Args: "print(\"hello world\")",
Injected: false,
},
@@ -261,6 +292,34 @@ func TestResponsesOutputMatchesUpstream(t *testing.T) {
},
expectedClient: aibridge.ClientUnknown,
},
{
// web_search_call is a hosted tool executed server-side by the
// provider. It carries an item id but no call_id, so the recorded
// ToolCallID must be empty and the ItemID must be the output item's
// id.
name: "streaming_web_search",
fixture: fixtures.OaiResponsesStreamingWebSearch,
streaming: true,
expectModel: "gpt-5.4",
expectPromptRecorded: "Search the web for the Example domain.",
expectToolRecorded: &recorder.ToolUsageRecord{
MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b",
Tool: "web_search_call",
ToolCallID: "",
ItemID: "ws_0b8f5f61bf0dee5f016a43ac7947bc819c945bff3bf2bcdbc9",
Injected: false,
},
expectTokenUsage: &recorder.TokenUsageRecord{
MsgID: "resp_0b8f5f61bf0dee5f016a43ac7294d8819ca794d13e1744ac2b",
Input: 50,
Output: 30,
ExtraTokenTypes: map[string]int64{
"output_reasoning": 0,
"total_tokens": 80,
},
},
expectedClient: aibridge.ClientUnknown,
},
{
name: "streaming_conversation",
fixture: fixtures.OaiResponsesStreamingConversation,
+14 -4
View File
@@ -87,10 +87,20 @@ type PromptUsageRecord struct {
}
type ToolUsageRecord struct {
InterceptionID string
MsgID string
Tool string
ToolCallID string
InterceptionID string
MsgID string
Tool string
// ToolCallID is the correlation ID used to match a tool call to its
// result (call_id in the Responses API, id in chat completions and
// Anthropic messages). It is empty for hosted Responses tools (e.g.
// web_search_call) which the provider executes internally.
ToolCallID string
// ItemID is the provider's unique ID for the output item that carried
// the tool call. It is specific to the OpenAI Responses API, where an
// output item has both an id and a call_id. It is empty for the chat
// completions and Anthropic messages APIs, which have no separate item
// ID concept.
ItemID string
ServerURL *string
Args ToolArgs
Injected bool