feat(site): add chat debug API layer and panel utilities (#23919)

## Summary

Add API client methods, React Query builders, and unit tests for the chat debug endpoints. Add `debugPanelUtils` with coercion helpers that transform raw debug step data into structured display models for the Debug panel, and wire debug run streaming into the chat store.

This is PR 7/9 in the chat debug logging stack.

### Changes

- **API client** (`site/src/api/api.ts`): typed methods for all debug endpoints — list runs, list steps, get/set deployment logging, get/set user logging, set per-chat override.
- **React Query builders** (`site/src/api/queries/chats.ts`): `chatDebugRuns`, `chatDebugSteps`, `chatDebugLoggingConfig`, `userDebugLoggingConfig` query/mutation factories with `refetchInterval: 5000` for live polling.
- **Debug panel utilities** (`site/src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts`): `coerceStepRequest` / `coerceStepResponse` that recursively parse nested JSONB into `StepRequestViewModel` / `StepResponseViewModel` with `MessagePart`, `ToolDef`, and `ToolCallPart` types. Includes `formatTokenSummary` (compact `3→5 tok` notation) and `compactDuration` helpers.
- **Unit tests**: coverage for coercion edge cases and formatting utilities.

### Stack overview

1. Database schema & SDK types
2. Types, context, and model normalization
3. Recorder, transport, and redaction
4. Service and summary aggregation
5. Chat lifecycle wiring
6. HTTP handlers and API docs
7. **→ Frontend API layer and panel utilities** (this PR)
8. Debug panel components and settings
9. Storybook stories

---
_Generated with [`mux`](https://github.com/coder/mux) • Model: `anthropic:claude-opus-4-6` • Thinking: `xhigh`_
This commit is contained in:
Thomas Kosiewski
2026-04-22 15:24:32 +02:00
committed by GitHub
parent b7c2c59931
commit 8c0fe6d5f2
7 changed files with 2357 additions and 30 deletions
+12 -1
View File
@@ -7,7 +7,18 @@
"./test/**/*.ts",
"./e2e/**/*.ts"
],
"ignore": ["**/*Generated.ts", "src/api/chatModelOptions.ts"],
"ignore": [
"**/*Generated.ts",
"src/api/chatModelOptions.ts",
// TODO(devtools): debugPanelUtils.ts is staged in PR 7; its exports are
// consumed by the Debug panel components in PRs 8 and 9. Remove this
// exclusion once the panel components land.
"src/pages/AgentsPage/components/RightPanel/DebugPanel/debugPanelUtils.ts",
// TODO(devtools): chatDebugLogging.ts queries are staged in PR 7;
// they are consumed by the Debug settings UI in PR 8. Remove this
// exclusion once the settings page lands.
"src/api/queries/chatDebugLogging.ts"
],
"ignoreBinaries": ["protoc"],
"ignoreDependencies": [
"@babel/plugin-syntax-typescript",
+51
View File
@@ -3301,6 +3301,57 @@ class ExperimentalApiMethods {
);
};
getChatDebugLogging =
async (): Promise<TypesGen.ChatDebugLoggingAdminSettings> => {
const response =
await this.axios.get<TypesGen.ChatDebugLoggingAdminSettings>(
"/api/experimental/chats/config/debug-logging",
);
return response.data;
};
updateChatDebugLogging = async (
req: TypesGen.UpdateChatDebugLoggingAllowUsersRequest,
): Promise<void> => {
await this.axios.put("/api/experimental/chats/config/debug-logging", req);
};
getUserChatDebugLogging =
async (): Promise<TypesGen.UserChatDebugLoggingSettings> => {
const response =
await this.axios.get<TypesGen.UserChatDebugLoggingSettings>(
"/api/experimental/chats/config/user-debug-logging",
);
return response.data;
};
updateUserChatDebugLogging = async (
req: TypesGen.UpdateUserChatDebugLoggingRequest,
): Promise<void> => {
await this.axios.put(
"/api/experimental/chats/config/user-debug-logging",
req,
);
};
getChatDebugRuns = async (
chatId: string,
): Promise<TypesGen.ChatDebugRunSummary[]> => {
const response = await this.axios.get<TypesGen.ChatDebugRunSummary[]>(
`/api/experimental/chats/${chatId}/debug/runs`,
);
return response.data;
};
getChatDebugRun = async (
chatId: string,
runId: string,
): Promise<TypesGen.ChatDebugRun> => {
const response = await this.axios.get<TypesGen.ChatDebugRun>(
`/api/experimental/chats/${chatId}/debug/runs/${runId}`,
);
return response.data;
};
getChatDesktopEnabled =
async (): Promise<TypesGen.ChatDesktopEnabledResponse> => {
const response =
+36
View File
@@ -0,0 +1,36 @@
import type { QueryClient } from "react-query";
import { API } from "#/api/api";
const chatDebugLoggingKey = ["chat-debug-logging"] as const;
const userChatDebugLoggingKey = ["user-chat-debug-logging"] as const;
export const chatDebugLogging = () => ({
queryKey: chatDebugLoggingKey,
queryFn: () => API.experimental.getChatDebugLogging(),
});
export const userChatDebugLogging = () => ({
queryKey: userChatDebugLoggingKey,
queryFn: () => API.experimental.getUserChatDebugLogging(),
});
export const updateChatDebugLogging = (queryClient: QueryClient) => ({
mutationFn: API.experimental.updateChatDebugLogging,
onSuccess: async () => {
await queryClient.invalidateQueries({
queryKey: chatDebugLoggingKey,
});
await queryClient.invalidateQueries({
queryKey: userChatDebugLoggingKey,
});
},
});
export const updateUserChatDebugLogging = (queryClient: QueryClient) => ({
mutationFn: API.experimental.updateUserChatDebugLogging,
onSuccess: async () => {
await queryClient.invalidateQueries({
queryKey: userChatDebugLoggingKey,
});
},
});
+56 -17
View File
@@ -9,6 +9,7 @@ import {
cancelChatListRefetches,
chatCostSummary,
chatCostSummaryKey,
chatDebugRunsKey,
chatDiffContentsKey,
chatKey,
chatMessagesKey,
@@ -737,6 +738,8 @@ describe("mutation invalidation scope", () => {
queryClient.setQueryData(chatKey(chatId), makeChat(chatId));
// Messages: ["chats", chatId, "messages"]
queryClient.setQueryData(chatMessagesKey(chatId), []);
// Debug runs: ["chats", chatId, "debug-runs"]
queryClient.setQueryData(chatDebugRunsKey(chatId), []);
// Diff contents: ["chats", chatId, "diff-contents"]
queryClient.setQueryData(chatDiffContentsKey(chatId), { files: [] });
// Cost summary: ["chats", "costSummary", "me", undefined]
@@ -758,13 +761,9 @@ describe("mutation invalidation scope", () => {
const chatId = "chat-1";
seedAllActiveQueries(queryClient, chatId);
// createChatMessage has no onSuccess handler — the WebSocket
// stream covers all real-time updates. Verify that constructing
// the mutation config does not define one.
const mutation = createChatMessage(queryClient, chatId);
expect(mutation).not.toHaveProperty("onSuccess");
await mutation.onSuccess?.();
// Since there is no onSuccess, no queries should be invalidated.
for (const { label, key } of unrelatedKeys(chatId)) {
const state = queryClient.getQueryState(key);
expect(
@@ -774,14 +773,18 @@ describe("mutation invalidation scope", () => {
}
});
it("createChatMessage does not invalidate chat detail or messages (WebSocket handles these)", async () => {
it("createChatMessage invalidates only debug runs, not chat detail or messages", async () => {
const queryClient = createTestQueryClient();
const chatId = "chat-1";
seedAllActiveQueries(queryClient, chatId);
// No onSuccess handler exists.
const mutation = createChatMessage(queryClient, chatId);
expect(mutation).not.toHaveProperty("onSuccess");
await mutation.onSuccess?.();
expect(
queryClient.getQueryState(chatDebugRunsKey(chatId))?.isInvalidated,
"chatDebugRunsKey should be invalidated",
).toBe(true);
const chatState = queryClient.getQueryState(chatKey(chatId));
expect(
@@ -815,7 +818,7 @@ describe("mutation invalidation scope", () => {
}
});
it("editChatMessage invalidates only chat detail and messages", async () => {
it("editChatMessage invalidates chat detail, messages, and debug runs", async () => {
const queryClient = createTestQueryClient();
const chatId = "chat-1";
seedAllActiveQueries(queryClient, chatId);
@@ -825,8 +828,9 @@ describe("mutation invalidation scope", () => {
await new Promise((r) => setTimeout(r, 0));
// These two should still be invalidated editing changes
// message content and potentially the chat's updated_at.
// These queries should be invalidated -- editing changes
// message content, may update the chat record, and can start
// a new debug run.
const chatState = queryClient.getQueryState(chatKey(chatId));
expect(chatState?.isInvalidated, "chatKey should be invalidated").toBe(
true,
@@ -837,6 +841,11 @@ describe("mutation invalidation scope", () => {
messagesState?.isInvalidated,
"chatMessagesKey should be invalidated",
).toBe(true);
expect(
queryClient.getQueryState(chatDebugRunsKey(chatId))?.isInvalidated,
"chatDebugRunsKey should be invalidated",
).toBe(true);
});
// Shared type for the infinite messages cache shape used by
@@ -1170,15 +1179,18 @@ describe("mutation invalidation scope", () => {
);
});
it("interruptChat does not invalidate unrelated queries", async () => {
it("interruptChat invalidates debug runs without touching unrelated queries", async () => {
const queryClient = createTestQueryClient();
const chatId = "chat-1";
seedAllActiveQueries(queryClient, chatId);
// interruptChat has no onSuccess handler — the WebSocket
// delivers status changes in real-time.
const mutation = interruptChat(queryClient, chatId);
expect(mutation).not.toHaveProperty("onSuccess");
await mutation.onSuccess?.();
expect(
queryClient.getQueryState(chatDebugRunsKey(chatId))?.isInvalidated,
"chatDebugRunsKey should be invalidated",
).toBe(true);
for (const { label, key } of unrelatedKeys(chatId)) {
const state = queryClient.getQueryState(key);
@@ -1189,13 +1201,18 @@ describe("mutation invalidation scope", () => {
}
});
it("promoteChatQueuedMessage does not invalidate unrelated queries", async () => {
it("promoteChatQueuedMessage invalidates debug runs without touching unrelated queries", async () => {
const queryClient = createTestQueryClient();
const chatId = "chat-1";
seedAllActiveQueries(queryClient, chatId);
const mutation = promoteChatQueuedMessage(queryClient, chatId);
expect(mutation).not.toHaveProperty("onSuccess");
await mutation.onSuccess?.();
expect(
queryClient.getQueryState(chatDebugRunsKey(chatId))?.isInvalidated,
"chatDebugRunsKey should be invalidated",
).toBe(true);
for (const { label, key } of unrelatedKeys(chatId)) {
const state = queryClient.getQueryState(key);
@@ -1206,6 +1223,28 @@ describe("mutation invalidation scope", () => {
}
});
it("regenerateChatTitle invalidates debug runs so the title_generation run surfaces immediately", async () => {
const queryClient = createTestQueryClient();
const chatId = "chat-1";
seedAllActiveQueries(queryClient, chatId);
const mutation = regenerateChatTitle(queryClient);
await mutation.onSettled(undefined, undefined, chatId);
expect(
queryClient.getQueryState(chatDebugRunsKey(chatId))?.isInvalidated,
"chatDebugRunsKey should be invalidated",
).toBe(true);
for (const { label, key } of unrelatedKeys(chatId)) {
const state = queryClient.getQueryState(key);
expect(
state?.isInvalidated,
`${label} should NOT be invalidated by regenerateChatTitle`,
).not.toBe(true);
}
});
it("createChat invalidates only sidebar queries on success", async () => {
const queryClient = createTestQueryClient();
const chatId = "chat-1";
+25 -12
View File
@@ -823,6 +823,7 @@ export const regenerateChatTitle = (queryClient: QueryClient) => ({
queryKey: chatKey(chatId),
exact: true,
});
void invalidateChatDebugRuns(queryClient, chatId);
},
});
@@ -858,6 +859,15 @@ export const updateChatTitle = (queryClient: QueryClient) => ({
},
});
export const chatDebugRunsKey = (chatId: string) =>
["chats", chatId, "debug-runs"] as const;
const invalidateChatDebugRuns = (queryClient: QueryClient, chatId: string) => {
return queryClient.invalidateQueries({
queryKey: chatDebugRunsKey(chatId),
});
};
export const createChat = (queryClient: QueryClient) => ({
mutationFn: (req: TypesGen.CreateChatRequest) =>
API.experimental.createChat(req),
@@ -870,14 +880,14 @@ export const createChat = (queryClient: QueryClient) => ({
});
export const createChatMessage = (
_queryClient: QueryClient,
queryClient: QueryClient,
chatId: string,
) => ({
mutationFn: (req: CreateChatMessageRequestWithClearablePlanMode) =>
API.experimental.createChatMessage(chatId, req),
// No onSuccess invalidation needed: the per-chat WebSocket delivers
// the response message via upsertDurableMessage, and the global
// watchChats() WebSocket updates the sidebar sort order.
onSuccess: () => {
void invalidateChatDebugRuns(queryClient, chatId);
},
});
type EditChatMessageMutationArgs = {
@@ -961,14 +971,15 @@ export const editChatMessage = (queryClient: QueryClient, chatId: string) => ({
queryKey: chatMessagesKey(chatId),
exact: true,
});
void invalidateChatDebugRuns(queryClient, chatId);
},
});
export const interruptChat = (_queryClient: QueryClient, chatId: string) => ({
export const interruptChat = (queryClient: QueryClient, chatId: string) => ({
mutationFn: () => API.experimental.interruptChat(chatId),
// No onSuccess invalidation needed: the per-chat WebSocket
// delivers the status change via setChatStatus, and the global
// watchChats() WebSocket updates the sidebar.
onSuccess: () => {
void invalidateChatDebugRuns(queryClient, chatId);
},
});
export const deleteChatQueuedMessage = (
@@ -990,14 +1001,14 @@ export const deleteChatQueuedMessage = (
});
export const promoteChatQueuedMessage = (
_queryClient: QueryClient,
queryClient: QueryClient,
chatId: string,
) => ({
mutationFn: (queuedMessageId: number) =>
API.experimental.promoteChatQueuedMessage(chatId, queuedMessageId),
// No onSuccess invalidation needed: the caller upserts the
// promoted message from the response, and the per-chat
// WebSocket delivers queue and status updates in real-time.
onSuccess: () => {
void invalidateChatDebugRuns(queryClient, chatId);
},
});
export const chatDiffContentsKey = (chatId: string) =>
@@ -1075,6 +1086,8 @@ export const updateChatDesktopEnabled = (queryClient: QueryClient) => ({
},
});
export * from "./chatDebugLogging";
const chatWorkspaceTTLKey = ["chat-workspace-ttl"] as const;
export const chatWorkspaceTTL = () => ({
@@ -0,0 +1,928 @@
import {
clampContent,
coerceRunSummary,
coerceStepRequest,
coerceStepResponse,
coerceUsageRecord,
compactDuration,
computeDurationMs,
extractTokenCounts,
formatTokenSummary,
getRoleBadgeVariant,
getRunKindLabel,
getStatusBadgeVariant,
isActiveStatus,
normalizeAttempts,
} from "./debugPanelUtils";
describe("coerceStepResponse", () => {
it("keeps tool-result content emitted in normalized response parts", () => {
const response = coerceStepResponse({
content: [
{
type: "tool-result",
tool_call_id: "call-1",
tool_name: "search_docs",
result: {
matches: ["model.go", "debugPanelUtils.ts"],
},
},
],
});
const parsed = JSON.parse(response.content);
expect(parsed).toEqual({
matches: ["model.go", "debugPanelUtils.ts"],
});
expect(response.toolCalls).toEqual([]);
expect(response.usage).toEqual({});
});
it.each([
["numeric zero", 0, "0"],
["boolean false", false, "false"],
["explicit null", null, "null"],
])("preserves primitive tool-result %s in debug payloads", (_label, result, expected) => {
const response = coerceStepResponse({
content: [
{
type: "tool-result",
tool_call_id: "call-1",
tool_name: "probe",
result,
},
],
});
expect(response.content).toBe(expected);
});
it("extracts tool_input streaming deltas as tool calls", () => {
// Interrupted streams emit `tool_input` parts with the accumulated
// arguments before a final `tool_call` summary exists.
const response = coerceStepResponse({
content: [
{
type: "tool_input",
tool_call_id: "call-42",
tool_name: "search_docs",
arguments: '{"query":"foo"}',
},
],
});
expect(response.toolCalls).toEqual([
{
id: "call-42",
name: "search_docs",
arguments: '{\n "query": "foo"\n}',
},
]);
});
it("prefers finalized tool_call over the streaming tool_input delta for the same call ID", () => {
const response = coerceStepResponse({
content: [
{
type: "tool_input",
tool_call_id: "call-42",
tool_name: "search_docs",
arguments: '{"query":"f',
},
{
type: "tool_call",
tool_call_id: "call-42",
tool_name: "search_docs",
arguments: '{"query":"foo"}',
},
],
});
expect(response.toolCalls).toEqual([
{
id: "call-42",
name: "search_docs",
arguments: '{\n "query": "foo"\n}',
},
]);
});
it("keeps the finalized payload when tool_call precedes a stray tool_input for the same ID", () => {
const response = coerceStepResponse({
content: [
{
type: "tool_call",
tool_call_id: "call-42",
tool_name: "search_docs",
arguments: '{"query":"foo"}',
},
{
type: "tool_input",
tool_call_id: "call-42",
tool_name: "search_docs",
arguments: '{"query":"bar"}',
},
],
});
expect(response.toolCalls).toEqual([
{
id: "call-42",
name: "search_docs",
arguments: '{\n "query": "foo"\n}',
},
]);
});
it("keeps distinct tool calls with empty tool_call_ids instead of collapsing them", () => {
// Go's zero value for string is "" and ChatStreamToolCall.tool_call_id
// has no `omitempty`, so unset IDs marshal as "" on the wire. Treat
// them as "no id" so two distinct calls don't collide on the same
// dedup Map key.
const response = coerceStepResponse({
content: [
{
type: "tool_call",
tool_call_id: "",
tool_name: "search_docs",
arguments: '{"query":"a"}',
},
{
type: "tool_call",
tool_call_id: "",
tool_name: "calc",
arguments: '{"op":"add"}',
},
],
});
expect(response.toolCalls).toEqual([
{
id: undefined,
name: "search_docs",
arguments: '{\n "query": "a"\n}',
},
{
id: undefined,
name: "calc",
arguments: '{\n "op": "add"\n}',
},
]);
});
it("keeps per-call entries when multiple distinct tool calls are emitted", () => {
const response = coerceStepResponse({
content: [
{
type: "tool_input",
tool_call_id: "call-1",
tool_name: "search_docs",
arguments: '{"query":"a"}',
},
{
type: "tool_input",
tool_call_id: "call-2",
tool_name: "calc",
arguments: '{"op":"add"}',
},
],
});
expect(response.toolCalls).toEqual([
{
id: "call-1",
name: "search_docs",
arguments: '{\n "query": "a"\n}',
},
{
id: "call-2",
name: "calc",
arguments: '{\n "op": "add"\n}',
},
]);
});
it("falls back to OpenAI choices when content is absent", () => {
// Raw OpenAI-format response: no top-level `content`, the data
// lives in `choices[0].message`.
const response = coerceStepResponse({
choices: [
{
message: {
content: "hello from openai",
tool_calls: [
{
id: "call-1",
function: {
name: "search_docs",
arguments: '{"query":"foo"}',
},
},
],
},
finish_reason: "stop",
},
],
});
expect(response.content).toBe("hello from openai");
expect(response.toolCalls).toEqual([
{
id: "call-1",
name: "search_docs",
arguments: '{\n "query": "foo"\n}',
},
]);
expect(response.finishReason).toBe("stop");
});
it("reads OpenAI choices content from array text parts", () => {
// Providers sometimes emit `content` as a structured array on the
// choice message instead of a plain string.
const response = coerceStepResponse({
choices: [
{
message: {
content: [
{ type: "text", text: "part one " },
{ type: "text", text: "part two" },
],
},
finish_reason: "length",
},
],
});
expect(response.content).toBe("part one part two");
expect(response.finishReason).toBe("length");
});
it("collects tool_calls from the OpenAI choice fallback when none come from content", () => {
const response = coerceStepResponse({
choices: [
{
message: {
content: "ok",
tool_calls: [
{
id: "call-9",
function: { name: "lookup", arguments: '{"q":"x"}' },
},
],
},
finish_reason: "tool_calls",
},
],
});
expect(response.toolCalls).toEqual([
{
id: "call-9",
name: "lookup",
arguments: '{\n "q": "x"\n}',
},
]);
expect(response.finishReason).toBe("tool_calls");
});
it("coerces top-level tool_calls when content is a plain string", () => {
const response = coerceStepResponse({
content: "hello",
tool_calls: [{ id: "c-1", name: "alpha", arguments: '{"q":"a"}' }],
});
expect(response.content).toBe("hello");
expect(response.toolCalls).toEqual([
{
id: "c-1",
name: "alpha",
arguments: '{\n "q": "a"\n}',
},
]);
});
it("captures usage, warnings, and model from the response body", () => {
const response = coerceStepResponse({
content: "done",
usage: { prompt_tokens: "11", completion_tokens: 22 },
warnings: [
"string warning",
{ message: "object warning" },
{ details: "object details" },
{ other: "ignored" },
],
model: "gpt-4o",
});
expect(response.usage).toEqual({
prompt_tokens: 11,
completion_tokens: 22,
});
expect(response.warnings).toEqual([
"string warning",
"object warning",
"object details",
]);
expect(response.model).toBe("gpt-4o");
});
it("returns defaults for non-object input", () => {
const response = coerceStepResponse(null);
expect(response).toEqual({
content: "",
toolCalls: [],
finishReason: undefined,
usage: {},
warnings: [],
model: undefined,
});
});
it("unwraps JSON-string payloads before coercing", () => {
const response = coerceStepResponse(
JSON.stringify({
content: "via json wrapper",
finish_reason: "stop",
}),
);
expect(response.content).toBe("via json wrapper");
expect(response.finishReason).toBe("stop");
});
});
describe("getRunKindLabel", () => {
it.each([
["chat_turn", "Chat Turn"],
["title_generation", "Title Generation"],
["compaction", "Compaction"],
["quickgen", "Quick Gen"],
["quick_gen", "Quick Gen"],
["llm_call", "LLM Call"],
["post_process", "Post-process"],
["tool_call", "Tool Call"],
])("maps %s to the canonical label", (kind, label) => {
expect(getRunKindLabel(kind)).toBe(label);
});
it("humanizes unknown kinds with title casing", () => {
expect(getRunKindLabel("custom_kind")).toBe("Custom Kind");
});
it("returns Unknown for blank input", () => {
expect(getRunKindLabel(" ")).toBe("Unknown");
});
});
describe("getStatusBadgeVariant", () => {
it.each([
["completed", "green"],
["SUCCESS", "green"],
["failed", "destructive"],
["interrupted", "destructive"],
["cancelled", "destructive"],
["canceled", "destructive"],
["running", "info"],
["in_progress", "info"],
["pending", "warning"],
["queued", "warning"],
["mystery", "default"],
])("maps %s to %s", (status, expected) => {
expect(getStatusBadgeVariant(status)).toBe(expected);
});
});
describe("isActiveStatus", () => {
it.each([
["running", true],
["in_progress", true],
["processing", true],
["started", true],
["completed", false],
["pending", false],
])("returns %s-active=%s", (status, expected) => {
expect(isActiveStatus(status)).toBe(expected);
});
});
describe("getRoleBadgeVariant", () => {
it.each([
["system", "purple"],
["user", "info"],
["assistant", "green"],
["tool", "warning"],
["function", "warning"],
["unknown", "default"],
])("maps %s to %s", (role, expected) => {
expect(getRoleBadgeVariant(role)).toBe(expected);
});
});
describe("normalizeAttempts", () => {
it("parses array input and sorts by attempt_number", () => {
const result = normalizeAttempts([
{ number: 2, status: "completed" },
{ attempt_number: 1, status: "error" },
]);
expect(result.rawFallback).toBeUndefined();
expect(result.parsed.map((a) => a.attempt_number)).toEqual([1, 2]);
expect(result.parsed.map((a) => a.status)).toEqual(["error", "completed"]);
});
it("parses JSON strings that wrap an array of attempts", () => {
const result = normalizeAttempts(
JSON.stringify([
{ attempt_number: 1, status: "completed", method: "POST" },
]),
);
expect(result.rawFallback).toBeUndefined();
expect(result.parsed).toEqual([
expect.objectContaining({
attempt_number: 1,
status: "completed",
method: "POST",
}),
]);
});
it("returns an empty array for empty input without a raw fallback", () => {
expect(normalizeAttempts([])).toEqual({ parsed: [] });
expect(normalizeAttempts({})).toEqual({ parsed: [] });
});
it("parses record-shaped attempts keyed by index", () => {
const result = normalizeAttempts({
"1": { attempt_number: 1, status: "completed" },
"2": { attempt_number: 2, status: "error" },
});
expect(result.rawFallback).toBeUndefined();
expect(result.parsed.map((a) => a.attempt_number)).toEqual([1, 2]);
});
it("returns raw fallback for unparsable strings", () => {
const result = normalizeAttempts("not json");
expect(result.parsed).toEqual([]);
expect(result.rawFallback).toBe("not json");
});
it("returns raw fallback for unsupported types", () => {
const result = normalizeAttempts(42);
expect(result.parsed).toEqual([]);
expect(result.rawFallback).toBe("42");
});
it("decodes base64-encoded request bodies into JSON", () => {
// {"prompt":"hi"} encoded as base64.
const encodedBody = btoa('{"prompt":"hi"}');
const [attempt] = normalizeAttempts([
{
attempt_number: 1,
status: "completed",
request_body: encodedBody,
},
]).parsed;
expect(attempt?.raw_request).toEqual({ body: { prompt: "hi" } });
});
it("preserves plain-text bodies that happen to be base64-alphabet", () => {
// "test" is in the base64 alphabet and has length 4, but it is
// almost certainly a literal payload. Decoding it would produce
// mojibake (0xB5 0xEB 0x2D is not valid UTF-8).
const [attempt] = normalizeAttempts([
{
attempt_number: 1,
status: "completed",
request_body: "test",
response_body: "abcd",
},
]).parsed;
expect(attempt?.raw_request).toEqual({ body: "test" });
expect(attempt?.raw_response).toEqual({ body: "abcd" });
});
it("decodes base64-encoded non-JSON text", () => {
// Go can emit non-JSON []byte payloads (e.g. plain-text error
// bodies). Once step 2 fails JSON parsing, step 3 should return
// the decoded UTF-8 text.
const encodedBody = btoa("hello world");
const [attempt] = normalizeAttempts([
{
attempt_number: 1,
status: "completed",
response_body: encodedBody,
},
]).parsed;
expect(attempt?.raw_response).toEqual({ body: "hello world" });
});
it("captures string and object-shaped errors", () => {
const [stringAttempt, objectAttempt] = normalizeAttempts([
{ attempt_number: 1, status: "error", error: "boom" },
{
attempt_number: 2,
status: "error",
error: { code: "ETIMEDOUT", detail: "slow" },
},
]).parsed;
expect(stringAttempt?.error).toBe("boom");
expect(objectAttempt?.error).toEqual({ code: "ETIMEDOUT", detail: "slow" });
});
it("preserves pre-built raw_request/raw_response records without rebuilding", () => {
const [attempt] = normalizeAttempts([
{
attempt_number: 1,
status: "completed",
raw_request: { method: "POST", url: "https://api.example/llm" },
raw_response: { status: 200, body: { ok: true } },
// These scalar fields should be ignored when raw_request/raw_response
// are already provided.
method: "IGNORED",
request_body: "ignored",
},
]).parsed;
expect(attempt?.raw_request).toEqual({
method: "POST",
url: "https://api.example/llm",
});
expect(attempt?.raw_response).toEqual({
status: 200,
body: { ok: true },
});
});
it("falls back to the positional index when no attempt_number is provided", () => {
const parsed = normalizeAttempts([
{ status: "completed" },
{ status: "error" },
]).parsed;
expect(parsed.map((a) => a.attempt_number)).toEqual([1, 2]);
});
});
describe("computeDurationMs", () => {
it("computes elapsed time between two ISO timestamps", () => {
expect(
computeDurationMs("2024-01-01T00:00:00.000Z", "2024-01-01T00:00:02.500Z"),
).toBe(2500);
});
it("returns null when startedAt is not parseable", () => {
expect(computeDurationMs("not-a-date")).toBeNull();
});
it("returns null when finishedAt is provided but not parseable", () => {
expect(
computeDurationMs("2024-01-01T00:00:00.000Z", "also-not-a-date"),
).toBeNull();
});
it("clamps negative durations to zero", () => {
expect(
computeDurationMs("2024-01-01T00:00:10.000Z", "2024-01-01T00:00:05.000Z"),
).toBe(0);
});
it("falls back to current time when finishedAt is omitted", () => {
vi.useFakeTimers();
vi.setSystemTime(new Date("2024-01-01T00:00:05.000Z"));
try {
expect(computeDurationMs("2024-01-01T00:00:00.000Z")).toBe(5000);
} finally {
vi.useRealTimers();
}
});
});
describe("compactDuration", () => {
it.each([
[0, "0ms"],
[999, "999ms"],
[1000, "1.0s"],
[1500, "1.5s"],
[59999, "60.0s"],
[60000, "1m"],
[61000, "1m 1s"],
[125000, "2m 5s"],
])("formats %sms as %s", (ms, expected) => {
expect(compactDuration(ms)).toBe(expected);
});
});
describe("formatTokenSummary", () => {
it("renders both input and output counts", () => {
expect(formatTokenSummary(1200, 340)).toBe("1,200→340 tok");
});
it("renders input-only when output is undefined", () => {
expect(formatTokenSummary(1200, undefined)).toBe("1,200 in");
});
it("renders output-only when input is undefined", () => {
expect(formatTokenSummary(undefined, 340)).toBe("340 out");
});
it("returns an empty string when both counts are undefined", () => {
expect(formatTokenSummary()).toBe("");
});
});
describe("extractTokenCounts", () => {
it("prefers prompt/completion keys when present", () => {
expect(
extractTokenCounts({
prompt_tokens: 10,
completion_tokens: 20,
total_tokens: 30,
input_tokens: 99,
output_tokens: 99,
}),
).toEqual({ input: 10, output: 20, total: 30 });
});
it("falls back to input/output_tokens when prompt/completion are absent", () => {
expect(
extractTokenCounts({
input_tokens: 5,
output_tokens: 7,
}),
).toEqual({ input: 5, output: 7, total: undefined });
});
it("returns undefined fields for an empty usage record", () => {
expect(extractTokenCounts({})).toEqual({
input: undefined,
output: undefined,
total: undefined,
});
});
});
describe("coerceUsageRecord", () => {
it("coerces string numeric values to numbers", () => {
expect(
coerceUsageRecord({ prompt_tokens: "10", completion_tokens: 20 }),
).toEqual({ prompt_tokens: 10, completion_tokens: 20 });
});
it("drops non-finite values", () => {
expect(coerceUsageRecord({ a: "abc", b: null, c: 5 })).toEqual({ c: 5 });
});
it("parses usage embedded as a JSON string", () => {
expect(coerceUsageRecord('{"prompt_tokens": 3}')).toEqual({
prompt_tokens: 3,
});
});
it("returns an empty record for non-object input", () => {
expect(coerceUsageRecord(null)).toEqual({});
expect(coerceUsageRecord(42)).toEqual({});
});
});
describe("coerceRunSummary", () => {
it("extracts the primary label and token counts from snake_case fields", () => {
const summary = coerceRunSummary({
first_message: "Hello",
endpoint_label: "openai/chat",
model: "gpt-4",
provider: "openai",
step_count: 3,
total_input_tokens: 120,
total_output_tokens: 45,
});
expect(summary).toEqual({
primaryLabel: "Hello",
endpointLabel: "openai/chat",
model: "gpt-4",
provider: "openai",
stepCount: 3,
totalInputTokens: 120,
totalOutputTokens: 45,
warnings: [],
});
});
it("falls back to camelCase and alternate token names", () => {
const summary = coerceRunSummary({
primaryLabel: "Fallback",
promptTokens: "90",
completionTokens: "30",
});
expect(summary.primaryLabel).toBe("Fallback");
expect(summary.totalInputTokens).toBe(90);
expect(summary.totalOutputTokens).toBe(30);
});
it("returns defaults for non-object input", () => {
expect(coerceRunSummary(null)).toEqual({
primaryLabel: "",
endpointLabel: undefined,
model: undefined,
provider: undefined,
stepCount: undefined,
totalInputTokens: undefined,
totalOutputTokens: undefined,
warnings: [],
});
});
it("unwraps JSON-string payloads before coercing", () => {
const summary = coerceRunSummary(
JSON.stringify({
first_message: "wrapped hello",
provider: "openai",
stepCount: 4,
}),
);
expect(summary.primaryLabel).toBe("wrapped hello");
expect(summary.provider).toBe("openai");
expect(summary.stepCount).toBe(4);
});
});
describe("coerceStepRequest", () => {
it("coerces messages, tools, and options nested under options/policy", () => {
const request = coerceStepRequest({
model: "gpt-4",
messages: [
{ role: "system", content: "Be helpful" },
{ role: "user", parts: [{ type: "text", text: "Hi" }] },
],
tools: [
{
type: "function",
function: {
name: "search_docs",
description: "Search the docs",
parameters: { type: "object" },
},
},
],
options: {
temperature: 0.2,
max_output_tokens: 512,
ignored_field: "drop me",
},
policy: {
tool_choice: "auto",
parallel_tool_calls: true,
},
});
expect(request.model).toBe("gpt-4");
expect(request.messages).toHaveLength(2);
expect(request.messages[0]).toMatchObject({
role: "system",
content: "Be helpful",
});
expect(request.messages[1]).toMatchObject({ role: "user", content: "Hi" });
expect(request.tools).toEqual([
{
name: "search_docs",
description: "Search the docs",
inputSchema: expect.any(String),
},
]);
expect(request.options).toEqual({
temperature: 0.2,
max_output_tokens: 512,
});
expect(request.policy).toEqual({
tool_choice: "auto",
parallel_tool_calls: true,
});
});
it("falls back to top-level option fields when no options wrapper is present", () => {
const request = coerceStepRequest({
temperature: 0.7,
top_p: 0.9,
});
expect(request.options).toEqual({ temperature: 0.7, top_p: 0.9 });
});
it("returns defaults for non-object input", () => {
expect(coerceStepRequest(null)).toEqual({
model: undefined,
messages: [],
tools: [],
options: {},
policy: {},
});
});
it("drops tool definitions without a name", () => {
const request = coerceStepRequest({
tools: [
{ type: "function", function: { description: "nameless" } },
{
type: "function",
function: { name: "valid", description: "kept" },
},
],
});
expect(request.tools).toEqual([
expect.objectContaining({ name: "valid", description: "kept" }),
]);
});
it("surfaces tool-call message parts with structured kind metadata", () => {
const request = coerceStepRequest({
messages: [
{
role: "assistant",
parts: [
{
type: "tool-call",
tool_call_id: "call-42",
tool_name: "search_docs",
arguments: '{"query":"foo"}',
},
],
},
{
role: "tool",
parts: [
{
type: "tool-result",
tool_call_id: "call-42",
tool_name: "search_docs",
result: { matches: 3 },
},
],
},
],
});
expect(request.messages).toHaveLength(2);
expect(request.messages[0]).toMatchObject({
role: "assistant",
kind: "tool-call",
toolCallId: "call-42",
toolName: "search_docs",
arguments: '{\n "query": "foo"\n}',
});
expect(request.messages[1]).toMatchObject({
role: "tool",
kind: "tool-result",
toolCallId: "call-42",
toolName: "search_docs",
result: expect.stringContaining('"matches"'),
});
});
it("unwraps JSON-string payloads including nested options", () => {
const request = coerceStepRequest(
JSON.stringify({
model: "gpt-4",
messages: [{ role: "user", content: "hi" }],
options: JSON.stringify({ temperature: 0.5 }),
policy: JSON.stringify({ tool_choice: "none" }),
}),
);
expect(request.model).toBe("gpt-4");
expect(request.messages).toHaveLength(1);
expect(request.options).toEqual({ temperature: 0.5 });
expect(request.policy).toEqual({ tool_choice: "none" });
});
});
describe("clampContent", () => {
it("returns the trimmed text when under the limit", () => {
expect(clampContent(" hello ", 20)).toBe("hello");
});
it("truncates and appends an ellipsis when over the limit", () => {
expect(clampContent("hello world", 5)).toBe("hello…");
});
it("returns an empty string for whitespace-only input", () => {
expect(clampContent(" ", 10)).toBe("");
});
it("keeps text exactly at the limit unchanged", () => {
expect(clampContent("abcde", 5)).toBe("abcde");
});
it("strips trailing whitespace before appending the ellipsis", () => {
expect(clampContent("abc defghij", 6)).toBe("abc…");
});
});
File diff suppressed because it is too large Load Diff