mirror of
https://github.com/cline/cline.git
synced 2026-10-24 02:02:27 +08:00
refactor: report terminal model failures at operation boundaries
This commit is contained in:
@@ -572,6 +572,14 @@ mark the operation as failed. Exhausted stream recovery disables the agent's
|
||||
whole-turn retry so the two retry budgets do not multiply. Previously streamed
|
||||
text and reasoning follow AI SDK's append semantics and may repeat on recovery.
|
||||
|
||||
Terminal `sdk.error` reporting belongs to the outermost operation. Agent-created
|
||||
gateway models return structured `finish.failure` details without reporting;
|
||||
the agent run reports once after recovery is exhausted. Standalone `Gateway.stream`
|
||||
calls report at their own terminal boundary. Providers maintain tracing spans
|
||||
but never emit terminal diagnostic events. Failures retain the display message,
|
||||
original cause, classification, retry decision, and correlation context through
|
||||
the model boundary; no reporting-ownership flags are carried in events.
|
||||
|
||||
### 7. Storage Adapters
|
||||
|
||||
Stateful persistence should be isolated behind adapter/service layers.
|
||||
|
||||
+12
@@ -84,6 +84,18 @@ Callback errors preserve their original type; they do not prove an earlier POST
|
||||
`CloudHandoffSeedRejectedError` means no seed was dispatched; clear only the seed marker before retrying the saved target.
|
||||
`verifyHandoffTranscript` checks the seeded history; `waitUntilReady(id)` waits for provisioning.
|
||||
|
||||
## Terminal model failures
|
||||
|
||||
`AgentModelEvent` finish events carry an optional `failure: AgentModelFailure`.
|
||||
It contains the display `message`, original `cause`, `classification`, turn-level
|
||||
`retryable` decision, and diagnostic `context`. Successful finish events omit it.
|
||||
Custom models return failure details and leave terminal reporting to their caller.
|
||||
|
||||
`AgentRuntime` owns terminal `sdk.error` reports for agent operations, including
|
||||
models from `Gateway.createAgentModel`. Standalone `Gateway.stream` calls report
|
||||
at the stream boundary. Provider tracing remains responsible for marking failed
|
||||
Langfuse spans. Recovered attempts do not emit terminal error reports.
|
||||
|
||||
## Voice input models
|
||||
|
||||
`getLocalTranscriptionModels(providerId, config?)` from `@cline/core` returns the
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import { createGateway } from "@cline/llms";
|
||||
import type {
|
||||
AgentMessage,
|
||||
AgentModel,
|
||||
@@ -715,14 +716,18 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
errorRetryable: kind === "provider",
|
||||
error:
|
||||
kind === "provider"
|
||||
? "Upstream returned HTTP 429"
|
||||
: "Context window exceeded",
|
||||
...(kind === "context-overflow"
|
||||
? { errorClass: "context_window_exceeded" as const }
|
||||
: {}),
|
||||
|
||||
failure: {
|
||||
message:
|
||||
kind === "provider"
|
||||
? "Upstream returned HTTP 429"
|
||||
: "Context window exceeded",
|
||||
classification:
|
||||
kind === "context-overflow"
|
||||
? "context_window_exceeded"
|
||||
: undefined,
|
||||
retryable: kind === "provider",
|
||||
},
|
||||
},
|
||||
],
|
||||
() => [
|
||||
@@ -1193,7 +1198,13 @@ describe("AgentRuntime", () => {
|
||||
{ type: "text-delta", text: "truncated..." },
|
||||
{ type: "finish", reason: "max-tokens" },
|
||||
],
|
||||
() => [{ type: "finish", reason: "error", error: "stream dropped" }],
|
||||
() => [
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: { message: "stream dropped" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
const compactedMessages: AgentMessage[] = [
|
||||
{ role: "user", content: [{ type: "text", text: "compacted" }] },
|
||||
@@ -1336,7 +1347,11 @@ describe("AgentRuntime", () => {
|
||||
}
|
||||
events.push(
|
||||
finish === "error"
|
||||
? { type: "finish", reason: "error", error: "stream dropped" }
|
||||
? {
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: { message: "stream dropped" },
|
||||
}
|
||||
: { type: "finish", reason: finish },
|
||||
);
|
||||
return events;
|
||||
@@ -1413,7 +1428,11 @@ describe("AgentRuntime", () => {
|
||||
output: { results: [{ url: "https://example.com" }] },
|
||||
},
|
||||
{ type: "text-delta", text: "The weather is" },
|
||||
{ type: "finish", reason: "error", error: "stream dropped" },
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: { message: "stream dropped" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
const compactedMessages: AgentMessage[] = [
|
||||
@@ -1458,7 +1477,11 @@ describe("AgentRuntime", () => {
|
||||
],
|
||||
// The compacted request hits a transient provider error...
|
||||
() => [
|
||||
{ type: "finish", reason: "error", error: "Provider returned error" },
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
],
|
||||
// ...and succeeds when re-issued.
|
||||
() => [
|
||||
@@ -1521,7 +1544,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish" as const,
|
||||
reason: "error" as const,
|
||||
error: "Provider returned error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
]),
|
||||
]);
|
||||
@@ -1558,7 +1581,13 @@ describe("AgentRuntime", () => {
|
||||
|
||||
it("does not persist an empty assistant message when the model stream fails", async () => {
|
||||
const model = new ScriptedModel([
|
||||
() => [{ type: "finish", reason: "error", error: "upstream failed" }],
|
||||
() => [
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: { message: "upstream failed" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
const addedMessages: AgentMessage[] = [];
|
||||
const runtime = new AgentRuntime({ model });
|
||||
@@ -1582,8 +1611,10 @@ describe("AgentRuntime", () => {
|
||||
const overflowEvent: AgentModelEvent = {
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
errorClass: "context_window_exceeded",
|
||||
failure: {
|
||||
message: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
classification: "context_window_exceeded",
|
||||
},
|
||||
};
|
||||
const model = new ScriptedModel([
|
||||
() => [overflowEvent],
|
||||
@@ -1639,7 +1670,9 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
failure: {
|
||||
message: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
},
|
||||
},
|
||||
],
|
||||
() => [
|
||||
@@ -1672,7 +1705,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "fetch failed: socket closed",
|
||||
failure: { message: "fetch failed: socket closed" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -1695,7 +1728,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish" as const,
|
||||
reason: "error" as const,
|
||||
error: "Provider returned error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
]),
|
||||
);
|
||||
@@ -1721,7 +1754,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish" as const,
|
||||
reason: "error" as const,
|
||||
error: "Provider returned error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
],
|
||||
() => [
|
||||
@@ -1729,7 +1762,8 @@ describe("AgentRuntime", () => {
|
||||
{ type: "finish" as const, reason: "stop" as const },
|
||||
],
|
||||
]);
|
||||
const runtime = new AgentRuntime({ model });
|
||||
const { telemetry, capture } = createTelemetryMock();
|
||||
const runtime = new AgentRuntime({ model, telemetry });
|
||||
|
||||
const runPromise = runtime.run("Hi");
|
||||
await vi.runAllTimersAsync();
|
||||
@@ -1738,21 +1772,23 @@ describe("AgentRuntime", () => {
|
||||
expect(result.status).toBe("completed");
|
||||
expect(result.outputText).toBe("recovered");
|
||||
expect(model.requests).toHaveLength(2);
|
||||
expect(
|
||||
capture.mock.calls.filter(
|
||||
([event]) => event.event === SDK_ERROR_TELEMETRY_EVENT,
|
||||
),
|
||||
).toHaveLength(0);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not retry a non-transient provider error (honors errorRetryable=false)", async () => {
|
||||
it("does not retry a non-transient provider error (honors failure.retryable=false)", async () => {
|
||||
const model = new ScriptedModel([
|
||||
() => [
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Provider returned error",
|
||||
// Boundary says this specific failure is not retryable; the flag
|
||||
// wins over the message-based fallback.
|
||||
errorRetryable: false,
|
||||
failure: { message: "Provider returned error", retryable: false },
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -1771,7 +1807,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Provider returned error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -1797,7 +1833,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Provider returned error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -1818,7 +1854,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Provider returned error",
|
||||
failure: { message: "Provider returned error" },
|
||||
},
|
||||
],
|
||||
// Valid per the AgentModel contract: an error finish with no payload.
|
||||
@@ -1844,8 +1880,10 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "input is too long for requested model",
|
||||
errorClass: "context_window_exceeded",
|
||||
failure: {
|
||||
message: "input is too long for requested model",
|
||||
classification: "context_window_exceeded",
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -1871,8 +1909,10 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long",
|
||||
errorClass: "context_window_exceeded",
|
||||
failure: {
|
||||
message: "prompt is too long",
|
||||
classification: "context_window_exceeded",
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -1892,8 +1932,10 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long",
|
||||
errorClass: "context_window_exceeded",
|
||||
failure: {
|
||||
message: "prompt is too long",
|
||||
classification: "context_window_exceeded",
|
||||
},
|
||||
},
|
||||
];
|
||||
const model = new ScriptedModel([overflow, overflow]);
|
||||
@@ -4136,7 +4178,7 @@ describe("AgentRuntime", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Invalid input for tool echo",
|
||||
failure: { message: "Invalid input for tool echo" },
|
||||
},
|
||||
],
|
||||
(request) => {
|
||||
@@ -4372,7 +4414,13 @@ describe("AgentRuntime", () => {
|
||||
},
|
||||
{ type: "finish", reason: "tool-calls" },
|
||||
],
|
||||
() => [{ type: "finish", reason: "error", error: "model failed" }],
|
||||
() => [
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: { message: "model failed" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
const runtime = new AgentRuntime({
|
||||
model,
|
||||
@@ -4563,21 +4611,92 @@ describe("AgentRuntime", () => {
|
||||
});
|
||||
|
||||
describe("AgentRuntime sdk.error reporting", () => {
|
||||
it.each([
|
||||
"agent",
|
||||
"standalone",
|
||||
])("gives the %s operation sole ownership of gateway failures", async (owner) => {
|
||||
const { telemetry, capture } = createTelemetryMock();
|
||||
const cause = Object.assign(new Error("transport closed"), {
|
||||
code: "ConnectionClosed",
|
||||
statusCode: 504,
|
||||
isRetryable: true,
|
||||
});
|
||||
const gateway = createGateway({
|
||||
builtins: false,
|
||||
telemetry,
|
||||
providers: [
|
||||
{
|
||||
manifest: {
|
||||
id: "test-provider",
|
||||
name: "Test",
|
||||
defaultModelId: "test",
|
||||
models: [{ id: "test", name: "Test", providerId: "test-provider" }],
|
||||
},
|
||||
createProvider: async () => ({
|
||||
async *stream() {
|
||||
yield {
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
failure: {
|
||||
message: "Display error",
|
||||
cause,
|
||||
retryable: false,
|
||||
context: { error_phase: "stream_read" },
|
||||
},
|
||||
};
|
||||
},
|
||||
}),
|
||||
},
|
||||
],
|
||||
});
|
||||
if (owner === "agent") {
|
||||
const runtime = new AgentRuntime({
|
||||
model: gateway.createAgentModel({
|
||||
providerId: "test-provider",
|
||||
modelId: "test",
|
||||
}),
|
||||
telemetry,
|
||||
sessionId: "session-test",
|
||||
messageModelInfo: { id: "test", provider: "test-provider" },
|
||||
});
|
||||
const result = await runtime.run("test");
|
||||
expect(result.status).toBe("failed");
|
||||
expect(result.error?.cause).toBe(cause);
|
||||
} else {
|
||||
for await (const _event of await gateway.stream({
|
||||
providerId: "test-provider",
|
||||
modelId: "test",
|
||||
messages: [],
|
||||
})) {
|
||||
}
|
||||
}
|
||||
const errors = capture.mock.calls
|
||||
.map(([call]) => call)
|
||||
.filter((call) => call.event === SDK_ERROR_TELEMETRY_EVENT);
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].properties).toMatchObject({
|
||||
component: owner === "agent" ? "agents" : "llms",
|
||||
error_message: "Display error",
|
||||
error_code: "ConnectionClosed",
|
||||
error_status: 504,
|
||||
error_retryable: true,
|
||||
error_phase: "stream_read",
|
||||
});
|
||||
});
|
||||
function sdkErrorEvents(capture: ReturnType<typeof vi.fn>) {
|
||||
return capture.mock.calls
|
||||
.map(([call]) => call as { event: string; properties?: unknown })
|
||||
.filter((call) => call.event === SDK_ERROR_TELEMETRY_EVENT);
|
||||
}
|
||||
|
||||
it("does not re-report a failure the model layer marked as reported", async () => {
|
||||
it("reports a structured model failure once at the run boundary", async () => {
|
||||
const { telemetry, capture } = createTelemetryMock();
|
||||
const model = new ScriptedModel([
|
||||
() => [
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Upstream returned HTTP 429",
|
||||
errorReported: true,
|
||||
failure: { message: "Upstream returned HTTP 429" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -4586,7 +4705,7 @@ describe("AgentRuntime sdk.error reporting", () => {
|
||||
const result = await runtime.run("Hi");
|
||||
|
||||
expect(result.status).toBe("failed");
|
||||
expect(sdkErrorEvents(capture)).toHaveLength(0);
|
||||
expect(sdkErrorEvents(capture)).toHaveLength(1);
|
||||
// The structured lifecycle event still records the failure.
|
||||
expect(capture).toHaveBeenCalledWith(
|
||||
expect.objectContaining({ event: "agent.run-failed" }),
|
||||
@@ -4596,14 +4715,14 @@ describe("AgentRuntime sdk.error reporting", () => {
|
||||
it("reports exactly one sdk.error for custom models that do not record their own telemetry", async () => {
|
||||
const { telemetry, capture } = createTelemetryMock();
|
||||
// A custom `AgentModel` flattens its failure into the finish event and
|
||||
// never calls `captureSdkError`; without `errorReported` the run loop
|
||||
// never calls `captureSdkError`; the run boundary
|
||||
// must own the report.
|
||||
const model = new ScriptedModel([
|
||||
() => [
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "backend unavailable",
|
||||
failure: { message: "backend unavailable" },
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -4809,8 +4928,10 @@ describe("AgentRuntime sdk.error reporting", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
errorClass: "context_window_exceeded",
|
||||
failure: {
|
||||
message: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
classification: "context_window_exceeded",
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
@@ -4844,8 +4965,10 @@ describe("AgentRuntime sdk.error reporting", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
errorClass: "context_window_exceeded",
|
||||
failure: {
|
||||
message: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
classification: "context_window_exceeded",
|
||||
},
|
||||
},
|
||||
],
|
||||
]);
|
||||
|
||||
@@ -12,6 +12,7 @@ import type {
|
||||
AgentMessagePart,
|
||||
AgentModel,
|
||||
AgentModelEvent,
|
||||
AgentModelFailure,
|
||||
AgentModelFinishReason,
|
||||
AgentModelRequest,
|
||||
AgentModelToolActivity,
|
||||
@@ -566,19 +567,13 @@ export class AgentRuntime {
|
||||
lastRequestInputTokens: 0,
|
||||
/**
|
||||
* Whether the last provider failure was transient and worth retrying,
|
||||
* carried from the model boundary via `errorRetryable` on the `finish`
|
||||
* carried from the model boundary via `failure.retryable` on the `finish`
|
||||
* event (the AI SDK's typed `isRetryable` flag). Undefined when no such
|
||||
* signal was provided, in which case the agent loop classifies from the
|
||||
* flattened `lastError` message instead.
|
||||
*/
|
||||
lastErrorRetryable: undefined as boolean | undefined,
|
||||
/**
|
||||
* Whether the model layer already recorded `sdk.error` telemetry for
|
||||
* `lastError` (from `errorReported` on the stream's `finish` event).
|
||||
* Custom `AgentModel` implementations that do not record their own
|
||||
* telemetry leave this false, so their failures still get reported.
|
||||
*/
|
||||
lastErrorReported: false,
|
||||
lastModelFailure: undefined as AgentModelFailure | undefined,
|
||||
/**
|
||||
* Finish reason carried into the run-failed `sdk.error` event. Set
|
||||
* only at the throw sites where a finish reason IS the failure's
|
||||
@@ -680,7 +675,7 @@ export class AgentRuntime {
|
||||
this.state.lastError = undefined;
|
||||
this.state.lastErrorClass = undefined;
|
||||
this.state.lastErrorRetryable = undefined;
|
||||
this.state.lastErrorReported = false;
|
||||
this.state.lastModelFailure = undefined;
|
||||
this.state.messages = cloneMessages(messages);
|
||||
this.config = {
|
||||
...this.config,
|
||||
@@ -800,7 +795,7 @@ export class AgentRuntime {
|
||||
this.state.lastError = undefined;
|
||||
this.state.lastErrorClass = undefined;
|
||||
this.state.lastErrorRetryable = undefined;
|
||||
this.state.lastErrorReported = false;
|
||||
this.state.lastModelFailure = undefined;
|
||||
this.state.lastFinishReason = undefined;
|
||||
this.state.usage = cloneUsage(DEFAULT_USAGE);
|
||||
this.overflowRecoveryAttempted = false;
|
||||
@@ -872,7 +867,7 @@ export class AgentRuntime {
|
||||
// field unset and the attribute is omitted.
|
||||
if (finishReason === "error") {
|
||||
this.state.lastFinishReason = finishReason;
|
||||
throw new Error(this.state.lastError ?? "Model stream failed");
|
||||
throw this.modelFailureError();
|
||||
}
|
||||
// Provider-executed tool activity lives in message metadata, not
|
||||
// content (projecting it into content would replay tool_use blocks
|
||||
@@ -938,7 +933,7 @@ export class AgentRuntime {
|
||||
}
|
||||
if (finishReason === "error" && toolCalls.length === 0) {
|
||||
this.state.lastFinishReason = finishReason;
|
||||
throw new Error(this.state.lastError ?? "Model stream failed");
|
||||
throw this.modelFailureError();
|
||||
}
|
||||
// A turn that yields tool calls is progress: reset the cut-off streak.
|
||||
if (toolCalls.length > 0) {
|
||||
@@ -1039,15 +1034,12 @@ export class AgentRuntime {
|
||||
: normalized.message === this.state.lastError
|
||||
? this.state.lastErrorClass
|
||||
: undefined;
|
||||
// Same guard: the model layer's telemetry only covers this failure
|
||||
// if the run failed on that exact recorded error.
|
||||
const errorAlreadyReported =
|
||||
normalized.message === this.state.lastError &&
|
||||
this.state.lastErrorReported;
|
||||
// Retain provider details only when this is the failure that ended the run.
|
||||
if (normalized.message !== this.state.lastError)
|
||||
this.state.lastModelFailure = undefined;
|
||||
this.state.status = status;
|
||||
this.state.lastError = normalized.message;
|
||||
this.state.lastErrorClass = errorClass;
|
||||
this.state.lastErrorReported = errorAlreadyReported;
|
||||
const lastAssistantMessage = this.findLastAssistantMessage();
|
||||
const result: AgentRunResult = {
|
||||
agentId: this.state.agentId,
|
||||
@@ -1315,7 +1307,7 @@ export class AgentRuntime {
|
||||
|
||||
/**
|
||||
* Clear the last-error fields. Called at the start of every turn and before
|
||||
* every provider-error retry, so a `finish` event that omits `error` (allowed
|
||||
* every provider-error retry, so a `finish` event that omits `failure` (allowed
|
||||
* by the public AgentModel contract) cannot inherit the class or retryability
|
||||
* of an earlier attempt. Deliberately not called inside overflow recovery,
|
||||
* whose "nothing to compact" error reports the first attempt's provider
|
||||
@@ -1325,7 +1317,7 @@ export class AgentRuntime {
|
||||
this.state.lastError = undefined;
|
||||
this.state.lastErrorClass = undefined;
|
||||
this.state.lastErrorRetryable = undefined;
|
||||
this.state.lastErrorReported = false;
|
||||
this.state.lastModelFailure = undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -1578,7 +1570,7 @@ export class AgentRuntime {
|
||||
) {
|
||||
await this.recordAssistantMessage(retry.message, retry.finishReason);
|
||||
}
|
||||
throw new Error(this.state.lastError ?? "Model stream failed");
|
||||
throw this.modelFailureError();
|
||||
}
|
||||
// A retry that came back with nothing is no replacement either: handing it
|
||||
// on would trip the loop's empty-response guard, which throws before
|
||||
@@ -1988,21 +1980,14 @@ export class AgentRuntime {
|
||||
case "finish": {
|
||||
finishReason = event.reason;
|
||||
requestId = event.requestId;
|
||||
if (event.error) {
|
||||
this.state.lastError = event.error;
|
||||
// Models that classify at their own error boundary (where the
|
||||
// raw provider error is still structured) win. Anything else —
|
||||
// custom `AgentModel` implementations, adapters that carry only
|
||||
// a flattened message — is classified from the message so it
|
||||
// stays eligible for overflow recovery.
|
||||
if (event.failure) {
|
||||
this.state.lastModelFailure = event.failure;
|
||||
this.state.lastError = event.failure.message;
|
||||
const error = event.failure.cause ?? event.failure;
|
||||
this.state.lastErrorClass =
|
||||
event.errorClass ?? classifyProviderError(event.error);
|
||||
// Prefer the boundary's typed `isRetryable` signal; fall back to
|
||||
// classifying the flattened message for models that do not carry
|
||||
// it.
|
||||
event.failure.classification ?? classifyProviderError(error);
|
||||
this.state.lastErrorRetryable =
|
||||
event.errorRetryable ?? isRetryableProviderError(event.error);
|
||||
this.state.lastErrorReported = event.errorReported === true;
|
||||
event.failure.retryable ?? isRetryableProviderError(error);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -2678,6 +2663,12 @@ export class AgentRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
private modelFailureError(): Error {
|
||||
return new Error(this.state.lastError ?? "Model stream failed", {
|
||||
cause: this.state.lastModelFailure?.cause,
|
||||
});
|
||||
}
|
||||
|
||||
private normalizeAbortError(): Error {
|
||||
const reason = this.abortController?.signal.reason;
|
||||
if (reason instanceof Error) {
|
||||
@@ -2720,31 +2711,28 @@ export class AgentRuntime {
|
||||
...metadata,
|
||||
error: event.error,
|
||||
});
|
||||
// Failures the model layer already recorded at its own error
|
||||
// boundary (`provider.stream`, carried across the stream's
|
||||
// string-flattening boundary as `finish.errorReported`) must not
|
||||
// be re-reported here — that exactly doubled `sdk.error` volume.
|
||||
// Everything else still reports: loop-originated failures, and
|
||||
// failures from model implementations that do not record their
|
||||
// own telemetry.
|
||||
if (!this.state.lastErrorReported) {
|
||||
captureSdkError(this.config.telemetry, {
|
||||
component: "agents",
|
||||
operation: "agent.run",
|
||||
error: event.error,
|
||||
severity: "error",
|
||||
handled: false,
|
||||
context: {
|
||||
...(metadata as TelemetryProperties),
|
||||
providerId: this.getTelemetryProviderId(),
|
||||
modelId: this.getTelemetryModelId(),
|
||||
modelRequestId: this.modelRequestId,
|
||||
...(this.state.lastFinishReason
|
||||
? { finishReason: this.state.lastFinishReason }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
}
|
||||
captureSdkError(this.config.telemetry, {
|
||||
component: "agents",
|
||||
operation: "agent.run",
|
||||
error: this.state.lastModelFailure?.cause ?? event.error,
|
||||
errorMessage: event.error.message,
|
||||
severity: "error",
|
||||
handled: false,
|
||||
context: {
|
||||
...(metadata as TelemetryProperties),
|
||||
...this.state.lastModelFailure?.context,
|
||||
providerId:
|
||||
this.getTelemetryProviderId() ??
|
||||
this.state.lastModelFailure?.context?.providerId,
|
||||
modelId:
|
||||
this.getTelemetryModelId() ??
|
||||
this.state.lastModelFailure?.context?.modelId,
|
||||
modelRequestId: this.modelRequestId,
|
||||
...(this.state.lastFinishReason
|
||||
? { finishReason: this.state.lastFinishReason }
|
||||
: {}),
|
||||
},
|
||||
});
|
||||
break;
|
||||
default:
|
||||
this.config.logger?.debug?.("Agent event", metadata);
|
||||
|
||||
@@ -1815,7 +1815,7 @@ describe("SessionRuntime real AgentRuntime smoke", () => {
|
||||
yield {
|
||||
type: "finish" as const,
|
||||
reason: "error" as const,
|
||||
error: "upstream failed",
|
||||
failure: { message: "upstream failed" },
|
||||
};
|
||||
return;
|
||||
}
|
||||
@@ -1900,7 +1900,7 @@ describe("SessionRuntime real AgentRuntime smoke", () => {
|
||||
yield {
|
||||
type: "finish" as const,
|
||||
reason: "error" as const,
|
||||
error: "upstream failed after tool result",
|
||||
failure: { message: "upstream failed after tool result" },
|
||||
};
|
||||
})();
|
||||
},
|
||||
|
||||
@@ -188,7 +188,7 @@ describe("createAgentModelFromApiHandler", () => {
|
||||
expect(finishes[0]).toEqual({
|
||||
type: "finish",
|
||||
reason: "stop",
|
||||
error: undefined,
|
||||
failure: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
@@ -209,7 +209,7 @@ describe("createAgentModelFromApiHandler", () => {
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "boom",
|
||||
failure: { message: "boom" },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -231,11 +231,7 @@ describe("createAgentModelFromApiHandler", () => {
|
||||
});
|
||||
const model = createAgentModelFromApiHandler(handler);
|
||||
const events = await collect(model.stream(baseRequest));
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
errorClass: "context_window_exceeded",
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({ type: "finish", reason: "error" });
|
||||
});
|
||||
|
||||
it("classifies a context-window overflow reported by a failed done chunk", async () => {
|
||||
@@ -249,11 +245,7 @@ describe("createAgentModelFromApiHandler", () => {
|
||||
]);
|
||||
const model = createAgentModelFromApiHandler(handler);
|
||||
const events = await collect(model.stream(baseRequest));
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
errorClass: "context_window_exceeded",
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({ type: "finish", reason: "error" });
|
||||
});
|
||||
|
||||
it("leaves unrelated handler failures unclassified", async () => {
|
||||
@@ -262,11 +254,7 @@ describe("createAgentModelFromApiHandler", () => {
|
||||
});
|
||||
const model = createAgentModelFromApiHandler(handler);
|
||||
const events = await collect(model.stream(baseRequest));
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
errorClass: "unknown",
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({ type: "finish", reason: "error" });
|
||||
});
|
||||
|
||||
it("does not emit a second finish when the handler throws after a done chunk", async () => {
|
||||
@@ -303,8 +291,10 @@ describe("createAgentModelFromApiHandler", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "no vscode.lm",
|
||||
errorClass: "unknown",
|
||||
failure: expect.objectContaining({
|
||||
message: "no vscode.lm",
|
||||
classification: "unknown",
|
||||
}),
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
@@ -85,14 +85,12 @@ function toAgentModelEvents(chunk: ApiStreamChunk): AgentModelEvent[] {
|
||||
{
|
||||
type: "finish",
|
||||
reason: doneFinishReason(chunk),
|
||||
error: chunk.error,
|
||||
// Handlers report failures as an already-flattened message, so
|
||||
// classification works from the text alone here. Keeps registered
|
||||
// handlers (e.g. VS Code LM) eligible for the runtime's
|
||||
// context-overflow recovery.
|
||||
...(chunk.error
|
||||
? { errorClass: classifyProviderError(chunk.error) }
|
||||
: {}),
|
||||
failure: chunk.error
|
||||
? {
|
||||
message: chunk.error,
|
||||
classification: classifyProviderError(chunk.error),
|
||||
}
|
||||
: undefined,
|
||||
},
|
||||
];
|
||||
default:
|
||||
@@ -182,12 +180,17 @@ export function createAgentModelFromApiHandler(
|
||||
yield {
|
||||
type: "finish",
|
||||
reason: aborted ? "aborted" : "error",
|
||||
error: error instanceof Error ? error.message : String(error),
|
||||
failure: {
|
||||
message: error instanceof Error ? error.message : String(error),
|
||||
cause: error,
|
||||
...(aborted
|
||||
? {}
|
||||
: { classification: classifyProviderError(error) }),
|
||||
},
|
||||
// Classify while the raw error is still structured — the
|
||||
// message alone can hide the provider's detail (status codes,
|
||||
// response bodies). Without this, an overflow rejection from a
|
||||
// registered handler would never reach recovery.
|
||||
...(aborted ? {} : { errorClass: classifyProviderError(error) }),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { LanguageModelV4 } from "@ai-sdk/provider";
|
||||
import type {
|
||||
AgentMessage,
|
||||
AgentModelEvent,
|
||||
AgentModelFailure,
|
||||
AgentModelFinishReason,
|
||||
GatewayProviderContext,
|
||||
GatewayProviderFactory,
|
||||
@@ -18,7 +19,6 @@ import type {
|
||||
import {
|
||||
type AiSdkFormatterMessage,
|
||||
type AiSdkFormatterPart,
|
||||
captureSdkError,
|
||||
createMediaBudgetState,
|
||||
formatMessagesForAiSdk,
|
||||
GeneratedMediaSchema,
|
||||
@@ -1460,6 +1460,7 @@ function extractGoogleThoughtMetadata(
|
||||
*/
|
||||
interface CapturedStreamError {
|
||||
error: unknown;
|
||||
aiSdkCallId?: string;
|
||||
phase:
|
||||
| "stream_callback"
|
||||
| "stream_part"
|
||||
@@ -1472,19 +1473,12 @@ interface CapturedStreamError {
|
||||
/**
|
||||
* Whether the agent loop's turn-level retry may re-run this turn, decided
|
||||
* while the structured error is still in hand and forwarded as
|
||||
* `errorRetryable` on the `finish` event (the flattened message the agent
|
||||
* loop receives cannot carry it). Transient by the AI SDK's own typed
|
||||
* `failure.retryable` on the `finish` event. Transient by the AI SDK's own typed
|
||||
* `isRetryable` flag, except that request-start RetryErrors and terminal
|
||||
* stream error parts have already spent the SDK's retry budget. The
|
||||
* turn-level retry must not multiply those attempts.
|
||||
*/
|
||||
retryable: boolean;
|
||||
/**
|
||||
* This layer already recorded `sdk.error` telemetry for the failure.
|
||||
* Forwarded as `errorReported` on the `finish` event so the agent loop
|
||||
* does not report the same failure a second time.
|
||||
*/
|
||||
reported?: boolean;
|
||||
requestId?: string;
|
||||
baseMessage: string;
|
||||
}
|
||||
@@ -1511,6 +1505,40 @@ function captureStreamError(
|
||||
};
|
||||
}
|
||||
|
||||
function modelFailure(
|
||||
captured: CapturedStreamError,
|
||||
request: GatewayStreamRequest,
|
||||
context: GatewayProviderContext,
|
||||
): AgentModelFailure {
|
||||
const runtimeContext = buildAiSdkRuntimeContext(request, context);
|
||||
const correlation = Object.fromEntries(
|
||||
[
|
||||
"runId",
|
||||
"agentId",
|
||||
"sessionId",
|
||||
"conversationId",
|
||||
"modelRequestId",
|
||||
"iteration",
|
||||
].flatMap((key) =>
|
||||
runtimeContext[key] === undefined ? [] : [[key, runtimeContext[key]]],
|
||||
),
|
||||
);
|
||||
return {
|
||||
message: captured.message,
|
||||
cause: captured.error,
|
||||
classification: captured.errorClass,
|
||||
retryable: captured.retryable,
|
||||
context: {
|
||||
...correlation,
|
||||
aiSdkCallId: captured.aiSdkCallId,
|
||||
error_phase: captured.phase,
|
||||
error_turn_retryable: captured.retryable,
|
||||
providerId: request.providerId,
|
||||
modelId: request.modelId,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function* emitAiSdkEvents(
|
||||
stream: AiSdkStreamResult,
|
||||
request: GatewayStreamRequest,
|
||||
@@ -1518,7 +1546,7 @@ async function* emitAiSdkEvents(
|
||||
pricingValue?: unknown,
|
||||
capturedError?: { current: CapturedStreamError | undefined },
|
||||
modelToolAdapters?: BuiltModelTools,
|
||||
reportError?: (error: CapturedStreamError) => Promise<void>,
|
||||
traceError?: (error: CapturedStreamError) => Promise<void>,
|
||||
): AsyncIterable<AgentModelEvent> {
|
||||
const emittedToolCallIds = new Set<string>();
|
||||
let finishReason: unknown;
|
||||
@@ -2035,17 +2063,16 @@ async function* emitAiSdkEvents(
|
||||
};
|
||||
}
|
||||
|
||||
if (streamError) await reportError?.(streamError);
|
||||
if (streamError) await traceError?.(streamError);
|
||||
|
||||
const finishRequestId = requestId ?? streamError?.requestId;
|
||||
yield {
|
||||
type: "finish",
|
||||
reason: streamError ? "error" : mapFinishReason(finishReason),
|
||||
...(finishRequestId ? { requestId: finishRequestId } : {}),
|
||||
error: streamError?.message,
|
||||
errorClass: streamError?.errorClass,
|
||||
errorRetryable: streamError?.retryable,
|
||||
errorReported: streamError?.reported,
|
||||
failure: streamError
|
||||
? modelFailure(streamError, request, context)
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2173,73 +2200,15 @@ function createAiSdkProvider(
|
||||
// Multi-protocol HTTP gateways declare model adapters in models.dev.
|
||||
// Keep native and local CLI transports authoritative for their models.
|
||||
const kind = resolveModelProviderKind(defaultKind, context);
|
||||
const log = context.logger;
|
||||
let stream: AiSdkStreamResult | undefined;
|
||||
const capturedError: { current: CapturedStreamError | undefined } = {
|
||||
current: undefined,
|
||||
};
|
||||
let getTraceCallId: (() => string | undefined) | undefined;
|
||||
let recordTraceError: ((error: unknown) => Promise<void>) | undefined;
|
||||
let getAiSdkCallId: (() => string | undefined) | undefined;
|
||||
let reportedError: CapturedStreamError | undefined;
|
||||
const reportError = async (
|
||||
captured: CapturedStreamError,
|
||||
operation = "provider.stream",
|
||||
) => {
|
||||
if (reportedError === captured) return;
|
||||
// Explicit user/steering cancellation is not a provider failure.
|
||||
if (
|
||||
request.signal?.aborted &&
|
||||
!isTimeoutReason(request.signal.reason) &&
|
||||
!isTimeoutReason(captured.error)
|
||||
)
|
||||
return;
|
||||
reportedError = captured;
|
||||
const runtimeContext = buildAiSdkRuntimeContext(request, context);
|
||||
const correlation = Object.fromEntries(
|
||||
[
|
||||
"runId",
|
||||
"agentId",
|
||||
"sessionId",
|
||||
"conversationId",
|
||||
"modelRequestId",
|
||||
"iteration",
|
||||
].flatMap((key) =>
|
||||
runtimeContext[key] === undefined
|
||||
? []
|
||||
: [[key, runtimeContext[key]]],
|
||||
),
|
||||
);
|
||||
if (log?.error) {
|
||||
log.error("[ai-sdk] provider error", {
|
||||
...correlation,
|
||||
providerId: request.providerId,
|
||||
error: captured.error,
|
||||
severity: "error",
|
||||
});
|
||||
} else {
|
||||
log?.log(`[ai-sdk] provider error: ${captured.message}`, {
|
||||
providerId: request.providerId,
|
||||
severity: "error",
|
||||
});
|
||||
}
|
||||
captured.reported = captureSdkError(context.telemetry, {
|
||||
component: "llms",
|
||||
operation,
|
||||
error: captured.error,
|
||||
errorMessage: captured.baseMessage,
|
||||
severity: "error",
|
||||
handled: true,
|
||||
context: {
|
||||
...correlation,
|
||||
aiSdkCallId: getAiSdkCallId?.(),
|
||||
errorPhase: captured.phase,
|
||||
errorTurnRetryable: captured.retryable,
|
||||
providerId: request.providerId,
|
||||
modelId: request.modelId,
|
||||
providerKind: kind,
|
||||
requestId: captured.requestId,
|
||||
},
|
||||
});
|
||||
|
||||
const traceError = async (captured: CapturedStreamError) => {
|
||||
captured.aiSdkCallId = getTraceCallId?.();
|
||||
await recordTraceError?.(captured.error);
|
||||
};
|
||||
try {
|
||||
@@ -2374,7 +2343,8 @@ function createAiSdkProvider(
|
||||
);
|
||||
const aiSdkTelemetry = streamTelemetry.telemetry;
|
||||
recordTraceError = streamTelemetry.recordError;
|
||||
getAiSdkCallId = streamTelemetry.getCallId;
|
||||
getTraceCallId = streamTelemetry.getCallId;
|
||||
|
||||
const externalToolExecutionDisabled =
|
||||
providerDisablesExternalToolExecution(context);
|
||||
const toolCallingDisabled =
|
||||
@@ -2486,7 +2456,7 @@ function createAiSdkProvider(
|
||||
if (!isTimeoutReason(reason)) return;
|
||||
const captured = captureStreamError(reason, "timeout");
|
||||
capturedError.current = captured;
|
||||
await reportError(captured);
|
||||
await traceError(captured);
|
||||
},
|
||||
}) as unknown as AiSdkStreamResult,
|
||||
);
|
||||
@@ -2504,7 +2474,7 @@ function createAiSdkProvider(
|
||||
context.model.metadata?.pricing,
|
||||
capturedError,
|
||||
modelToolAdapters,
|
||||
reportError,
|
||||
traceError,
|
||||
);
|
||||
} catch (error) {
|
||||
suppressDanglingStreamPromises(stream);
|
||||
@@ -2512,14 +2482,11 @@ function createAiSdkProvider(
|
||||
// NoOutputGeneratedError that the AI SDK throws when 0 steps are recorded.
|
||||
const captured =
|
||||
capturedError.current ?? captureStreamError(error, "request_start");
|
||||
await reportError(captured, "provider.create_or_stream");
|
||||
await traceError(captured);
|
||||
yield {
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: captured.message,
|
||||
errorClass: captured.errorClass,
|
||||
errorRetryable: captured.retryable,
|
||||
errorReported: captured.reported,
|
||||
failure: modelFailure(captured, request, context),
|
||||
};
|
||||
}
|
||||
},
|
||||
|
||||
@@ -593,7 +593,7 @@ function toApiStreamChunk(
|
||||
type: "done",
|
||||
id,
|
||||
success: event.reason !== "error",
|
||||
error: event.error,
|
||||
error: event.failure?.message,
|
||||
incompleteReason:
|
||||
event.reason === "max-tokens"
|
||||
? "max_tokens"
|
||||
|
||||
@@ -1037,8 +1037,10 @@ describe("sdk-gateway", () => {
|
||||
expect.objectContaining({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error:
|
||||
'Provider adapter for "compatible-with-search" does not implement requested model tool(s): web_search.',
|
||||
failure: expect.objectContaining({
|
||||
message:
|
||||
'Provider adapter for "compatible-with-search" does not implement requested model tool(s): web_search.',
|
||||
}),
|
||||
}),
|
||||
);
|
||||
expect(streamTextSpy).not.toHaveBeenCalled();
|
||||
@@ -1406,7 +1408,7 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Generated media must contain valid base64",
|
||||
failure: { message: "Generated media must contain valid base64" },
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1506,7 +1508,9 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: `Image media exceeds the ${DEFAULT_MAX_IMAGE_ENCODED_BYTES} byte encoded limit`,
|
||||
failure: {
|
||||
message: `Image media exceeds the ${DEFAULT_MAX_IMAGE_ENCODED_BYTES} byte encoded limit`,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1646,10 +1650,11 @@ describe("sdk-gateway", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: `Image media exceeds the ${DEFAULT_MAX_IMAGE_ENCODED_BYTES} byte encoded limit`,
|
||||
errorClass: "unknown",
|
||||
errorRetryable: false,
|
||||
errorReported: false,
|
||||
failure: expect.objectContaining({
|
||||
message: `Image media exceeds the ${DEFAULT_MAX_IMAGE_ENCODED_BYTES} byte encoded limit`,
|
||||
classification: "unknown",
|
||||
retryable: false,
|
||||
}),
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -2426,9 +2431,11 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error:
|
||||
'Model tool "image_generation" call is missing a valid tool-call ID',
|
||||
errorClass: "unknown",
|
||||
failure: {
|
||||
message:
|
||||
'Model tool "image_generation" call is missing a valid tool-call ID',
|
||||
classification: "unknown",
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2481,10 +2488,12 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toEqual({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "OpenAI image generation tool returned no supported image output",
|
||||
errorClass: "unknown",
|
||||
errorRetryable: false,
|
||||
errorReported: false,
|
||||
failure: expect.objectContaining({
|
||||
message:
|
||||
"OpenAI image generation tool returned no supported image output",
|
||||
classification: "unknown",
|
||||
retryable: false,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2701,10 +2710,11 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toEqual({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Invalid API key",
|
||||
errorClass: "unknown",
|
||||
errorRetryable: false,
|
||||
errorReported: false,
|
||||
failure: expect.objectContaining({
|
||||
message: "Invalid API key",
|
||||
classification: "unknown",
|
||||
retryable: false,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -2767,8 +2777,7 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: error.message,
|
||||
errorReported: true,
|
||||
failure: { message: error.message },
|
||||
});
|
||||
expect(
|
||||
capture.mock.calls.filter(([call]) => call.event === "sdk.error"),
|
||||
@@ -2848,10 +2857,7 @@ describe("sdk-gateway", () => {
|
||||
expect(streamTextSpy.mock.calls.at(-1)?.[0]).toMatchObject({
|
||||
streamRetries: 3,
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
reason: "error",
|
||||
errorRetryable: false,
|
||||
});
|
||||
expect(events.at(-1)).toMatchObject({ reason: "error" });
|
||||
});
|
||||
|
||||
it("records the extracted provider message for AI SDK stream errors", async () => {
|
||||
@@ -2898,7 +2904,7 @@ describe("sdk-gateway", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("marks the finish event as reported so the agent loop skips the same failure", async () => {
|
||||
it("reports standalone gateway failures once at the terminal boundary", async () => {
|
||||
const rawError = Object.assign(new Error("Upstream returned HTTP 429"), {
|
||||
statusCode: 429,
|
||||
});
|
||||
@@ -2927,8 +2933,7 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toMatchObject({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Upstream returned HTTP 429",
|
||||
errorReported: true,
|
||||
failure: { message: "Upstream returned HTTP 429" },
|
||||
});
|
||||
const sdkErrors = capture.mock.calls.filter(
|
||||
([call]) => (call as { event: string }).event === "sdk.error",
|
||||
@@ -2936,7 +2941,7 @@ describe("sdk-gateway", () => {
|
||||
expect(sdkErrors).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not mark the finish event as reported when telemetry is unavailable", async () => {
|
||||
it("preserves the original structured error when telemetry is unavailable", async () => {
|
||||
const rawError = Object.assign(new Error("Upstream returned HTTP 429"), {
|
||||
statusCode: 429,
|
||||
});
|
||||
@@ -2960,8 +2965,8 @@ describe("sdk-gateway", () => {
|
||||
}),
|
||||
);
|
||||
|
||||
const finish = events.at(-1) as { errorReported?: boolean };
|
||||
expect(finish.errorReported).not.toBe(true);
|
||||
const finish = events.at(-1);
|
||||
expect(finish?.type === "finish" && finish.failure?.cause).toBe(rawError);
|
||||
});
|
||||
|
||||
it("rate-limits identical stream failures per process", async () => {
|
||||
@@ -3105,10 +3110,11 @@ describe("sdk-gateway", () => {
|
||||
{
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Invalid API key",
|
||||
errorClass: "unknown",
|
||||
errorRetryable: false,
|
||||
errorReported: false,
|
||||
failure: expect.objectContaining({
|
||||
message: "Invalid API key",
|
||||
classification: "unknown",
|
||||
retryable: false,
|
||||
}),
|
||||
},
|
||||
]);
|
||||
});
|
||||
@@ -3147,10 +3153,11 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toEqual({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
errorClass: "context_window_exceeded",
|
||||
errorRetryable: false,
|
||||
errorReported: false,
|
||||
failure: expect.objectContaining({
|
||||
message: "prompt is too long: 213462 tokens > 200000 maximum",
|
||||
classification: "context_window_exceeded",
|
||||
retryable: false,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3185,10 +3192,11 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toEqual({
|
||||
type: "finish",
|
||||
reason: "error",
|
||||
error: "Instructions are required",
|
||||
errorClass: "unknown",
|
||||
errorRetryable: false,
|
||||
errorReported: false,
|
||||
failure: expect.objectContaining({
|
||||
message: "Instructions are required",
|
||||
classification: "unknown",
|
||||
retryable: false,
|
||||
}),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5149,7 +5157,7 @@ describe("sdk-gateway", () => {
|
||||
expect(events.at(-1)).toEqual({
|
||||
type: "finish",
|
||||
reason: "tool-calls",
|
||||
error: undefined,
|
||||
failure: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import type {
|
||||
AgentModel,
|
||||
AgentModelEvent,
|
||||
AgentModelFailure,
|
||||
AgentModelRequest,
|
||||
BasicLogger,
|
||||
GatewayConfig,
|
||||
@@ -13,9 +14,11 @@ import type {
|
||||
ReasoningEffort,
|
||||
} from "@cline/shared";
|
||||
import {
|
||||
captureSdkError,
|
||||
estimateRequestInputTokens,
|
||||
ReasoningEffortSchema,
|
||||
} from "@cline/shared";
|
||||
import { isTimeoutReason } from "../services/stream-error-telemetry";
|
||||
import { toAsyncIterable } from "./async";
|
||||
import { BUILTIN_PROVIDER_REGISTRATIONS } from "./builtins-runtime";
|
||||
import { providerManifestSupportsModelOperation } from "./model-operations";
|
||||
@@ -125,7 +128,9 @@ export interface Gateway {
|
||||
|
||||
class GatewayModelAdapter implements AgentModel {
|
||||
constructor(
|
||||
private readonly gateway: DefaultGateway,
|
||||
private readonly streamModel: (
|
||||
request: GatewayStreamRequest,
|
||||
) => Promise<AsyncIterable<AgentModelEvent>>,
|
||||
private readonly selection: GatewayModelSelection,
|
||||
private readonly defaults: GatewayModelHandleOptions | undefined,
|
||||
) {}
|
||||
@@ -162,7 +167,7 @@ class GatewayModelAdapter implements AgentModel {
|
||||
budgetTokens: legacyBudgetTokens,
|
||||
}
|
||||
: undefined;
|
||||
return this.gateway.stream({
|
||||
return this.streamModel({
|
||||
providerId: this.selection.providerId,
|
||||
modelId: this.selection.modelId ?? "",
|
||||
systemPrompt: request.systemPrompt,
|
||||
@@ -327,11 +332,115 @@ export class DefaultGateway implements Gateway {
|
||||
selection: GatewayModelSelection,
|
||||
options?: GatewayModelHandleOptions,
|
||||
): AgentModel {
|
||||
return new GatewayModelAdapter(this, selection, options);
|
||||
return new GatewayModelAdapter(
|
||||
(request) => this.streamModel(request),
|
||||
selection,
|
||||
options,
|
||||
);
|
||||
}
|
||||
|
||||
async stream(
|
||||
request: GatewayStreamRequest,
|
||||
): Promise<AsyncIterable<AgentModelEvent>> {
|
||||
let stream: AsyncIterable<AgentModelEvent>;
|
||||
try {
|
||||
stream = await this.streamModel(request);
|
||||
} catch (cause) {
|
||||
this.reportFailure(
|
||||
request,
|
||||
{
|
||||
message: cause instanceof Error ? cause.message : String(cause),
|
||||
cause,
|
||||
},
|
||||
"provider.create_or_stream",
|
||||
);
|
||||
throw cause;
|
||||
}
|
||||
return this.streamWithTerminalReporting(request, stream);
|
||||
}
|
||||
|
||||
private async *streamWithTerminalReporting(
|
||||
request: GatewayStreamRequest,
|
||||
stream: AsyncIterable<AgentModelEvent>,
|
||||
): AsyncIterable<AgentModelEvent> {
|
||||
try {
|
||||
for await (const event of stream) {
|
||||
if (event.type === "finish" && event.reason === "error") {
|
||||
this.reportFailure(
|
||||
request,
|
||||
event.failure ?? { message: "Model stream failed" },
|
||||
);
|
||||
yield event;
|
||||
return;
|
||||
}
|
||||
yield event;
|
||||
}
|
||||
} catch (cause) {
|
||||
this.reportFailure(request, {
|
||||
message: cause instanceof Error ? cause.message : String(cause),
|
||||
cause,
|
||||
});
|
||||
throw cause;
|
||||
}
|
||||
}
|
||||
|
||||
private reportFailure(
|
||||
request: GatewayStreamRequest,
|
||||
failure: AgentModelFailure,
|
||||
operation = "provider.stream",
|
||||
): void {
|
||||
if (
|
||||
request.signal?.aborted &&
|
||||
!isTimeoutReason(request.signal.reason) &&
|
||||
!isTimeoutReason(failure.cause)
|
||||
)
|
||||
return;
|
||||
const context = Object.fromEntries(
|
||||
[
|
||||
"runId",
|
||||
"agentId",
|
||||
"sessionId",
|
||||
"conversationId",
|
||||
"modelRequestId",
|
||||
"iteration",
|
||||
].flatMap((key) => {
|
||||
const value = request.metadata?.[key];
|
||||
return typeof value === "string" || typeof value === "number"
|
||||
? [[key, value]]
|
||||
: [];
|
||||
}),
|
||||
);
|
||||
const error = failure.cause ?? failure;
|
||||
const details = {
|
||||
...context,
|
||||
...failure.context,
|
||||
providerId: request.providerId,
|
||||
modelId: request.modelId,
|
||||
error,
|
||||
severity: "error" as const,
|
||||
};
|
||||
if (this.logger?.error) this.logger.error("Gateway stream failed", details);
|
||||
else this.logger?.log(`Gateway stream failed: ${failure.message}`, details);
|
||||
captureSdkError(this.telemetry, {
|
||||
component: "llms",
|
||||
operation:
|
||||
failure.context?.error_phase === "request_start"
|
||||
? "provider.create_or_stream"
|
||||
: operation,
|
||||
error,
|
||||
errorMessage: failure.message,
|
||||
context: {
|
||||
...context,
|
||||
...failure.context,
|
||||
providerId: request.providerId,
|
||||
modelId: request.modelId,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/** Internal model boundary; the agent run owns terminal reporting. */
|
||||
private async streamModel(
|
||||
request: GatewayStreamRequest,
|
||||
): Promise<AsyncIterable<AgentModelEvent>> {
|
||||
const resolved = this.registry.resolveModel({
|
||||
providerId: request.providerId,
|
||||
|
||||
@@ -13,7 +13,10 @@ import type {
|
||||
ToolPolicy,
|
||||
} from "./llms/tools";
|
||||
import type { BasicLogger } from "./logging/logger";
|
||||
import type { ITelemetryService } from "./services/telemetry";
|
||||
import type {
|
||||
ITelemetryService,
|
||||
TelemetryProperties,
|
||||
} from "./services/telemetry";
|
||||
|
||||
// =============================================================================
|
||||
// Lightweight telemetry surface used by AgentRuntime
|
||||
@@ -303,6 +306,15 @@ export type AgentModelFinishReason =
|
||||
*/
|
||||
export type ProviderErrorClass = "context_window_exceeded" | "auth" | "unknown";
|
||||
|
||||
/** Structured terminal model failure, reported by the outermost operation. */
|
||||
export interface AgentModelFailure {
|
||||
message: string;
|
||||
cause?: unknown;
|
||||
classification?: ProviderErrorClass;
|
||||
retryable?: boolean;
|
||||
context?: TelemetryProperties;
|
||||
}
|
||||
|
||||
export type AgentModelEvent =
|
||||
| { type: "text-delta"; text: string }
|
||||
| { type: "media"; media: GeneratedMedia }
|
||||
@@ -345,25 +357,7 @@ export type AgentModelEvent =
|
||||
reason: AgentModelFinishReason;
|
||||
/** HTTP X-Request-ID of the surfaced response, not the provider's generation ID. */
|
||||
requestId?: string;
|
||||
error?: string;
|
||||
errorClass?: ProviderErrorClass;
|
||||
/**
|
||||
* Whether the underlying provider error was transient and worth
|
||||
* retrying, decided at the model boundary from the AI SDK's typed
|
||||
* `isRetryable` flag while the structured error is still in hand
|
||||
* (`error` is a flattened string, so the agent loop cannot re-derive
|
||||
* this). When absent, the agent loop classifies from the message.
|
||||
*/
|
||||
errorRetryable?: boolean;
|
||||
/**
|
||||
* The model layer already recorded `sdk.error` telemetry for this
|
||||
* failure at its own error boundary. `error` is a flattened string,
|
||||
* so this bit carries reporting ownership across the boundary: the
|
||||
* agent loop skips re-reporting when it is set, and still reports
|
||||
* failures from model implementations that do not record their own
|
||||
* telemetry.
|
||||
*/
|
||||
errorReported?: boolean;
|
||||
failure?: AgentModelFailure;
|
||||
};
|
||||
|
||||
export interface AgentModel {
|
||||
|
||||
@@ -313,10 +313,7 @@ export function captureTaskLifecycleEvent(
|
||||
*
|
||||
* Returns `true` when the failure is recorded — emitted, or counted toward
|
||||
* `suppressed_count` by the volume cap — and `false` when telemetry is
|
||||
* unavailable. Reporters that sit on a layer boundary forward the return
|
||||
* value (see `errorReported` on the model stream's `finish` event) so outer
|
||||
* layers know the failure is already accounted for and one underlying
|
||||
* failure produces one event, not one per layer it propagates through.
|
||||
* unavailable. Terminal reporting is owned by the outermost public operation.
|
||||
*/
|
||||
export function captureSdkError(
|
||||
telemetry: ITelemetryService | undefined,
|
||||
|
||||
Reference in New Issue
Block a user