mirror of
https://github.com/coder/coder.git
synced 2026-09-24 15:04:27 +08:00
fix: preserve Anthropic replay fidelity (#25377)
Anthropic is strict about replaying the latest assistant turn once it contains signed or redacted reasoning. We were still mutating that turn in a few Coder-owned places: dropping empty reasoning blocks on replay, rewriting provider-tool history during sanitization, and in the worst case sending a prompt we already knew Anthropic would reject. This patch keeps the latest signed assistant immutable through Coder's replay and sanitization paths, preserves empty signed or redacted reasoning anywhere Coder owns the ledger, and fails before the provider call if the prompt is still unsafe. It also bumps the existing `coder/fantasy` `coder_2_33` fork that `main` already uses to the commit containing coder/fantasy#35. These fixes have also been upstreamed to charmbracelet/fantasy. Closes CODAGT-409.
This commit is contained in:
Ethan
@@ -12,8 +12,11 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"cdr.dev/slog/v3/sloggers/slogtest"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chatopenai"
|
||||
"github.com/coder/coder/v2/coderd/x/chatd/chattest"
|
||||
"github.com/coder/coder/v2/codersdk"
|
||||
)
|
||||
|
||||
func TestRun_ChainBrokenRecovers(t *testing.T) {
|
||||
@@ -418,6 +421,64 @@ func TestRun_ChainBrokenReloadFailureStillClearsChain(t *testing.T) {
|
||||
requireTextPrompt(t, secondPrompt, "prepared")
|
||||
}
|
||||
|
||||
func TestRun_ChainBrokenRecoveryPrepareFailureReturnsPreparePhaseError(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
var streamCalls int
|
||||
model := &chattest.FakeModel{
|
||||
ProviderName: fantasyanthropic.Name,
|
||||
ModelName: "claude-test",
|
||||
StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
|
||||
streamCalls++
|
||||
return nil, xerrors.New(chainBrokenErrorMessage)
|
||||
},
|
||||
}
|
||||
|
||||
reloadCalls := 0
|
||||
err := Run(context.Background(), RunOptions{
|
||||
Model: model,
|
||||
Logger: slogtest.Make(t, &slogtest.Options{IgnoreErrors: true}),
|
||||
MaxSteps: 1,
|
||||
ContextLimitFallback: 4096,
|
||||
Messages: []fantasy.Message{
|
||||
textMessage(fantasy.MessageRoleUser, "chain-filtered"),
|
||||
},
|
||||
ProviderOptions: chainModeProviderOptions("resp_poisoned"),
|
||||
PersistStep: func(_ context.Context, _ PersistedStep) error {
|
||||
return nil
|
||||
},
|
||||
DisableChainMode: func() {},
|
||||
ReloadMessages: func(_ context.Context) ([]fantasy.Message, error) {
|
||||
reloadCalls++
|
||||
return []fantasy.Message{
|
||||
textMessage(fantasy.MessageRoleUser, "search"),
|
||||
{
|
||||
Role: fantasy.MessageRoleAssistant,
|
||||
Content: []fantasy.MessagePart{
|
||||
fantasy.ReasoningPart{ProviderOptions: fantasy.ProviderOptions{fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{RedactedData: "redacted-payload"}}},
|
||||
fantasy.ToolCallPart{ToolCallID: "ws-orphan", ToolName: "web_search", Input: `{"query":"coder"}`, ProviderExecuted: true},
|
||||
fantasy.TextPart{Text: "partial"},
|
||||
},
|
||||
},
|
||||
textMessage(fantasy.MessageRoleUser, "continue"),
|
||||
}, nil
|
||||
},
|
||||
})
|
||||
|
||||
require.Error(t, err)
|
||||
require.Equal(t, 1, reloadCalls)
|
||||
require.Equal(t, 1, streamCalls, "retry must fail before issuing another provider call")
|
||||
require.ErrorContains(t, err, "prepare prompt:")
|
||||
require.NotContains(t, err.Error(), "stream response:")
|
||||
require.Equal(t, chaterror.ClassifiedError{
|
||||
Message: "The chat continuation failed due to an internal state mismatch. This is not a configuration or billing issue. Start a new chat to continue.",
|
||||
Detail: "Anthropic replay diagnostic: match=provider_tool_guard_postcondition_failed.",
|
||||
Kind: codersdk.ChatErrorKindGeneric,
|
||||
Provider: fantasyanthropic.Name,
|
||||
Retryable: false,
|
||||
}, chaterror.Classify(err))
|
||||
}
|
||||
|
||||
func TestRun_ChainBrokenWithoutChainModeIsSafe(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
@@ -512,6 +573,132 @@ func TestRun_NonChainBrokenRetryDoesNotTouchChainState(t *testing.T) {
|
||||
)
|
||||
}
|
||||
|
||||
func TestProcessStepStreamPreservesReasoningMetadataAcrossNilDelta(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningStart, ID: "0"})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0", Delta: "thinking"})
|
||||
yield(fantasy.StreamPart{
|
||||
Type: fantasy.StreamPartTypeReasoningDelta,
|
||||
ID: "0",
|
||||
ProviderMetadata: fantasy.ProviderMetadata{
|
||||
fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{
|
||||
Signature: "sig",
|
||||
},
|
||||
},
|
||||
})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0", ProviderMetadata: fantasy.ProviderMetadata{}})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningDelta, ID: "0"})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeReasoningEnd, ID: "0", ProviderMetadata: fantasy.ProviderMetadata{}})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
||||
})
|
||||
|
||||
result, err := processStepStream(context.Background(), stream, func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) {})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.content, 1)
|
||||
reasoning, ok := fantasy.AsContentType[fantasy.ReasoningContent](result.content[0])
|
||||
require.True(t, ok)
|
||||
require.Equal(t, "thinking", reasoning.Text)
|
||||
metadata := fantasyanthropic.GetReasoningMetadata(fantasy.ProviderOptions(reasoning.ProviderMetadata))
|
||||
require.NotNil(t, metadata)
|
||||
require.Equal(t, "sig", metadata.Signature)
|
||||
}
|
||||
|
||||
func TestProcessStepStreamPersistsRedactedThinkingOnEnd(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
stream := iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) {
|
||||
reasoningMetadata := fantasy.ProviderMetadata{
|
||||
fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{
|
||||
RedactedData: "redacted-payload",
|
||||
},
|
||||
}
|
||||
yield(fantasy.StreamPart{
|
||||
Type: fantasy.StreamPartTypeReasoningStart,
|
||||
ID: "0",
|
||||
ProviderMetadata: reasoningMetadata,
|
||||
})
|
||||
yield(fantasy.StreamPart{
|
||||
Type: fantasy.StreamPartTypeReasoningEnd,
|
||||
ID: "0",
|
||||
ProviderMetadata: reasoningMetadata,
|
||||
})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "1"})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextDelta, ID: "1", Delta: "done"})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextEnd, ID: "1"})
|
||||
yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop})
|
||||
})
|
||||
|
||||
result, err := processStepStream(context.Background(), stream, func(codersdk.ChatMessageRole, codersdk.ChatMessagePart) {})
|
||||
require.NoError(t, err)
|
||||
require.Len(t, result.content, 2)
|
||||
reasoning, ok := fantasy.AsContentType[fantasy.ReasoningContent](result.content[0])
|
||||
require.True(t, ok)
|
||||
require.Empty(t, reasoning.Text)
|
||||
metadata := fantasyanthropic.GetReasoningMetadata(fantasy.ProviderOptions(reasoning.ProviderMetadata))
|
||||
require.NotNil(t, metadata)
|
||||
require.Equal(t, "redacted-payload", metadata.RedactedData)
|
||||
}
|
||||
|
||||
func TestStepResultToResponseMessagesPreservesEmptySignedReasoning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result := stepResult{
|
||||
content: []fantasy.Content{
|
||||
fantasy.ReasoningContent{
|
||||
ProviderMetadata: fantasy.ProviderMetadata{
|
||||
fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{
|
||||
RedactedData: "redacted-payload",
|
||||
},
|
||||
},
|
||||
},
|
||||
fantasy.TextContent{Text: "done"},
|
||||
},
|
||||
}
|
||||
|
||||
messages := result.toResponseMessages()
|
||||
|
||||
require.Len(t, messages, 1)
|
||||
require.Len(t, messages[0].Content, 2)
|
||||
reasoning, ok := fantasy.AsMessagePart[fantasy.ReasoningPart](messages[0].Content[0])
|
||||
require.True(t, ok)
|
||||
require.Empty(t, reasoning.Text)
|
||||
metadata := fantasyanthropic.GetReasoningMetadata(reasoning.ProviderOptions)
|
||||
require.NotNil(t, metadata)
|
||||
require.Equal(t, "redacted-payload", metadata.RedactedData)
|
||||
}
|
||||
|
||||
func TestFlushActiveStatePreservesEmptySignedReasoning(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
result := &stepResult{}
|
||||
flushActiveState(
|
||||
result,
|
||||
map[string]string{},
|
||||
map[string]reasoningState{
|
||||
"signed": {
|
||||
options: fantasy.ProviderMetadata{
|
||||
fantasyanthropic.Name: &fantasyanthropic.ReasoningOptionMetadata{
|
||||
RedactedData: "redacted-payload",
|
||||
},
|
||||
},
|
||||
},
|
||||
"empty": {},
|
||||
},
|
||||
map[string]*fantasy.ToolCallContent{},
|
||||
map[string]string{},
|
||||
)
|
||||
|
||||
require.Len(t, result.content, 1)
|
||||
reasoning, ok := fantasy.AsContentType[fantasy.ReasoningContent](result.content[0])
|
||||
require.True(t, ok)
|
||||
require.Empty(t, reasoning.Text)
|
||||
metadata := fantasyanthropic.GetReasoningMetadata(fantasy.ProviderOptions(reasoning.ProviderMetadata))
|
||||
require.NotNil(t, metadata)
|
||||
require.Equal(t, "redacted-payload", metadata.RedactedData)
|
||||
}
|
||||
|
||||
// chainBrokenError is what OpenAI returns when previous_response_id
|
||||
// points at a response it does not have stored.
|
||||
const chainBrokenErrorMessage = "Previous response with id 'resp_abc' not found."
|
||||
|
||||
Reference in New Issue
Block a user