mirror of
https://github.com/coder/coder.git
synced 2026-09-21 20:51:01 +08:00
fix(aibridge): handle sonnet 5 adaptive thinking in bedrock (#27339)
Adds sonnet 5 to the list of models that require adaptive thinking for Bedrock InvokeModel. Smoke-tested locally. > Obligatory disclosure: a Coder agent helped with this.
This commit is contained in:
@@ -424,7 +424,7 @@ func (i *interceptionBase) withBedrockMantleOptions(ctx context.Context) ([]opti
|
||||
// augmentRequestForBedrockInvokeModel changes the model used for the request since AWS Bedrock doesn't support
|
||||
// Anthropics' model names. It also converts adaptive thinking to enabled with a budget for models that
|
||||
// don't support adaptive thinking natively, or enabled thinking to adaptive for models that only support
|
||||
// adaptive (Opus 4.7+).
|
||||
// adaptive.
|
||||
func (i *interceptionBase) augmentRequestForBedrockInvokeModel() {
|
||||
if i.bedrock == nil {
|
||||
return
|
||||
@@ -440,7 +440,7 @@ func (i *interceptionBase) augmentRequestForBedrockInvokeModel() {
|
||||
|
||||
switch {
|
||||
case bedrockModelRequiresAdaptiveThinking(model):
|
||||
// Symmetric conversion for adaptive-only models (Opus 4.7+): rewrite
|
||||
// Symmetric conversion for adaptive-only models: rewrite
|
||||
// thinking.type "enabled" with budget_tokens to the "adaptive" shape,
|
||||
// since Bedrock returns 400 for these models when the legacy shape is
|
||||
// used. Claude Code falls back to the legacy shape when it cannot
|
||||
@@ -468,7 +468,7 @@ func (i *interceptionBase) augmentRequestForBedrockInvokeModel() {
|
||||
}
|
||||
|
||||
// Strip body fields that Bedrock does not accept. Adaptive-only models
|
||||
// (Opus 4.7+) support output_config natively without a beta flag, so
|
||||
// support output_config natively without a beta flag, so
|
||||
// keep it for those models even when the effort-2025-11-24 flag is
|
||||
// absent from the request.
|
||||
var exemptFields []string
|
||||
@@ -496,8 +496,8 @@ func (i *interceptionBase) augmentRequestForBedrockInvokeModel() {
|
||||
}
|
||||
|
||||
// bedrockModelSupportsAdaptiveThinking returns true if the given Bedrock model ID
|
||||
// supports the "adaptive" thinking type natively (i.e. Claude 4.6 models, and
|
||||
// adaptive-only models such as Opus 4.7+).
|
||||
// supports the "adaptive" thinking type natively (i.e. Claude 4.6 models and
|
||||
// adaptive-only models).
|
||||
// See https://docs.aws.amazon.com/bedrock/latest/userguide/claude-messages-adaptive-thinking.html
|
||||
func bedrockModelSupportsAdaptiveThinking(model string) bool {
|
||||
return strings.Contains(model, "anthropic.claude-opus-4-6") ||
|
||||
@@ -507,13 +507,13 @@ func bedrockModelSupportsAdaptiveThinking(model string) bool {
|
||||
|
||||
// bedrockModelRequiresAdaptiveThinking returns true if the given Bedrock model
|
||||
// ID only supports the "adaptive" thinking type and rejects the legacy
|
||||
// "enabled" + budget_tokens shape with a 400. Claude Opus 4.7 was the first
|
||||
// model in this category.
|
||||
// "enabled" + budget_tokens shape with a 400.
|
||||
//
|
||||
// See https://docs.aws.amazon.com/bedrock/latest/userguide/model-card-anthropic-claude-opus-4-7.html
|
||||
func bedrockModelRequiresAdaptiveThinking(model string) bool {
|
||||
return strings.Contains(model, "anthropic.claude-opus-4-7") ||
|
||||
strings.Contains(model, "anthropic.claude-opus-4-8")
|
||||
strings.Contains(model, "anthropic.claude-opus-4-8") ||
|
||||
strings.Contains(model, "anthropic.claude-sonnet-5")
|
||||
}
|
||||
|
||||
// filterBedrockBetaFlags removes unsupported beta flags from the Anthropic-Beta
|
||||
|
||||
@@ -716,7 +716,7 @@ func TestAugmentRequestForBedrock_AdaptiveThinking(t *testing.T) {
|
||||
expectRemovedFields: []string{"output_config", "metadata", "service_tier", "container", "inference_geo", "context_management"},
|
||||
},
|
||||
|
||||
// Adaptive-only models (Opus 4.7+), see coder/aibridge#280. The
|
||||
// Adaptive-only models, see coder/aibridge#280. The
|
||||
// conversion drops budget_tokens and flips the type; an explicit
|
||||
// output_config.effort from the caller is preserved, but none is
|
||||
// fabricated when absent.
|
||||
@@ -765,6 +765,21 @@ func TestAugmentRequestForBedrock_AdaptiveThinking(t *testing.T) {
|
||||
requestBody: `{"max_tokens":10000,"thinking":{"type":"enabled","budget_tokens":5000}}`,
|
||||
expectThinkingType: "adaptive",
|
||||
},
|
||||
{
|
||||
name: "sonnet_5_model_with_enabled_thinking_is_converted_to_adaptive_and_drops_budget",
|
||||
bedrockModel: "anthropic.claude-sonnet-5",
|
||||
requestBody: `{"max_tokens":10000,"thinking":{"type":"enabled","budget_tokens":5000}}`,
|
||||
expectThinkingType: "adaptive",
|
||||
},
|
||||
{
|
||||
name: "regional_sonnet_5_model_keeps_adaptive_thinking_and_effort_and_strips_output_config_format",
|
||||
bedrockModel: "us.anthropic.claude-sonnet-5",
|
||||
requestBody: `{"max_tokens":10000,"thinking":{"type":"adaptive"},"output_config":{"effort":"medium","format":{"type":"json_schema","schema":{"type":"object"}}}}`,
|
||||
expectThinkingType: "adaptive",
|
||||
expectEffort: "medium",
|
||||
expectKeptFields: []string{"output_config", "output_config.effort"},
|
||||
expectRemovedFields: []string{"output_config.format"},
|
||||
},
|
||||
{
|
||||
// Opus 4.7 on Bedrock rejects output_config.format (structured
|
||||
// outputs) with a 400 even though it accepts output_config.effort.
|
||||
|
||||
@@ -76,7 +76,7 @@ var (
|
||||
// If the beta flag is present in the (already-filtered) Anthropic-Beta header,
|
||||
// the field is kept; otherwise it is stripped. Model-specific beta flags must
|
||||
// be removed from the header before this check (see filterBedrockBetaFlags).
|
||||
// Adaptive-only models (Opus 4.7+) are exempt for output_config since they
|
||||
// Adaptive-only models are exempt for output_config since they
|
||||
// support it natively without a beta flag, see
|
||||
// bedrockModelRequiresAdaptiveThinking.
|
||||
bedrockBetaGatedFields = map[string]string{
|
||||
@@ -342,8 +342,9 @@ func (RequestPayload) resultToRawMessage(items []gjson.Result) []json.RawMessage
|
||||
// The two Bedrock thinking-type conversions below are a temporary shim.
|
||||
// AI Gateway relays the Anthropic Messages API shape to Bedrock, whose Claude
|
||||
// models accept a disjoint subset on each generation (older models reject
|
||||
// "adaptive"; Opus 4.7+ rejects "enabled"). A planned native Bedrock provider
|
||||
// removes the impedance mismatch and lets us delete this whole block. Hopefully.
|
||||
// "adaptive"; adaptive-only models reject "enabled"). A planned native
|
||||
// Bedrock provider removes the impedance mismatch and lets us delete this
|
||||
// whole block. Hopefully.
|
||||
|
||||
// bedrockThinkingEffortRatios maps an output_config.effort hint to the fraction
|
||||
// of max_tokens to allocate as thinking budget. The mapping is a heuristic
|
||||
@@ -402,7 +403,7 @@ func (p RequestPayload) convertAdaptiveThinkingForBedrock() (RequestPayload, err
|
||||
|
||||
// convertEnabledThinkingForBedrock rewrites thinking.type "enabled" to plain
|
||||
// "adaptive", dropping budget_tokens. Needed for Bedrock models that only
|
||||
// support adaptive thinking (Opus 4.7+).
|
||||
// support adaptive thinking.
|
||||
//
|
||||
// We deliberately do not derive output_config.effort from the budget. Any
|
||||
// such mapping would be invented (no canonical budget-to-effort relationship
|
||||
@@ -421,7 +422,7 @@ func (p RequestPayload) convertEnabledThinkingForBedrock() (RequestPayload, erro
|
||||
|
||||
// removeBedrockUnsupportedOutputConfigSubFields drops sub-fields of
|
||||
// output_config that Bedrock rejects even on models where the parent
|
||||
// output_config object is accepted. Adaptive-only models (Opus 4.7+) accept
|
||||
// output_config object is accepted. Adaptive-only models accept
|
||||
// output_config.effort but reject output_config.format (structured outputs)
|
||||
// with a 400 "Extra inputs are not permitted." The generic field-strip pass
|
||||
// (removeUnsupportedBedrockFields) operates at top-level granularity only, so
|
||||
@@ -444,7 +445,7 @@ func (p RequestPayload) removeBedrockUnsupportedOutputConfigSubFields() (Request
|
||||
// calling this method (see filterBedrockBetaFlags).
|
||||
//
|
||||
// Fields exempted by exemptFields are always kept regardless of beta flag
|
||||
// state. Adaptive-only Bedrock models (Opus 4.7+) require output_config
|
||||
// state. Adaptive-only Bedrock models require output_config
|
||||
// without a beta flag, so callers pass the field through this set to bypass
|
||||
// the effort-2025-11-24 gate.
|
||||
func (p RequestPayload) removeUnsupportedBedrockFields(headers http.Header, exemptFields ...string) (RequestPayload, error) {
|
||||
|
||||
@@ -460,13 +460,15 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
}
|
||||
|
||||
cases := []struct {
|
||||
name string
|
||||
model string
|
||||
smallFastModel string
|
||||
expectThinkingType string
|
||||
expectBudgetTokens int64 // 0 means budget_tokens should not be present
|
||||
expectKeptFields []string // fields from strippableFields expected to survive
|
||||
expectedBetaFlags []string // values expected in the anthropic_beta array in the forwarded body
|
||||
name string
|
||||
model string
|
||||
smallFastModel string
|
||||
expectThinkingType string
|
||||
expectEffort string
|
||||
expectBudgetTokens int64 // 0 means budget_tokens should not be present
|
||||
sendThinkingEnabled bool // send enabled thinking with budget_tokens instead of the fixture's adaptive thinking
|
||||
expectKeptFields []string // fields from strippableFields expected to survive
|
||||
expectedBetaFlags []string // values expected in the anthropic_beta array in the forwarded body
|
||||
}{
|
||||
// "beddel" matches no model prefix, so adaptive thinking is converted
|
||||
// to enabled with budget, and all model-gated beta flags are stripped.
|
||||
@@ -474,6 +476,7 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
name: "beddel",
|
||||
model: "beddel",
|
||||
smallFastModel: "modrock",
|
||||
expectEffort: "",
|
||||
expectThinkingType: "enabled",
|
||||
expectBudgetTokens: 16000, // 32000 * 0.5 (medium effort)
|
||||
expectedBetaFlags: []string{"interleaved-thinking-2025-05-14"},
|
||||
@@ -483,6 +486,7 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
name: "opus-4.5",
|
||||
model: "anthropic.claude-opus-4-5-20250514-v1:0",
|
||||
smallFastModel: "anthropic.claude-haiku-4-5-20241022-v1:0",
|
||||
expectEffort: "medium",
|
||||
expectThinkingType: "enabled",
|
||||
expectBudgetTokens: 16000,
|
||||
expectKeptFields: []string{"output_config"},
|
||||
@@ -493,6 +497,7 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
name: "sonnet-4.5",
|
||||
model: "anthropic.claude-sonnet-4-5-20241022-v2:0",
|
||||
smallFastModel: "anthropic.claude-haiku-4-5-20241022-v1:0",
|
||||
expectEffort: "",
|
||||
expectThinkingType: "enabled",
|
||||
expectBudgetTokens: 16000,
|
||||
expectKeptFields: []string{"context_management"},
|
||||
@@ -503,10 +508,23 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
{
|
||||
name: "opus-4.6",
|
||||
model: "anthropic.claude-opus-4-6-20260619-v1:0",
|
||||
expectEffort: "",
|
||||
smallFastModel: "anthropic.claude-haiku-4-5-20241022-v1:0",
|
||||
expectThinkingType: "adaptive",
|
||||
expectedBetaFlags: []string{"interleaved-thinking-2025-05-14"},
|
||||
},
|
||||
// Sonnet 5 requires adaptive thinking, so legacy enabled thinking is
|
||||
// converted and output_config.effort is preserved.
|
||||
{
|
||||
name: "sonnet-5",
|
||||
model: "us.anthropic.claude-sonnet-5",
|
||||
smallFastModel: "anthropic.claude-haiku-4-5-20241022-v1:0",
|
||||
expectEffort: "medium",
|
||||
expectThinkingType: "adaptive",
|
||||
sendThinkingEnabled: true,
|
||||
expectKeptFields: []string{"output_config"},
|
||||
expectedBetaFlags: []string{"interleaved-thinking-2025-05-14"},
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range cases {
|
||||
@@ -535,6 +553,13 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
|
||||
reqBody, err := sjson.SetBytes(fix.Request(), "stream", streaming)
|
||||
require.NoError(t, err)
|
||||
if tc.sendThinkingEnabled {
|
||||
reqBody, err = sjson.SetBytes(reqBody, "thinking", map[string]any{
|
||||
"type": "enabled",
|
||||
"budget_tokens": 16000,
|
||||
})
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
// Send with Anthropic-Beta header containing flags that should be filtered.
|
||||
resp, err := bridgeServer.makeRequest(t, http.MethodPost, pathAnthropicMessages, reqBody, http.Header{
|
||||
@@ -562,6 +587,7 @@ func TestAWSBedrockIntegration(t *testing.T) {
|
||||
} else {
|
||||
assert.False(t, gjson.GetBytes(body, "thinking.budget_tokens").Exists(), "budget_tokens should not be present")
|
||||
}
|
||||
assert.Equal(t, tc.expectEffort, gjson.GetBytes(body, "output_config.effort").String(), "effort mismatch")
|
||||
|
||||
// The Bedrock SDK middleware moves Anthropic-Beta from the header
|
||||
// into the body as "anthropic_beta".
|
||||
|
||||
Reference in New Issue
Block a user