feat: surface model content-filter refusals as a blocked chat error (#27118)

Blocked turns from a provider's content filter (Anthropic's `refusal`
stop reason with empty content) previously ended silently on the
"Thinking" spinner. They now end as a terminal `content_filter` error
that renders as a "Response blocked" message with the provider's
category and explanation.

<img width="888" height="335" alt="image"
src="https://github.com/user-attachments/assets/cef85a59-4091-4e62-9d45-1eb06748db48"
/>


Closes CODAGT-611

Follow-ups will involve implementing fallbacks, but this alone is pretty
important

coderd/apidoc/docs.go

@@ -17122,7 +17122,8 @@ const docTemplate = `{
                 "config",
                 "usage_limit",
                 "missing_key",
-                "provider_disabled"
+                "provider_disabled",
+                "content_filter"
             ],
             "x-enum-varnames": [
                 "ChatErrorKindGeneric",
@@ -17134,7 +17135,8 @@ const docTemplate = `{
                 "ChatErrorKindConfig",
                 "ChatErrorKindUsageLimit",
                 "ChatErrorKindMissingKey",
-                "ChatErrorKindProviderDisabled"
+                "ChatErrorKindProviderDisabled",
+                "ChatErrorKindContentFilter"
             ]
         },
         "codersdk.ChatFileMetadata": {

coderd/apidoc/swagger.json

@@ -15396,7 +15396,8 @@
 				"config",
 				"usage_limit",
 				"missing_key",
-				"provider_disabled"
+				"provider_disabled",
+				"content_filter"
 			],
 			"x-enum-varnames": [
 				"ChatErrorKindGeneric",
@@ -15408,7 +15409,8 @@
 				"ChatErrorKindConfig",
 				"ChatErrorKindUsageLimit",
 				"ChatErrorKindMissingKey",
-				"ChatErrorKindProviderDisabled"
+				"ChatErrorKindProviderDisabled",
+				"ChatErrorKindContentFilter"
 			]
 		},
 		"codersdk.ChatFileMetadata": {

coderd/x/chatd/chaterror/message.go

@@ -64,6 +64,8 @@ func terminalMessage(classified ClassifiedError) string {
 				" Contact your Coder administrator.",
 			displayName,
 		)
+	case codersdk.ChatErrorKindContentFilter:
+		return ContentFilterMessage(classified.Provider, "")
 	default:
 		if !classified.Retryable && classified.StatusCode == 0 {
 			return "The chat request failed unexpectedly."
@@ -116,6 +118,20 @@ func retryMessage(classified ClassifiedError) string {
 	}
 }
 
+// ContentFilterMessage is the user-facing message for a response blocked by
+// the provider's content filter.
+func ContentFilterMessage(provider, category string) string {
+	subject := providerSubject(provider)
+	if category = strings.TrimSpace(category); category != "" {
+		return stringutil.Capitalize(fmt.Sprintf(
+			"%s blocked this response under its content policy (%s).", subject, category,
+		))
+	}
+	return stringutil.Capitalize(fmt.Sprintf(
+		"%s blocked this response under its content policy.", subject,
+	))
+}
+
 func providerSubject(provider string) string {
 	if displayName := providerDisplayName(provider); displayName != "AI" && displayName != "" {
 		return displayName

coderd/x/chatd/chatloop/chatloop.go

@@ -47,6 +47,10 @@ var (
 	// StopAfterTools produces a successful result, indicating
 	// the run should terminate cleanly after persistence.
 	ErrStopAfterTool = xerrors.New("stop after tool")
+	// ErrContentFiltered is returned when the provider's safety
+	// classifiers blocked the response and the model produced no
+	// content, e.g. Anthropic's stop_reason "refusal".
+	ErrContentFiltered = xerrors.New("response blocked by provider content filter")
 
 	errStreamSilenceTimeout = xerrors.New(
 		"chat stream was silent for longer than the configured timeout",
@@ -427,6 +431,12 @@ func GenerateAssistant(ctx context.Context, opts GenerateAssistantOptions) (Assi
 		ctx, opts.Logger, provider, modelName,
 		"assistant_helper", 0, result.finishReason, result.content,
 	)
+	// A content-filter finish with no content means the provider's
+	// safety classifiers blocked the whole response (e.g. Anthropic
+	// stop_reason "refusal").
+	if len(result.content) == 0 && result.finishReason == fantasy.FinishReasonContentFilter {
+		return AssistantOutcome{}, contentFilterError(errorProvider, result.providerMetadata)
+	}
 	step := PersistedStep{
 		Content:              result.content,
 		Usage:                result.usage,
@@ -461,6 +471,19 @@ func wrapProviderStreamError(provider string, err error) error {
 	return xerrors.Errorf("stream response: %w", chaterror.WithClassification(err, classified))
 }
 
+func contentFilterError(provider string, metadata fantasy.ProviderMetadata) error {
+	classified := chaterror.ClassifiedError{
+		Kind:      codersdk.ChatErrorKindContentFilter,
+		Provider:  provider,
+		Retryable: false,
+	}
+	if refusal := fantasyanthropic.GetRefusalMetadata(metadata); refusal != nil {
+		classified.Message = chaterror.ContentFilterMessage(provider, refusal.Category)
+		classified.Detail = strings.TrimSpace(refusal.Explanation)
+	}
+	return chaterror.WithClassification(ErrContentFiltered, classified)
+}
+
 // ExecuteLocalTools runs local tool calls and returns durable tool results. It
 // does not retry or persist.
 func ExecuteLocalTools(ctx context.Context, opts ExecuteLocalToolsOptions) (ToolExecutionOutcome, error) {

coderd/x/chatd/chatloop/contentfilter_internal_test.go

@@ -0,0 +1,152 @@
+package chatloop
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	"charm.land/fantasy"
+	fantasyanthropic "charm.land/fantasy/providers/anthropic"
+	"github.com/stretchr/testify/require"
+
+	"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
+	"github.com/coder/coder/v2/coderd/x/chatd/chattest"
+	"github.com/coder/coder/v2/codersdk"
+)
+
+func refusalProviderMetadataForTest(category, explanation string) fantasy.ProviderMetadata {
+	return fantasy.ProviderMetadata{
+		fantasyanthropic.Name: &fantasyanthropic.RefusalMetadata{
+			Category:    category,
+			Explanation: explanation,
+		},
+	}
+}
+
+func TestContentFilterError(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		provider    string
+		metadata    fantasy.ProviderMetadata
+		wantMessage string
+		wantDetail  string
+	}{
+		{
+			name:     "CategoryVerbatim",
+			provider: "anthropic",
+			metadata: refusalProviderMetadataForTest(
+				"harmful_content", "The response was blocked. See https://example.com for help.",
+			),
+			wantMessage: "Anthropic blocked this response under its content policy (harmful_content).",
+			wantDetail:  "The response was blocked. See https://example.com for help.",
+		},
+		{
+			name:        "NoMetadataFallsBackToDefault",
+			provider:    "anthropic",
+			metadata:    nil,
+			wantMessage: "Anthropic blocked this response under its content policy.",
+		},
+		{
+			name:        "WhitespaceCategory",
+			provider:    "anthropic",
+			metadata:    refusalProviderMetadataForTest("   ", ""),
+			wantMessage: "Anthropic blocked this response under its content policy.",
+		},
+		{
+			name:        "UnknownProvider",
+			provider:    "",
+			metadata:    refusalProviderMetadataForTest("cyber", ""),
+			wantMessage: "The AI provider blocked this response under its content policy (cyber).",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			err := contentFilterError(tt.provider, tt.metadata)
+			require.ErrorIs(t, err, ErrContentFiltered)
+
+			classified := chaterror.Classify(err)
+			require.Equal(t, codersdk.ChatErrorKindContentFilter, classified.Kind)
+			require.Equal(t, tt.provider, classified.Provider)
+			require.False(t, classified.Retryable)
+			require.Equal(t, tt.wantMessage, classified.Message)
+			require.Equal(t, tt.wantDetail, classified.Detail)
+
+			payload := chaterror.TerminalErrorPayload(classified)
+			require.NotNil(t, payload)
+			require.Equal(t, codersdk.ChatErrorKindContentFilter, payload.Kind)
+		})
+	}
+}
+
+func TestGenerateAssistant_ContentFilterRefusal(t *testing.T) {
+	t.Parallel()
+
+	t.Run("EmptyContentSurfacesTerminalError", func(t *testing.T) {
+		t.Parallel()
+
+		model := &chattest.FakeModel{
+			ProviderName: "anthropic",
+			ModelName:    "test-model",
+			StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
+				return streamFromParts([]fantasy.StreamPart{{
+					Type:         fantasy.StreamPartTypeFinish,
+					FinishReason: fantasy.FinishReasonContentFilter,
+					ProviderMetadata: refusalProviderMetadataForTest(
+						"harmful_content", "The response was blocked.",
+					),
+				}}), nil
+			},
+		}
+
+		outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
+			Model: model,
+			Messages: []fantasy.Message{
+				textMessage(fantasy.MessageRoleUser, "hello"),
+			},
+		})
+		require.ErrorIs(t, err, ErrContentFiltered)
+		require.Empty(t, outcome.Step.Content)
+
+		classified := chaterror.Classify(err)
+		require.Equal(t, codersdk.ChatErrorKindContentFilter, classified.Kind)
+		require.Equal(t, "anthropic", classified.Provider)
+		require.False(t, classified.Retryable)
+		require.Equal(t, "Anthropic blocked this response under its content policy (harmful_content).", classified.Message)
+		require.Equal(t, "The response was blocked.", classified.Detail)
+	})
+
+	t.Run("PartialContentIsPersistedNotErrored", func(t *testing.T) {
+		t.Parallel()
+
+		model := &chattest.FakeModel{
+			ProviderName: "anthropic",
+			ModelName:    "test-model",
+			StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
+				return streamFromParts([]fantasy.StreamPart{
+					{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
+					{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "partial"},
+					{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
+					{
+						Type:         fantasy.StreamPartTypeFinish,
+						FinishReason: fantasy.FinishReasonContentFilter,
+					},
+				}), nil
+			},
+		}
+
+		outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
+			Model: model,
+			Messages: []fantasy.Message{
+				textMessage(fantasy.MessageRoleUser, "hello"),
+			},
+		})
+		require.NoError(t, err)
+		require.False(t, errors.Is(err, ErrContentFiltered))
+		require.NotEmpty(t, outcome.Step.Content)
+	})
+}