coder/coderd/x/chatd/chatloop/nooutput_internal_test.go

package chatloop

import (
	"context"
	"testing"

	"charm.land/fantasy"
	"github.com/prometheus/client_golang/prometheus"
	promtestutil "github.com/prometheus/client_golang/prometheus/testutil"
	"github.com/stretchr/testify/require"

	"github.com/coder/coder/v2/coderd/x/chatd/chaterror"
	"github.com/coder/coder/v2/coderd/x/chatd/chattest"
	"github.com/coder/coder/v2/codersdk"
)

func TestGenerateAssistant_SilentNoOutputFinish(t *testing.T) {
	t.Parallel()

	generate := func(t *testing.T, parts []fantasy.StreamPart) (AssistantOutcome, *Metrics, error) {
		t.Helper()
		model := &chattest.FakeModel{
			ProviderName: "google",
			ModelName:    "test-model",
			StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) {
				return streamFromParts(parts), nil
			},
		}
		metrics := NewMetrics(prometheus.NewRegistry())
		outcome, err := GenerateAssistant(context.Background(), GenerateAssistantOptions{
			Model:   model,
			Metrics: metrics,
			Messages: []fantasy.Message{
				textMessage(fantasy.MessageRoleUser, "hello"),
			},
		})
		return outcome, metrics, err
	}

	unterminatedReasoning := func(finish fantasy.FinishReason) []fantasy.StreamPart {
		return []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeReasoningStart, ID: "reasoning-1"},
			{Type: fantasy.StreamPartTypeReasoningDelta, ID: "reasoning-1", Delta: "planning"},
			{Type: fantasy.StreamPartTypeFinish, FinishReason: finish},
		}
	}

	t.Run("UnknownFinishWithoutOutputErrors", func(t *testing.T) {
		t.Parallel()

		outcome, metrics, err := generate(t, unterminatedReasoning(fantasy.FinishReasonUnknown))
		require.ErrorIs(t, err, ErrNoModelOutput)
		require.Empty(t, outcome.Step.Content)

		classified := chaterror.Classify(err)
		require.True(t, classified.Retryable)
		require.Equal(t, codersdk.ChatErrorKindGeneric, classified.Kind)
		require.Equal(t, "google", classified.Provider)
		require.Equal(t, "The model ended its response without producing any output.", classified.Message)
		require.Equal(t, "finish reason: unknown", classified.Detail)

		retries := promtestutil.ToFloat64(metrics.StreamRetriesTotal.WithLabelValues(
			"google", "test-model", string(codersdk.ChatErrorKindGeneric),
		))
		require.Equal(t, float64(1), retries)
	})

	t.Run("ReasoningOnlyContentWithUnknownFinishErrors", func(t *testing.T) {
		t.Parallel()

		_, _, err := generate(t, []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeReasoningStart, ID: "reasoning-1"},
			{Type: fantasy.StreamPartTypeReasoningDelta, ID: "reasoning-1", Delta: "planning"},
			{Type: fantasy.StreamPartTypeReasoningEnd, ID: "reasoning-1"},
			{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonUnknown},
		})
		require.ErrorIs(t, err, ErrNoModelOutput)
	})

	t.Run("ToolCallsFinishWithoutToolCallsErrors", func(t *testing.T) {
		t.Parallel()

		_, _, err := generate(t, []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonToolCalls},
		})
		require.ErrorIs(t, err, ErrNoModelOutput)
		require.Equal(t, "finish reason: tool-calls", chaterror.Classify(err).Detail)
	})

	t.Run("StopFinishWithoutOutputCompletes", func(t *testing.T) {
		t.Parallel()

		outcome, metrics, err := generate(t, unterminatedReasoning(fantasy.FinishReasonStop))
		require.NoError(t, err)
		require.Empty(t, outcome.Step.Content)
		require.True(t, outcome.ModelStopped)

		retries := promtestutil.ToFloat64(metrics.StreamRetriesTotal.WithLabelValues(
			"google", "test-model", string(codersdk.ChatErrorKindGeneric),
		))
		require.Equal(t, float64(0), retries)
	})

	t.Run("LengthFinishWithoutOutputCompletes", func(t *testing.T) {
		t.Parallel()

		_, _, err := generate(t, unterminatedReasoning(fantasy.FinishReasonLength))
		require.NoError(t, err)
	})

	t.Run("EmptyTextWithUnknownFinishErrors", func(t *testing.T) {
		t.Parallel()

		_, _, err := generate(t, []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
			{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
			{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonUnknown},
		})
		require.ErrorIs(t, err, ErrNoModelOutput)
	})

	t.Run("WhitespaceTextWithUnknownFinishErrors", func(t *testing.T) {
		t.Parallel()

		_, _, err := generate(t, []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
			{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "  \n"},
			{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
			{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonUnknown},
		})
		require.ErrorIs(t, err, ErrNoModelOutput)
	})

	t.Run("UnknownFinishWithTextCompletes", func(t *testing.T) {
		t.Parallel()

		outcome, _, err := generate(t, []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"},
			{Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "answer"},
			{Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"},
			{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonUnknown},
		})
		require.NoError(t, err)
		require.NotEmpty(t, outcome.Step.Content)
	})

	t.Run("ToolCallsFinishWithToolCallCompletes", func(t *testing.T) {
		t.Parallel()

		outcome, _, err := generate(t, []fantasy.StreamPart{
			{Type: fantasy.StreamPartTypeToolCall, ID: "call-1", ToolCallName: "do_thing", ToolCallInput: "{}"},
			{Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonToolCalls},
		})
		require.NoError(t, err)
		require.Len(t, outcome.ToolCalls, 1)
	})
}