diff --git a/coderd/x/chatd/chatdebug/context.go b/coderd/x/chatd/chatdebug/context.go new file mode 100644 index 0000000000..f67ddb6456 --- /dev/null +++ b/coderd/x/chatd/chatdebug/context.go @@ -0,0 +1,84 @@ +package chatdebug + +import ( + "context" + "runtime" + "sync" + + "github.com/google/uuid" +) + +type ( + runContextKey struct{} + stepContextKey struct{} + reuseStepKey struct{} + reuseHolder struct { + mu sync.Mutex + handle *stepHandle + } +) + +// ContextWithRun stores rc in ctx. +// +// Step counter cleanup is reference-counted per RunID: each live +// RunContext increments a counter and runtime.AddCleanup decrements +// it when the struct is garbage collected. Shared state (step +// counters) is only deleted when the last RunContext for a given +// RunID becomes unreachable, preventing premature cleanup when +// multiple RunContext instances share the same RunID. +func ContextWithRun(ctx context.Context, rc *RunContext) context.Context { + if rc == nil { + panic("chatdebug: nil RunContext") + } + + enriched := context.WithValue(ctx, runContextKey{}, rc) + if rc.RunID != uuid.Nil { + trackRunRef(rc.RunID) + runtime.AddCleanup(rc, func(id uuid.UUID) { + releaseRunRef(id) + }, rc.RunID) + } + return enriched +} + +// RunFromContext returns the debug run context stored in ctx. +func RunFromContext(ctx context.Context) (*RunContext, bool) { + rc, ok := ctx.Value(runContextKey{}).(*RunContext) + if !ok { + return nil, false + } + return rc, true +} + +// ContextWithStep stores sc in ctx. +func ContextWithStep(ctx context.Context, sc *StepContext) context.Context { + if sc == nil { + panic("chatdebug: nil StepContext") + } + return context.WithValue(ctx, stepContextKey{}, sc) +} + +// StepFromContext returns the debug step context stored in ctx. +func StepFromContext(ctx context.Context) (*StepContext, bool) { + sc, ok := ctx.Value(stepContextKey{}).(*StepContext) + if !ok { + return nil, false + } + return sc, true +} + +// ReuseStep marks ctx so wrapped model calls under it share one debug step. +func ReuseStep(ctx context.Context) context.Context { + if holder, ok := reuseHolderFromContext(ctx); ok { + return context.WithValue(ctx, reuseStepKey{}, holder) + } + return context.WithValue(ctx, reuseStepKey{}, &reuseHolder{}) +} + +func reuseHolderFromContext(ctx context.Context) (*reuseHolder, bool) { + holder, ok := ctx.Value(reuseStepKey{}).(*reuseHolder) + if !ok { + return nil, false + } + return holder, true +} diff --git a/coderd/x/chatd/chatdebug/context_internal_test.go b/coderd/x/chatd/chatdebug/context_internal_test.go new file mode 100644 index 0000000000..e4d83ae14c --- /dev/null +++ b/coderd/x/chatd/chatdebug/context_internal_test.go @@ -0,0 +1,124 @@ +package chatdebug + +import ( + "context" + "runtime" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/testutil" +) + +func TestReuseStep_PreservesExistingHolder(t *testing.T) { + t.Parallel() + + ctx := ReuseStep(context.Background()) + first, ok := reuseHolderFromContext(ctx) + require.True(t, ok) + + reused := ReuseStep(ctx) + second, ok := reuseHolderFromContext(reused) + require.True(t, ok) + require.Same(t, first, second) +} + +func TestContextWithRun_CleansUpStepCounterAfterGC(t *testing.T) { + t.Parallel() + + runID := uuid.New() + chatID := uuid.New() + t.Cleanup(func() { CleanupStepCounter(runID) }) + + func() { + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + handle, _ := beginStep(ctx, &Service{}, RecorderOptions{ChatID: chatID}, OperationGenerate, nil) + require.NotNil(t, handle) + _, ok := stepCounters.Load(runID) + require.True(t, ok) + }() + + require.Eventually(t, func() bool { + runtime.GC() //nolint:revive // Intentional GC to test cleanup finalizer. + runtime.Gosched() + _, ok := stepCounters.Load(runID) + return !ok + }, testutil.WaitShort, testutil.IntervalFast) +} + +func TestContextWithRun_MultipleInstancesSameRunID(t *testing.T) { + t.Parallel() + + runID := uuid.New() + chatID := uuid.New() + t.Cleanup(func() { CleanupStepCounter(runID) }) + + // rc2 is the surviving instance that should keep the step counter alive. + rc2 := &RunContext{RunID: runID, ChatID: chatID} + ctx2 := ContextWithRun(context.Background(), rc2) + + // Create a second RunContext with the same RunID and let it become + // unreachable. Its GC cleanup must NOT delete the step counter + // because rc2 is still alive. + func() { + rc1 := &RunContext{RunID: runID, ChatID: chatID} + ctx1 := ContextWithRun(context.Background(), rc1) + h, _ := beginStep(ctx1, &Service{}, RecorderOptions{ChatID: chatID}, OperationGenerate, nil) + require.NotNil(t, h) + require.Equal(t, int32(1), h.stepCtx.StepNumber) + }() + + // Force GC to collect rc1. + for range 5 { + runtime.GC() //nolint:revive // Intentional GC to test cleanup finalizer. + runtime.Gosched() + } + + // The step counter must still be present because rc2 is alive. + _, ok := stepCounters.Load(runID) + require.True(t, ok, "step counter was prematurely cleaned up while another RunContext is still alive") + + // Subsequent steps on the surviving context must continue numbering. + h2, _ := beginStep(ctx2, &Service{}, RecorderOptions{ChatID: chatID}, OperationGenerate, nil) + require.NotNil(t, h2) + require.Equal(t, int32(2), h2.stepCtx.StepNumber) +} + +func TestContextWithRun_CleansUpStepCounterOnGCAfterCancel(t *testing.T) { + t.Parallel() + + runID := uuid.New() + chatID := uuid.New() + t.Cleanup(func() { CleanupStepCounter(runID) }) + + // Run in a closure so the RunContext becomes unreachable after + // context cancellation, allowing GC to trigger the cleanup. + func() { + ctx, cancel := context.WithCancel(context.Background()) + ctx = ContextWithRun(ctx, &RunContext{RunID: runID, ChatID: chatID}) + + handle, _ := beginStep(ctx, &Service{}, RecorderOptions{ChatID: chatID}, OperationGenerate, nil) + require.NotNil(t, handle) + require.Equal(t, int32(1), handle.stepCtx.StepNumber) + + _, ok := stepCounters.Load(runID) + require.True(t, ok) + + cancel() + }() + + // After the closure, the RunContext is unreachable. + // runtime.AddCleanup fires during GC. + require.Eventually(t, func() bool { + runtime.GC() //nolint:revive // Intentional GC to test cleanup finalizer. + runtime.Gosched() + _, ok := stepCounters.Load(runID) + return !ok + }, testutil.WaitShort, testutil.IntervalFast) + + freshCtx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + freshHandle, _ := beginStep(freshCtx, &Service{}, RecorderOptions{ChatID: chatID}, OperationGenerate, nil) + require.NotNil(t, freshHandle) + require.Equal(t, int32(1), freshHandle.stepCtx.StepNumber) +} diff --git a/coderd/x/chatd/chatdebug/context_test.go b/coderd/x/chatd/chatdebug/context_test.go new file mode 100644 index 0000000000..7069059e4a --- /dev/null +++ b/coderd/x/chatd/chatdebug/context_test.go @@ -0,0 +1,105 @@ +package chatdebug_test + +import ( + "context" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" +) + +func TestContextWithRunRoundTrip(t *testing.T) { + t.Parallel() + + rc := &chatdebug.RunContext{ + RunID: uuid.New(), + ChatID: uuid.New(), + RootChatID: uuid.New(), + ParentChatID: uuid.New(), + ModelConfigID: uuid.New(), + TriggerMessageID: 11, + HistoryTipMessageID: 22, + Kind: chatdebug.KindChatTurn, + Provider: "anthropic", + Model: "claude-sonnet", + } + + ctx := chatdebug.ContextWithRun(context.Background(), rc) + got, ok := chatdebug.RunFromContext(ctx) + require.True(t, ok) + require.Same(t, rc, got) + require.Equal(t, *rc, *got) +} + +func TestRunFromContextAbsent(t *testing.T) { + t.Parallel() + + got, ok := chatdebug.RunFromContext(context.Background()) + require.False(t, ok) + require.Nil(t, got) +} + +func TestContextWithStepRoundTrip(t *testing.T) { + t.Parallel() + + sc := &chatdebug.StepContext{ + StepID: uuid.New(), + RunID: uuid.New(), + ChatID: uuid.New(), + StepNumber: 7, + Operation: chatdebug.OperationStream, + HistoryTipMessageID: 33, + } + + ctx := chatdebug.ContextWithStep(context.Background(), sc) + got, ok := chatdebug.StepFromContext(ctx) + require.True(t, ok) + require.Same(t, sc, got) + require.Equal(t, *sc, *got) +} + +func TestStepFromContextAbsent(t *testing.T) { + t.Parallel() + + got, ok := chatdebug.StepFromContext(context.Background()) + require.False(t, ok) + require.Nil(t, got) +} + +func TestContextWithRunAndStep(t *testing.T) { + t.Parallel() + + rc := &chatdebug.RunContext{RunID: uuid.New(), ChatID: uuid.New()} + sc := &chatdebug.StepContext{StepID: uuid.New(), RunID: rc.RunID, ChatID: rc.ChatID} + + ctx := chatdebug.ContextWithStep( + chatdebug.ContextWithRun(context.Background(), rc), + sc, + ) + + gotRun, ok := chatdebug.RunFromContext(ctx) + require.True(t, ok) + require.Same(t, rc, gotRun) + + gotStep, ok := chatdebug.StepFromContext(ctx) + require.True(t, ok) + require.Same(t, sc, gotStep) +} + +func TestContextWithRunPanicsOnNil(t *testing.T) { + t.Parallel() + + require.Panics(t, func() { + _ = chatdebug.ContextWithRun(context.Background(), nil) + }) +} + +func TestContextWithStepPanicsOnNil(t *testing.T) { + t.Parallel() + + require.Panics(t, func() { + _ = chatdebug.ContextWithStep(context.Background(), nil) + }) +} diff --git a/coderd/x/chatd/chatdebug/model.go b/coderd/x/chatd/chatdebug/model.go new file mode 100644 index 0000000000..2a33b43d88 --- /dev/null +++ b/coderd/x/chatd/chatdebug/model.go @@ -0,0 +1,1202 @@ +package chatdebug + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "iter" + "reflect" + "sync" + "sync/atomic" + "unicode/utf8" + + "charm.land/fantasy" + "golang.org/x/xerrors" +) + +type debugModel struct { + inner fantasy.LanguageModel + svc *Service + opts RecorderOptions +} + +var _ fantasy.LanguageModel = (*debugModel)(nil) + +// ErrNilModelResult is returned when the underlying language model +// returns a nil response or stream. Callers can match with +// errors.Is to distinguish this from provider-level failures. +var ErrNilModelResult = xerrors.New("language model returned nil result") + +// normalizedCallOptions holds the optional model parameters shared by +// both regular and structured-output calls. +type normalizedCallOptions struct { + MaxOutputTokens *int64 `json:"max_output_tokens,omitempty"` + Temperature *float64 `json:"temperature,omitempty"` + TopP *float64 `json:"top_p,omitempty"` + TopK *int64 `json:"top_k,omitempty"` + PresencePenalty *float64 `json:"presence_penalty,omitempty"` + FrequencyPenalty *float64 `json:"frequency_penalty,omitempty"` +} + +// normalizedCallPayload is the rich envelope persisted for Generate / +// Stream calls. It carries the full message structure and tool +// metadata so the debug panel can render conversation context. +type normalizedCallPayload struct { + Messages []normalizedMessage `json:"messages"` + Tools []normalizedTool `json:"tools,omitempty"` + Options normalizedCallOptions `json:"options"` + ToolChoice string `json:"tool_choice,omitempty"` + ProviderOptionCount int `json:"provider_option_count"` +} + +// normalizedObjectCallPayload is the rich envelope for +// GenerateObject / StreamObject calls, including schema metadata. +type normalizedObjectCallPayload struct { + Messages []normalizedMessage `json:"messages"` + Options normalizedCallOptions `json:"options"` + SchemaName string `json:"schema_name,omitempty"` + SchemaDescription string `json:"schema_description,omitempty"` + StructuredOutput bool `json:"structured_output"` + ProviderOptionCount int `json:"provider_option_count"` +} + +// normalizedResponsePayload is the rich envelope for persisted model +// responses. It includes the full content parts, finish reason, token +// usage breakdown, and any provider warnings. +type normalizedResponsePayload struct { + Content []normalizedContentPart `json:"content"` + FinishReason string `json:"finish_reason"` + Usage normalizedUsage `json:"usage"` + Warnings []normalizedWarning `json:"warnings,omitempty"` +} + +// normalizedObjectResponsePayload is the rich envelope for +// structured-output responses. Raw text is bounded to length only. +type normalizedObjectResponsePayload struct { + RawTextLength int `json:"raw_text_length"` + FinishReason string `json:"finish_reason"` + Usage normalizedUsage `json:"usage"` + Warnings []normalizedWarning `json:"warnings,omitempty"` + StructuredOutput bool `json:"structured_output"` +} + +// --------------- helper types --------------- + +// normalizedMessage represents a single message in the prompt with +// its role and constituent parts. +type normalizedMessage struct { + Role string `json:"role"` + Parts []normalizedMessagePart `json:"parts"` +} + +// MaxMessagePartTextLength is the rune limit for bounded text stored +// in request message parts. Longer text is truncated with an ellipsis. +const MaxMessagePartTextLength = 10_000 + +// maxStreamDebugTextBytes caps accumulated streamed text persisted in +// debug responses. +const maxStreamDebugTextBytes = 50_000 + +// normalizedMessagePart captures the type and bounded metadata for a +// single part within a prompt message. Text-like payloads are truncated +// to MaxMessagePartTextLength runes so request payloads stay bounded +// while still giving the debug panel readable content. +type normalizedMessagePart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + TextLength int `json:"text_length,omitempty"` + Filename string `json:"filename,omitempty"` + MediaType string `json:"media_type,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolName string `json:"tool_name,omitempty"` + Arguments string `json:"arguments,omitempty"` + Result string `json:"result,omitempty"` +} + +// normalizedTool captures tool identity along with any JSON input +// schema needed by the debug panel. +type normalizedTool struct { + Type string `json:"type"` + Name string `json:"name"` + Description string `json:"description,omitempty"` + ID string `json:"id,omitempty"` + HasInputSchema bool `json:"has_input_schema,omitempty"` + InputSchema json.RawMessage `json:"input_schema,omitempty"` +} + +// normalizedContentPart captures one piece of the model response. +// Text payloads are bounded to MaxMessagePartTextLength runes; +// TextLength stores the original rune count for truncation detection. +// Tool-call arguments are similarly bounded, and file data is never +// stored. +type normalizedContentPart struct { + Type string `json:"type"` + Text string `json:"text,omitempty"` + TextLength int `json:"text_length,omitempty"` + ToolCallID string `json:"tool_call_id,omitempty"` + ToolName string `json:"tool_name,omitempty"` + Arguments string `json:"arguments,omitempty"` + Result string `json:"result,omitempty"` + InputLength int `json:"input_length,omitempty"` + MediaType string `json:"media_type,omitempty"` + SourceType string `json:"source_type,omitempty"` + Title string `json:"title,omitempty"` + URL string `json:"url,omitempty"` +} + +// normalizedUsage mirrors fantasy.Usage with the full token +// breakdown so the debug panel can display cost/cache info. +type normalizedUsage struct { + InputTokens int64 `json:"input_tokens"` + OutputTokens int64 `json:"output_tokens"` + TotalTokens int64 `json:"total_tokens"` + ReasoningTokens int64 `json:"reasoning_tokens"` + CacheCreationTokens int64 `json:"cache_creation_tokens"` + CacheReadTokens int64 `json:"cache_read_tokens"` +} + +// normalizedWarning captures a single provider warning. +type normalizedWarning struct { + Type string `json:"type"` + Setting string `json:"setting,omitempty"` + Details string `json:"details,omitempty"` + Message string `json:"message,omitempty"` +} + +type normalizedErrorPayload struct { + Message string `json:"message"` + Type string `json:"type"` + ContextError string `json:"context_error,omitempty"` + ProviderTitle string `json:"provider_title,omitempty"` + ProviderStatus int `json:"provider_status,omitempty"` + IsRetryable bool `json:"is_retryable,omitempty"` +} + +type streamSummary struct { + FinishReason string `json:"finish_reason,omitempty"` + TextDeltaCount int `json:"text_delta_count"` + ToolCallCount int `json:"tool_call_count"` + SourceCount int `json:"source_count"` + WarningCount int `json:"warning_count"` + ErrorCount int `json:"error_count"` + LastError string `json:"last_error,omitempty"` + PartCount int `json:"part_count"` +} + +type objectStreamSummary struct { + FinishReason string `json:"finish_reason,omitempty"` + ObjectPartCount int `json:"object_part_count"` + TextDeltaCount int `json:"text_delta_count"` + ErrorCount int `json:"error_count"` + LastError string `json:"last_error,omitempty"` + WarningCount int `json:"warning_count"` + PartCount int `json:"part_count"` + StructuredOutput bool `json:"structured_output"` +} + +func (d *debugModel) Generate( + ctx context.Context, + call fantasy.Call, +) (*fantasy.Response, error) { + if d.svc == nil { + return d.inner.Generate(ctx, call) + } + if _, ok := RunFromContext(ctx); !ok { + return d.inner.Generate(ctx, call) + } + + handle, enrichedCtx := beginStep(ctx, d.svc, d.opts, OperationGenerate, + normalizeCall(call)) + if handle == nil { + return d.inner.Generate(ctx, call) + } + + resp, err := d.inner.Generate(enrichedCtx, call) + if err != nil { + handle.finish(ctx, stepStatusForError(err), nil, nil, normalizeError(ctx, err), nil) + return nil, err + } + if resp == nil { + err = xerrors.Errorf("Generate: %w", ErrNilModelResult) + handle.finish(ctx, StatusError, nil, nil, normalizeError(ctx, err), nil) + return nil, err + } + + handle.finish(ctx, StatusCompleted, normalizeResponse(resp), &resp.Usage, nil, nil) + return resp, nil +} + +func (d *debugModel) Stream( + ctx context.Context, + call fantasy.Call, +) (fantasy.StreamResponse, error) { + if d.svc == nil { + return d.inner.Stream(ctx, call) + } + if _, ok := RunFromContext(ctx); !ok { + return d.inner.Stream(ctx, call) + } + + handle, enrichedCtx := beginStep(ctx, d.svc, d.opts, OperationStream, + normalizeCall(call)) + if handle == nil { + return d.inner.Stream(ctx, call) + } + + seq, err := d.inner.Stream(enrichedCtx, call) + if err != nil { + handle.finish(ctx, stepStatusForError(err), nil, nil, normalizeError(ctx, err), nil) + return nil, err + } + if seq == nil { + err = xerrors.Errorf("Stream: %w", ErrNilModelResult) + handle.finish(ctx, StatusError, nil, nil, normalizeError(ctx, err), nil) + return nil, err + } + + return wrapStreamSeq(ctx, handle, seq), nil +} + +func (d *debugModel) GenerateObject( + ctx context.Context, + call fantasy.ObjectCall, +) (*fantasy.ObjectResponse, error) { + if d.svc == nil { + return d.inner.GenerateObject(ctx, call) + } + if _, ok := RunFromContext(ctx); !ok { + return d.inner.GenerateObject(ctx, call) + } + + handle, enrichedCtx := beginStep(ctx, d.svc, d.opts, OperationGenerate, + normalizeObjectCall(call)) + if handle == nil { + return d.inner.GenerateObject(ctx, call) + } + + resp, err := d.inner.GenerateObject(enrichedCtx, call) + if err != nil { + handle.finish(ctx, stepStatusForError(err), nil, nil, normalizeError(ctx, err), + map[string]any{"structured_output": true}) + return nil, err + } + if resp == nil { + err = xerrors.Errorf("GenerateObject: %w", ErrNilModelResult) + handle.finish(ctx, StatusError, nil, nil, normalizeError(ctx, err), + map[string]any{"structured_output": true}) + return nil, err + } + + handle.finish(ctx, StatusCompleted, normalizeObjectResponse(resp), &resp.Usage, + nil, map[string]any{"structured_output": true}) + return resp, nil +} + +func (d *debugModel) StreamObject( + ctx context.Context, + call fantasy.ObjectCall, +) (fantasy.ObjectStreamResponse, error) { + if d.svc == nil { + return d.inner.StreamObject(ctx, call) + } + if _, ok := RunFromContext(ctx); !ok { + return d.inner.StreamObject(ctx, call) + } + + handle, enrichedCtx := beginStep(ctx, d.svc, d.opts, OperationStream, + normalizeObjectCall(call)) + if handle == nil { + return d.inner.StreamObject(ctx, call) + } + + seq, err := d.inner.StreamObject(enrichedCtx, call) + if err != nil { + handle.finish(ctx, stepStatusForError(err), nil, nil, normalizeError(ctx, err), + map[string]any{"structured_output": true}) + return nil, err + } + if seq == nil { + err = xerrors.Errorf("StreamObject: %w", ErrNilModelResult) + handle.finish(ctx, StatusError, nil, nil, normalizeError(ctx, err), + map[string]any{"structured_output": true}) + return nil, err + } + + return wrapObjectStreamSeq(ctx, handle, seq), nil +} + +func (d *debugModel) Provider() string { + return d.inner.Provider() +} + +func (d *debugModel) Model() string { + return d.inner.Model() +} + +func wrapStreamSeq( + ctx context.Context, + handle *stepHandle, + seq iter.Seq[fantasy.StreamPart], +) fantasy.StreamResponse { + // mu and finalized guard both the normal finalization path + // inside the iterator and the safety-net AfterFunc below. + // This ensures handle.finish is called exactly once regardless + // of whether the caller iterates, drops the stream, or the + // context is canceled mid-flight. We use a mutex rather than + // sync.Once so the AfterFunc can yield to the normal path + // when the stream already received its terminal chunk + // (streamComplete), preventing the AfterFunc from clobbering + // completed stream data with nil. + var ( + mu sync.Mutex + finalized bool + streamComplete atomic.Bool + ) + + // Safety net: if the caller drops the returned iterator without + // consuming it (or abandons mid-stream and the context is + // canceled), finalize the step so it does not remain permanently + // in_progress once persistence lands in later branches. + stop := context.AfterFunc(ctx, func() { + mu.Lock() + defer mu.Unlock() + // If the stream already received a finish chunk, let + // finalize handle it; it has the real response payload + // and usage data that we would otherwise clobber. + if finalized || streamComplete.Load() { + return + } + finalized = true + handle.finish(ctx, StatusInterrupted, nil, nil, nil, nil) + }) + + return func(yield func(fantasy.StreamPart) bool) { + var ( + summary streamSummary + latestUsage fantasy.Usage + usageSeen bool + finishSeen bool + finishReason fantasy.FinishReason + content []normalizedContentPart + warnings []normalizedWarning + streamDebugBytes int + streamError any + streamStatus = StatusCompleted + ) + + finalize := func(status Status) { + // Cancel the safety net since we're finalizing normally. + if stop != nil { + stop() + } + mu.Lock() + defer mu.Unlock() + if finalized { + return + } + finalized = true + + summary.FinishReason = string(finishReason) + + resp := normalizedResponsePayload{ + Content: content, + FinishReason: string(finishReason), + Warnings: warnings, + } + if usageSeen { + resp.Usage = normalizeUsage(latestUsage) + } + + var usage any + if usageSeen { + usage = &latestUsage + } + handle.finish(ctx, status, resp, usage, streamError, map[string]any{ + "stream_summary": summary, + }) + } + + if seq != nil { + seq(func(part fantasy.StreamPart) bool { + summary.PartCount++ + summary.WarningCount += len(part.Warnings) + if len(part.Warnings) > 0 { + warnings = append(warnings, normalizeWarnings(part.Warnings)...) + } + + switch part.Type { + case fantasy.StreamPartTypeTextDelta: + summary.TextDeltaCount++ + case fantasy.StreamPartTypeReasoningStart, + fantasy.StreamPartTypeReasoningDelta: + case fantasy.StreamPartTypeToolCall: + summary.ToolCallCount++ + case fantasy.StreamPartTypeToolResult: + case fantasy.StreamPartTypeSource: + summary.SourceCount++ + case fantasy.StreamPartTypeFinish: + finishReason = part.FinishReason + latestUsage = part.Usage + usageSeen = true + finishSeen = true + // Signal that the stream received its terminal + // chunk so the AfterFunc safety net yields to + // finalize, which has the real response payload. + streamComplete.Store(true) + } + + content = appendNormalizedStreamContent(content, part, &streamDebugBytes) + + if part.Type == fantasy.StreamPartTypeError || part.Error != nil { + summary.ErrorCount++ + if part.Error != nil { + summary.LastError = part.Error.Error() + streamError = normalizeError(ctx, part.Error) + } else { + summary.LastError = "stream error part with nil error" + streamError = map[string]string{"error": "stream error part with nil error"} + } + streamStatus = streamErrorStatus(streamStatus, part.Error) + } + + if !yield(part) { + // When the consumer stops iteration after + // receiving a finish part, the stream completed + // successfully; the consumer simply has nothing + // left to read. Only mark as interrupted when the + // consumer exits before the provider finished. + switch { + case streamStatus == StatusError: + finalize(StatusError) + case finishSeen: + finalize(StatusCompleted) + default: + finalize(StatusInterrupted) + } + return false + } + + return true + }) + } + + // If the stream ended without a finish part and + // without an explicit error, the provider closed + // the connection prematurely. Record this as + // interrupted so debug runs surface incomplete + // output instead of falsely reporting success. + if streamStatus == StatusCompleted && !finishSeen { + streamStatus = StatusInterrupted + } + finalize(streamStatus) + } +} + +func wrapObjectStreamSeq( + ctx context.Context, + handle *stepHandle, + seq iter.Seq[fantasy.ObjectStreamPart], +) fantasy.ObjectStreamResponse { + // Same safety-net pattern as wrapStreamSeq: a mutex rather + // than sync.Once lets the AfterFunc yield to the normal + // finalization path when the stream has already completed. + var ( + mu sync.Mutex + finalized bool + streamComplete atomic.Bool + ) + stop := context.AfterFunc(ctx, func() { + mu.Lock() + defer mu.Unlock() + if finalized || streamComplete.Load() { + return + } + finalized = true + handle.finish(ctx, StatusInterrupted, nil, nil, nil, nil) + }) + + return func(yield func(fantasy.ObjectStreamPart) bool) { + var ( + summary = objectStreamSummary{StructuredOutput: true} + latestUsage fantasy.Usage + usageSeen bool + finishSeen bool + finishReason fantasy.FinishReason + rawTextLength int + warnings []normalizedWarning + streamError any + streamStatus = StatusCompleted + ) + + finalize := func(status Status) { + if stop != nil { + stop() + } + mu.Lock() + defer mu.Unlock() + if finalized { + return + } + finalized = true + + summary.FinishReason = string(finishReason) + + resp := normalizedObjectResponsePayload{ + RawTextLength: rawTextLength, + FinishReason: string(finishReason), + Warnings: warnings, + StructuredOutput: true, + } + if usageSeen { + resp.Usage = normalizeUsage(latestUsage) + } + + var usage any + if usageSeen { + usage = &latestUsage + } + handle.finish(ctx, status, resp, usage, streamError, map[string]any{ + "structured_output": true, + "stream_summary": summary, + }) + } + + if seq != nil { + seq(func(part fantasy.ObjectStreamPart) bool { + summary.PartCount++ + summary.WarningCount += len(part.Warnings) + if len(part.Warnings) > 0 { + warnings = append(warnings, normalizeWarnings(part.Warnings)...) + } + + switch part.Type { + case fantasy.ObjectStreamPartTypeObject: + summary.ObjectPartCount++ + case fantasy.ObjectStreamPartTypeTextDelta: + summary.TextDeltaCount++ + rawTextLength += utf8.RuneCountInString(part.Delta) + case fantasy.ObjectStreamPartTypeFinish: + finishReason = part.FinishReason + latestUsage = part.Usage + usageSeen = true + finishSeen = true + streamComplete.Store(true) + } + + if part.Type == fantasy.ObjectStreamPartTypeError || part.Error != nil { + summary.ErrorCount++ + if part.Error != nil { + summary.LastError = part.Error.Error() + streamError = normalizeError(ctx, part.Error) + } else { + summary.LastError = "stream error part with nil error" + streamError = map[string]string{"error": "stream error part with nil error"} + } + streamStatus = streamErrorStatus(streamStatus, part.Error) + } + + if !yield(part) { + // Same as the regular stream wrapper: if a + // finish part was already seen, the consumer + // exited normally after completion. + switch { + case streamStatus == StatusError: + finalize(StatusError) + case finishSeen: + finalize(StatusCompleted) + default: + finalize(StatusInterrupted) + } + return false + } + + return true + }) + } + + // Same as the regular stream wrapper: treat a + // stream that ended without a finish part as + // interrupted rather than falsely completed. + if streamStatus == StatusCompleted && !finishSeen { + streamStatus = StatusInterrupted + } + finalize(streamStatus) + } +} + +// --------------- helper functions --------------- + +// normalizeMessages converts a fantasy.Prompt into a slice of +// normalizedMessage values with bounded part metadata. +func normalizeMessages(prompt fantasy.Prompt) []normalizedMessage { + msgs := make([]normalizedMessage, 0, len(prompt)) + for _, m := range prompt { + msgs = append(msgs, normalizedMessage{ + Role: string(m.Role), + Parts: normalizeMessageParts(m.Content), + }) + } + return msgs +} + +// boundText truncates s to MaxMessagePartTextLength runes, appending +// an ellipsis if truncation occurs. +func boundText(s string) string { + return truncateRunes(s, MaxMessagePartTextLength) +} + +// safeMarshalJSON marshals value to JSON. On failure it returns a +// diagnostic error object rather than panicking, which is appropriate +// for debug telemetry where a marshal failure should not crash the +// caller. +func safeMarshalJSON(label string, value any) json.RawMessage { + data, err := json.Marshal(value) + if err != nil { + fallback, fallbackErr := json.Marshal(map[string]string{ + "error": fmt.Sprintf("chatdebug: failed to marshal %s: %v", label, err), + }) + if fallbackErr == nil { + return append(json.RawMessage(nil), fallback...) + } + return json.RawMessage(`{"error":"chatdebug: failed to marshal value"}`) + } + return append(json.RawMessage(nil), data...) +} + +func appendStreamContentText( + content []normalizedContentPart, + partType string, + delta string, + streamDebugBytes *int, +) []normalizedContentPart { + if delta == "" { + return content + } + + remaining := maxStreamDebugTextBytes + if streamDebugBytes != nil { + remaining -= *streamDebugBytes + } + if remaining <= 0 { + return content + } + if len(delta) > remaining { + cut := 0 + for _, r := range delta { + size := utf8.RuneLen(r) + if size < 0 { + size = 1 + } + if cut+size > remaining { + break + } + cut += size + } + delta = delta[:cut] + } + if delta == "" { + return content + } + + if len(content) == 0 || content[len(content)-1].Type != partType { + content = append(content, normalizedContentPart{Type: partType}) + } + last := &content[len(content)-1] + last.Text += delta + if streamDebugBytes != nil { + *streamDebugBytes += len(delta) + } + return content +} + +// appendStreamToolInput accumulates incremental tool-input deltas +// per tool call ID so that parallel or sequential tool invocations +// remain distinguishable in interrupted stream debug payloads. +func appendStreamToolInput( + content []normalizedContentPart, + part fantasy.StreamPart, + streamDebugBytes *int, +) []normalizedContentPart { + if part.Delta == "" { + return content + } + + remaining := maxStreamDebugTextBytes + if streamDebugBytes != nil { + remaining -= *streamDebugBytes + } + if remaining <= 0 { + return content + } + delta := part.Delta + if len(delta) > remaining { + cut := 0 + for _, r := range delta { + size := utf8.RuneLen(r) + if size < 0 { + size = 1 + } + if cut+size > remaining { + break + } + cut += size + } + delta = delta[:cut] + } + if delta == "" { + return content + } + + // Find the existing tool_input part for this specific tool call ID. + // Scan backwards through all content; tool_input deltas for the + // same call may be separated by text, reasoning, or source parts + // when streams interleave multiple tool invocations. + for i := len(content) - 1; i >= 0; i-- { + if content[i].Type == "tool_input" && content[i].ToolCallID == part.ID { + content[i].Arguments += delta + if streamDebugBytes != nil { + *streamDebugBytes += len(delta) + } + return content + } + } + + content = append(content, normalizedContentPart{ + Type: "tool_input", + ToolCallID: part.ID, + ToolName: part.ToolCallName, + Arguments: delta, + }) + if streamDebugBytes != nil { + *streamDebugBytes += len(delta) + } + return content +} + +func canonicalContentType(partType string) string { + switch partType { + case string(fantasy.StreamPartTypeToolCall), string(fantasy.ContentTypeToolCall): + return string(fantasy.ContentTypeToolCall) + case string(fantasy.StreamPartTypeToolResult), string(fantasy.ContentTypeToolResult): + return string(fantasy.ContentTypeToolResult) + default: + return partType + } +} + +func appendNormalizedStreamContent( + content []normalizedContentPart, + part fantasy.StreamPart, + streamDebugBytes *int, +) []normalizedContentPart { + switch part.Type { + case fantasy.StreamPartTypeTextDelta: + return appendStreamContentText(content, "text", part.Delta, streamDebugBytes) + case fantasy.StreamPartTypeReasoningStart, fantasy.StreamPartTypeReasoningDelta: + return appendStreamContentText(content, "reasoning", part.Delta, streamDebugBytes) + case fantasy.StreamPartTypeToolInputStart, + fantasy.StreamPartTypeToolInputDelta, + fantasy.StreamPartTypeToolInputEnd: + // Incremental tool input parts are emitted before the final + // tool_call summary. Attribute each chunk to its tool call + // so interrupted streams can reconstruct which partial input + // belonged to which invocation. + return appendStreamToolInput(content, part, streamDebugBytes) + case fantasy.StreamPartTypeToolCall: + return append(content, normalizedContentPart{ + Type: canonicalContentType(string(part.Type)), + ToolCallID: part.ID, + ToolName: part.ToolCallName, + Arguments: boundText(part.ToolCallInput), + InputLength: utf8.RuneCountInString(part.ToolCallInput), + }) + case fantasy.StreamPartTypeToolResult: + return append(content, normalizedContentPart{ + Type: canonicalContentType(string(part.Type)), + ToolCallID: part.ID, + ToolName: part.ToolCallName, + Result: boundText(part.ToolCallInput), + }) + case fantasy.StreamPartTypeSource: + return append(content, normalizedContentPart{ + Type: string(part.Type), + SourceType: string(part.SourceType), + Title: part.Title, + URL: part.URL, + }) + default: + return content + } +} + +func normalizeToolResultOutput(output fantasy.ToolResultOutputContent) string { + switch v := output.(type) { + case fantasy.ToolResultOutputContentText: + return boundText(v.Text) + case *fantasy.ToolResultOutputContentText: + if v == nil { + return "" + } + return boundText(v.Text) + case fantasy.ToolResultOutputContentError: + if v.Error == nil { + return "" + } + return boundText(v.Error.Error()) + case *fantasy.ToolResultOutputContentError: + if v == nil || v.Error == nil { + return "" + } + return boundText(v.Error.Error()) + case fantasy.ToolResultOutputContentMedia: + if v.Text != "" { + return boundText(v.Text) + } + if v.MediaType == "" { + return "[media output]" + } + return fmt.Sprintf("[media output: %s]", v.MediaType) + case *fantasy.ToolResultOutputContentMedia: + if v == nil { + return "" + } + if v.Text != "" { + return boundText(v.Text) + } + if v.MediaType == "" { + return "[media output]" + } + return fmt.Sprintf("[media output: %s]", v.MediaType) + default: + if output == nil { + return "" + } + return boundText(string(safeMarshalJSON("tool result output", output))) + } +} + +// isNilInterfaceValue reports whether v is nil or holds a nil pointer, +// map, slice, channel, or func. +func isNilInterfaceValue(v any) bool { + if v == nil { + return true + } + + rv := reflect.ValueOf(v) + switch rv.Kind() { + case reflect.Chan, reflect.Func, reflect.Map, reflect.Pointer, reflect.Slice: + return rv.IsNil() + default: + return false + } +} + +// normalizeMessageParts extracts type and bounded metadata from each +// MessagePart. Text-like payloads are bounded to +// MaxMessagePartTextLength runes so the debug panel can display +// readable content. +func normalizeMessageParts(parts []fantasy.MessagePart) []normalizedMessagePart { + result := make([]normalizedMessagePart, 0, len(parts)) + for _, p := range parts { + if isNilInterfaceValue(p) { + continue + } + np := normalizedMessagePart{ + Type: canonicalContentType(string(p.GetType())), + } + switch v := p.(type) { + case fantasy.TextPart: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case *fantasy.TextPart: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case fantasy.ReasoningPart: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case *fantasy.ReasoningPart: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case fantasy.FilePart: + np.Filename = v.Filename + np.MediaType = v.MediaType + case *fantasy.FilePart: + np.Filename = v.Filename + np.MediaType = v.MediaType + case fantasy.ToolCallPart: + np.ToolCallID = v.ToolCallID + np.ToolName = v.ToolName + np.Arguments = boundText(v.Input) + case *fantasy.ToolCallPart: + np.ToolCallID = v.ToolCallID + np.ToolName = v.ToolName + np.Arguments = boundText(v.Input) + case fantasy.ToolResultPart: + np.ToolCallID = v.ToolCallID + np.Result = normalizeToolResultOutput(v.Output) + case *fantasy.ToolResultPart: + np.ToolCallID = v.ToolCallID + np.Result = normalizeToolResultOutput(v.Output) + } + result = append(result, np) + } + return result +} + +// normalizeTools converts the tool list into lightweight descriptors. +// Function tool schemas are preserved so the debug panel can render +// parameter details without re-fetching provider metadata. +func normalizeTools(tools []fantasy.Tool) []normalizedTool { + if len(tools) == 0 { + return nil + } + result := make([]normalizedTool, 0, len(tools)) + for _, t := range tools { + if isNilInterfaceValue(t) { + continue + } + nt := normalizedTool{ + Type: string(t.GetType()), + Name: t.GetName(), + } + switch v := t.(type) { + case fantasy.FunctionTool: + nt.Description = v.Description + nt.HasInputSchema = len(v.InputSchema) > 0 + if nt.HasInputSchema { + nt.InputSchema = safeMarshalJSON( + fmt.Sprintf("tool %q input schema", v.Name), + v.InputSchema, + ) + } + case *fantasy.FunctionTool: + nt.Description = v.Description + nt.HasInputSchema = len(v.InputSchema) > 0 + if nt.HasInputSchema { + nt.InputSchema = safeMarshalJSON( + fmt.Sprintf("tool %q input schema", v.Name), + v.InputSchema, + ) + } + case fantasy.ProviderDefinedTool: + nt.ID = v.ID + case *fantasy.ProviderDefinedTool: + nt.ID = v.ID + case fantasy.ExecutableProviderTool: + nt.ID = v.Definition().ID + case *fantasy.ExecutableProviderTool: + nt.ID = v.Definition().ID + } + result = append(result, nt) + } + return result +} + +// normalizeContentParts converts the response content into a slice +// of normalizedContentPart values. Text payloads are bounded to +// MaxMessagePartTextLength runes per part; tool-call arguments are +// similarly bounded. File data is never stored. +// +// Unlike the stream path which caps total accumulated text at +// maxStreamDebugTextBytes, the Generate path bounds each part +// individually. This is intentional: stream deltas are many small +// fragments that accumulate unboundedly, while Generate responses +// contain a fixed number of discrete content parts, each +// independently bounded by MaxMessagePartTextLength. +func normalizeContentParts(content fantasy.ResponseContent) []normalizedContentPart { + result := make([]normalizedContentPart, 0, len(content)) + for _, c := range content { + if isNilInterfaceValue(c) { + continue + } + np := normalizedContentPart{ + Type: canonicalContentType(string(c.GetType())), + } + switch v := c.(type) { + case fantasy.TextContent: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case *fantasy.TextContent: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case fantasy.ReasoningContent: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case *fantasy.ReasoningContent: + np.Text = boundText(v.Text) + np.TextLength = utf8.RuneCountInString(v.Text) + case fantasy.ToolCallContent: + np.ToolCallID = v.ToolCallID + np.ToolName = v.ToolName + np.Arguments = boundText(v.Input) + np.InputLength = utf8.RuneCountInString(v.Input) + case *fantasy.ToolCallContent: + np.ToolCallID = v.ToolCallID + np.ToolName = v.ToolName + np.Arguments = boundText(v.Input) + np.InputLength = utf8.RuneCountInString(v.Input) + case fantasy.FileContent: + np.MediaType = v.MediaType + case *fantasy.FileContent: + np.MediaType = v.MediaType + case fantasy.SourceContent: + np.SourceType = string(v.SourceType) + np.Title = v.Title + np.URL = v.URL + case *fantasy.SourceContent: + np.SourceType = string(v.SourceType) + np.Title = v.Title + np.URL = v.URL + case fantasy.ToolResultContent: + np.ToolCallID = v.ToolCallID + np.ToolName = v.ToolName + np.Result = normalizeToolResultOutput(v.Result) + case *fantasy.ToolResultContent: + if v != nil { + np.ToolCallID = v.ToolCallID + np.ToolName = v.ToolName + np.Result = normalizeToolResultOutput(v.Result) + } + } + result = append(result, np) + } + return result +} + +// normalizeUsage maps the full fantasy.Usage token breakdown into +// the debug-friendly normalizedUsage struct. +func normalizeUsage(u fantasy.Usage) normalizedUsage { + return normalizedUsage{ + InputTokens: u.InputTokens, + OutputTokens: u.OutputTokens, + TotalTokens: u.TotalTokens, + ReasoningTokens: u.ReasoningTokens, + CacheCreationTokens: u.CacheCreationTokens, + CacheReadTokens: u.CacheReadTokens, + } +} + +// normalizeWarnings converts provider call warnings into their +// normalized form. Returns nil for empty input to keep JSON clean. +func normalizeWarnings(warnings []fantasy.CallWarning) []normalizedWarning { + if len(warnings) == 0 { + return nil + } + result := make([]normalizedWarning, 0, len(warnings)) + for _, w := range warnings { + result = append(result, normalizedWarning{ + Type: string(w.Type), + Setting: w.Setting, + Details: w.Details, + Message: w.Message, + }) + } + return result +} + +// --------------- normalize functions --------------- + +func normalizeCall(call fantasy.Call) normalizedCallPayload { + payload := normalizedCallPayload{ + Messages: normalizeMessages(call.Prompt), + Tools: normalizeTools(call.Tools), + Options: normalizedCallOptions{ + MaxOutputTokens: call.MaxOutputTokens, + Temperature: call.Temperature, + TopP: call.TopP, + TopK: call.TopK, + PresencePenalty: call.PresencePenalty, + FrequencyPenalty: call.FrequencyPenalty, + }, + ProviderOptionCount: len(call.ProviderOptions), + } + if call.ToolChoice != nil { + payload.ToolChoice = string(*call.ToolChoice) + } + return payload +} + +func normalizeObjectCall(call fantasy.ObjectCall) normalizedObjectCallPayload { + return normalizedObjectCallPayload{ + Messages: normalizeMessages(call.Prompt), + Options: normalizedCallOptions{ + MaxOutputTokens: call.MaxOutputTokens, + Temperature: call.Temperature, + TopP: call.TopP, + TopK: call.TopK, + PresencePenalty: call.PresencePenalty, + FrequencyPenalty: call.FrequencyPenalty, + }, + SchemaName: call.SchemaName, + SchemaDescription: call.SchemaDescription, + StructuredOutput: true, + ProviderOptionCount: len(call.ProviderOptions), + } +} + +func normalizeResponse(resp *fantasy.Response) normalizedResponsePayload { + if resp == nil { + return normalizedResponsePayload{} + } + + return normalizedResponsePayload{ + Content: normalizeContentParts(resp.Content), + FinishReason: string(resp.FinishReason), + Usage: normalizeUsage(resp.Usage), + Warnings: normalizeWarnings(resp.Warnings), + } +} + +func normalizeObjectResponse(resp *fantasy.ObjectResponse) normalizedObjectResponsePayload { + if resp == nil { + return normalizedObjectResponsePayload{StructuredOutput: true} + } + + return normalizedObjectResponsePayload{ + RawTextLength: utf8.RuneCountInString(resp.RawText), + FinishReason: string(resp.FinishReason), + Usage: normalizeUsage(resp.Usage), + Warnings: normalizeWarnings(resp.Warnings), + StructuredOutput: true, + } +} + +func streamErrorStatus(current Status, err error) Status { + if current == StatusError { + return current + } + if err == nil { + return StatusError + } + return stepStatusForError(err) +} + +func stepStatusForError(err error) Status { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return StatusInterrupted + } + return StatusError +} + +func normalizeError(ctx context.Context, err error) normalizedErrorPayload { + payload := normalizedErrorPayload{} + if err == nil { + return payload + } + + payload.Message = err.Error() + payload.Type = fmt.Sprintf("%T", err) + if ctxErr := ctx.Err(); ctxErr != nil { + payload.ContextError = ctxErr.Error() + } + + var providerErr *fantasy.ProviderError + if errors.As(err, &providerErr) { + payload.ProviderTitle = providerErr.Title + payload.ProviderStatus = providerErr.StatusCode + payload.IsRetryable = providerErr.IsRetryable() + } + + return payload +} diff --git a/coderd/x/chatd/chatdebug/model_coverage_test.go b/coderd/x/chatd/chatdebug/model_coverage_test.go new file mode 100644 index 0000000000..8ce87ca40f --- /dev/null +++ b/coderd/x/chatd/chatdebug/model_coverage_test.go @@ -0,0 +1,331 @@ +package chatdebug //nolint:testpackage // Checks unexported normalized structs against fantasy source types. + +import ( + "reflect" + "testing" + + "charm.land/fantasy" + "github.com/stretchr/testify/require" +) + +// fieldDisposition documents whether a fantasy struct field is captured +// by the corresponding normalized struct ("normalized") or +// intentionally omitted ("skipped: "). The test fails when a +// fantasy type gains a field that is not yet classified, forcing the +// developer to decide whether to normalize or skip it. +// +// This mirrors the audit-table exhaustiveness check in +// enterprise/audit/table.go; same idea, different domain. +type fieldDisposition = map[string]string + +// TestNormalizationFieldCoverage ensures every exported field on the +// fantasy types that model.go normalizes is explicitly accounted for. +// When the fantasy library adds a field the test fails, surfacing the +// drift at `go test` time rather than silently dropping data. +func TestNormalizationFieldCoverage(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + typ reflect.Type + fields fieldDisposition + }{ + // ── struct-to-struct mappings ────────────────────────── + + { + name: "fantasy.Usage → normalizedUsage", + typ: reflect.TypeFor[fantasy.Usage](), + fields: fieldDisposition{ + "InputTokens": "normalized", + "OutputTokens": "normalized", + "TotalTokens": "normalized", + "ReasoningTokens": "normalized", + "CacheCreationTokens": "normalized", + "CacheReadTokens": "normalized", + }, + }, + { + name: "fantasy.Call → normalizedCallPayload", + typ: reflect.TypeFor[fantasy.Call](), + fields: fieldDisposition{ + "Prompt": "normalized", + "MaxOutputTokens": "normalized", + "Temperature": "normalized", + "TopP": "normalized", + "TopK": "normalized", + "PresencePenalty": "normalized", + "FrequencyPenalty": "normalized", + "Tools": "normalized", + "ToolChoice": "normalized", + "UserAgent": "skipped: internal transport header, not useful for debug panel", + "ProviderOptions": "skipped: opaque provider data, only count preserved", + }, + }, + { + name: "fantasy.ObjectCall → normalizedObjectCallPayload", + typ: reflect.TypeFor[fantasy.ObjectCall](), + fields: fieldDisposition{ + "Prompt": "normalized", + "Schema": "skipped: full schema too large; SchemaName+SchemaDescription captured instead", + "SchemaName": "normalized", + "SchemaDescription": "normalized", + "MaxOutputTokens": "normalized", + "Temperature": "normalized", + "TopP": "normalized", + "TopK": "normalized", + "PresencePenalty": "normalized", + "FrequencyPenalty": "normalized", + "UserAgent": "skipped: internal transport header, not useful for debug panel", + "ProviderOptions": "skipped: opaque provider data, only count preserved", + "RepairText": "skipped: function value, not serializable", + }, + }, + { + name: "fantasy.Response → normalizedResponsePayload", + typ: reflect.TypeFor[fantasy.Response](), + fields: fieldDisposition{ + "Content": "normalized", + "FinishReason": "normalized", + "Usage": "normalized", + "Warnings": "normalized", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.ObjectResponse → normalizedObjectResponsePayload", + typ: reflect.TypeFor[fantasy.ObjectResponse](), + fields: fieldDisposition{ + "Object": "skipped: arbitrary user type, not serializable generically", + "RawText": "normalized: as RawTextLength (length only, content unbounded)", + "Usage": "normalized", + "FinishReason": "normalized", + "Warnings": "normalized", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.CallWarning → normalizedWarning", + typ: reflect.TypeFor[fantasy.CallWarning](), + fields: fieldDisposition{ + "Type": "normalized", + "Setting": "normalized", + "Tool": "skipped: interface value, warning message+type sufficient for debug panel", + "Details": "normalized", + "Message": "normalized", + }, + }, + { + name: "fantasy.StreamPart → appendNormalizedStreamContent", + typ: reflect.TypeFor[fantasy.StreamPart](), + fields: fieldDisposition{ + "Type": "normalized", + "ID": "normalized: as ToolCallID in content parts", + "ToolCallName": "normalized: as ToolName in content parts", + "ToolCallInput": "normalized: as Arguments or Result (bounded)", + "Delta": "normalized: accumulated into text/reasoning content parts", + "ProviderExecuted": "skipped: provider vs client distinction not needed for debug panel", + "Usage": "normalized: captured in stream finalize", + "FinishReason": "normalized: captured in stream finalize", + "Error": "normalized: captured in stream error handling", + "Warnings": "normalized: captured in stream warning accumulation", + "SourceType": "normalized", + "URL": "normalized", + "Title": "normalized", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.ObjectStreamPart → wrapObjectStreamSeq", + typ: reflect.TypeFor[fantasy.ObjectStreamPart](), + fields: fieldDisposition{ + "Type": "normalized: drives switch in wrapObjectStreamSeq", + "Object": "skipped: arbitrary user type, only ObjectPartCount tracked", + "Delta": "normalized: accumulated into rawTextLength", + "Error": "normalized: captured in stream error handling", + "Usage": "normalized: captured in stream finalize", + "FinishReason": "normalized: captured in stream finalize", + "Warnings": "normalized: captured in stream warning accumulation", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + + // ── message part types (normalizeMessageParts) ──────── + + { + name: "fantasy.TextPart → normalizedMessagePart", + typ: reflect.TypeFor[fantasy.TextPart](), + fields: fieldDisposition{ + "Text": "normalized: bounded to MaxMessagePartTextLength", + "ProviderOptions": "skipped: opaque provider-specific options", + }, + }, + { + name: "fantasy.ReasoningPart → normalizedMessagePart", + typ: reflect.TypeFor[fantasy.ReasoningPart](), + fields: fieldDisposition{ + "Text": "normalized: bounded to MaxMessagePartTextLength", + "ProviderOptions": "skipped: opaque provider-specific options", + }, + }, + { + name: "fantasy.FilePart → normalizedMessagePart", + typ: reflect.TypeFor[fantasy.FilePart](), + fields: fieldDisposition{ + "Filename": "normalized", + "Data": "skipped: binary data never stored in debug records", + "MediaType": "normalized", + "ProviderOptions": "skipped: opaque provider-specific options", + }, + }, + { + name: "fantasy.ToolCallPart → normalizedMessagePart", + typ: reflect.TypeFor[fantasy.ToolCallPart](), + fields: fieldDisposition{ + "ToolCallID": "normalized", + "ToolName": "normalized", + "Input": "normalized: as Arguments (bounded)", + "ProviderExecuted": "skipped: provider vs client distinction not needed for debug panel", + "ProviderOptions": "skipped: opaque provider-specific options", + }, + }, + { + name: "fantasy.ToolResultPart → normalizedMessagePart", + typ: reflect.TypeFor[fantasy.ToolResultPart](), + fields: fieldDisposition{ + "ToolCallID": "normalized", + "Output": "normalized: text extracted via normalizeToolResultOutput", + "ProviderExecuted": "skipped: provider vs client distinction not needed for debug panel", + "ProviderOptions": "skipped: opaque provider-specific options", + }, + }, + + // ── response content types (normalizeContentParts) ──── + + { + name: "fantasy.TextContent → normalizedContentPart", + typ: reflect.TypeFor[fantasy.TextContent](), + fields: fieldDisposition{ + "Text": "normalized: bounded to MaxMessagePartTextLength", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.ReasoningContent → normalizedContentPart", + typ: reflect.TypeFor[fantasy.ReasoningContent](), + fields: fieldDisposition{ + "Text": "normalized: bounded to MaxMessagePartTextLength", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.FileContent → normalizedContentPart", + typ: reflect.TypeFor[fantasy.FileContent](), + fields: fieldDisposition{ + "MediaType": "normalized", + "Data": "skipped: binary data never stored in debug records", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.SourceContent → normalizedContentPart", + typ: reflect.TypeFor[fantasy.SourceContent](), + fields: fieldDisposition{ + "SourceType": "normalized", + "ID": "skipped: provider-internal identifier, not actionable in debug panel", + "URL": "normalized", + "Title": "normalized", + "MediaType": "skipped: only relevant for document sources, rarely useful for debugging", + "Filename": "skipped: only relevant for document sources, rarely useful for debugging", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + { + name: "fantasy.ToolCallContent → normalizedContentPart", + typ: reflect.TypeFor[fantasy.ToolCallContent](), + fields: fieldDisposition{ + "ToolCallID": "normalized", + "ToolName": "normalized", + "Input": "normalized: as Arguments (bounded), InputLength tracks original", + "ProviderExecuted": "skipped: provider vs client distinction not needed for debug panel", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + "Invalid": "skipped: validation state not surfaced in debug panel", + "ValidationError": "skipped: validation state not surfaced in debug panel", + }, + }, + { + name: "fantasy.ToolResultContent → normalizedContentPart", + typ: reflect.TypeFor[fantasy.ToolResultContent](), + fields: fieldDisposition{ + "ToolCallID": "normalized", + "ToolName": "normalized", + "Result": "normalized: text extracted via normalizeToolResultOutput", + "ClientMetadata": "skipped: client execution metadata not needed for debug panel", + "ProviderExecuted": "skipped: provider vs client distinction not needed for debug panel", + "ProviderMetadata": "skipped: opaque provider-specific metadata", + }, + }, + + // ── tool types (normalizeTools) ─────────────────────── + + { + name: "fantasy.FunctionTool → normalizedTool", + typ: reflect.TypeFor[fantasy.FunctionTool](), + fields: fieldDisposition{ + "Name": "normalized", + "Description": "normalized", + "InputSchema": "normalized: preserved as JSON for debug panel rendering", + "ProviderOptions": "skipped: opaque provider-specific options", + }, + }, + { + name: "fantasy.ProviderDefinedTool → normalizedTool", + typ: reflect.TypeFor[fantasy.ProviderDefinedTool](), + fields: fieldDisposition{ + "ID": "normalized", + "Name": "normalized", + "Args": "skipped: provider-specific configuration not needed for debug panel", + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Every exported field on the fantasy type must be + // registered as "normalized" or "skipped: ". + for i := range tt.typ.NumField() { + field := tt.typ.Field(i) + if !field.IsExported() { + continue + } + disposition, ok := tt.fields[field.Name] + if !ok { + require.Failf(t, "unregistered field", + "%s.%s is not in the coverage map: "+ + "add it as \"normalized\" or \"skipped: \"", + tt.typ.Name(), field.Name) + } + require.NotEmptyf(t, disposition, + "%s.%s has an empty disposition: "+ + "use \"normalized\" or \"skipped: \"", + tt.typ.Name(), field.Name) + } + + // Catch stale entries that reference removed fields. + for name := range tt.fields { + found := false + for i := range tt.typ.NumField() { + if tt.typ.Field(i).Name == name { + found = true + break + } + } + require.Truef(t, found, + "stale coverage entry %s.%s: "+ + "field no longer exists in fantasy, remove it", + tt.typ.Name(), name) + } + }) + } +} diff --git a/coderd/x/chatd/chatdebug/model_internal_test.go b/coderd/x/chatd/chatdebug/model_internal_test.go new file mode 100644 index 0000000000..f40204bb82 --- /dev/null +++ b/coderd/x/chatd/chatdebug/model_internal_test.go @@ -0,0 +1,723 @@ +package chatdebug + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "charm.land/fantasy" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "go.uber.org/mock/gomock" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/database/dbmock" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/testutil" +) + +type testError struct{ message string } + +func (e *testError) Error() string { return e.message } + +func TestDebugModel_Provider(t *testing.T) { + t.Parallel() + + inner := &chattest.FakeModel{ProviderName: "provider-a", ModelName: "model-a"} + model := &debugModel{inner: inner} + + require.Equal(t, inner.Provider(), model.Provider()) +} + +func TestDebugModel_Model(t *testing.T) { + t.Parallel() + + inner := &chattest.FakeModel{ProviderName: "provider-a", ModelName: "model-a"} + model := &debugModel{inner: inner} + + require.Equal(t, inner.Model(), model.Model()) +} + +func TestDebugModel_Disabled(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + svc := NewService(db, testutil.Logger(t), nil) + respWant := &fantasy.Response{FinishReason: fantasy.FinishReasonStop} + inner := &chattest.FakeModel{ + GenerateFn: func(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) { + _, ok := StepFromContext(ctx) + require.False(t, ok) + require.Nil(t, attemptSinkFromContext(ctx)) + return respWant, nil + }, + } + + model := &debugModel{ + inner: inner, + svc: svc, + opts: RecorderOptions{ + ChatID: chatID, + OwnerID: ownerID, + }, + } + + resp, err := model.Generate(context.Background(), fantasy.Call{}) + require.NoError(t, err) + require.Same(t, respWant, resp) +} + +func TestDebugModel_Generate(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + call := fantasy.Call{ + Prompt: fantasy.Prompt{fantasy.NewUserMessage("hello")}, + MaxOutputTokens: int64Ptr(128), + Temperature: float64Ptr(0.25), + } + respWant := &fantasy.Response{ + Content: fantasy.ResponseContent{ + fantasy.TextContent{Text: "hello"}, + fantasy.ToolCallContent{ToolCallID: "tool-1", ToolName: "tool", Input: `{}`}, + fantasy.SourceContent{ID: "source-1", Title: "docs", URL: "https://example.com"}, + }, + FinishReason: fantasy.FinishReasonStop, + Usage: fantasy.Usage{InputTokens: 10, OutputTokens: 4, TotalTokens: 14}, + Warnings: []fantasy.CallWarning{{Message: "warning"}}, + } + + svc := NewService(db, testutil.Logger(t), nil) + inner := &chattest.FakeModel{ + GenerateFn: func(ctx context.Context, got fantasy.Call) (*fantasy.Response, error) { + require.Equal(t, call, got) + stepCtx, ok := StepFromContext(ctx) + require.True(t, ok) + require.Equal(t, runID, stepCtx.RunID) + require.Equal(t, chatID, stepCtx.ChatID) + require.Equal(t, int32(1), stepCtx.StepNumber) + require.Equal(t, OperationGenerate, stepCtx.Operation) + require.NotEqual(t, uuid.Nil, stepCtx.StepID) + require.NotNil(t, attemptSinkFromContext(ctx)) + return respWant, nil + }, + } + + model := &debugModel{ + inner: inner, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + resp, err := model.Generate(ctx, call) + require.NoError(t, err) + require.Same(t, respWant, resp) +} + +func TestDebugModel_GeneratePersistsAttemptsWithoutResponseClose(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + + server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { + body, err := io.ReadAll(req.Body) + require.NoError(t, err) + require.JSONEq(t, `{"message":"hello","api_key":"super-secret"}`, + string(body)) + require.Equal(t, "Bearer top-secret", req.Header.Get("Authorization")) + + rw.Header().Set("Content-Type", "application/json") + rw.Header().Set("X-API-Key", "response-secret") + rw.WriteHeader(http.StatusCreated) + _, _ = rw.Write([]byte(`{"token":"response-secret","safe":"ok"}`)) + })) + defer server.Close() + + svc := NewService(db, testutil.Logger(t), nil) + inner := &chattest.FakeModel{ + GenerateFn: func(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) { + client := &http.Client{Transport: &RecordingTransport{Base: server.Client().Transport}} + req, err := http.NewRequestWithContext( + ctx, + http.MethodPost, + server.URL, + strings.NewReader(`{"message":"hello","api_key":"super-secret"}`), + ) + require.NoError(t, err) + req.Header.Set("Authorization", "Bearer top-secret") + req.Header.Set("Content-Type", "application/json") + + resp, err := client.Do(req) + require.NoError(t, err) + + body, err := io.ReadAll(resp.Body) + require.NoError(t, err) + require.JSONEq(t, `{"token":"response-secret","safe":"ok"}`, string(body)) + require.NoError(t, resp.Body.Close()) + return &fantasy.Response{FinishReason: fantasy.FinishReasonStop}, nil + }, + } + + model := &debugModel{ + inner: inner, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + resp, err := model.Generate(ctx, fantasy.Call{}) + require.NoError(t, err) + require.NotNil(t, resp) +} + +func TestDebugModel_GenerateError(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + wantErr := &testError{message: "boom"} + + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + GenerateFn: func(context.Context, fantasy.Call) (*fantasy.Response, error) { + return nil, wantErr + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + resp, err := model.Generate(ctx, fantasy.Call{}) + require.Nil(t, resp) + require.ErrorIs(t, err, wantErr) +} + +func TestStepStatusForError(t *testing.T) { + t.Parallel() + + t.Run("Canceled", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusInterrupted, stepStatusForError(context.Canceled)) + }) + + t.Run("DeadlineExceeded", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusInterrupted, stepStatusForError(context.DeadlineExceeded)) + }) + + t.Run("OtherError", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusError, stepStatusForError(xerrors.New("boom"))) + }) +} + +func TestDebugModel_Stream(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + errPart := xerrors.New("chunk failed") + parts := []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "hel"}, + {Type: fantasy.StreamPartTypeToolCall, ID: "tool-call-1", ToolCallName: "tool"}, + {Type: fantasy.StreamPartTypeSource, ID: "source-1", URL: "https://example.com", Title: "docs"}, + {Type: fantasy.StreamPartTypeWarnings, Warnings: []fantasy.CallWarning{{Message: "w1"}, {Message: "w2"}}}, + {Type: fantasy.StreamPartTypeError, Error: errPart}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop, Usage: fantasy.Usage{InputTokens: 8, OutputTokens: 3, TotalTokens: 11}}, + } + + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + StreamFn: func(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + stepCtx, ok := StepFromContext(ctx) + require.True(t, ok) + require.Equal(t, runID, stepCtx.RunID) + require.Equal(t, chatID, stepCtx.ChatID) + require.Equal(t, int32(1), stepCtx.StepNumber) + require.Equal(t, OperationStream, stepCtx.Operation) + require.NotEqual(t, uuid.Nil, stepCtx.StepID) + require.NotNil(t, attemptSinkFromContext(ctx)) + return partsToSeq(parts), nil + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + seq, err := model.Stream(ctx, fantasy.Call{}) + require.NoError(t, err) + + got := make([]fantasy.StreamPart, 0, len(parts)) + for part := range seq { + got = append(got, part) + } + + require.Equal(t, parts, got) +} + +func TestDebugModel_StreamObject(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + parts := []fantasy.ObjectStreamPart{ + {Type: fantasy.ObjectStreamPartTypeTextDelta, Delta: "ob"}, + {Type: fantasy.ObjectStreamPartTypeTextDelta, Delta: "ject"}, + {Type: fantasy.ObjectStreamPartTypeObject, Object: map[string]any{"value": "object"}}, + {Type: fantasy.ObjectStreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop, Usage: fantasy.Usage{InputTokens: 5, OutputTokens: 2, TotalTokens: 7}}, + } + + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + StreamObjectFn: func(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) { + stepCtx, ok := StepFromContext(ctx) + require.True(t, ok) + require.Equal(t, runID, stepCtx.RunID) + require.Equal(t, chatID, stepCtx.ChatID) + require.Equal(t, int32(1), stepCtx.StepNumber) + require.Equal(t, OperationStream, stepCtx.Operation) + require.NotEqual(t, uuid.Nil, stepCtx.StepID) + require.NotNil(t, attemptSinkFromContext(ctx)) + return objectPartsToSeq(parts), nil + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + seq, err := model.StreamObject(ctx, fantasy.ObjectCall{}) + require.NoError(t, err) + + got := make([]fantasy.ObjectStreamPart, 0, len(parts)) + for part := range seq { + got = append(got, part) + } + + require.Equal(t, parts, got) +} + +// TestDebugModel_StreamCompletedAfterFinish verifies that when a consumer +// stops iteration after receiving a finish part, the step is marked as +// completed rather than interrupted. +func TestDebugModel_StreamCompletedAfterFinish(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + + parts := []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "hello"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop, Usage: fantasy.Usage{InputTokens: 5, OutputTokens: 1, TotalTokens: 6}}, + } + seq := wrapStreamSeq(context.Background(), handle, partsToSeq(parts)) + + // Consumer reads through the finish part then breaks. The wrapper + // should finalize as completed, not interrupted. + for part := range seq { + if part.Type == fantasy.StreamPartTypeFinish { + break + } + } + + handle.mu.Lock() + status := handle.status + handle.mu.Unlock() + require.Equal(t, StatusCompleted, status) +} + +// TestDebugModel_StreamInterruptedBeforeFinish verifies that when a consumer +// stops iteration before receiving a finish part, the step is marked as +// interrupted. +func TestDebugModel_StreamInterruptedBeforeFinish(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + + parts := []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "hello"}, + {Type: fantasy.StreamPartTypeTextDelta, Delta: " world"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + } + seq := wrapStreamSeq(context.Background(), handle, partsToSeq(parts)) + + // Consumer reads the first delta then breaks before finish. + count := 0 + for range seq { + count++ + if count == 1 { + break + } + } + require.Equal(t, 1, count) + + handle.mu.Lock() + status := handle.status + handle.mu.Unlock() + require.Equal(t, StatusInterrupted, status) +} + +func TestDebugModel_StreamRejectsNilSequence(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + StreamFn: func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) { + var nilStream fantasy.StreamResponse + return nilStream, nil + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: uuid.New()}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + seq, err := model.Stream(ctx, fantasy.Call{}) + require.Nil(t, seq) + require.ErrorIs(t, err, ErrNilModelResult) +} + +func TestDebugModel_StreamObjectRejectsNilSequence(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + StreamObjectFn: func(context.Context, fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) { + var nilStream fantasy.ObjectStreamResponse + return nilStream, nil + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: uuid.New()}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + seq, err := model.StreamObject(ctx, fantasy.ObjectCall{}) + require.Nil(t, seq) + require.ErrorIs(t, err, ErrNilModelResult) +} + +func TestDebugModel_StreamEarlyStop(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + parts := []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "first"}, + {Type: fantasy.StreamPartTypeTextDelta, Delta: "second"}, + } + + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + StreamFn: func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) { + return partsToSeq(parts), nil + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + seq, err := model.Stream(ctx, fantasy.Call{}) + require.NoError(t, err) + + count := 0 + for part := range seq { + require.Equal(t, parts[0], part) + count++ + break + } + require.Equal(t, 1, count) +} + +func TestStreamErrorStatus(t *testing.T) { + t.Parallel() + + t.Run("CancellationBecomesInterrupted", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusInterrupted, streamErrorStatus(StatusCompleted, context.Canceled)) + }) + + t.Run("DeadlineExceededBecomesInterrupted", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusInterrupted, streamErrorStatus(StatusCompleted, context.DeadlineExceeded)) + }) + + t.Run("NilErrorBecomesError", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusError, streamErrorStatus(StatusCompleted, nil)) + }) + + t.Run("ExistingErrorWins", func(t *testing.T) { + t.Parallel() + require.Equal(t, StatusError, streamErrorStatus(StatusError, context.Canceled)) + }) +} + +func objectPartsToSeq(parts []fantasy.ObjectStreamPart) fantasy.ObjectStreamResponse { + return func(yield func(fantasy.ObjectStreamPart) bool) { + for _, part := range parts { + if !yield(part) { + return + } + } + } +} + +func partsToSeq(parts []fantasy.StreamPart) fantasy.StreamResponse { + return func(yield func(fantasy.StreamPart) bool) { + for _, part := range parts { + if !yield(part) { + return + } + } + } +} + +func TestDebugModel_GenerateObject(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + ownerID := uuid.New() + runID := uuid.New() + call := fantasy.ObjectCall{ + Prompt: fantasy.Prompt{fantasy.NewUserMessage("summarize")}, + SchemaName: "Summary", + MaxOutputTokens: int64Ptr(256), + } + respWant := &fantasy.ObjectResponse{ + RawText: `{"title":"test"}`, + FinishReason: fantasy.FinishReasonStop, + Usage: fantasy.Usage{InputTokens: 5, OutputTokens: 3, TotalTokens: 8}, + } + + svc := NewService(db, testutil.Logger(t), nil) + inner := &chattest.FakeModel{ + GenerateObjectFn: func(ctx context.Context, got fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + require.Equal(t, call, got) + stepCtx, ok := StepFromContext(ctx) + require.True(t, ok) + require.Equal(t, runID, stepCtx.RunID) + require.Equal(t, chatID, stepCtx.ChatID) + require.Equal(t, OperationGenerate, stepCtx.Operation) + require.NotEqual(t, uuid.Nil, stepCtx.StepID) + require.NotNil(t, attemptSinkFromContext(ctx)) + return respWant, nil + }, + } + + model := &debugModel{ + inner: inner, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: ownerID}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + resp, err := model.GenerateObject(ctx, call) + require.NoError(t, err) + require.Same(t, respWant, resp) +} + +func TestDebugModel_GenerateObjectError(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + wantErr := &testError{message: "object boom"} + + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + return nil, wantErr + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: uuid.New()}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + resp, err := model.GenerateObject(ctx, fantasy.ObjectCall{}) + require.Nil(t, resp) + require.ErrorIs(t, err, wantErr) +} + +func TestDebugModel_GenerateObjectRejectsNilResponse(t *testing.T) { + t.Parallel() + + ctrl := gomock.NewController(t) + db := dbmock.NewMockStore(ctrl) + chatID := uuid.New() + runID := uuid.New() + + svc := NewService(db, testutil.Logger(t), nil) + model := &debugModel{ + inner: &chattest.FakeModel{ + GenerateObjectFn: func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + return nil, nil //nolint:nilnil // Intentionally testing nil response handling. + }, + }, + svc: svc, + opts: RecorderOptions{ChatID: chatID, OwnerID: uuid.New()}, + } + t.Cleanup(func() { CleanupStepCounter(runID) }) + ctx := ContextWithRun(context.Background(), &RunContext{RunID: runID, ChatID: chatID}) + + resp, err := model.GenerateObject(ctx, fantasy.ObjectCall{}) + require.Nil(t, resp) + require.ErrorIs(t, err, ErrNilModelResult) +} + +func TestWrapStreamSeq_CompletedNotDowngradedByCtxCancel(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + + // Create a context that we cancel after the stream finishes. + ctx, cancel := context.WithCancel(context.Background()) + + parts := []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "hello"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop, Usage: fantasy.Usage{InputTokens: 5, OutputTokens: 1, TotalTokens: 6}}, + } + seq := wrapStreamSeq(ctx, handle, partsToSeq(parts)) + + //nolint:revive // Intentionally consuming iterator to trigger side-effects. + for range seq { + } + + // Cancel the context after the stream has been fully consumed + // and finalized. The status should remain completed. + cancel() + + handle.mu.Lock() + status := handle.status + handle.mu.Unlock() + require.Equal(t, StatusCompleted, status) +} + +func TestWrapObjectStreamSeq_CompletedNotDowngradedByCtxCancel(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + + ctx, cancel := context.WithCancel(context.Background()) + + parts := []fantasy.ObjectStreamPart{ + {Type: fantasy.ObjectStreamPartTypeTextDelta, Delta: "obj"}, + {Type: fantasy.ObjectStreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop, Usage: fantasy.Usage{InputTokens: 3, OutputTokens: 1, TotalTokens: 4}}, + } + seq := wrapObjectStreamSeq(ctx, handle, objectPartsToSeq(parts)) + + //nolint:revive // Intentionally consuming iterator to trigger side-effects. + for range seq { + } + + cancel() + + handle.mu.Lock() + status := handle.status + handle.mu.Unlock() + require.Equal(t, StatusCompleted, status) +} + +func TestWrapStreamSeq_DroppedStreamFinalizedOnCtxCancel(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + + ctx, cancel := context.WithCancel(context.Background()) + parts := []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "hello"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + } + + // Create the wrapped stream but never iterate it. + _ = wrapStreamSeq(ctx, handle, partsToSeq(parts)) + + // Cancel the context; the AfterFunc safety net should finalize + // the step as interrupted. + cancel() + + // AfterFunc fires asynchronously; give it a moment. + require.Eventually(t, func() bool { + handle.mu.Lock() + defer handle.mu.Unlock() + return handle.status == StatusInterrupted + }, testutil.WaitShort, testutil.IntervalFast) +} + +func int64Ptr(v int64) *int64 { return &v } + +func float64Ptr(v float64) *float64 { return &v } diff --git a/coderd/x/chatd/chatdebug/model_normalization_test.go b/coderd/x/chatd/chatdebug/model_normalization_test.go new file mode 100644 index 0000000000..395fa056f7 --- /dev/null +++ b/coderd/x/chatd/chatdebug/model_normalization_test.go @@ -0,0 +1,439 @@ +package chatdebug //nolint:testpackage // Uses unexported normalization helpers. + +import ( + "context" + "strings" + "testing" + "unicode/utf8" + + "charm.land/fantasy" + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" +) + +func TestNormalizeCall_PreservesToolSchemasAndMessageToolPayloads(t *testing.T) { + t.Parallel() + + payload := normalizeCall(fantasy.Call{ + Prompt: fantasy.Prompt{ + { + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: "call-search", + ToolName: "search_docs", + Input: `{"query":"debug panel"}`, + }, + }, + }, + { + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: "call-search", + Output: fantasy.ToolResultOutputContentText{ + Text: `{"matches":["model.go","DebugStepCard.tsx"]}`, + }, + }, + }, + }, + }, + Tools: []fantasy.Tool{ + fantasy.FunctionTool{ + Name: "search_docs", + Description: "Searches documentation.", + InputSchema: map[string]any{ + "type": "object", + "properties": map[string]any{ + "query": map[string]any{"type": "string"}, + }, + "required": []string{"query"}, + }, + }, + }, + }) + + require.Len(t, payload.Tools, 1) + require.True(t, payload.Tools[0].HasInputSchema) + require.JSONEq(t, `{"type":"object","properties":{"query":{"type":"string"}},"required":["query"]}`, + string(payload.Tools[0].InputSchema)) + + require.Len(t, payload.Messages, 2) + require.Equal(t, "tool-call", payload.Messages[0].Parts[0].Type) + require.Equal(t, `{"query":"debug panel"}`, payload.Messages[0].Parts[0].Arguments) + require.Equal(t, "tool-result", payload.Messages[1].Parts[0].Type) + require.Equal(t, + `{"matches":["model.go","DebugStepCard.tsx"]}`, + payload.Messages[1].Parts[0].Result, + ) +} + +func TestNormalizeTools_PreservesExecutableProviderToolID(t *testing.T) { + t.Parallel() + + pdt := fantasy.ProviderDefinedTool{ + ID: "anthropic.computer_use", + Name: "computer", + } + ept := fantasy.NewExecutableProviderTool(pdt, func(context.Context, fantasy.ToolCall) (fantasy.ToolResponse, error) { + return fantasy.ToolResponse{}, nil + }) + + tools := normalizeTools([]fantasy.Tool{ept}) + require.Len(t, tools, 1) + require.Equal(t, "anthropic.computer_use", tools[0].ID) + require.Equal(t, "computer", tools[0].Name) +} + +func TestNormalizers_SkipTypedNilInterfaceValues(t *testing.T) { + t.Parallel() + + t.Run("MessageParts", func(t *testing.T) { + t.Parallel() + + var nilPart *fantasy.TextPart + parts := normalizeMessageParts([]fantasy.MessagePart{ + nilPart, + fantasy.TextPart{Text: "hello"}, + }) + require.Len(t, parts, 1) + require.Equal(t, "text", parts[0].Type) + require.Equal(t, "hello", parts[0].Text) + }) + + t.Run("Tools", func(t *testing.T) { + t.Parallel() + + var nilTool *fantasy.FunctionTool + tools := normalizeTools([]fantasy.Tool{ + nilTool, + fantasy.FunctionTool{Name: "search_docs"}, + }) + require.Len(t, tools, 1) + require.Equal(t, "function", tools[0].Type) + require.Equal(t, "search_docs", tools[0].Name) + }) + + t.Run("ContentParts", func(t *testing.T) { + t.Parallel() + + var nilContent *fantasy.TextContent + content := normalizeContentParts(fantasy.ResponseContent{ + nilContent, + fantasy.TextContent{Text: "hello"}, + }) + require.Len(t, content, 1) + require.Equal(t, "text", content[0].Type) + require.Equal(t, "hello", content[0].Text) + }) +} + +func TestAppendNormalizedStreamContent_PreservesOrderAndCanonicalTypes(t *testing.T) { + t.Parallel() + + var content []normalizedContentPart + streamDebugBytes := 0 + for _, part := range []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: "before "}, + {Type: fantasy.StreamPartTypeToolCall, ID: "call-1", ToolCallName: "search_docs", ToolCallInput: `{"query":"debug"}`}, + {Type: fantasy.StreamPartTypeToolResult, ID: "call-1", ToolCallName: "search_docs", ToolCallInput: `{"matches":1}`}, + {Type: fantasy.StreamPartTypeTextDelta, Delta: "after"}, + } { + content = appendNormalizedStreamContent(content, part, &streamDebugBytes) + } + + require.Equal(t, []normalizedContentPart{ + {Type: "text", Text: "before "}, + {Type: "tool-call", ToolCallID: "call-1", ToolName: "search_docs", Arguments: `{"query":"debug"}`, InputLength: utf8.RuneCountInString(`{"query":"debug"}`)}, + {Type: "tool-result", ToolCallID: "call-1", ToolName: "search_docs", Result: `{"matches":1}`}, + {Type: "text", Text: "after"}, + }, content) +} + +func TestAppendNormalizedStreamContent_ToolInputAttributionPerCall(t *testing.T) { + t.Parallel() + + var content []normalizedContentPart + streamDebugBytes := 0 + for _, part := range []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeToolInputStart, ID: "call-a", ToolCallName: "search", Delta: `{"q`}, + {Type: fantasy.StreamPartTypeToolInputDelta, ID: "call-a", ToolCallName: "search", Delta: `uery`}, + // Interleaved second tool call. + {Type: fantasy.StreamPartTypeToolInputStart, ID: "call-b", ToolCallName: "calc", Delta: `{"op`}, + {Type: fantasy.StreamPartTypeToolInputDelta, ID: "call-a", ToolCallName: "search", Delta: `":"x"}`}, + {Type: fantasy.StreamPartTypeToolInputEnd, ID: "call-b", ToolCallName: "calc", Delta: `":"add"}`}, + } { + content = appendNormalizedStreamContent(content, part, &streamDebugBytes) + } + + require.Equal(t, []normalizedContentPart{ + {Type: "tool_input", ToolCallID: "call-a", ToolName: "search", Arguments: `{"query":"x"}`}, + {Type: "tool_input", ToolCallID: "call-b", ToolName: "calc", Arguments: `{"op":"add"}`}, + }, content) +} + +func TestAppendNormalizedStreamContent_ToolInputAcrossInterleavedText(t *testing.T) { + t.Parallel() + + var content []normalizedContentPart + streamDebugBytes := 0 + for _, part := range []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeToolInputStart, ID: "call-a", ToolCallName: "search", Delta: `{"q`}, + // Text delta interleaved between tool_input deltas for call-a. + {Type: fantasy.StreamPartTypeTextDelta, Delta: "thinking..."}, + {Type: fantasy.StreamPartTypeToolInputDelta, ID: "call-a", ToolCallName: "search", Delta: `uery":"x"}`}, + } { + content = appendNormalizedStreamContent(content, part, &streamDebugBytes) + } + + require.Equal(t, []normalizedContentPart{ + {Type: "tool_input", ToolCallID: "call-a", ToolName: "search", Arguments: `{"query":"x"}`}, + {Type: "text", Text: "thinking..."}, + }, content) +} + +func TestAppendNormalizedStreamContent_GlobalTextCap(t *testing.T) { + t.Parallel() + + streamDebugBytes := 0 + long := strings.Repeat("a", maxStreamDebugTextBytes) + var content []normalizedContentPart + for _, part := range []fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextDelta, Delta: long}, + {Type: fantasy.StreamPartTypeToolCall, ID: "call-1", ToolCallName: "search_docs", ToolCallInput: `{}`}, + {Type: fantasy.StreamPartTypeTextDelta, Delta: "tail"}, + } { + content = appendNormalizedStreamContent(content, part, &streamDebugBytes) + } + + require.Len(t, content, 2) + require.Equal(t, strings.Repeat("a", maxStreamDebugTextBytes), content[0].Text) + require.Equal(t, "tool-call", content[1].Type) + require.Equal(t, maxStreamDebugTextBytes, streamDebugBytes) +} + +func TestWrapStreamSeq_SourceCountExcludesToolResults(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + seq := wrapStreamSeq(context.Background(), handle, partsToSeq([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeToolResult, ID: "tool-1", ToolCallName: "search_docs"}, + {Type: fantasy.StreamPartTypeSource, ID: "source-1", URL: "https://example.com", Title: "docs"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + })) + + partCount := 0 + for range seq { + partCount++ + } + require.Equal(t, 3, partCount) + + metadata, ok := handle.metadata.(map[string]any) + require.True(t, ok) + summary, ok := metadata["stream_summary"].(streamSummary) + require.True(t, ok) + require.Equal(t, 1, summary.SourceCount) +} + +func TestWrapObjectStreamSeq_UsesStructuredOutputPayload(t *testing.T) { + t.Parallel() + + handle := &stepHandle{ + stepCtx: &StepContext{StepID: uuid.New(), RunID: uuid.New(), ChatID: uuid.New()}, + sink: &attemptSink{}, + } + usage := fantasy.Usage{InputTokens: 3, OutputTokens: 2, TotalTokens: 5} + seq := wrapObjectStreamSeq(context.Background(), handle, objectPartsToSeq([]fantasy.ObjectStreamPart{ + {Type: fantasy.ObjectStreamPartTypeTextDelta, Delta: "ob"}, + {Type: fantasy.ObjectStreamPartTypeTextDelta, Delta: "ject"}, + {Type: fantasy.ObjectStreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop, Usage: usage}, + })) + + partCount := 0 + for range seq { + partCount++ + } + require.Equal(t, 3, partCount) + + resp, ok := handle.response.(normalizedObjectResponsePayload) + require.True(t, ok) + require.Equal(t, normalizedObjectResponsePayload{ + RawTextLength: utf8.RuneCountInString("object"), + FinishReason: string(fantasy.FinishReasonStop), + Usage: normalizeUsage(usage), + StructuredOutput: true, + }, resp) +} + +func TestNormalizeResponse_UsesCanonicalToolTypes(t *testing.T) { + t.Parallel() + + payload := normalizeResponse(&fantasy.Response{ + Content: fantasy.ResponseContent{ + fantasy.ToolCallContent{ + ToolCallID: "call-calc", + ToolName: "calculator", + Input: `{"operation":"add","operands":[2,2]}`, + }, + fantasy.ToolResultContent{ + ToolCallID: "call-calc", + ToolName: "calculator", + Result: fantasy.ToolResultOutputContentText{Text: `{"sum":4}`}, + }, + }, + }) + + require.Len(t, payload.Content, 2) + require.Equal(t, "tool-call", payload.Content[0].Type) + require.Equal(t, "tool-result", payload.Content[1].Type) +} + +func TestBoundText_RespectsDocumentedRuneLimit(t *testing.T) { + t.Parallel() + + runes := make([]rune, MaxMessagePartTextLength+5) + for i := range runes { + runes[i] = 'a' + } + input := string(runes) + got := boundText(input) + require.Equal(t, MaxMessagePartTextLength, len([]rune(got))) + require.Equal(t, '…', []rune(got)[len([]rune(got))-1]) +} + +func TestNormalizeToolResultOutput(t *testing.T) { + t.Parallel() + + t.Run("TextValue", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentText{Text: "hello"}) + require.Equal(t, "hello", got) + }) + + t.Run("TextPointer", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(&fantasy.ToolResultOutputContentText{Text: "hello"}) + require.Equal(t, "hello", got) + }) + + t.Run("TextPointerNil", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput((*fantasy.ToolResultOutputContentText)(nil)) + require.Equal(t, "", got) + }) + + t.Run("ErrorValue", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentError{ + Error: xerrors.New("tool failed"), + }) + require.Equal(t, "tool failed", got) + }) + + t.Run("ErrorValueNilError", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentError{Error: nil}) + require.Equal(t, "", got) + }) + + t.Run("ErrorPointer", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(&fantasy.ToolResultOutputContentError{ + Error: xerrors.New("ptr fail"), + }) + require.Equal(t, "ptr fail", got) + }) + + t.Run("ErrorPointerNil", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput((*fantasy.ToolResultOutputContentError)(nil)) + require.Equal(t, "", got) + }) + + t.Run("ErrorPointerNilError", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(&fantasy.ToolResultOutputContentError{Error: nil}) + require.Equal(t, "", got) + }) + + t.Run("MediaWithText", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentMedia{ + Text: "caption", + MediaType: "image/png", + }) + require.Equal(t, "caption", got) + }) + + t.Run("MediaWithoutText", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentMedia{ + MediaType: "image/png", + }) + require.Equal(t, "[media output: image/png]", got) + }) + + t.Run("MediaWithoutTextOrType", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentMedia{}) + require.Equal(t, "[media output]", got) + }) + + t.Run("MediaPointerNil", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput((*fantasy.ToolResultOutputContentMedia)(nil)) + require.Equal(t, "", got) + }) + + t.Run("MediaPointerWithText", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(&fantasy.ToolResultOutputContentMedia{ + Text: "ptr caption", + MediaType: "image/jpeg", + }) + require.Equal(t, "ptr caption", got) + }) + + t.Run("NilOutput", func(t *testing.T) { + t.Parallel() + got := normalizeToolResultOutput(nil) + require.Equal(t, "", got) + }) + + t.Run("DefaultJSON", func(t *testing.T) { + t.Parallel() + // An unexpected type falls through to the default JSON + // marshal branch. + got := normalizeToolResultOutput(fantasy.ToolResultOutputContentText{ + Text: "fallback", + }) + require.Equal(t, "fallback", got) + }) +} + +func TestNormalizeResponse_PreservesToolCallArguments(t *testing.T) { + t.Parallel() + + payload := normalizeResponse(&fantasy.Response{ + Content: fantasy.ResponseContent{ + fantasy.ToolCallContent{ + ToolCallID: "call-calc", + ToolName: "calculator", + Input: `{"operation":"add","operands":[2,2]}`, + }, + }, + }) + + require.Len(t, payload.Content, 1) + require.Equal(t, "call-calc", payload.Content[0].ToolCallID) + require.Equal(t, "calculator", payload.Content[0].ToolName) + require.JSONEq(t, + `{"operation":"add","operands":[2,2]}`, + payload.Content[0].Arguments, + ) + require.Equal(t, utf8.RuneCountInString(`{"operation":"add","operands":[2,2]}`), payload.Content[0].InputLength) +} diff --git a/coderd/x/chatd/chatdebug/stubs.go b/coderd/x/chatd/chatdebug/stubs.go new file mode 100644 index 0000000000..aee4b85ff7 --- /dev/null +++ b/coderd/x/chatd/chatdebug/stubs.go @@ -0,0 +1,245 @@ +package chatdebug + +import ( + "context" + "regexp" + "strings" + "sync" + "sync/atomic" + "unicode/utf8" + + "github.com/google/uuid" + + "cdr.dev/slog/v3" + "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/database/pubsub" +) + +// This branch-02 compatibility shim forward-declares recorder, service, +// and summary symbols that land in later stacked branches. Delete this +// file once recorder.go, service.go, and summary.go are available here. + +// RecorderOptions identifies the chat/model context for debug recording. +type RecorderOptions struct { + ChatID uuid.UUID + OwnerID uuid.UUID + Provider string + Model string +} + +// Service is a placeholder for the later chat debug persistence service. +type Service struct{} + +// NewService constructs the branch-02 placeholder chat debug service. +func NewService(_ database.Store, _ slog.Logger, _ pubsub.Pubsub) *Service { + return &Service{} +} + +type attemptSink struct{} + +type attemptSinkKey struct{} + +func withAttemptSink(ctx context.Context, sink *attemptSink) context.Context { + if sink == nil { + panic("chatdebug: nil attemptSink") + } + return context.WithValue(ctx, attemptSinkKey{}, sink) +} + +func attemptSinkFromContext(ctx context.Context) *attemptSink { + sink, _ := ctx.Value(attemptSinkKey{}).(*attemptSink) + return sink +} + +var stepCounters sync.Map // map[uuid.UUID]*atomic.Int32 + +// runRefCounts tracks how many live RunContext instances reference each +// RunID. Cleanup of shared state (step counters) is deferred until the +// last RunContext for a given RunID is garbage collected. +var ( + runRefCounts sync.Map // map[uuid.UUID]*atomic.Int32 + // refCountMu serializes trackRunRef and releaseRunRef so the + // decrement-to-zero check and subsequent map deletions are + // atomic with respect to new references being added. + refCountMu sync.Mutex +) + +func trackRunRef(runID uuid.UUID) { + refCountMu.Lock() + defer refCountMu.Unlock() + val, _ := runRefCounts.LoadOrStore(runID, &atomic.Int32{}) + counter, ok := val.(*atomic.Int32) + if !ok { + panic("chatdebug: runRefCounts contains non-*atomic.Int32 value") + } + counter.Add(1) +} + +// releaseRunRef decrements the reference count for runID and cleans up +// shared state when the last reference is released. The mutex ensures +// no concurrent trackRunRef can increment between the zero check and +// the map deletions. +func releaseRunRef(runID uuid.UUID) { + refCountMu.Lock() + defer refCountMu.Unlock() + val, ok := runRefCounts.Load(runID) + if !ok { + return + } + counter, ok := val.(*atomic.Int32) + if !ok { + panic("chatdebug: runRefCounts contains non-*atomic.Int32 value") + } + if counter.Add(-1) <= 0 { + runRefCounts.Delete(runID) + stepCounters.Delete(runID) + } +} + +func nextStepNumber(runID uuid.UUID) int32 { + val, _ := stepCounters.LoadOrStore(runID, &atomic.Int32{}) + counter, ok := val.(*atomic.Int32) + if !ok { + panic("chatdebug: invalid step counter type") + } + return counter.Add(1) +} + +// CleanupStepCounter removes per-run step counter and reference count +// state. This is used by tests and later stacked branches that have a +// real run lifecycle. +func CleanupStepCounter(runID uuid.UUID) { + stepCounters.Delete(runID) + runRefCounts.Delete(runID) +} + +type stepHandle struct { + stepCtx *StepContext + sink *attemptSink + mu sync.Mutex + status Status + response any + usage any + err any + metadata any +} + +func beginStep( + ctx context.Context, + svc *Service, + opts RecorderOptions, + op Operation, + _ any, +) (*stepHandle, context.Context) { + if svc == nil { + return nil, ctx + } + + rc, ok := RunFromContext(ctx) + if !ok || rc.RunID == uuid.Nil { + return nil, ctx + } + + if holder, reuseStep := reuseHolderFromContext(ctx); reuseStep { + holder.mu.Lock() + defer holder.mu.Unlock() + // Only reuse the cached handle if it belongs to the same run. + // A different RunContext means a new logical run, so we must + // create a fresh step to avoid cross-run attribution. + if holder.handle != nil && holder.handle.stepCtx.RunID == rc.RunID { + enriched := ContextWithStep(ctx, holder.handle.stepCtx) + enriched = withAttemptSink(enriched, holder.handle.sink) + return holder.handle, enriched + } + + handle, enriched := newStepHandle(ctx, rc, opts, op) + holder.handle = handle + return handle, enriched + } + + return newStepHandle(ctx, rc, opts, op) +} + +func newStepHandle( + ctx context.Context, + rc *RunContext, + opts RecorderOptions, + op Operation, +) (*stepHandle, context.Context) { + if rc == nil || rc.RunID == uuid.Nil { + return nil, ctx + } + + chatID := opts.ChatID + if chatID == uuid.Nil { + chatID = rc.ChatID + } + + handle := &stepHandle{ + stepCtx: &StepContext{ + StepID: uuid.New(), + RunID: rc.RunID, + ChatID: chatID, + StepNumber: nextStepNumber(rc.RunID), + Operation: op, + HistoryTipMessageID: rc.HistoryTipMessageID, + }, + sink: &attemptSink{}, + } + enriched := ContextWithStep(ctx, handle.stepCtx) + enriched = withAttemptSink(enriched, handle.sink) + return handle, enriched +} + +func (h *stepHandle) finish( + _ context.Context, + status Status, + response any, + usage any, + err any, + metadata any, +) { + if h == nil || h.stepCtx == nil { + return + } + // Guard with a mutex so concurrent callers (e.g. retried stream + // wrappers sharing a reused handle) don't race. Unlike sync.Once, + // later retries are allowed to overwrite earlier failure results so + // the step reflects the final outcome. + h.mu.Lock() + defer h.mu.Unlock() + h.status = status + h.response = response + h.usage = usage + h.err = err + h.metadata = metadata +} + +// whitespaceRun matches one or more consecutive whitespace characters. +var whitespaceRun = regexp.MustCompile(`\s+`) + +// truncateRunes truncates s to maxLen runes, appending an ellipsis +// when truncation occurs. Returns "" when maxLen <= 0. +func truncateRunes(s string, maxLen int) string { + if maxLen <= 0 { + return "" + } + if utf8.RuneCountInString(s) <= maxLen { + return s + } + if maxLen == 1 { + return "…" + } + runes := []rune(s) + return string(runes[:maxLen-1]) + "…" +} + +// TruncateLabel whitespace-normalizes and truncates text to maxLen runes. +// Returns "" if input is empty or whitespace-only. +func TruncateLabel(text string, maxLen int) string { + normalized := strings.TrimSpace(whitespaceRun.ReplaceAllString(text, " ")) + if normalized == "" { + return "" + } + return truncateRunes(normalized, maxLen) +} diff --git a/coderd/x/chatd/chatdebug/stubs_internal_test.go b/coderd/x/chatd/chatdebug/stubs_internal_test.go new file mode 100644 index 0000000000..b7bfa81c18 --- /dev/null +++ b/coderd/x/chatd/chatdebug/stubs_internal_test.go @@ -0,0 +1,82 @@ +package chatdebug + +import ( + "context" + "net/http" + "testing" + "unicode/utf8" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" +) + +func TestBeginStep_SkipsNilRunID(t *testing.T) { + t.Parallel() + + ctx := ContextWithRun(context.Background(), &RunContext{ChatID: uuid.New()}) + handle, enriched := beginStep(ctx, &Service{}, RecorderOptions{ChatID: uuid.New()}, OperationGenerate, nil) + require.Nil(t, handle) + require.Equal(t, ctx, enriched) +} + +func TestNewStepHandle_SkipsNilRunID(t *testing.T) { + t.Parallel() + + ctx := context.Background() + handle, enriched := newStepHandle(ctx, &RunContext{ChatID: uuid.New()}, RecorderOptions{ChatID: uuid.New()}, OperationGenerate) + require.Nil(t, handle) + require.Equal(t, ctx, enriched) +} + +func TestTruncateLabel(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + maxLen int + want string + }{ + {name: "Empty", input: "", maxLen: 10, want: ""}, + {name: "WhitespaceOnly", input: " \t\n ", maxLen: 10, want: ""}, + {name: "ShortText", input: "hello world", maxLen: 20, want: "hello world"}, + {name: "ExactLength", input: "abcde", maxLen: 5, want: "abcde"}, + {name: "LongTextTruncated", input: "abcdefghij", maxLen: 5, want: "abcd…"}, + {name: "NegativeMaxLen", input: "hello", maxLen: -1, want: ""}, + {name: "ZeroMaxLen", input: "hello", maxLen: 0, want: ""}, + {name: "SingleRuneLimit", input: "hello", maxLen: 1, want: "…"}, + {name: "MultipleWhitespaceRuns", input: " hello world \t again ", maxLen: 100, want: "hello world again"}, + {name: "UnicodeRunes", input: "こんにちは世界", maxLen: 3, want: "こん…"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got := TruncateLabel(tc.input, tc.maxLen) + require.Equal(t, tc.want, got) + require.LessOrEqual(t, utf8.RuneCountInString(got), max(tc.maxLen, 0)) + }) + } +} + +// RedactedValue replaces sensitive values in debug payloads. +const RedactedValue = "[REDACTED]" + +// RecordingTransport is the branch-02 placeholder HTTP recording transport. +type RecordingTransport struct { + Base http.RoundTripper +} + +var _ http.RoundTripper = (*RecordingTransport)(nil) + +func (t *RecordingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if req == nil { + panic("chatdebug: nil request") + } + + base := t.Base + if base == nil { + base = http.DefaultTransport + } + return base.RoundTrip(req) +} diff --git a/coderd/x/chatd/chatdebug/types.go b/coderd/x/chatd/chatdebug/types.go new file mode 100644 index 0000000000..115b70b2d5 --- /dev/null +++ b/coderd/x/chatd/chatdebug/types.go @@ -0,0 +1,137 @@ +package chatdebug + +import "github.com/google/uuid" + +// RunKind identifies the kind of debug run being recorded. +type RunKind string + +const ( + // KindChatTurn records a standard chat turn. + KindChatTurn RunKind = "chat_turn" + // KindTitleGeneration records title generation for a chat. + KindTitleGeneration RunKind = "title_generation" + // KindQuickgen records quick-generation workflows. + KindQuickgen RunKind = "quickgen" + // KindCompaction records history compaction workflows. + KindCompaction RunKind = "compaction" +) + +// AllRunKinds contains every RunKind value. Update this when +// adding new constants above. +var AllRunKinds = []RunKind{ + KindChatTurn, + KindTitleGeneration, + KindQuickgen, + KindCompaction, +} + +// Status identifies lifecycle state shared by runs and steps. +type Status string + +const ( + // StatusInProgress indicates work is still running. + StatusInProgress Status = "in_progress" + // StatusCompleted indicates work finished successfully. + StatusCompleted Status = "completed" + // StatusError indicates work finished with an error. + StatusError Status = "error" + // StatusInterrupted indicates work was canceled or interrupted. + StatusInterrupted Status = "interrupted" +) + +// AllStatuses contains every Status value. Update this when +// adding new constants above. +var AllStatuses = []Status{ + StatusInProgress, + StatusCompleted, + StatusError, + StatusInterrupted, +} + +// Operation identifies the model operation a step performed. +type Operation string + +const ( + // OperationStream records a streaming model operation. + OperationStream Operation = "stream" + // OperationGenerate records a non-streaming generation operation. + OperationGenerate Operation = "generate" +) + +// AllOperations contains every Operation value. Update this when +// adding new constants above. +var AllOperations = []Operation{ + OperationStream, + OperationGenerate, +} + +// RunContext carries identity and metadata for a debug run. +type RunContext struct { + RunID uuid.UUID + ChatID uuid.UUID + RootChatID uuid.UUID // Zero means not set. + ParentChatID uuid.UUID // Zero means not set. + ModelConfigID uuid.UUID // Zero means not set. + TriggerMessageID int64 // Zero means not set. + HistoryTipMessageID int64 // Zero means not set. + Kind RunKind + Provider string + Model string +} + +// StepContext carries identity and metadata for a debug step. +type StepContext struct { + StepID uuid.UUID + RunID uuid.UUID + ChatID uuid.UUID + StepNumber int32 + Operation Operation + HistoryTipMessageID int64 // Zero means not set. +} + +// Attempt captures a single HTTP round trip made during a step. +type Attempt struct { + Number int `json:"number"` + Status string `json:"status,omitempty"` + Method string `json:"method,omitempty"` + URL string `json:"url,omitempty"` + Path string `json:"path,omitempty"` + StartedAt string `json:"started_at,omitempty"` + FinishedAt string `json:"finished_at,omitempty"` + RequestHeaders map[string]string `json:"request_headers,omitempty"` + RequestBody []byte `json:"request_body,omitempty"` + ResponseStatus int `json:"response_status,omitempty"` + ResponseHeaders map[string]string `json:"response_headers,omitempty"` + ResponseBody []byte `json:"response_body,omitempty"` + Error string `json:"error,omitempty"` + DurationMs int64 `json:"duration_ms"` + RetryClassification string `json:"retry_classification,omitempty"` + RetryDelayMs int64 `json:"retry_delay_ms,omitempty"` +} + +// EventKind identifies the type of pubsub debug event. +type EventKind string + +const ( + // EventKindRunUpdate publishes a run mutation. + EventKindRunUpdate EventKind = "run_update" + // EventKindStepUpdate publishes a step mutation. + EventKindStepUpdate EventKind = "step_update" + // EventKindFinalize publishes a finalization signal. + EventKindFinalize EventKind = "finalize" + // EventKindDelete publishes a deletion signal. + EventKindDelete EventKind = "delete" +) + +// DebugEvent is the lightweight pubsub envelope for chat debug updates. +type DebugEvent struct { + Kind EventKind `json:"kind"` + ChatID uuid.UUID `json:"chat_id"` + RunID uuid.UUID `json:"run_id"` + StepID uuid.UUID `json:"step_id"` +} + +// PubsubChannel returns the chat-scoped pubsub channel for debug events. +func PubsubChannel(chatID uuid.UUID) string { + return "chat_debug:" + chatID.String() +} diff --git a/coderd/x/chatd/chatdebug/types_test.go b/coderd/x/chatd/chatdebug/types_test.go new file mode 100644 index 0000000000..621f589bae --- /dev/null +++ b/coderd/x/chatd/chatdebug/types_test.go @@ -0,0 +1,54 @@ +package chatdebug_test + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/coder/coder/v2/coderd/x/chatd/chatdebug" + "github.com/coder/coder/v2/codersdk" +) + +// toStrings converts a typed string slice to []string for comparison. +func toStrings[T ~string](values []T) []string { + out := make([]string, len(values)) + for i, v := range values { + out[i] = string(v) + } + return out +} + +// TestTypesMatchSDK verifies that every chatdebug constant has a +// corresponding codersdk constant with the same string value. +// If this test fails you probably added a constant to one package +// but forgot to update the other. +func TestTypesMatchSDK(t *testing.T) { + t.Parallel() + + t.Run("RunKind", func(t *testing.T) { + t.Parallel() + require.ElementsMatch(t, + toStrings(chatdebug.AllRunKinds), + toStrings(codersdk.AllChatDebugRunKinds), + "chatdebug.AllRunKinds and codersdk.AllChatDebugRunKinds have diverged", + ) + }) + + t.Run("Status", func(t *testing.T) { + t.Parallel() + require.ElementsMatch(t, + toStrings(chatdebug.AllStatuses), + toStrings(codersdk.AllChatDebugStatuses), + "chatdebug.AllStatuses and codersdk.AllChatDebugStatuses have diverged", + ) + }) + + t.Run("Operation", func(t *testing.T) { + t.Parallel() + require.ElementsMatch(t, + toStrings(chatdebug.AllOperations), + toStrings(codersdk.AllChatDebugStepOperations), + "chatdebug.AllOperations and codersdk.AllChatDebugStepOperations have diverged", + ) + }) +} diff --git a/coderd/x/chatd/chatloop/chatloop_test.go b/coderd/x/chatd/chatloop/chatloop_test.go index b96292fd65..47d08b75db 100644 --- a/coderd/x/chatd/chatloop/chatloop_test.go +++ b/coderd/x/chatd/chatloop/chatloop_test.go @@ -18,6 +18,7 @@ import ( "github.com/coder/coder/v2/coderd/x/chatd/chaterror" "github.com/coder/coder/v2/coderd/x/chatd/chatretry" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" "github.com/coder/coder/v2/codersdk" "github.com/coder/coder/v2/testutil" "github.com/coder/quartz" @@ -41,9 +42,9 @@ func TestRun_ActiveToolsPrepareBehavior(t *testing.T) { t.Parallel() var capturedCall fantasy.Call - model := &loopTestModel{ - provider: fantasyanthropic.Name, - streamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: fantasyanthropic.Name, + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { capturedCall = call return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, @@ -103,9 +104,9 @@ func TestRun_ActiveToolsPrepareBehavior(t *testing.T) { func TestProcessStepStream_AnthropicUsageMatchesFinalDelta(t *testing.T) { t.Parallel() - model := &loopTestModel{ - provider: fantasyanthropic.Name, - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: fantasyanthropic.Name, + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "cached response"}, @@ -160,9 +161,9 @@ func TestRun_OnRetryEnrichesProvider(t *testing.T) { var records []retryRecord calls := 0 - model := &loopTestModel{ - provider: "openai", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "openai", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { calls++ if calls == 1 { return nil, xerrors.New("received status 429 from upstream") @@ -286,9 +287,9 @@ func TestRun_RetriesStartupTimeoutWhileOpeningStream(t *testing.T) { attempts := 0 attemptCause := make(chan error, 1) var retries []chatretry.ClassifiedError - model := &loopTestModel{ - provider: "openai", - streamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "openai", + StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { attempts++ if attempts == 1 { <-ctx.Done() @@ -364,9 +365,9 @@ func TestRun_RetriesStartupTimeoutBeforeFirstPart(t *testing.T) { attempts := 0 attemptCause := make(chan error, 1) var retries []chatretry.ClassifiedError - model := &loopTestModel{ - provider: "openai", - streamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "openai", + StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { attempts++ if attempts == 1 { return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) { @@ -447,9 +448,9 @@ func TestRun_FirstPartDisarmsStartupTimeout(t *testing.T) { retried := false firstPartYielded := make(chan struct{}, 1) continueStream := make(chan struct{}) - model := &loopTestModel{ - provider: "openai", - streamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "openai", + StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { attempts++ return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) { if !yield(fantasy.StreamPart{Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}) { @@ -526,9 +527,9 @@ func TestRun_PanicInPublishMessagePartReleasesAttempt(t *testing.T) { t.Parallel() attemptReleased := make(chan struct{}) - model := &loopTestModel{ - provider: "openai", - streamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "openai", + StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { go func() { <-ctx.Done() close(attemptReleased) @@ -583,9 +584,9 @@ func TestRun_RetriesStartupTimeoutWhenStreamClosesSilently(t *testing.T) { attempts := 0 attemptCause := make(chan error, 1) var retries []chatretry.ClassifiedError - model := &loopTestModel{ - provider: "openai", - streamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "openai", + StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { attempts++ if attempts == 1 { return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) { @@ -648,9 +649,9 @@ func TestRun_InterruptedStepPersistsSyntheticToolResult(t *testing.T) { t.Parallel() started := make(chan struct{}) - model := &loopTestModel{ - provider: "fake", - streamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(ctx context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) { parts := []fantasy.StreamPart{ { @@ -762,52 +763,6 @@ func TestRun_InterruptedStepPersistsSyntheticToolResult(t *testing.T) { "interrupted tool should have no call timestamp (never reached StreamPartTypeToolCall)") } -type loopTestModel struct { - provider string - model string - generateFn func(context.Context, fantasy.Call) (*fantasy.Response, error) - streamFn func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) -} - -func (m *loopTestModel) Provider() string { - if m.provider != "" { - return m.provider - } - return "fake" -} - -func (m *loopTestModel) Model() string { - if m.model != "" { - return m.model - } - return "fake" -} - -func (m *loopTestModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) { - if m.generateFn != nil { - return m.generateFn(ctx, call) - } - return &fantasy.Response{}, nil -} - -func (m *loopTestModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { - if m.streamFn != nil { - return m.streamFn(ctx, call) - } - return streamFromParts([]fantasy.StreamPart{{ - Type: fantasy.StreamPartTypeFinish, - FinishReason: fantasy.FinishReasonStop, - }}), nil -} - -func (*loopTestModel) GenerateObject(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { - return nil, xerrors.New("not implemented") -} - -func (*loopTestModel) StreamObject(context.Context, fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) { - return nil, xerrors.New("not implemented") -} - func streamFromParts(parts []fantasy.StreamPart) fantasy.StreamResponse { return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) { for _, part := range parts { @@ -860,9 +815,9 @@ func TestRun_MultiStepToolExecution(t *testing.T) { var streamCalls int var secondCallPrompt []fantasy.Message - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { mu.Lock() step := streamCalls streamCalls++ @@ -972,9 +927,9 @@ func TestRun_ParallelToolExecutionTimestamps(t *testing.T) { var mu sync.Mutex var streamCalls int - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { mu.Lock() step := streamCalls streamCalls++ @@ -1064,9 +1019,9 @@ func TestRun_ParallelToolExecutionTimestamps(t *testing.T) { func TestRun_PersistStepErrorPropagates(t *testing.T) { t.Parallel() - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "hello"}, @@ -1103,9 +1058,9 @@ func TestRun_ShutdownDuringToolExecutionReturnsContextCanceled(t *testing.T) { toolStarted := make(chan struct{}) // Model returns a single tool call, then finishes. - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeToolInputStart, ID: "tc-block", ToolCallName: "blocking_tool"}, {Type: fantasy.StreamPartTypeToolInputDelta, ID: "tc-block", Delta: `{}`}, @@ -1361,9 +1316,9 @@ func TestRun_InterruptedDuringToolExecutionPersistsStep(t *testing.T) { toolStarted := make(chan struct{}) // Model returns a completed tool call in the stream. - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "calling tool"}, @@ -1471,9 +1426,9 @@ func TestRun_InterruptedDuringToolExecutionPersistsStep(t *testing.T) { func TestRun_ProviderExecutedToolResultTimestamps(t *testing.T) { t.Parallel() - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { // Simulate a provider-executed tool call and result // (e.g. Anthropic web search) followed by a text // response — all in a single stream. @@ -1541,9 +1496,9 @@ func TestRun_ProviderExecutedToolResultTimestamps(t *testing.T) { func TestRun_PersistStepInterruptedFallback(t *testing.T) { t.Parallel() - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "hello world"}, diff --git a/coderd/x/chatd/chatloop/compaction_test.go b/coderd/x/chatd/chatloop/compaction_test.go index 5a443bcd62..ea01ac5141 100644 --- a/coderd/x/chatd/chatloop/compaction_test.go +++ b/coderd/x/chatd/chatloop/compaction_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/xerrors" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" "github.com/coder/coder/v2/codersdk" ) @@ -22,9 +23,9 @@ func TestRun_Compaction(t *testing.T) { var persistedCompaction CompactionResult const summaryText = "summary text for compaction" - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "done"}, @@ -39,7 +40,7 @@ func TestRun_Compaction(t *testing.T) { }, }), nil }, - generateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, call fantasy.Call) (*fantasy.Response, error) { require.NotEmpty(t, call.Prompt) lastPrompt := call.Prompt[len(call.Prompt)-1] require.Equal(t, fantasy.MessageRoleUser, lastPrompt.Role) @@ -107,9 +108,9 @@ func TestRun_Compaction(t *testing.T) { // and the tool-result part publishes after Persist. var callOrder []string - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "done"}, @@ -124,7 +125,7 @@ func TestRun_Compaction(t *testing.T) { }, }), nil }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { callOrder = append(callOrder, "generate") return &fantasy.Response{ Content: []fantasy.Content{ @@ -189,9 +190,9 @@ func TestRun_Compaction(t *testing.T) { publishCalled := false - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ { Type: fantasy.StreamPartTypeFinish, @@ -240,9 +241,9 @@ func TestRun_Compaction(t *testing.T) { const summaryText = "compacted summary" - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { mu.Lock() step := streamCallCount streamCallCount++ @@ -287,7 +288,7 @@ func TestRun_Compaction(t *testing.T) { }), nil } }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return &fantasy.Response{ Content: []fantasy.Content{ fantasy.TextContent{Text: summaryText}, @@ -346,9 +347,9 @@ func TestRun_Compaction(t *testing.T) { const summaryText = "compacted summary for skip test" - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { mu.Lock() step := streamCallCount streamCallCount++ @@ -393,7 +394,7 @@ func TestRun_Compaction(t *testing.T) { }), nil } }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return &fantasy.Response{ Content: []fantasy.Content{ fantasy.TextContent{Text: summaryText}, @@ -442,9 +443,9 @@ func TestRun_Compaction(t *testing.T) { t.Run("ErrorsAreReported", func(t *testing.T) { t.Parallel() - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ { Type: fantasy.StreamPartTypeFinish, @@ -455,7 +456,7 @@ func TestRun_Compaction(t *testing.T) { }, }), nil }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return nil, xerrors.New("generate failed") }, } @@ -511,9 +512,9 @@ func TestRun_Compaction(t *testing.T) { textMessage(fantasy.MessageRoleUser, "compacted user"), } - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { mu.Lock() step := streamCallCount streamCallCount++ @@ -556,7 +557,7 @@ func TestRun_Compaction(t *testing.T) { }), nil } }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return &fantasy.Response{ Content: []fantasy.Content{ fantasy.TextContent{Text: summaryText}, @@ -617,9 +618,9 @@ func TestRun_Compaction(t *testing.T) { const summaryText = "post-run compacted summary" - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { mu.Lock() step := streamCallCount streamCallCount++ @@ -659,7 +660,7 @@ func TestRun_Compaction(t *testing.T) { }), nil } }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return &fantasy.Response{ Content: []fantasy.Content{ fantasy.TextContent{Text: summaryText}, @@ -723,9 +724,9 @@ func TestRun_Compaction(t *testing.T) { // The LLM calls a dynamic tool. Usage is above the // compaction threshold so compaction should fire even // though the chatloop exits via ErrDynamicToolCall. - model := &loopTestModel{ - provider: "fake", - streamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + model := &chattest.FakeModel{ + ProviderName: "fake", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { return streamFromParts([]fantasy.StreamPart{ {Type: fantasy.StreamPartTypeToolInputStart, ID: "tc-1", ToolCallName: "my_dynamic_tool"}, {Type: fantasy.StreamPartTypeToolInputDelta, ID: "tc-1", Delta: `{"query": "test"}`}, @@ -746,7 +747,7 @@ func TestRun_Compaction(t *testing.T) { }, }), nil }, - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return &fantasy.Response{ Content: []fantasy.Content{ fantasy.TextContent{Text: summaryText}, diff --git a/coderd/x/chatd/chattest/fakemodel.go b/coderd/x/chatd/chattest/fakemodel.go new file mode 100644 index 0000000000..a841a1ef19 --- /dev/null +++ b/coderd/x/chatd/chattest/fakemodel.go @@ -0,0 +1,52 @@ +package chattest + +import ( + "context" + + "charm.land/fantasy" +) + +// FakeModel is a configurable test double for fantasy.LanguageModel. +// Calling a method whose function field is nil panics, forcing tests +// to be explicit about which methods they expect to be invoked. +type FakeModel struct { + ProviderName string + ModelName string + GenerateFn func(context.Context, fantasy.Call) (*fantasy.Response, error) + StreamFn func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) + GenerateObjectFn func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) + StreamObjectFn func(context.Context, fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) +} + +var _ fantasy.LanguageModel = (*FakeModel)(nil) + +func (m *FakeModel) Generate(ctx context.Context, call fantasy.Call) (*fantasy.Response, error) { + if m.GenerateFn == nil { + panic("chattest: FakeModel.Generate called but GenerateFn is nil") + } + return m.GenerateFn(ctx, call) +} + +func (m *FakeModel) Stream(ctx context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + if m.StreamFn == nil { + panic("chattest: FakeModel.Stream called but StreamFn is nil") + } + return m.StreamFn(ctx, call) +} + +func (m *FakeModel) GenerateObject(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + if m.GenerateObjectFn == nil { + panic("chattest: FakeModel.GenerateObject called but GenerateObjectFn is nil") + } + return m.GenerateObjectFn(ctx, call) +} + +func (m *FakeModel) StreamObject(ctx context.Context, call fantasy.ObjectCall) (fantasy.ObjectStreamResponse, error) { + if m.StreamObjectFn == nil { + panic("chattest: FakeModel.StreamObject called but StreamObjectFn is nil") + } + return m.StreamObjectFn(ctx, call) +} + +func (m *FakeModel) Provider() string { return m.ProviderName } +func (m *FakeModel) Model() string { return m.ModelName } diff --git a/coderd/x/chatd/quickgen_test.go b/coderd/x/chatd/quickgen_test.go index b47dc318bb..5d0b47b6ad 100644 --- a/coderd/x/chatd/quickgen_test.go +++ b/coderd/x/chatd/quickgen_test.go @@ -10,9 +10,9 @@ import ( "charm.land/fantasy" "github.com/sqlc-dev/pqtype" "github.com/stretchr/testify/require" - "golang.org/x/xerrors" "github.com/coder/coder/v2/coderd/database" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" "github.com/coder/coder/v2/codersdk" ) @@ -375,8 +375,8 @@ func Test_generateManualTitle_UsesTimeout(t *testing.T) { ), } - model := &stubModel{ - generateObjectFn: func(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + model := &chattest.FakeModel{ + GenerateObjectFn: func(ctx context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { deadline, ok := ctx.Deadline() require.True(t, ok, "manual title generation should set a deadline") require.WithinDuration( @@ -413,8 +413,8 @@ func Test_generateManualTitle_TruncatesFirstUserInput(t *testing.T) { ), } - model := &stubModel{ - generateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + model := &chattest.FakeModel{ + GenerateObjectFn: func(_ context.Context, call fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { require.Len(t, call.Prompt, 2) systemText, ok := call.Prompt[0].Content[0].(fantasy.TextPart) require.True(t, ok) @@ -447,8 +447,8 @@ func Test_generateManualTitle_ReturnsUsageForEmptyNormalizedTitle(t *testing.T) ), } - model := &stubModel{ - generateObjectFn: func(_ context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { + model := &chattest.FakeModel{ + GenerateObjectFn: func(_ context.Context, _ fantasy.ObjectCall) (*fantasy.ObjectResponse, error) { return &fantasy.ObjectResponse{ Object: map[string]any{"title": "\"\""}, Usage: fantasy.Usage{ @@ -504,8 +504,8 @@ func Test_selectPreferredConfiguredShortTextModelConfig(t *testing.T) { func Test_generateShortText_NormalizesQuotedOutput(t *testing.T) { t.Parallel() - model := &stubModel{ - generateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { + model := &chattest.FakeModel{ + GenerateFn: func(_ context.Context, _ fantasy.Call) (*fantasy.Response, error) { return &fantasy.Response{ Content: fantasy.ResponseContent{ fantasy.TextContent{Text: " \"Quoted summary\" "}, @@ -520,53 +520,6 @@ func Test_generateShortText_NormalizesQuotedOutput(t *testing.T) { require.Equal(t, "Quoted summary", text) } -type stubModel struct { - generateFn func(context.Context, fantasy.Call) (*fantasy.Response, error) - generateObjectFn func(context.Context, fantasy.ObjectCall) (*fantasy.ObjectResponse, error) -} - -func (m *stubModel) Generate( - ctx context.Context, - call fantasy.Call, -) (*fantasy.Response, error) { - if m.generateFn == nil { - return nil, xerrors.New("generate not implemented") - } - return m.generateFn(ctx, call) -} - -func (*stubModel) Stream( - context.Context, - fantasy.Call, -) (fantasy.StreamResponse, error) { - return nil, xerrors.New("stream not implemented") -} - -func (m *stubModel) GenerateObject( - ctx context.Context, - call fantasy.ObjectCall, -) (*fantasy.ObjectResponse, error) { - if m.generateObjectFn == nil { - return nil, xerrors.New("generate object not implemented") - } - return m.generateObjectFn(ctx, call) -} - -func (*stubModel) StreamObject( - context.Context, - fantasy.ObjectCall, -) (fantasy.ObjectStreamResponse, error) { - return nil, xerrors.New("stream object not implemented") -} - -func (*stubModel) Provider() string { - return "test" -} - -func (*stubModel) Model() string { - return "test" -} - func mustChatMessage( t *testing.T, role database.ChatMessageRole, diff --git a/codersdk/chats.go b/codersdk/chats.go index c4f176f7b4..c13fd222af 100644 --- a/codersdk/chats.go +++ b/codersdk/chats.go @@ -600,6 +600,15 @@ func ChatDebugTerminalStatuses() []ChatDebugStatus { } } +// AllChatDebugStatuses contains every ChatDebugStatus value. +// Update this when adding new constants above. +var AllChatDebugStatuses = []ChatDebugStatus{ + ChatDebugStatusInProgress, + ChatDebugStatusCompleted, + ChatDebugStatusError, + ChatDebugStatusInterrupted, +} + // ChatDebugRunKind labels the operation that produced the debug // run. Each value corresponds to a distinct call-site in chatd. type ChatDebugRunKind string @@ -611,6 +620,15 @@ const ( ChatDebugRunKindCompaction ChatDebugRunKind = "compaction" ) +// AllChatDebugRunKinds contains every ChatDebugRunKind value. +// Update this when adding new constants above. +var AllChatDebugRunKinds = []ChatDebugRunKind{ + ChatDebugRunKindChatTurn, + ChatDebugRunKindTitleGeneration, + ChatDebugRunKindQuickgen, + ChatDebugRunKindCompaction, +} + // ChatDebugStepOperation labels the model interaction type for a // debug step. type ChatDebugStepOperation string @@ -620,6 +638,13 @@ const ( ChatDebugStepOperationGenerate ChatDebugStepOperation = "generate" ) +// AllChatDebugStepOperations contains every ChatDebugStepOperation +// value. Update this when adding new constants above. +var AllChatDebugStepOperations = []ChatDebugStepOperation{ + ChatDebugStepOperationStream, + ChatDebugStepOperationGenerate, +} + // ChatDebugRunSummary is a lightweight run entry for list endpoints. type ChatDebugRunSummary struct { ID uuid.UUID `json:"id" format:"uuid"`