diff --git a/coderd/x/chatd/chatadvisor/guidance.go b/coderd/x/chatd/chatadvisor/guidance.go new file mode 100644 index 0000000000..3a733d0406 --- /dev/null +++ b/coderd/x/chatd/chatadvisor/guidance.go @@ -0,0 +1,24 @@ +package chatadvisor + +const ( + // AdvisorSystemPrompt steers the nested advisor model to help the parent + // agent rather than speaking directly to the end user. + AdvisorSystemPrompt = `You are an internal advisor for another AI coding agent. +You are advising the parent agent, not the end user. +Give concise strategic guidance that helps the parent decide what to do next. +Focus on planning ambiguity, architecture tradeoffs, debugging strategy, +and risk reduction. +Do not address the user directly. +Do not suggest using tools yourself because this nested run has no tools. +Respond with practical guidance only.` + + // ParentGuidanceBlock is a reusable prompt block for teaching parent agents + // when to invoke the built-in advisor tool. + ParentGuidanceBlock = ` +Use the built-in advisor tool when you need strategic guidance on planning +ambiguity, architectural tradeoffs, debugging strategy, or repeated failures. +The advisor sees recent conversation context, runs as a single-step nested model +call with no tools, and returns concise guidance for the parent agent rather +than the end user. +` +) diff --git a/coderd/x/chatd/chatadvisor/handoff.go b/coderd/x/chatd/chatadvisor/handoff.go new file mode 100644 index 0000000000..3fe311a808 --- /dev/null +++ b/coderd/x/chatd/chatadvisor/handoff.go @@ -0,0 +1,208 @@ +package chatadvisor + +import ( + "encoding/json" + "maps" + "slices" + "strings" + + "charm.land/fantasy" +) + +const ( + // advisorRecentMessageLimit caps how many recent non-system messages + // from the parent conversation are forwarded to the advisor. The + // advisor only needs enough tail to ground its guidance, not the full + // history. + advisorRecentMessageLimit = 20 + // advisorConversationJSONByteBudget caps the combined size of the + // forwarded recent messages, measured as JSON-serialized bytes (not + // raw text runes). The JSON wrapping inflates the count relative to + // user-visible text, so the effective text budget is smaller than the + // number suggests. The walk stops at the first message that would + // overflow, trading breadth for contiguity. + advisorConversationJSONByteBudget = 12000 + // advisorSystemJSONByteBudget caps the combined size of inherited + // system messages forwarded to the advisor. Without a cap, a large + // parent system prompt (long injected instructions, accumulated + // context) could push the advisor call past the model's context + // window on top of the advisor contract, the recent tail, and the + // question, surfacing as a provider error instead of advice. + advisorSystemJSONByteBudget = 12000 + defaultAdvisorQuestion = "Provide concise strategic guidance for the parent agent." +) + +// BuildAdvisorMessages prepares a nested advisor prompt using the recent chat +// context plus the explicit advisor question. +func BuildAdvisorMessages( + question string, + conversationSnapshot []fantasy.Message, +) []fantasy.Message { + trimmedQuestion := strings.TrimSpace(question) + if trimmedQuestion == "" { + trimmedQuestion = defaultAdvisorQuestion + } + + messages := make([]fantasy.Message, 0, len(conversationSnapshot)+2) + + // Place inherited system messages before AdvisorSystemPrompt so the + // advisor contract is the final system instruction the model sees. + // Later system directives win when they conflict, and the parent's + // prompt may tell the model to address the end user directly or use + // tools. The advisor must override those behaviors, not be overridden + // by them. + // + // Walk system messages newest-to-oldest when consuming the byte + // budget so that truncation preserves the most recent directives. + // The parent may have injected recent safety or user-instruction + // blocks that should win over older foundational prompts, and later + // directives override earlier ones anyway. After selection, restore + // the original order before appending so the advisor still sees the + // parent's intended directive sequence. + inheritedSystem := make([]fantasy.Message, 0) + remainingSystemBudget := advisorSystemJSONByteBudget + for i := len(conversationSnapshot) - 1; i >= 0; i-- { + msg := conversationSnapshot[i] + if msg.Role != fantasy.MessageRoleSystem { + continue + } + messageBytes := messageJSONByteCount(msg) + if messageBytes > remainingSystemBudget { + // Skip oversized inherited system messages rather + // than forwarding them wholesale. A single massive + // parent system prompt could otherwise push the + // advisor prompt past the model's context window, + // returning a provider error instead of advice. + // Continue walking so smaller older directives can + // still contribute; stopping here would drop them + // solely because a newer sibling was oversized. + continue + } + inheritedSystem = append(inheritedSystem, cloneMessage(msg)) + remainingSystemBudget -= messageBytes + } + slices.Reverse(inheritedSystem) + messages = append(messages, inheritedSystem...) + messages = append(messages, textMessage(fantasy.MessageRoleSystem, AdvisorSystemPrompt)) + + recent := make([]fantasy.Message, 0, min(len(conversationSnapshot), advisorRecentMessageLimit)) + remainingBudget := advisorConversationJSONByteBudget + for i := len(conversationSnapshot) - 1; i >= 0; i-- { + msg := conversationSnapshot[i] + if msg.Role == fantasy.MessageRoleSystem { + continue + } + if len(recent) >= advisorRecentMessageLimit { + break + } + + messageBytes := messageJSONByteCount(msg) + if messageBytes > remainingBudget { + // Stop at the first message that doesn't fit so the + // advisor window stays contiguous from most recent + // backward. Skipping an oversized message would leave + // the advisor with an invisible hole in the history, + // where later messages reference context that is no + // longer present. + break + } + + recent = append(recent, cloneMessage(msg)) + remainingBudget -= messageBytes + } + slices.Reverse(recent) + recent = dropOrphanToolMessages(recent) + messages = append(messages, recent...) + messages = append(messages, textMessage(fantasy.MessageRoleUser, trimmedQuestion)) + return messages +} + +// dropOrphanToolMessages removes tool-role messages whose tool-call references +// have been truncated out of the recent window. Providers reject prompts with +// tool_result blocks that do not have a matching tool_use, so a truncation cut +// that lands between an assistant tool-call message and its tool-result message +// would otherwise produce a provider error rather than advice. The backward +// walk always picks up tool results before their originating assistant +// message, so orphan results can only appear at the leading edge of the +// recent window. A single forward pass tracking known tool-call IDs is +// sufficient to drop them. +func dropOrphanToolMessages(recent []fantasy.Message) []fantasy.Message { + if len(recent) == 0 { + return recent + } + known := make(map[string]struct{}) + result := make([]fantasy.Message, 0, len(recent)) + for _, msg := range recent { + if msg.Role == fantasy.MessageRoleAssistant { + for _, part := range msg.Content { + call, ok := fantasy.AsMessagePart[fantasy.ToolCallPart](part) + if !ok { + continue + } + known[call.ToolCallID] = struct{}{} + } + result = append(result, msg) + continue + } + if msg.Role != fantasy.MessageRoleTool { + result = append(result, msg) + continue + } + + kept := make([]fantasy.MessagePart, 0, len(msg.Content)) + for _, part := range msg.Content { + tr, ok := fantasy.AsMessagePart[fantasy.ToolResultPart](part) + if !ok { + kept = append(kept, part) + continue + } + if _, matched := known[tr.ToolCallID]; matched { + kept = append(kept, part) + } + } + if len(kept) == 0 { + continue + } + trimmed := msg + trimmed.Content = kept + result = append(result, trimmed) + } + return result +} + +func textMessage(role fantasy.MessageRole, text string) fantasy.Message { + return fantasy.Message{ + Role: role, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: text}, + }, + } +} + +func cloneMessage(msg fantasy.Message) fantasy.Message { + cloned := msg + cloned.Content = append([]fantasy.MessagePart(nil), msg.Content...) + cloned.ProviderOptions = maps.Clone(msg.ProviderOptions) + return cloned +} + +// messageJSONByteCount approximates the message's contribution to the +// advisor prompt using the length of its JSON serialization. The JSON +// wrapping ({"role":"...","content":[{"type":"text","text":"..."}]}) is +// counted alongside the user-visible text; the measurement is intended +// for budget accounting, not for reporting visible character counts. +func messageJSONByteCount(msg fantasy.Message) int { + data, err := json.Marshal(msg) + if err == nil { + return len(data) + } + + total := 0 + for _, part := range msg.Content { + partData, partErr := json.Marshal(part) + if partErr == nil { + total += len(partData) + } + } + return total +} diff --git a/coderd/x/chatd/chatadvisor/runner.go b/coderd/x/chatd/chatadvisor/runner.go new file mode 100644 index 0000000000..a3d144967c --- /dev/null +++ b/coderd/x/chatd/chatadvisor/runner.go @@ -0,0 +1,98 @@ +package chatadvisor + +import ( + "context" + "strings" + + "charm.land/fantasy" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/x/chatd/chatloop" +) + +// RunAdvisor executes a single, tool-less nested advisor call. +func (rt *Runtime) RunAdvisor( + ctx context.Context, + question string, + conversationSnapshot []fantasy.Message, +) (AdvisorResult, error) { + // Model, MaxUsesPerRun, and MaxOutputTokens are validated by NewRuntime. + // Runtime fields are unexported so callers cannot bypass that. + if strings.TrimSpace(question) == "" { + return AdvisorResult{}, xerrors.New("advisor question is required") + } + + if !rt.tryAcquire() { + return AdvisorResult{ + Type: ResultTypeLimitReached, + RemainingUses: 0, + }, nil + } + + // Clone per invocation and reset inherited state so chatloop cannot + // mutate the Runtime's stored options across calls, and so the nested + // call never runs as a chain-mode continuation against stale parent + // state or persists an orphan stored response on the provider side. + nestedProviderOptions := cloneProviderOptions(rt.cfg.ProviderOptions) + resetProviderOptionsForNestedCall(nestedProviderOptions) + + var persistedStep chatloop.PersistedStep + runOpts := chatloop.RunOptions{ + Model: rt.cfg.Model, + Messages: BuildAdvisorMessages(question, conversationSnapshot), + MaxSteps: 1, + ModelConfig: rt.cfg.ModelConfig, + ProviderOptions: nestedProviderOptions, + PersistStep: func(_ context.Context, step chatloop.PersistedStep) error { + persistedStep = step + return nil + }, + } + + if err := chatloop.Run(ctx, runOpts); err != nil { + // Refund the use so a transient provider failure does not + // permanently exhaust the per-run advisor budget. + rt.release() + return AdvisorResult{ + Type: ResultTypeError, + Error: err.Error(), + RemainingUses: rt.RemainingUses(), + }, nil + } + + advice := extractAdvisorText(persistedStep) + if advice == "" { + // Refund: the run did not produce advice, so the contract + // "increments on every successful advisor call" treats this + // as not consuming a use. + rt.release() + return AdvisorResult{ + Type: ResultTypeError, + Error: "advisor produced no text output", + RemainingUses: rt.RemainingUses(), + }, nil + } + + return AdvisorResult{ + Type: ResultTypeAdvice, + Advice: advice, + AdvisorModel: rt.cfg.Model.Provider() + "/" + rt.cfg.Model.Model(), + RemainingUses: rt.RemainingUses(), + }, nil +} + +func extractAdvisorText(step chatloop.PersistedStep) string { + parts := make([]string, 0, len(step.Content)) + for _, content := range step.Content { + text, ok := fantasy.AsContentType[fantasy.TextContent](content) + if !ok { + continue + } + trimmed := strings.TrimSpace(text.Text) + if trimmed == "" { + continue + } + parts = append(parts, trimmed) + } + return strings.TrimSpace(strings.Join(parts, "\n\n")) +} diff --git a/coderd/x/chatd/chatadvisor/runner_test.go b/coderd/x/chatd/chatadvisor/runner_test.go new file mode 100644 index 0000000000..ec81328274 --- /dev/null +++ b/coderd/x/chatd/chatadvisor/runner_test.go @@ -0,0 +1,585 @@ +package chatadvisor_test + +import ( + "context" + "fmt" + "iter" + "strings" + "testing" + + "charm.land/fantasy" + fantasyopenai "charm.land/fantasy/providers/openai" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/x/chatd/chatadvisor" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" + "github.com/coder/coder/v2/codersdk" +) + +func TestAdvisorRunAdvice(t *testing.T) { + t.Parallel() + + const ( + question = "What is the smallest safe change?" + maxOutputTokens = int64(321) + ) + + var capturedCall fantasy.Call + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + capturedCall = call + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "Take the smallest safe change."}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + MaxUsesPerRun: 2, + MaxOutputTokens: maxOutputTokens, + }) + require.NoError(t, err) + + result, err := runtime.RunAdvisor(t.Context(), question, []fantasy.Message{ + textMessage(fantasy.MessageRoleSystem, "existing system"), + textMessage(fantasy.MessageRoleUser, "hello"), + }) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type) + require.Equal(t, "Take the smallest safe change.", result.Advice) + require.Equal(t, "test-provider/test-model", result.AdvisorModel) + require.Equal(t, 1, result.RemainingUses) + + require.Empty(t, capturedCall.Tools) + require.NotNil(t, capturedCall.MaxOutputTokens) + require.Equal(t, maxOutputTokens, *capturedCall.MaxOutputTokens) + require.NotEmpty(t, capturedCall.Prompt) + require.Equal(t, fantasy.MessageRoleUser, capturedCall.Prompt[len(capturedCall.Prompt)-1].Role) + require.Equal(t, question, singleText(t, capturedCall.Prompt[len(capturedCall.Prompt)-1])) +} + +func TestAdvisorRunLimitReached(t *testing.T) { + t.Parallel() + + var calls int + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + calls++ + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "first answer"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + first, err := runtime.RunAdvisor(t.Context(), "first?", nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeAdvice, first.Type) + require.Equal(t, 0, first.RemainingUses) + + second, err := runtime.RunAdvisor(t.Context(), "second?", nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeLimitReached, second.Type) + require.Equal(t, 0, second.RemainingUses) + require.Equal(t, 1, calls) +} + +func TestAdvisorRunError(t *testing.T) { + t.Parallel() + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return nil, xerrors.New("boom") + }, + }, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + result, err := runtime.RunAdvisor(t.Context(), "what failed?", nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeError, result.Type) + require.Contains(t, result.Error, "boom") + // A transient nested run failure must not consume quota: callers + // can retry up to MaxUsesPerRun times despite the failure. + require.Equal(t, 1, result.RemainingUses) + + // Confirm the refund left the runtime in a usable state by issuing + // a successful call after the failure, even though MaxUsesPerRun=1. + runtime2, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func() func(context.Context, fantasy.Call) (fantasy.StreamResponse, error) { + var calls int + return func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + calls++ + if calls == 1 { + return nil, xerrors.New("boom") + } + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "recovered"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + } + }(), + }, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + failed, err := runtime2.RunAdvisor(t.Context(), "first?", nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeError, failed.Type) + require.Equal(t, 1, failed.RemainingUses) + + retried, err := runtime2.RunAdvisor(t.Context(), "retry?", nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeAdvice, retried.Type) + require.Equal(t, "recovered", retried.Advice) + require.Equal(t, 0, retried.RemainingUses) +} + +func TestNewRuntimeValidation(t *testing.T) { + t.Parallel() + + matchingTokens := int64(64) + mismatchedTokens := int64(32) + model := &chattest.FakeModel{ProviderName: "test-provider", ModelName: "test-model"} + + tests := []struct { + name string + cfg chatadvisor.RuntimeConfig + errText string + }{ + { + name: "NilModel", + cfg: chatadvisor.RuntimeConfig{MaxUsesPerRun: 1, MaxOutputTokens: 64}, + errText: "advisor model is required", + }, + { + name: "NonPositiveMaxUses", + cfg: chatadvisor.RuntimeConfig{ + Model: model, + MaxUsesPerRun: 0, + MaxOutputTokens: 64, + }, + errText: "advisor max uses per run must be positive", + }, + { + name: "NonPositiveMaxOutputTokens", + cfg: chatadvisor.RuntimeConfig{ + Model: model, + MaxUsesPerRun: 1, + MaxOutputTokens: 0, + }, + errText: "advisor max output tokens must be positive", + }, + { + name: "MismatchedModelConfigMaxOutputTokens", + cfg: chatadvisor.RuntimeConfig{ + Model: model, + MaxUsesPerRun: 1, + MaxOutputTokens: matchingTokens, + ModelConfig: codersdk.ChatModelCallConfig{ + MaxOutputTokens: &mismatchedTokens, + }, + }, + errText: "must match runtime max output tokens", + }, + } + + for _, testCase := range tests { + t.Run(testCase.name, func(t *testing.T) { + t.Parallel() + + _, err := chatadvisor.NewRuntime(testCase.cfg) + require.Error(t, err) + require.ErrorContains(t, err, testCase.errText) + }) + } +} + +func TestNewRuntimeDeepClonesOpenAIResponsesProviderOptions(t *testing.T) { + t.Parallel() + + parentPrevID := "resp_parent_abc123" + parentOpts := &fantasyopenai.ResponsesProviderOptions{ + PreviousResponseID: &parentPrevID, + } + parentProviderOpts := fantasy.ProviderOptions{ + fantasyopenai.Name: parentOpts, + } + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "advice"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + ProviderOptions: parentProviderOpts, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + result, err := runtime.RunAdvisor(t.Context(), "anything?", nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type) + + // Parent's OpenAI Responses entry must still carry its PreviousResponseID; + // the advisor's nested chatloop run must not have mutated the shared pointer. + require.NotNil(t, parentOpts.PreviousResponseID) + require.Equal(t, parentPrevID, *parentOpts.PreviousResponseID) +} + +func TestAdvisorRunStripsChainStateAndIsConsistentAcrossCalls(t *testing.T) { + t.Parallel() + + parentPrevID := "resp_parent_xyz" + parentOpts := &fantasyopenai.ResponsesProviderOptions{ + PreviousResponseID: &parentPrevID, + } + parentProviderOpts := fantasy.ProviderOptions{ + fantasyopenai.Name: parentOpts, + } + + // Snapshot PreviousResponseID and Store at stream time, before chatloop + // has any chance to clear them on the shared map. Comparing across calls + // proves the advisor observes consistent (non-chained, non-persisted) + // options each invocation. + type observedOpts struct { + prevID *string + store *bool + } + var observed []observedOpts + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, call fantasy.Call) (fantasy.StreamResponse, error) { + openaiOpts, ok := call.ProviderOptions[fantasyopenai.Name].(*fantasyopenai.ResponsesProviderOptions) + if !ok { + observed = append(observed, observedOpts{}) + } else { + snap := observedOpts{} + if openaiOpts.PreviousResponseID != nil { + copied := *openaiOpts.PreviousResponseID + snap.prevID = &copied + } + if openaiOpts.Store != nil { + copied := *openaiOpts.Store + snap.store = &copied + } + observed = append(observed, snap) + } + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "advice"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + ProviderOptions: parentProviderOpts, + MaxUsesPerRun: 2, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + for i := range 2 { + result, err := runtime.RunAdvisor(t.Context(), fmt.Sprintf("q%d", i), nil) + require.NoError(t, err) + require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type) + } + + require.Len(t, observed, 2) + for i, snap := range observed { + // Each nested call must run without chain mode so prompts built + // from full history by BuildAdvisorMessages are accepted. + require.Nil(t, snap.prevID, "call %d unexpectedly ran in chain mode", i) + // Store must be explicitly disabled so the provider does not + // persist an orphan response that later chain-mode calls would + // fail to resume. + require.NotNil(t, snap.store, "call %d did not disable Store", i) + require.False(t, *snap.store, "call %d ran with Store enabled", i) + } + + // The parent's pointer must be untouched across repeated advisor runs. + require.NotNil(t, parentOpts.PreviousResponseID) + require.Equal(t, parentPrevID, *parentOpts.PreviousResponseID) +} + +func TestBuildAdvisorMessagesTruncatesToRecentMessageLimit(t *testing.T) { + t.Parallel() + + snapshot := []fantasy.Message{textMessage(fantasy.MessageRoleSystem, "existing system")} + for i := range 25 { + snapshot = append(snapshot, textMessage(fantasy.MessageRoleUser, fmt.Sprintf("msg-%02d", i))) + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + // cloned existing system + advisor system + 20 most recent user messages + question. + require.Len(t, messages, 23) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Equal(t, "existing system", singleText(t, messages[0])) + require.Equal(t, fantasy.MessageRoleSystem, messages[1].Role) + require.Contains(t, singleText(t, messages[1]), "parent agent") + require.Equal(t, "msg-05", singleText(t, messages[2])) + require.Equal(t, "msg-24", singleText(t, messages[len(messages)-2])) + require.Equal(t, "Need advice", singleText(t, messages[len(messages)-1])) +} + +func TestBuildAdvisorMessagesStopsAtOversizedMessage(t *testing.T) { + t.Parallel() + + // The walk is backward from the end of the snapshot. user-late fits, + // the oversized assistant message breaks the walk, and user-early is + // never reached. This preserves contiguity: the advisor never sees a + // message that references missing context. + snapshot := []fantasy.Message{ + textMessage(fantasy.MessageRoleSystem, "existing system"), + textMessage(fantasy.MessageRoleUser, "user-early"), + textMessage(fantasy.MessageRoleAssistant, strings.Repeat("x", 20000)), + textMessage(fantasy.MessageRoleUser, "user-late"), + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + require.Len(t, messages, 4) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Equal(t, "existing system", singleText(t, messages[0])) + require.Equal(t, fantasy.MessageRoleSystem, messages[1].Role) + require.Contains(t, singleText(t, messages[1]), "parent agent") + require.Equal(t, "user-late", singleText(t, messages[2])) + require.Equal(t, "Need advice", singleText(t, messages[3])) + + for _, msg := range messages { + require.NotContains(t, singleText(t, msg), strings.Repeat("x", 100)) + } +} + +func TestBuildAdvisorMessagesPlacesAdvisorPromptAfterInheritedSystem(t *testing.T) { + t.Parallel() + + snapshot := []fantasy.Message{ + textMessage(fantasy.MessageRoleSystem, "parent-first"), + textMessage(fantasy.MessageRoleSystem, "parent-second"), + textMessage(fantasy.MessageRoleUser, "hello"), + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + + // Inherited system messages come first in their original order, then + // the advisor contract, then the recent tail, then the question. + // This ordering makes the advisor prompt the last system directive + // so it wins over conflicting parent instructions. + require.Len(t, messages, 5) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Equal(t, "parent-first", singleText(t, messages[0])) + require.Equal(t, fantasy.MessageRoleSystem, messages[1].Role) + require.Equal(t, "parent-second", singleText(t, messages[1])) + require.Equal(t, fantasy.MessageRoleSystem, messages[2].Role) + require.Contains(t, singleText(t, messages[2]), "parent agent") + require.Equal(t, fantasy.MessageRoleUser, messages[3].Role) + require.Equal(t, "hello", singleText(t, messages[3])) + require.Equal(t, fantasy.MessageRoleUser, messages[4].Role) + require.Equal(t, "Need advice", singleText(t, messages[4])) +} + +func TestBuildAdvisorMessagesDropsOversizedInheritedSystem(t *testing.T) { + t.Parallel() + + // A single oversized parent system message is skipped so it cannot + // push the advisor prompt past the model's context window. Smaller + // system messages that fit the budget survive, as do later non-system + // messages. + snapshot := []fantasy.Message{ + textMessage(fantasy.MessageRoleSystem, "small-system"), + textMessage(fantasy.MessageRoleSystem, strings.Repeat("x", 20000)), + textMessage(fantasy.MessageRoleUser, "hello"), + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + + // small-system + advisor system + recent user + question. The + // oversized inherited system message must not appear. + require.Len(t, messages, 4) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Equal(t, "small-system", singleText(t, messages[0])) + require.Equal(t, fantasy.MessageRoleSystem, messages[1].Role) + require.Contains(t, singleText(t, messages[1]), "parent agent") + require.Equal(t, fantasy.MessageRoleUser, messages[2].Role) + require.Equal(t, "hello", singleText(t, messages[2])) + require.Equal(t, fantasy.MessageRoleUser, messages[3].Role) + require.Equal(t, "Need advice", singleText(t, messages[3])) + + for _, msg := range messages { + require.NotContains(t, singleText(t, msg), strings.Repeat("x", 100)) + } +} + +func TestBuildAdvisorMessagesPrefersNewestSystemDirectivesUnderBudget(t *testing.T) { + t.Parallel() + + // Two parent system messages together exceed the advisor system byte + // budget, so one must be dropped. Later directives override earlier + // ones when they conflict, so the advisor must receive the newest + // directive and drop the older one. Preserve original order among + // messages that survive so the parent's intended directive sequence + // is unchanged. + const payload = 9000 + snapshot := []fantasy.Message{ + textMessage(fantasy.MessageRoleSystem, "older-"+strings.Repeat("a", payload)), + textMessage(fantasy.MessageRoleSystem, "newer-"+strings.Repeat("b", payload)), + textMessage(fantasy.MessageRoleUser, "hello"), + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + + // newer parent system + advisor system + recent user + question. The + // older system message must be dropped because the newer directive + // consumed the remaining budget. + require.Len(t, messages, 4) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Contains(t, singleText(t, messages[0]), "newer-") + require.NotContains(t, singleText(t, messages[0]), "older-") + require.Equal(t, fantasy.MessageRoleSystem, messages[1].Role) + require.Contains(t, singleText(t, messages[1]), "parent agent") + require.Equal(t, fantasy.MessageRoleUser, messages[2].Role) + require.Equal(t, "hello", singleText(t, messages[2])) + require.Equal(t, fantasy.MessageRoleUser, messages[3].Role) + require.Equal(t, "Need advice", singleText(t, messages[3])) +} + +func TestBuildAdvisorMessagesDropsOrphanToolResults(t *testing.T) { + t.Parallel() + + // Simulate a truncation cut that lands between the assistant tool-call + // message and its tool-result. The resulting recent window should not + // contain an orphan tool_result referencing a missing tool_use block. + // Building the window with only [tool_result, assistant_reply] mimics + // the state produced by the backward walk hitting its byte budget right + // before the tool-call assistant message. + snapshot := []fantasy.Message{ + toolResultMessage("call-1", "ok"), + textMessage(fantasy.MessageRoleAssistant, "final reply"), + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + + // Advisor system + assistant reply + question. The orphan tool result + // must not appear in the advisor prompt. + require.Len(t, messages, 3) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Contains(t, singleText(t, messages[0]), "parent agent") + require.Equal(t, fantasy.MessageRoleAssistant, messages[1].Role) + require.Equal(t, "final reply", singleText(t, messages[1])) + require.Equal(t, fantasy.MessageRoleUser, messages[2].Role) + require.Equal(t, "Need advice", singleText(t, messages[2])) + + for _, msg := range messages { + require.NotEqual(t, fantasy.MessageRoleTool, msg.Role) + } +} + +func TestBuildAdvisorMessagesKeepsPairedToolCallAndResult(t *testing.T) { + t.Parallel() + + snapshot := []fantasy.Message{ + toolCallAssistantMessage("call-1", "search", `{"q":"x"}`), + toolResultMessage("call-1", "ok"), + textMessage(fantasy.MessageRoleAssistant, "done"), + } + + messages := chatadvisor.BuildAdvisorMessages("Need advice", snapshot) + + // Advisor system + assistant tool call + tool result + assistant reply + // + question. The matched pair must survive. + require.Len(t, messages, 5) + require.Equal(t, fantasy.MessageRoleSystem, messages[0].Role) + require.Equal(t, fantasy.MessageRoleAssistant, messages[1].Role) + require.Equal(t, fantasy.MessageRoleTool, messages[2].Role) + require.Equal(t, fantasy.MessageRoleAssistant, messages[3].Role) + require.Equal(t, "done", singleText(t, messages[3])) + require.Equal(t, fantasy.MessageRoleUser, messages[4].Role) +} + +func streamFromParts(parts []fantasy.StreamPart) fantasy.StreamResponse { + return iter.Seq[fantasy.StreamPart](func(yield func(fantasy.StreamPart) bool) { + for _, part := range parts { + if !yield(part) { + return + } + } + }) +} + +func textMessage(role fantasy.MessageRole, text string) fantasy.Message { + return fantasy.Message{ + Role: role, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: text}, + }, + } +} + +func toolCallAssistantMessage(callID, name, input string) fantasy.Message { + return fantasy.Message{ + Role: fantasy.MessageRoleAssistant, + Content: []fantasy.MessagePart{ + fantasy.ToolCallPart{ + ToolCallID: callID, + ToolName: name, + Input: input, + }, + }, + } +} + +func toolResultMessage(callID, text string) fantasy.Message { + return fantasy.Message{ + Role: fantasy.MessageRoleTool, + Content: []fantasy.MessagePart{ + fantasy.ToolResultPart{ + ToolCallID: callID, + Output: fantasy.ToolResultOutputContentText{Text: text}, + }, + }, + } +} + +func singleText(t *testing.T, msg fantasy.Message) string { + t.Helper() + require.NotEmpty(t, msg.Content) + text, ok := fantasy.AsMessagePart[fantasy.TextPart](msg.Content[0]) + require.True(t, ok) + return text.Text +} diff --git a/coderd/x/chatd/chatadvisor/runtime.go b/coderd/x/chatd/chatadvisor/runtime.go new file mode 100644 index 0000000000..e5ca864d28 --- /dev/null +++ b/coderd/x/chatd/chatadvisor/runtime.go @@ -0,0 +1,142 @@ +package chatadvisor + +import ( + "sync/atomic" + + "charm.land/fantasy" + fantasyopenai "charm.land/fantasy/providers/openai" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/codersdk" +) + +// RuntimeConfig configures a single advisor runtime instance. +type RuntimeConfig struct { + Model fantasy.LanguageModel + ModelConfig codersdk.ChatModelCallConfig + ProviderOptions fantasy.ProviderOptions + MaxUsesPerRun int + MaxOutputTokens int64 +} + +// Runtime executes nested, tool-less advisor runs against the configured +// language model. +// +// Each Runtime instance is scoped to a single outer chat run. The +// MaxUsesPerRun counter increments on every successful advisor call and +// is never reset, so callers must construct a fresh Runtime (via +// NewRuntime) for each outer run. There is intentionally no Reset method: +// the per-run quota is a safety bound on a single run, not a rolling +// window. +type Runtime struct { + cfg RuntimeConfig + used atomic.Int64 +} + +// NewRuntime validates and normalizes advisor runtime configuration. +func NewRuntime(cfg RuntimeConfig) (*Runtime, error) { + if cfg.Model == nil { + return nil, xerrors.New("advisor model is required") + } + if cfg.MaxUsesPerRun <= 0 { + return nil, xerrors.New("advisor max uses per run must be positive") + } + if cfg.MaxOutputTokens <= 0 { + return nil, xerrors.New("advisor max output tokens must be positive") + } + if cfg.ModelConfig.MaxOutputTokens != nil && + *cfg.ModelConfig.MaxOutputTokens != cfg.MaxOutputTokens { + return nil, xerrors.Errorf( + "advisor model_config.max_output_tokens (%d) must match runtime max output tokens (%d)", + *cfg.ModelConfig.MaxOutputTokens, + cfg.MaxOutputTokens, + ) + } + + normalized := cfg + normalized.ProviderOptions = cloneProviderOptions(cfg.ProviderOptions) + maxOutputTokens := cfg.MaxOutputTokens + normalized.ModelConfig.MaxOutputTokens = &maxOutputTokens + + return &Runtime{cfg: normalized}, nil +} + +// cloneProviderOptions returns a copy of opts with pointer entries for known, +// in-place mutated provider option types replaced by a shallow struct copy. +// chatloop mutates the OpenAI Responses entry (PreviousResponseID) on +// chain-mode exit, so sharing the pointer with the parent run would let an +// advisor call corrupt the parent's chain state. Value fields such as +// Metadata and Include are still shared with the parent; nothing in this +// package mutates them, but callers that need true deep-copy semantics must +// handle those fields explicitly. +func cloneProviderOptions(opts fantasy.ProviderOptions) fantasy.ProviderOptions { + if opts == nil { + return nil + } + cloned := make(fantasy.ProviderOptions, len(opts)) + for key, value := range opts { + switch typed := value.(type) { + case *fantasyopenai.ResponsesProviderOptions: + if typed == nil { + cloned[key] = value + continue + } + copied := *typed + cloned[key] = &copied + default: + cloned[key] = value + } + } + return cloned +} + +// resetProviderOptionsForNestedCall strips inherited state from opts that +// does not apply to an ephemeral advisor call. PreviousResponseID is +// cleared so the nested call is not sent as a chain-mode continuation +// (BuildAdvisorMessages sends the full history, not an incremental turn). +// Store is forced off so the advisor call does not persist an orphan +// response on the provider side. Must be called on a cloned map to avoid +// mutating shared parent state. +func resetProviderOptionsForNestedCall(opts fantasy.ProviderOptions) { + for _, value := range opts { + if typed, ok := value.(*fantasyopenai.ResponsesProviderOptions); ok && typed != nil { + storeDisabled := false + typed.PreviousResponseID = nil + typed.Store = &storeDisabled + } + } +} + +// RemainingUses reports how many advisor calls are still available for the +// current runtime. +func (rt *Runtime) RemainingUses() int { + if rt == nil || rt.cfg.MaxUsesPerRun <= 0 { + return 0 + } + + remaining := int64(rt.cfg.MaxUsesPerRun) - rt.used.Load() + if remaining < 0 { + return 0 + } + return int(remaining) +} + +func (rt *Runtime) tryAcquire() bool { + for { + used := rt.used.Load() + if used >= int64(rt.cfg.MaxUsesPerRun) { + return false + } + if rt.used.CompareAndSwap(used, used+1) { + return true + } + } +} + +// release returns a previously acquired use to the pool. Callers must +// invoke this at most once per successful tryAcquire when the advisor +// call did not complete successfully, so a transient provider failure +// does not permanently consume quota for the run. +func (rt *Runtime) release() { + rt.used.Add(-1) +} diff --git a/coderd/x/chatd/chatadvisor/tool.go b/coderd/x/chatd/chatadvisor/tool.go new file mode 100644 index 0000000000..bb1de5e01b --- /dev/null +++ b/coderd/x/chatd/chatadvisor/tool.go @@ -0,0 +1,60 @@ +package chatadvisor + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "unicode/utf8" + + "charm.land/fantasy" +) + +// advisorQuestionMaxRunes caps the parent agent's question at a length +// that leaves room in the advisor prompt for system preamble and recent +// conversation context. +const advisorQuestionMaxRunes = 2000 + +// ToolOptions configures the built-in advisor tool. +type ToolOptions struct { + Runtime *Runtime + GetConversationSnapshot func() []fantasy.Message +} + +// Tool returns a fantasy.AgentTool that asks a nested model for concise +// strategic guidance. The nested advisor sees recent conversation +// context, runs without tools, and is limited to a single model step. +func Tool(opts ToolOptions) fantasy.AgentTool { + return fantasy.NewAgentTool( + "advisor", + "Ask a separate advisor pass for strategic guidance about planning, architecture, tradeoffs, or debugging strategy. Provide a brief question. The advisor sees recent conversation context, runs without tools for a single step, and responds to the parent agent rather than the end user.", + func(ctx context.Context, args AdvisorArgs, _ fantasy.ToolCall) (fantasy.ToolResponse, error) { + if opts.Runtime == nil { + return fantasy.NewTextErrorResponse("advisor runtime is not configured"), nil + } + if opts.GetConversationSnapshot == nil { + return fantasy.NewTextErrorResponse("conversation snapshot provider is not configured"), nil + } + + question := strings.TrimSpace(args.Question) + if question == "" { + return fantasy.NewTextErrorResponse("question is required"), nil + } + if utf8.RuneCountInString(question) > advisorQuestionMaxRunes { + return fantasy.NewTextErrorResponse( + fmt.Sprintf("question must be %d runes or fewer", advisorQuestionMaxRunes), + ), nil + } + + result, err := opts.Runtime.RunAdvisor(ctx, question, opts.GetConversationSnapshot()) + if err != nil { + return fantasy.NewTextErrorResponse(err.Error()), nil + } + data, err := json.Marshal(result) + if err != nil { + return fantasy.NewTextResponse("{}"), nil + } + return fantasy.NewTextResponse(string(data)), nil + }, + ) +} diff --git a/coderd/x/chatd/chatadvisor/tool_test.go b/coderd/x/chatd/chatadvisor/tool_test.go new file mode 100644 index 0000000000..8208d054f8 --- /dev/null +++ b/coderd/x/chatd/chatadvisor/tool_test.go @@ -0,0 +1,266 @@ +package chatadvisor_test + +import ( + "context" + "encoding/json" + "strings" + "testing" + + "charm.land/fantasy" + "github.com/stretchr/testify/require" + "golang.org/x/xerrors" + + "github.com/coder/coder/v2/coderd/x/chatd/chatadvisor" + "github.com/coder/coder/v2/coderd/x/chatd/chattest" +) + +func TestAdvisorToolSuccess(t *testing.T) { + t.Parallel() + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "Use the smaller diff."}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + MaxUsesPerRun: 2, + MaxOutputTokens: 128, + }) + require.NoError(t, err) + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + Runtime: runtime, + GetConversationSnapshot: func() []fantasy.Message { + return []fantasy.Message{{ + Role: fantasy.MessageRoleUser, + Content: []fantasy.MessagePart{ + fantasy.TextPart{Text: "We need a safe fix."}, + }, + }} + }, + }) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "What's the safest next step?"}) + require.False(t, resp.IsError) + + var result chatadvisor.AdvisorResult + require.NoError(t, json.Unmarshal([]byte(resp.Content), &result)) + require.Equal(t, chatadvisor.ResultTypeAdvice, result.Type) + require.Equal(t, "Use the smaller diff.", result.Advice) + require.Equal(t, "test-provider/test-model", result.AdvisorModel) + require.Equal(t, 1, result.RemainingUses) +} + +func TestAdvisorToolRejectsEmptyQuestion(t *testing.T) { + t.Parallel() + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + Runtime: mustAdvisorRuntime(t), + GetConversationSnapshot: func() []fantasy.Message { + return nil + }, + }) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: " \t\n "}) + require.True(t, resp.IsError) + require.Contains(t, resp.Content, "question is required") +} + +func TestAdvisorToolRejectsLongQuestion(t *testing.T) { + t.Parallel() + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + Runtime: mustAdvisorRuntime(t), + GetConversationSnapshot: func() []fantasy.Message { + return nil + }, + }) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: strings.Repeat("x", 2001)}) + require.True(t, resp.IsError) + require.Contains(t, resp.Content, "2000 runes or fewer") +} + +func TestAdvisorToolRejectsMissingRuntime(t *testing.T) { + t.Parallel() + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + GetConversationSnapshot: func() []fantasy.Message { + return nil + }, + }) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "Need advice"}) + require.True(t, resp.IsError) + require.Contains(t, resp.Content, "advisor runtime is not configured") +} + +func TestAdvisorToolRejectsMissingSnapshotFunc(t *testing.T) { + t.Parallel() + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{Runtime: mustAdvisorRuntime(t)}) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "Need advice"}) + require.True(t, resp.IsError) + require.Contains(t, resp.Content, "conversation snapshot provider is not configured") +} + +func TestAdvisorToolReportsNestedError(t *testing.T) { + t.Parallel() + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return nil, xerrors.New("boom") + }, + }, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + Runtime: runtime, + GetConversationSnapshot: func() []fantasy.Message { return nil }, + }) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "why?"}) + require.False(t, resp.IsError) + + var result chatadvisor.AdvisorResult + require.NoError(t, json.Unmarshal([]byte(resp.Content), &result)) + require.Equal(t, chatadvisor.ResultTypeError, result.Type) + require.Contains(t, result.Error, "boom") + require.Empty(t, result.Advice) + require.Empty(t, result.AdvisorModel) + // A failed nested run does not consume the per-run quota. + require.Equal(t, 1, result.RemainingUses) +} + +func TestAdvisorToolReportsLimitReached(t *testing.T) { + t.Parallel() + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "first"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + Runtime: runtime, + GetConversationSnapshot: func() []fantasy.Message { return nil }, + }) + + first := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "first?"}) + require.False(t, first.IsError) + + second := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "second?"}) + require.False(t, second.IsError) + + var result chatadvisor.AdvisorResult + require.NoError(t, json.Unmarshal([]byte(second.Content), &result)) + require.Equal(t, chatadvisor.ResultTypeLimitReached, result.Type) + require.Equal(t, 0, result.RemainingUses) + require.Empty(t, result.Advice) + require.Empty(t, result.Error) + require.Empty(t, result.AdvisorModel) +} + +func TestAdvisorToolReportsEmptyModelOutput(t *testing.T) { + t.Parallel() + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + MaxUsesPerRun: 1, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + + tool := chatadvisor.Tool(chatadvisor.ToolOptions{ + Runtime: runtime, + GetConversationSnapshot: func() []fantasy.Message { return nil }, + }) + + resp := runAdvisorTool(t, tool, chatadvisor.AdvisorArgs{Question: "anything?"}) + require.False(t, resp.IsError) + + var result chatadvisor.AdvisorResult + require.NoError(t, json.Unmarshal([]byte(resp.Content), &result)) + require.Equal(t, chatadvisor.ResultTypeError, result.Type) + require.Contains(t, result.Error, "no text output") + require.Empty(t, result.Advice) + // An advisor call that produces no advice does not count as a + // successful use, so the quota must still be available. + require.Equal(t, 1, result.RemainingUses) +} + +func mustAdvisorRuntime(t *testing.T) *chatadvisor.Runtime { + t.Helper() + + runtime, err := chatadvisor.NewRuntime(chatadvisor.RuntimeConfig{ + Model: &chattest.FakeModel{ + ProviderName: "test-provider", + ModelName: "test-model", + StreamFn: func(_ context.Context, _ fantasy.Call) (fantasy.StreamResponse, error) { + return streamFromParts([]fantasy.StreamPart{ + {Type: fantasy.StreamPartTypeTextStart, ID: "text-1"}, + {Type: fantasy.StreamPartTypeTextDelta, ID: "text-1", Delta: "fallback advice"}, + {Type: fantasy.StreamPartTypeTextEnd, ID: "text-1"}, + {Type: fantasy.StreamPartTypeFinish, FinishReason: fantasy.FinishReasonStop}, + }), nil + }, + }, + MaxUsesPerRun: 2, + MaxOutputTokens: 64, + }) + require.NoError(t, err) + return runtime +} + +func runAdvisorTool( + t *testing.T, + tool fantasy.AgentTool, + args chatadvisor.AdvisorArgs, +) fantasy.ToolResponse { + t.Helper() + + data, err := json.Marshal(args) + require.NoError(t, err) + + resp, err := tool.Run(t.Context(), fantasy.ToolCall{ + ID: "call-1", + Name: "advisor", + Input: string(data), + }) + require.NoError(t, err) + return resp +} diff --git a/coderd/x/chatd/chatadvisor/types.go b/coderd/x/chatd/chatadvisor/types.go new file mode 100644 index 0000000000..c537e53f28 --- /dev/null +++ b/coderd/x/chatd/chatadvisor/types.go @@ -0,0 +1,28 @@ +package chatadvisor + +// ResultType is the tagged variant of AdvisorResult. Callers should +// compare against the exported constants rather than string literals. +type ResultType string + +const ( + // ResultTypeAdvice indicates the advisor returned guidance. + ResultTypeAdvice ResultType = "advice" + // ResultTypeLimitReached indicates the per-run advisor budget is exhausted. + ResultTypeLimitReached ResultType = "limit_reached" + // ResultTypeError indicates the nested advisor run failed. + ResultTypeError ResultType = "error" +) + +// AdvisorArgs contains the tool-visible advisor question. +type AdvisorArgs struct { + Question string `json:"question"` +} + +// AdvisorResult is the structured result returned by the advisor runtime. +type AdvisorResult struct { + Type ResultType `json:"type"` + Advice string `json:"advice,omitempty"` + Error string `json:"error,omitempty"` + AdvisorModel string `json:"advisor_model,omitempty"` + RemainingUses int `json:"remaining_uses"` +}