Files
WeKnora/internal/application/service/session_agent_qa.go
T
wizardchen ffcb10f953 feat(memory): add cross-session long-term memory
Persist extracted and explicit memories across sessions with topic
resolution, semantic recall, document affinity, consolidation, workspace
settings, and inbox/interests UI.
2026-08-13 18:00:02 +08:00

587 lines
23 KiB
Go

package service
import (
"context"
"encoding/json"
"errors"
"fmt"
"github.com/Tencent/WeKnora/internal/agent/tools"
"github.com/Tencent/WeKnora/internal/event"
"github.com/Tencent/WeKnora/internal/logger"
"github.com/Tencent/WeKnora/internal/models/chat"
"github.com/Tencent/WeKnora/internal/models/rerank"
"github.com/Tencent/WeKnora/internal/types"
)
// AgentQA performs agent-based question answering with conversation history and streaming support
// customAgent is optional - if provided, uses custom agent configuration instead of tenant defaults
// summaryModelID is optional - if provided, overrides the model from customAgent config
func (s *sessionService) AgentQA(
ctx context.Context,
req *types.QARequest,
eventBus *event.EventBus,
) error {
sessionID := req.Session.ID
// Propagate the session ID so stateful sandbox backends (CubeSandbox) can
// bind script execution to a per-session MicroVM instance.
ctx = types.WithSessionID(ctx, sessionID)
sessionJSON, err := json.Marshal(req.Session)
if err != nil {
logger.Errorf(ctx, "Failed to marshal session, session ID: %s, error: %v", sessionID, err)
return fmt.Errorf("failed to marshal session: %w", err)
}
// customAgent is required for AgentQA (handler has already done permission check for shared agent)
if req.CustomAgent == nil {
logger.Warnf(ctx, "Custom agent not provided for session: %s", sessionID)
return errors.New("custom agent configuration is required for agent QA")
}
// Resolve retrieval tenant using shared helper
agentTenantID := s.resolveRetrievalTenantID(ctx, req)
logger.Infof(ctx, "Start agent-based question answering, session ID: %s, agent tenant ID: %d, query: %s, session: %s",
sessionID, agentTenantID, req.Query, string(sessionJSON))
var tenantInfo *types.Tenant
if v := ctx.Value(types.TenantInfoContextKey); v != nil {
tenantInfo, _ = v.(*types.Tenant)
}
// When agent belongs to another tenant (shared agent), use agent's tenant for KB/model scope; load tenantInfo if needed
if tenantInfo == nil || tenantInfo.ID != agentTenantID {
if s.tenantService != nil {
if agentTenant, err := s.tenantService.GetTenantByID(ctx, agentTenantID); err == nil && agentTenant != nil {
tenantInfo = agentTenant
logger.Infof(ctx, "Using agent tenant info for retrieval scope, tenant ID: %d", agentTenantID)
}
}
}
if tenantInfo == nil {
logger.Warnf(ctx, "Tenant info not available for agent tenant %d, proceeding with defaults", agentTenantID)
tenantInfo = &types.Tenant{ID: agentTenantID}
}
// Ensure defaults are set
req.CustomAgent.EnsureDefaults()
// Build AgentConfig from custom agent and tenant info
agentConfig, err := s.buildAgentConfig(ctx, req, tenantInfo, agentTenantID)
if err != nil {
return err
}
// Set VLM model ID for tool result image analysis (runtime-only field)
if req.CustomAgent != nil && req.CustomAgent.Config.VLMModelID != "" {
agentConfig.VLMModelID = req.CustomAgent.Config.VLMModelID
}
// Resolve model ID using shared helper (AgentQA requires a model, so error if not found)
effectiveModelID, err := s.resolveChatModelID(ctx, req, agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs)
if err != nil {
return err
}
if effectiveModelID == "" {
logger.Warnf(ctx, "No summary model configured for custom agent %s", req.CustomAgent.ID)
return errors.New("summary model (model_id) is not configured in custom agent settings")
}
summaryModel, err := s.modelService.GetChatModel(ctx, effectiveModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get chat model: %v", err)
return fmt.Errorf("failed to get chat model: %w", err)
}
// Get rerank model from custom agent config only when knowledge_search can
// actually run. A disabled KB scope makes all KB tools ineffective, so it
// must not force users to configure an otherwise-unused rerank model.
var rerankModel rerank.Reranker
if agentRequiresRerankModel(req.CustomAgent) {
// Rerank model is resolved purely from the agent config now.
// We used to fall back to ConversationConfig.RerankModelID at
// the tenant level, but that path encouraged "leave rerank
// blank on the agent and inherit silently" which made debugging
// retrieval quality a guessing game across tenant settings vs
// agent settings. Forcing the agent to declare its own rerank
// model puts the configuration where the user actually edits
// the agent. If a Wiki-only agent doesn't need reranking,
// agentRequiresRerankModel() below already lets it pass.
rerankModelID := req.CustomAgent.Config.RerankModelID
if rerankModelID == "" {
logger.Warnf(ctx, "No rerank model configured for custom agent %s, but knowledge_search tool is enabled", req.CustomAgent.ID)
return errors.New("rerank model is not configured: please set rerank_model_id on the agent")
}
rerankModel, err = s.modelService.GetRerankModel(ctx, rerankModelID)
if err != nil {
logger.Warnf(ctx, "Failed to get rerank model: %v", err)
return fmt.Errorf("failed to get rerank model: %w", err)
}
} else {
logger.Infof(ctx, "knowledge_search is unavailable for the effective agent scope, skipping rerank model initialization")
}
// Load multi-turn history directly from DB (the single source of truth).
// AgentSteps on each historical assistant message are expanded into proper
// assistant_with_tool_calls + tool messages so the model can see what was
// tried last turn — except final_answer, which is replayed as the trailing
// canonical assistant message.
var llmContext []chat.Message
if agentConfig.MultiTurnEnabled {
historyTurns := agentConfig.HistoryTurns
if historyTurns <= 0 {
historyTurns = 5
}
llmContext, err = LoadAgentHistory(ctx, s.messageRepo, sessionID, historyTurns)
if err != nil {
logger.Warnf(ctx, "Failed to load agent history from DB: %v, continuing without history", err)
llmContext = []chat.Message{}
}
logger.Infof(ctx, "Loaded %d history messages from DB (turns=%d)", len(llmContext), historyTurns)
} else {
logger.Infof(ctx, "Multi-turn disabled for this agent, running without history")
llmContext = []chat.Message{}
}
// Reconcile all durable session attachments into the session's remote
// sandbox before the model can request shell or skill execution. The
// durable storage URL — not the ephemeral sandbox path — remains the
// source of truth. Gated on the sandbox manager advertising a session
// filesystem capability so provider-neutral remote wiring stays here.
var stagedAttachments []stagedSessionAttachment
stager, ok := s.agentService.(sessionAttachmentStager)
if !ok {
return errors.New("agent service does not support session attachment staging")
}
// Probe the backend this session's sandbox actually runs on. Gating on the
// process-wide manager instead could inspect a different backend than the
// named workspace config selected by this agent.
inputStore, storeErr := stager.sessionSandboxInputStore(ctx, sessionID, agentConfig.SandboxConfigID)
if storeErr != nil {
return fmt.Errorf("resolve sandbox file store for session %s: %w", sessionID, storeErr)
}
if inputStore != nil {
sessionAttachments, loadErr := s.messageRepo.GetSessionAttachments(ctx, sessionID)
if loadErr != nil {
return fmt.Errorf("load session attachments for sandbox staging: %w", loadErr)
}
stagedAttachments, err = stager.stageSessionAttachments(ctx, sessionID, agentConfig.SandboxConfigID, sessionAttachments)
if err != nil {
return fmt.Errorf("restore session attachments into sandbox: %w", err)
}
}
// Create agent engine with EventBus
logger.Info(ctx, "Creating agent engine")
engine, err := s.agentService.CreateAgentEngine(
ctx,
agentConfig,
summaryModel,
rerankModel,
eventBus,
sessionID,
req.AssistantMessageID,
)
if err != nil {
logger.Errorf(ctx, "Failed to create agent engine: %v", err)
return err
}
// Recall long-term memory for this turn. Like the RAG path this is a
// no-model read, and an agent may opt out of it entirely.
memoryCtx := types.ApplyAgentMemoryPreference(ctx, agentConfig.MemoryEnabled)
if s.memoryService != nil {
recall := s.memoryService.Recall(memoryCtx, req.Query)
if recall.Prompt != "" {
engine.SetMemoryPrompt(recall.Prompt)
used := types.UsedMemoriesFromItems(recall.Items)
if err := eventBus.Emit(ctx, event.Event{
Type: event.EventMemoryRecalled,
SessionID: sessionID,
Data: event.MemoryRecalledData{Memories: used},
}); err != nil {
logger.Warnf(ctx, "Failed to emit memory recalled event: %v", err)
}
logger.Infof(ctx, "Injected %d long-term memories into agent context", len(used))
}
}
// Route image data based on agent model's vision capability
var agentModelSupportsVision bool
if effectiveModelID != "" {
if modelInfo, err := s.modelService.GetModelByID(ctx, effectiveModelID); err == nil && modelInfo != nil {
agentModelSupportsVision = modelInfo.Parameters.SupportsVision
}
}
agentQuery := req.Query
var agentImageURLs []string
if agentModelSupportsVision && len(req.ImageURLs) > 0 {
agentImageURLs = req.ImageURLs
logger.Infof(ctx, "Agent model supports vision, passing %d image(s) directly", len(agentImageURLs))
} else if req.ImageDescription != "" {
agentQuery = req.Query + "\n\n[用户上传图片内容]\n" + req.ImageDescription
logger.Infof(ctx, "Agent model does not support vision, appending image description (%d chars)", len(req.ImageDescription))
}
if req.QuotedContext != "" {
agentQuery += "\n\n" + req.QuotedContext
}
// Inject attachment content (documents, audio transcripts, etc.) so the agent
// can see uploaded files. Mirrors the behavior of the KnowledgeQA pipeline
// (see chat_pipeline/into_chat_message.go).
if len(req.Attachments) > 0 {
agentQuery += req.Attachments.BuildPrompt()
logger.Infof(ctx, "Appended %d attachment(s) to agent query", len(req.Attachments))
}
if manifest := buildSandboxAttachmentsPrompt(stagedAttachments); manifest != "" {
agentQuery += manifest
logger.Infof(ctx, "Appended %d staged sandbox attachment path(s) to agent query", len(stagedAttachments))
}
// Scope envelopes (runtime_context / must_use) are injected per LLM call inside
// the agent engine only; we intentionally do not persist them on user messages
// so multi-turn history stays clean and is not skewed by stale @mention scope.
// Execute agent with streaming (asynchronously)
// Events will be emitted to EventBus and handled by the Handler layer
logger.Info(ctx, "Executing agent with streaming")
if _, err := engine.Execute(ctx, sessionID, req.AssistantMessageID, agentQuery, llmContext, agentImageURLs); err != nil {
logger.Errorf(ctx, "Agent execution failed: %v", err)
// Emit error event to the EventBus used by this agent
eventBus.Emit(ctx, event.Event{
Type: event.EventError,
SessionID: sessionID,
Data: event.ErrorData{
Error: err.Error(),
Stage: "agent_execution",
SessionID: sessionID,
},
})
}
// Return empty - events will be handled by Handler via EventBus subscription
return nil
}
// buildAgentConfig creates a runtime AgentConfig from the QARequest's custom agent configuration,
// tenant info, and resolved knowledge bases / search targets.
func (s *sessionService) buildAgentConfig(
ctx context.Context,
req *types.QARequest,
tenantInfo *types.Tenant,
agentTenantID uint64,
) (*types.AgentConfig, error) {
customAgent := req.CustomAgent
agentConfig := &types.AgentConfig{
MaxIterations: customAgent.Config.MaxIterations,
Temperature: customAgent.Config.Temperature,
WebSearchEnabled: customAgent.Config.WebSearchEnabled && req.WebSearchEnabled,
WebSearchMaxResults: customAgent.Config.WebSearchMaxResults,
WebSearchProviderID: customAgent.Config.WebSearchProviderID,
MultiTurnEnabled: customAgent.Config.MultiTurnEnabled,
HistoryTurns: customAgent.Config.HistoryTurns,
MCPSelectionMode: customAgent.Config.MCPSelectionMode,
MCPServices: customAgent.Config.MCPServices,
MCPAuthWaitTimeout: customAgent.Config.MCPAuthWaitTimeout,
Thinking: customAgent.Config.Thinking,
CitationEnabled: customAgent.Config.CitationEnabled,
RetrieveKBOnlyWhenMentioned: customAgent.Config.RetrieveKBOnlyWhenMentioned,
LLMCallTimeout: customAgent.Config.LLMCallTimeout,
RetainRetrievalHistory: customAgent.Config.RetainRetrievalHistory,
SharedAgentReadOnly: req.SharedAgentReadOnly,
}
// Falls back to global configuration if no specific timeout is set for the agent.
if agentConfig.LLMCallTimeout == 0 && s.cfg.Agent != nil && s.cfg.Agent.LLMCallTimeout > 0 {
agentConfig.LLMCallTimeout = s.cfg.Agent.LLMCallTimeout
}
// Configure skills based on CustomAgentConfig
s.configureSkillsFromAgent(ctx, agentConfig, customAgent)
// Resolve knowledge bases using shared helper
kbIDs, knowledgeIDs, err := s.resolveKnowledgeBases(ctx, req)
if err != nil {
return nil, err
}
agentConfig.KnowledgeBases = kbIDs
agentConfig.KnowledgeIDs = knowledgeIDs
// Use custom agent's allowed tools if specified, otherwise use defaults
if len(customAgent.Config.AllowedTools) > 0 {
agentConfig.AllowedTools = customAgent.Config.AllowedTools
} else {
agentConfig.AllowedTools = tools.DefaultAllowedTools()
}
// Apply per-turn @Skill / @MCP scope. Each helper narrows the agent's
// whitelist to the mentioned items and records the pinned set used for the
// <must_use> hint, keeping all scope logic in one place per resource type.
isSharedAgent := req.SharedAgentReadOnly
applyPerRequestSkillScope(ctx, agentConfig, customAgent.Config.SkillsSelectionMode, req.SkillNames)
applyPerRequestMCPScope(ctx, agentConfig, customAgent.Config.MCPServices, isSharedAgent, req.MCPServiceIDs)
// Use custom agent's system prompt if specified
if customAgent.Config.SystemPrompt != "" {
agentConfig.UseCustomSystemPrompt = true
agentConfig.SystemPrompt = customAgent.Config.SystemPrompt
}
logger.Infof(ctx, "Custom agent config applied: MaxIterations=%d, Temperature=%.2f, AllowedTools=%v, WebSearchEnabled=%v",
agentConfig.MaxIterations, agentConfig.Temperature, agentConfig.AllowedTools, agentConfig.WebSearchEnabled)
// Set web search max results from tenant config if not set (default: 5)
if agentConfig.WebSearchMaxResults == 0 {
agentConfig.WebSearchMaxResults = 5
if tenantInfo.WebSearchConfig != nil && tenantInfo.WebSearchConfig.MaxResults > 0 {
agentConfig.WebSearchMaxResults = tenantInfo.WebSearchConfig.MaxResults
}
}
// Resolve web search provider ID: agent-level > tenant default (is_default=true)
if agentConfig.WebSearchProviderID == "" {
if defaultProvider, err := s.webSearchProviderRepo.GetDefault(ctx, tenantInfo.ID); err == nil && defaultProvider != nil {
agentConfig.WebSearchProviderID = defaultProvider.ID
}
}
logger.Infof(ctx, "Merged agent config from tenant %d and session %s", tenantInfo.ID, req.Session.ID)
// Log knowledge bases if present
if len(agentConfig.KnowledgeBases) > 0 || len(req.TagScopes) > 0 {
if len(agentConfig.KnowledgeBases) > 0 {
logger.Infof(ctx, "Agent configured with %d knowledge base(s): %v",
len(agentConfig.KnowledgeBases), agentConfig.KnowledgeBases)
} else {
logger.Infof(ctx, "Agent configured with %d tag-scoped search target(s)", len(req.TagScopes))
}
} else {
logger.Infof(ctx, "No knowledge bases specified for agent, running in pure agent mode")
}
// Build search targets using agent's tenant (handler has validated access for shared agent)
searchTargets, err := s.buildSearchTargets(ctx, agentTenantID, agentConfig.KnowledgeBases, agentConfig.KnowledgeIDs, req.TagScopes)
if err != nil {
return nil, fmt.Errorf("build search targets: %w", err)
}
agentConfig.SearchTargets = searchTargets
// Document tags are stored in knowledge_tag_relations, so document-KB tag
// scopes are resolved to concrete knowledge IDs before retrieval. Preserve
// those resolved IDs as this turn's pinned documents as well: otherwise the
// Agent tools are correctly constrained behind the scenes, but the model only
// sees a bound KB and does not know which documents the user explicitly chose.
if len(req.TagScopes) > 0 {
agentConfig.KnowledgeIDs = mergeResolvedTagKnowledgeIDs(
agentConfig.KnowledgeIDs,
searchTargets,
req.TagScopes,
)
}
logger.Infof(ctx, "Agent search targets built: %d targets", len(searchTargets))
if agentConfig.MaxContextTokens <= 0 {
agentConfig.MaxContextTokens = types.DefaultMaxContextTokens
}
return agentConfig, nil
}
func mergeResolvedTagKnowledgeIDs(
existing []string,
searchTargets types.SearchTargets,
tagScopes []types.TagScope,
) []string {
tagKBs := make(map[string]bool, len(tagScopes))
for _, scope := range tagScopes {
if scope.KnowledgeBaseID != "" && len(scope.TagIDs) > 0 {
tagKBs[scope.KnowledgeBaseID] = true
}
}
if len(tagKBs) == 0 {
return uniqueNonEmptyStrings(existing)
}
merged := append([]string(nil), existing...)
for _, target := range searchTargets {
if target == nil || !tagKBs[target.KnowledgeBaseID] || target.Type != types.SearchTargetTypeKnowledge {
continue
}
merged = append(merged, target.KnowledgeIDs...)
}
return uniqueNonEmptyStrings(merged)
}
// applyPerRequestSkillScope narrows the agent's skill whitelist to the @Skill
// mentions for this turn and records the pinned set for the <must_use> hint.
// It is a no-op when no skills were mentioned or skills are disabled.
func applyPerRequestSkillScope(
ctx context.Context,
agentConfig *types.AgentConfig,
skillsMode string,
requested []string,
) {
if len(requested) == 0 {
return
}
if skillsMode == "none" || skillsMode == "" {
logger.Warnf(ctx, "Ignoring @skill mention: agent skills selection is disabled (mode=%s)", skillsMode)
return
}
if !agentConfig.SkillsEnabled {
return
}
switch skillsMode {
case "selected":
agentConfig.AllowedSkills = intersectPreservingRequestOrder(requested, agentConfig.AllowedSkills)
if len(agentConfig.AllowedSkills) == 0 {
agentConfig.SkillsEnabled = false
}
case "all":
agentConfig.AllowedSkills = dedupPreservingOrder(requested)
}
if agentConfig.SkillsEnabled && len(agentConfig.AllowedSkills) > 0 {
agentConfig.PinnedSkillNames = intersectPreservingRequestOrder(requested, agentConfig.AllowedSkills)
}
logger.Infof(ctx, "Applied per-request @skill scope: requested=%v effective=%v pinned=%v",
requested, agentConfig.AllowedSkills, agentConfig.PinnedSkillNames)
}
// applyPerRequestMCPScope narrows the agent's MCP services to the @MCP mentions
// for this turn and records the pinned set for the <must_use> hint. It is a
// no-op when no services were mentioned or MCP selection is disabled.
func applyPerRequestMCPScope(
ctx context.Context,
agentConfig *types.AgentConfig,
agentPresetMCPs []string,
isSharedAgent bool,
requested []string,
) {
if len(requested) == 0 {
return
}
if agentConfig.MCPSelectionMode == "none" {
logger.Warnf(ctx, "Ignoring @MCP mention: agent MCP selection is disabled (mode=none)")
return
}
mentioned := dedupPreservingOrder(requested)
effective, mode := resolvePerRequestMCPScope(mentioned, agentPresetMCPs, agentConfig.MCPSelectionMode, isSharedAgent)
if len(effective) == 0 {
logger.Warnf(ctx, "Ignoring @MCP scope outside agent preset: requested=%v agent=%v shared=%v",
requested, agentPresetMCPs, isSharedAgent)
return
}
agentConfig.MCPSelectionMode = mode
agentConfig.MCPServices = effective
agentConfig.PinnedMCPServiceIDs = intersectPreservingRequestOrder(requested, agentConfig.MCPServices)
logger.Infof(ctx, "Applied per-request @MCP scope: requested=%v mode=%s effective=%v",
requested, agentConfig.MCPSelectionMode, agentConfig.MCPServices)
}
// resolvePerRequestMCPScope narrows MCP registration for a per-turn @mention.
// selectionMode "none" rejects all mentions. Shared agents never register MCP
// services outside the agent preset.
func resolvePerRequestMCPScope(
mentioned, agentMCPs []string,
selectionMode string,
isSharedAgent bool,
) (effective []string, mode string) {
if len(mentioned) == 0 {
return nil, selectionMode
}
if isSharedAgent {
mentioned = intersectPreservingRequestOrder(mentioned, agentMCPs)
if len(mentioned) == 0 {
return nil, selectionMode
}
}
switch selectionMode {
case "none":
return nil, selectionMode
case "selected":
effective = intersectPreservingRequestOrder(mentioned, agentMCPs)
case "all", "":
effective = mentioned
default:
effective = mentioned
}
if len(effective) == 0 {
return nil, selectionMode
}
return effective, "selected"
}
func intersectPreservingRequestOrder(requested []string, allowed []string) []string {
allowedSet := make(map[string]bool, len(allowed))
for _, value := range allowed {
if value != "" {
allowedSet[value] = true
}
}
result := make([]string, 0, len(requested))
seen := make(map[string]bool, len(requested))
for _, value := range requested {
if value == "" || seen[value] || !allowedSet[value] {
continue
}
seen[value] = true
result = append(result, value)
}
return result
}
func dedupPreservingOrder(values []string) []string {
result := make([]string, 0, len(values))
seen := make(map[string]bool, len(values))
for _, value := range values {
if value == "" || seen[value] {
continue
}
seen[value] = true
result = append(result, value)
}
return result
}
// configureSkillsFromAgent configures skills settings in AgentConfig based on CustomAgentConfig
// Returns the skill directories and allowed skills based on the selection mode:
// - "all": uses all preloaded skills
// - "selected": uses the explicitly selected skills
// - "none" or "": skills are disabled
func (s *sessionService) configureSkillsFromAgent(
ctx context.Context,
agentConfig *types.AgentConfig,
customAgent *types.CustomAgent,
) {
if customAgent == nil {
return
}
agentConfig.SandboxConfigID = customAgent.Config.SandboxConfigID
dir := getPreloadedSkillsDir()
switch customAgent.Config.SkillsSelectionMode {
case "all":
// Enable all preloaded skills
agentConfig.SkillsEnabled = true
agentConfig.SkillDirs = []string{dir}
agentConfig.AllowedSkills = nil // Empty means all skills allowed
logger.Infof(ctx, "SkillsSelectionMode=all: enabled all preloaded skills")
case "selected":
// Enable only selected skills
if len(customAgent.Config.SelectedSkills) > 0 {
agentConfig.SkillsEnabled = true
agentConfig.SkillDirs = []string{dir}
agentConfig.AllowedSkills = customAgent.Config.SelectedSkills
logger.Infof(ctx, "SkillsSelectionMode=selected: enabled %d selected skills: %v",
len(customAgent.Config.SelectedSkills), customAgent.Config.SelectedSkills)
} else {
agentConfig.SkillsEnabled = false
logger.Infof(ctx, "SkillsSelectionMode=selected but no skills selected: skills disabled")
}
case "none", "":
// Skills disabled
agentConfig.SkillsEnabled = false
logger.Infof(ctx, "SkillsSelectionMode=%s: skills disabled", customAgent.Config.SkillsSelectionMode)
default:
// Unknown mode, disable skills
agentConfig.SkillsEnabled = false
logger.Warnf(ctx, "Unknown SkillsSelectionMode=%s: skills disabled", customAgent.Config.SkillsSelectionMode)
}
}