diff --git a/.gitignore b/.gitignore index bf41e7bd9..9f7c24191 100644 --- a/.gitignore +++ b/.gitignore @@ -51,6 +51,10 @@ docs/roadmap/* !docs/roadmap/harness-engine/diagrams.md !docs/roadmap/warp/ !docs/roadmap/warp/*.md +!docs/roadmap/agentui/ +!docs/roadmap/agentui/*.md +!docs/roadmap/voice/ +!docs/roadmap/voice/*.md docs/gongzonghao/ docs/bussniss/ docs/oem/ diff --git a/AGENTS.md b/AGENTS.md index b6d56bd15..a7d5bde94 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,10 +31,11 @@ 3. **协议改动必须同步四侧** - `safeInvoke(...)` / `invoke(...)`、`tauri::generate_handler!`、`agentCommandCatalog`、`mockPriorityCommands` / `defaultMocks` 必须保持一致，并执行 `npm run test:contracts` 4. **Lime 是 GUI 桌面产品** - 不能只以 `lint`、`typecheck`、单测通过作为“可交付”判断 5. **高风险 GUI 改动必须做最小冒烟** - 涉及 GUI 壳、DevBridge、Workspace、主路径时执行 `npm run verify:gui-smoke` -6. **用户可见 UI 改动必须补稳定回归** - 优先补现有 `*.test.tsx` 或 snapshot 断言 -7. **配置与依赖改动要成组更新** - schema、校验器、消费者、文档、锁文件保持同步 -8. **Rust 变更先小测后全量** - 先跑受影响 crate / 模块 / 定向测试；新增模块尽量控制在 `500 LoC` 内，文件接近 `800 LoC` 时优先拆新模块 -9. **Harness Engine 只认单一事实源** - handoff / evidence / replay / analysis / review / GUI 统一消费 `agent_runtime_export_evidence_pack`；`requestTelemetry` 需要按 `session/thread/turn` 真实关联导出，无匹配请求时输出空摘要，不再保留伪 `unlinked` +6. **Playwright 续测优先稳定桌面 Chrome 会话** - 真实交互验证优先复用已有 Lime 页签；需要新启浏览器时走持久化 Chrome 上下文，避免本地桌面出现自动化横幅或 `--no-sandbox` 安全横幅，细则见 `docs/aiprompts/playwright-e2e.md` +7. **用户可见 UI 改动必须补稳定回归** - 优先补现有 `*.test.tsx` 或 snapshot 断言 +8. **配置与依赖改动要成组更新** - schema、校验器、消费者、文档、锁文件保持同步 +9. **Rust 变更先小测后全量** - 先跑受影响 crate / 模块 / 定向测试；新增模块尽量控制在 `500 LoC` 内，文件接近 `800 LoC` 时优先拆新模块 +10. **Harness Engine 只认单一事实源** - handoff / evidence / replay / analysis / review / GUI 统一消费 `agent_runtime_export_evidence_pack`；`requestTelemetry` 需要按 `session/thread/turn` 真实关联导出，无匹配请求时输出空摘要，不再保留伪 `unlinked` ## 执行与路线图 diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 1d99276af..fb5df3469 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,88 +1,111 @@ -## Lime v1.24.0 +## Lime v1.25.0 发布日期：`2026-04-30` ### 发布概览 -- 本次发布目标 tag 为 `v1.24.0`。 -- 本次发布聚焦多模态运行合同底座、Browser / PDF / Voice / Research / Text Transform 合同闭环、任务中心与旧会话打开体验、工作台首页技能入口，以及版本发布配置同步。 -- 本轮待递交内容覆盖 Rust 后端、Tauri 配置、前端 Agent Workspace / Home / Settings / Provider / DevBridge、治理合同 registry、测试覆盖、版本锁文件、图片素材与执行计划文档。 +- 本次发布目标 tag 为 `v1.25.0`，重点把 Lime 的语音输入、音频转写、AgentUI 旧会话体验和多模态运行合同推进到同一条 current 主链。 +- 版本文件、Tauri 配置、Cargo / npm lockfile、CLI wrapper、浏览器 mock 与 release updater 测试样例已同步到 `1.25.0`。 +- 该版本继续坚持“一个事实源”：语音、音频、转写、任务轻卡、Evidence Pack、Replay 与 GUI 恢复层都消费统一的 runtime contract / task artifact / media task index，而不是新增平行协议。 -### 重点更新 +### 用户可见更新 -#### 1. 版本号同步到 v1.24.0 +#### 1. 离线语音模型与 Fn 听写 -- 应用版本已同步为 `1.24.0`： - - `package.json` - - `package-lock.json` - - `src-tauri/Cargo.toml` - - `src-tauri/Cargo.lock` - - `src-tauri/tauri.conf.json` - - `src-tauri/tauri.conf.headless.json` -- `packages/lime-cli-npm/package.json` 与 `packages/lime-cli-npm/README.md` 已同步到 `1.24.0`，保持 CLI wrapper 与桌面 release 版本一致。 -- 浏览器模式默认 mock 的 update current version 已同步为 `1.24.0`。 -- GitHub release asset staging 测试中的当前发布样例已同步到 `v1.24.0`。 +- 设置页新增 `SenseVoice Small` 本地语音模型主链：可查看安装状态、下载模型、删除本地模型、设为默认 ASR、执行 WAV 文件测试转写。 +- 本地模型按需下载到 Lime 数据目录，不进入桌面安装包；缺失模型文件时会给出明确引导，不会静默拉取大模型。 +- `voice-core` 接入 `sherpa-onnx` offline recognizer，支持通过 `model.int8.onnx` 与 `tokens.txt` 运行 SenseVoice 本地转写。 +- `voice_asr_service` 新增 `SenseVoiceLocal` 分支，录音采样率非 `16kHz` 时会转换为 `16kHz` mono float samples 后再识别。 +- macOS 新增 Fn 按住录音第一刀：按住 Fn 打开语音输入，松开 Fn 停止并转写；Fn 不可用或权限不足时保留普通全局快捷键 fallback。 +- 语音设置、快捷键设置、onboarding 语音测试与 ASR Provider UI 已补齐 SenseVoice / Fn 状态展示和回归覆盖。 -#### 2. 多模态运行合同底座 +#### 2. Audio Transcription 转写任务 -- 新增并扩展 `ModalityRuntimeContract` 前端 registry 与 Rust 侧合同常量，统一 `image_generation`、`browser_control`、`pdf_extract`、`voice_generation`、`web_research`、`text_transform` 六类底层运行合同。 -- `scripts/check-modality-runtime-contracts.mjs` 与 `npm run governance:modality-contracts` 继续作为合同 registry 的机器校验入口，保护 contract key、entry binding、executor binding 与 metadata path 不漂移。 -- 图片生成任务、媒体任务 API、DevBridge mock 与治理目录册同步合同字段，避免入口层直接拥有底层 task / model / artifact / evidence 事实源。 -- `docs/roadmap/warp/contract-schema.md` 与 `docs/exec-plans/multimodal-runtime-contract-plan.md` 更新合同 schema、执行阶段与 current / compat / deprecated / dead 分类。 +- `@转写 / @transcribe / @Audio Extractor` 归并到 `audio_transcription` 运行合同，入口只作为 binding，不再直接伪造 transcript 事实源。 +- 新增 `transcription_generate` task artifact 主链：任务会写入 contract snapshot、transcript 状态、provider 结果、错误码与统一媒体任务索引。 +- 最小 `lime-transcription-worker` 接入 OpenAI-compatible `/audio/transcriptions` provider：支持本地 `source_path` 与远程 `source_url`，成功后写入 `.lime/runtime/transcripts/*`。 +- 转写失败会保留结构化错误，例如 provider 未配置、source 不可读、provider 请求失败、空 transcript、输出写入失败等；不会退回前端直连 ASR 或普通文件读取。 +- 前端转写轻卡和 workspace viewer 可从媒体任务索引恢复完成态或失败态，完成态支持 transcript 文本、语言、输出格式、路径和时间轴信息展示。 +- Transcript parser 支持从 verbose JSON、VTT / SRT 内容中恢复分段、时间戳和说话人信息。 +- 转写校对产物新增 `transcriptCorrection*` metadata：保存修订版本时保留原始 transcript 不可变，并记录修订文本、分段数量、说话人数量和来源路径。 -#### 3. Browser / PDF / Voice / Research / Text Transform 合同闭环 +#### 3. 音频生成与多模态任务恢复 -- `@浏览器` / Browser Assist 发送链路注入 `browser_control` 合同快照，Rust 侧保留 Browser Assist runtime hint，并把 browser tool timeline metadata 纳入 evidence / replay。 -- `@读PDF` 入口收敛到 `pdf_extract` 合同，前端 metadata、Rust prompt 准备、Skill(pdf_read) timeline、evidence pack 与 replay 检查保持同一份合同事实源。 -- `@配音` 服务型入口收敛到 `voice_generation` 合同，ServiceSkill / voice runtime 的 request metadata、timeline args 与 replay grader 统一识别音频运行证据。 -- `@搜索`、`@深搜`、`@站点搜索`、`@研报`、`@竞品` 统一归入 `web_research` 合同，并补齐 evidence `snapshotIndex`，让联网研究可以按 contract/source/routing/tool trace 检索。 -- `@读文件`、`@总结`、`@翻译`、`@分析`、`@发布合规`、`@Logo拆解` 统一归入 `text_transform` 合同，禁止回退为前端直出、ToolSearch / WebSearch 或普通聊天摘要。 +- `audio_generate` 与 `transcription_generate` 继续统一到媒体任务索引，任务轻卡、运行时文档、DevBridge mock 和恢复层消费同一份 artifact。 +- 音频预览 runtime 会优先读取统一索引中的 audio output 状态，不再依赖隐藏 task JSON；provider 失败时不会保留旧音频路径或伪造可播放结果。 +- `TaskMessagePreview`、`taskPreviewFromToolResult`、媒体任务 API 与 mock 输出补齐 audio / transcript 状态字段，减少完成态、失败态、缺 trace 之间的误判。 -#### 4. Evidence Pack / Replay / Task Runtime +#### 4. AgentUI 首页、任务中心与旧会话性能 -- `runtime_evidence_pack_service` 与 `runtime_replay_case_service` 扩展多模态合同快照、索引、expected checks 与 grader 规则，回放样本能判断是否真实命中对应 executor trace。 -- 媒体 runtime 与 `media_task_cmd` 补齐合同字段归一化、能力缺口提示、artifact payload 与任务预览字段。 -- 前端 `agentRuntime` media/session 类型、`mediaTasks` API、ImageTask viewer 与工作台预览 runtime 同步消费合同字段，减少 viewer 自行猜测 artifact 类型。 -- `modalityRuntimeContracts` 的前端/Rust 双侧测试覆盖合同解析、fallback 与 mock 分发，保持浏览器模式不是假绿。 +- Agent Chat 首页补齐 Home Start Surface、引导卡、技能卡片、更多技能抽屉、Starter Chips 与稳定回归。 +- 新建任务、打开已有会话、任务中心 tab、侧栏会话列表之间的事件链收敛到 `taskCenterDraftTaskEvents`，减少旧导航和本地 tab 状态双写。 +- 旧会话打开继续降载：最近会话首屏请求从大窗口降到 `11`，任务中心 topics 初始请求降到 `21`，归档列表也改为可见数量 `+1` 的哨兵分页。 +- 侧栏 hover / focus 旧会话预取延迟到约 `900ms`，点击时取消未触发预取，避免鼠标扫过列表时抢占正式切换链路。 +- 侧栏打开旧会话后，路由追平会在短窗口内去重，避免同一用户意图触发两次 `switchTopic` / `agent_runtime_get_session`。 +- 旧会话恢复首帧优先正文和输入区，运行轨迹、thread items、queued turns、pending actions 等投影延迟到 idle；正在发送、聚焦 timeline 或存在 A2UI 待处理时不会延迟实时反馈。 +- `MessageList` 继续强化旧会话轻量渲染：历史窗口只先渲染最近消息，timeline 延迟构建，长历史 Markdown 使用轻量预览，Provider / Prompt Cache 配置扫描推迟到首帧后。 +- 新增 `window.__LIME_AGENTUI_PERF__` 性能采集器，可按 session 汇总 `clickToMessageListPaintMs`、`runtimeGetSessionDurationMs`、隐藏历史数量、最终渲染消息数量和可用堆内存指标。 -#### 5. 工作台首页、任务中心与旧会话体验 +#### 5. Browser Assist 与工作区展示 -- Agent Chat 新增 Home Start Surface、技能卡片画廊、更多技能抽屉与场景技能管理弹窗，并提交对应 home cover 素材。 -- EmptyState / Workspace / AppSidebar 接入新的首页入口、补充快捷入口与任务中心事件桥，一级“新建任务”与会话侧栏切换保持可取消草稿语义。 -- 旧会话打开性能继续收口：首屏优先最近消息、timeline 延后物化、历史消息与流式 delta overlap 合并，降低旧会话切换卡顿与布局跳动。 -- MessageList / Team Workspace / Workspace Send Actions 增补回归，覆盖新建草稿、旧会话切换、图片任务预览和合同 metadata 发送。 +- Browser Assist renderer 增强运行结果展示，支持更清晰地呈现浏览器会话、页面、操作结果与可恢复状态。 +- Workspace 的音频任务预览、转写任务预览、图片任务预览和 general workbench 恢复逻辑进一步对齐 task artifact / media index 事实源。 +- Inputbar 与消息渲染继续收紧多模态、运行态、队列态和任务文件展示，减少过程信息污染最终正文。 -#### 6. Provider / 设置页 / 云端能力边界 +#### 6. Provider、模型能力与设置体验 -- Provider 与 API Key 设置页进一步收口模型能力、Prompt Cache、云端套餐 / 权益 / API Key 展示口径，避免本地设置页重新承载用户中心商业工作台。 -- OEM cloud access、LimeHub provider sync、model registry service 与 provider list 工具函数补齐测试，保护登录态、权益摘要、模型支持和 provider capability 展示。 -- `withI18nPatch`、Settings Provider 页面与相关 hooks 做格式化和回归更新，保持当前设置入口与 mock / Rust 模型事实源一致。 +- API Key Provider 设置页补齐 provider 配置、模型能力和 Prompt Cache 相关展示与回归。 +- OpenAI-compatible provider 和 credential bridge 增强，为音频转写、模型能力推断和 provider 解析提供更稳定的后端事实源。 +- LimeHub local dev models 与本地 provider 模型列表同步更新，减少设置页和运行时模型认知漂移。 +- 外观配色扩展：在原有配色基础上新增 `霓虹`、`青柠`、`暮色`、`极简`、`鲜活`、`文学`、`奢华` 等主题，并调整现有配色名称与描述。 -### 待递交范围确认 +### 开发者与治理更新 -- 版本与发布：版本文件、lockfile、Tauri 配置、CLI wrapper、release updater 测试样例与 release notes。 -- Rust 主链：多模态合同、媒体任务、Browser Assist、PDF / Research / Summary / Translation / Analysis / ServiceSkill launch、evidence pack、replay case、model registry。 -- 前端主链：Agent Chat Workspace、Home Start Surface、MessageList、ImageTask Viewer、任务中心事件桥、Settings Provider、API Key Provider、DevBridge、tauri mock。 -- 治理与文档：`modalityRuntimeContracts` registry / 校验脚本、Warp contract schema、执行计划与旧会话性能计划。 -- 素材与测试：home cover 素材、新增/更新的 Vitest、Rust 回归与治理合同测试。 +#### 1. 运行合同与治理校验 + +- `modalityRuntimeContracts` 新增 / 更新 `audio_transcription`，并同步 Rust 合同常量、前端 registry、mock 和校验脚本。 +- 新增 `modalityArtifactGraph.json`，把 entry binding、executor binding、artifact、viewer 和 evidence / replay 关系显式化。 +- `scripts/check-modality-runtime-contracts.mjs` 扩展校验范围，覆盖 capability、model role、artifact kind、artifact graph 与 current contract 同步关系。 +- `npm run test:contracts` 现在覆盖 agent runtime client 生成检查、命令契约、harness 契约、modality contracts 与 cleanup report contract。 + +#### 2. Evidence Pack 与 Replay + +- Evidence Pack 可导出 audio transcription contract snapshot、transcript 索引、audio output 索引、provider 失败状态与 runtime verification 信息。 +- Replay case 可携带 `audio_transcription`、`voice_generation`、`pdf_extract`、`browser_control`、`web_research`、`text_transform` 等合同进入 grader checks。 +- Replay 对音频和转写失败做更细分类：可区分 provider failure、缺少输出、缺少 trace、合同不匹配等情况。 + +#### 3. 文档与路线图 + +- 新增 AgentUI 路线图文档，覆盖目标架构、代码地图、事件流、时序图、后端协作和实施路线。 +- 新增离线语音模型路线图，明确 SenseVoice Small、Fn 听写、模型下载、测试转写和后续 VAD / 历史规划。 +- Warp roadmap 补齐 artifact graph、contract schema、acceptance 与 evolution guide，用于多模态 / runtime contract 后续演进。 +- Playwright E2E 指南补充真实 GUI 续测约束，继续优先复用稳定桌面 Chrome / Lime 页签。 +- `AGENTS.md` 与执行计划文档同步更新，保留本轮关键决策和验证记录。 + +### 已知说明 + +- SenseVoice Small 当前按需从上游 release 下载；内置清单会记录下载摘要，但上游未在本地清单内提供可信 sha256 时不会声明强校验完成。 +- Fn 按住录音当前只作为 macOS 第一刀能力；第三方键盘或缺少系统权限时，用户应继续使用普通语音快捷键。 +- `audio_transcription` 执行器当前只接入 OpenAI-compatible transcription adapter；其他 provider runtime type 仍需要后续补 client。 +- 真实联网 web search 测试默认 gated，需要设置 `LIME_REAL_API_TEST=1` 才会执行。 ### 校验状态 -- 已执行： +- 已通过： - `cargo fmt --manifest-path "src-tauri/Cargo.toml" --all` - `npm run format` - - `npm run verify:app-version` - - `cargo test --manifest-path "src-tauri/Cargo.toml" --target-dir "src-tauri/target/codex-release-v124"` - - `cargo clippy --manifest-path "src-tauri/Cargo.toml" --target-dir "src-tauri/target/codex-release-v124" --all-targets --all-features` + - `npm run verify:app-version`（版本一致性检查通过：`1.25.0`） - `npm run lint` - - `npm test` + - `npm run typecheck` + - `npm test`（`44` 个 Vitest 批次通过） - `npm run test:contracts` - - `npm run verify:gui-smoke` - - `git diff --check` -- 结果：上述校验均已通过；`cargo test` 结果为 `1103 passed; 0 failed; 0 ignored`，另有 2 个真实联网测试按预期 ignored。 -- 备注：`cargo clippy` 仍保留既有 warning（`manual_repeat_n`、`too_many_arguments`、`needless_lifetimes`），本轮未扩大 warning 面。 -- GUI 主路径：`npm run verify:gui-smoke` 已通过，覆盖 DevBridge、默认 workspace、browser runtime、site adapter catalog、服务技能入口与 runtime tool surface 页面烟测。 + - `npm run lint:rust`（命令通过；仓库仍保留若干既有 Clippy warning，未配置为阻断） + - `cargo test --manifest-path "src-tauri/Cargo.toml" -p voice-core` + - `cargo test --manifest-path "src-tauri/Cargo.toml"`（主库 `1120` 项通过，DeepSeek runtime `2` 项通过，真实联网 web search `2` 项按环境变量门禁 ignored） + - `npm run verify:gui-smoke -- --reuse-running --timeout-ms 600000` +- 说明：完整 `cargo test` 首次因本机磁盘空间不足中断；清理已确认无进程占用的临时 GUI smoke target 后重跑通过。 +- GUI 主路径：已复用运行中的 headless Tauri 完成最小冒烟，覆盖 DevBridge、默认 workspace、browser runtime、site adapter、Agent service skill entry 与 runtime tool surface。 --- -**完整变更**: `v1.23.0` -> `v1.24.0` +**完整变更**: `v1.24.0` -> `v1.25.0` diff --git a/docs/aiprompts/playwright-e2e.md b/docs/aiprompts/playwright-e2e.md index 80f895499..6473eb949 100644 --- a/docs/aiprompts/playwright-e2e.md +++ b/docs/aiprompts/playwright-e2e.md @@ -24,6 +24,8 @@ - 优先使用 **Playwright MCP** 做交互验证，不优先编写新的本地 Playwright 测试文件 - 浏览器模式默认首页从 `http://127.0.0.1:1420/` 进入 +- 需要稳定的桌面 Chrome 观感时，优先复用已有 Lime 页签；必须新启 headed 浏览器时，使用固定 profile 的 Chrome 持久化上下文，例如 `channel: 'chrome'` + `launchPersistentContext(userDataDir, { headless: false, viewport: null })` +- 本地桌面续测不要传 `--no-sandbox` / `--disable-setuid-sandbox`；如需去掉 Chrome 顶部“自动测试软件控制”提示，只能在受控 launcher 中忽略 `--enable-automation`。CI、容器或 Linux sandbox 受限环境例外，但要在结论中说明原因 - 如果 Playwright 工具当前还在 deferred surface，优先用 `ToolSearch` 的精确选择名，例如 `select:mcp__playwright__browser_click`；不要把 `playwright_browser_click`、`browser click` 之类同义词反复丢给 `ToolSearch` - 能走真实后端就走真实后端；浏览器模式暂不支持或尚未桥接的能力，允许走 mock - `verify:gui-smoke` 内部的 browser runtime 校验默认走无界面浏览器会话；它只证明主链可启动，不替代后续真实页面交互验证 diff --git a/docs/exec-plans/agentui-implementation-progress.md b/docs/exec-plans/agentui-implementation-progress.md new file mode 100644 index 000000000..91e90d228 --- /dev/null +++ b/docs/exec-plans/agentui-implementation-progress.md @@ -0,0 +1,730 @@ +# AgentUI 实施进度 + +> 状态：进行中 +> 更新时间：2026-04-30 +> 路线图：`docs/roadmap/agentui/lime-agentui-implementation-roadmap.md` + +## 主目标 + +逐步实现 Lime AgentUI 下一阶段主线，优先解决旧会话恢复慢、首字慢、tab 卡顿、流式重复吐字和过程信息噪声。 + +## 当前阶段 + +P1：Tab 与 Task Capsule，先落旧会话切换期间的轻量投影。 + +## 本轮执行准则更新：旧 UI 顺路清理 + +主线仍以旧会话性能数值分析和首字/流式体验优化为优先级；遇到现有 UI 老旧、不适合继续承载 AgentUI 主链时，可以顺路清理，但清理必须直接服务当前交付，不能偏航成纯治理或视觉翻新。 + +### 准入条件 + +仅当旧 UI 满足以下任一条件时，本轮允许顺路清理： + +- 直接造成旧会话打开、切换、新建对话或流式输出卡顿，例如重复渲染非活跃会话、旧面板抢占 hydrate、旧入口触发额外查询。 +- 直接造成用户路径错误，例如新建对话被旧界面跳走、无法同时打开多个历史对话、输入区/对话区位置与 AgentUI 规划冲突。 +- 直接造成重复事实源，例如同一状态同时由旧 tab、旧任务面板和新 capsule 独立维护，导致状态不同步或额外 render。 +- 直接违反当前 Lime UI 规范并影响可用性，例如半透明主表面、多层套卡、重复标题、伪交互、中文排版被压缩。 + +### 分类与动作 + +- `current`：继续沿 AgentUI 当前主链演进；性能采样、消息窗口、tab/capsule、新建对话入口都应向这里收敛。 +- `compat`：只允许委托和适配，不新增状态、查询、渲染分支；如果旧 UI 仍被调用，必须写清退出条件。 +- `deprecated`：只允许迁移和下线；遇到影响主线的旧 UI，优先从主入口摘除，再登记后续删除。 +- `dead`：确认无入口或与 current 规划冲突时直接删除或补治理守卫，防止后续重新接回主链。 + +### 清理边界 + +- 每一刀最多顺路清理一个直接阻塞主线的旧 UI surface，其余旧面先登记，不连续深挖。 +- 不新增平行的新旧两套组件；能收敛到现有 AgentUI current 组件时，不再补新的 compat 包装层。 +- 清理后仍需保持桌面 GUI 气质：实体主表面、清晰边界、中文优先、按钮层级明确，避免半透明主体和过度嵌套套卡。 +- 如果清理会触及 Tauri command、Bridge、mock 或 session 数据事实源，必须另起命令边界检查，不把 UI 清理伪装成纯样式改动。 + +### 验收口径 + +- 性能侧：用 `window.__LIME_AGENTUI_PERF__.summary()` 对比清理前后，至少看 `clickToMessageListPaintMs`、`runtimeGetSessionDurationMs`、`finalRenderedMessagesCount`、`hiddenHistoryCount`。 +- 交互侧：覆盖新建对话、打开旧会话 A、打开旧会话 B、旧会话间切换、发送短句看首字和流式输出。 +- UI 侧：补稳定回归或 snapshot，必要时复用现有 Lime 页签做 Playwright E2E；不使用会新开 isolated profile 或 `--no-sandbox` 临时浏览器的验证方式。 +- 治理侧：如果删除或下线旧入口，补 `current / compat / deprecated / dead` 分类说明；必要时运行 `npm run governance:legacy-report` 或 `npm run test:contracts`。 + +## 进度日志 + +### 2026-04-30：P0 第一刀，流式主链观测与渲染降频 + +已完成： + +- 在 stream request state 中补齐首字慢分段字段： + - listener bound + - submit dispatched / accepted / failed + - first event + - first runtime status + - first text delta + - first text paint + - text render flush / backlog +- 在 `agentStreamTurnEventBinding.ts` 记录 listener bound 和 first event。 +- 在 `agentStreamSubmitExecution.ts` 记录 submit dispatched / accepted / failed。 +- 在 `agentStreamRuntimeHandler.ts` 记录 first runtime status、first text delta、first text paint、text render flush。 +- 保留并补齐已有 text_delta 低频刷新策略，避免每个字符都刷新消息树。 +- 保留并补齐已有 thinking 关闭策略，`thinking_delta` 不再污染最终正文。 +- 保留并补齐已有 `final_done` reconcile 防线，避免最终文本整段重复追加。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts" "src/components/agent/chat/hooks/agentStreamTurnEventBinding.test.ts" +npm exec -- vitest run "src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts" "src/components/agent/chat/hooks/agentStreamTurnEventBinding.test.ts" "src/components/agent/chat/components/MessageList.test.tsx" "src/components/agent/chat/components/StreamingRenderer.test.tsx" +npm run typecheck +npm run verify:gui-smoke +``` + +结果： + +- 流式定向测试：通过，`17` 个测试通过。 +- AgentUI P0 定向测试：通过，`131` 个测试通过。 +- TypeScript：通过。 +- GUI smoke：通过。 + +下一步： + +1. 跑 touched files 的 lint/type 校验。 +2. 继续 P0 旧会话渐进恢复：确认 active tab 与非 active tab 的 hydrate 边界，减少旧会话打开时 MessageList/timeline 同步负担。 +3. 再进入 P1 tab/capsule：避免多历史会话同时全量渲染。 + +### 2026-04-30：P0 第二刀复核，旧会话渐进恢复现状 + +已确认当前工作区已有以下旧会话恢复优化： + +- `useAgentSession.ts`： + - `getSession` 恢复路径统一带 `historyLimit: 40`。 + - recent session prefetch 使用 `SESSION_DETAIL_PREFETCH_HISTORY_LIMIT = 40`。 + - cached snapshot fresh 时延迟 detail hydrate。 + - detail hydrate 进入 `startTransition`，降低切换时主线程抢占。 + - “加载更多历史”改为分页：`SESSION_HISTORY_LOAD_PAGE_SIZE = 50`，使用 `historyOffset` / `historyBeforeMessageId`。 +- `MessageList.tsx`： + - 旧会话只先渲染最近消息窗口。 + - 旧会话隐藏历史不自动逐批补齐，必须用户点击展开。 + - 历史 timeline 延迟到 idle，完成历史 timeline 可先折叠为轻量摘要。 + - 超长历史助手消息和较长历史助手消息先展示纯文本预览。 + - 历史 Markdown 使用 light render mode，避免首帧挂载重 Markdown。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" "src/components/agent/chat/components/StreamingRenderer.test.tsx" +npm run typecheck +npm run verify:gui-smoke +``` + +结果：通过，`MessageList` / `StreamingRenderer` 定向测试共 `114` 个通过，TypeScript 检查通过，GUI smoke 通过。 + +下一步： + +1. 进入 P1 tab/capsule 前，先用 Playwright 复测真实多历史会话打开路径，采集 `runtimeGetSession.*`、`switchTopic.*`、`AgentStream.*` 日志。 +2. 若 E2E 仍出现切换卡顿，下一刀优先做非活跃 tab snapshot/freeze，而不是继续压 MessageList 单点。 + +### 2026-04-30：P1 第一刀，旧会话切换期间延迟运行轨迹投影 + +已完成： + +- 在 `useWorkspaceConversationSceneRuntime.tsx` 增加 session runtime projection defer： + - 旧会话恢复首帧继续立即透传 `messages`，保证正文和输入区先可交互。 + - `turns`、`threadItems`、`threadRead`、`pendingActions`、`queuedTurns`、`childSubagentSessions` 延迟到 idle 后再挂载，减少点击历史会话时的同步投影和 timeline 构建压力。 + - 延迟状态按 `sessionId + 首尾 message + 尾部 turn/item` 绑定，避免从一个历史会话切到另一个同长度历史会话时复用上一会话的“已投影”状态。 + - 正在发送、聚焦 timeline、存在 pending A2UI 表单时不延迟，避免影响运行中反馈和用户待处理动作。 +- 在 `useWorkspaceConversationSceneRuntime.test.ts` 补回归： + - 恢复旧会话首帧应先透传消息，并延迟运行轨迹投影。 + - 切换到另一条同长度旧会话时应重新延迟运行轨迹投影。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts" "src/components/agent/chat/workspace/WorkspaceConversationScene.test.tsx" "src/components/agent/chat/components/MessageList.test.tsx" +npm run typecheck +``` + +结果： + +- Workspace / MessageList 定向回归：通过，`96` 个测试通过。 +- TypeScript：通过。 + +未完成验证： + +- `npm run verify:gui-smoke` 本轮未完成。原因是本地 `DevBridge` 3030 未就绪，smoke 复用/拉起 headless 链路时触发新的 Rust 编译链，持续数分钟占用 CPU；继续等待会污染“旧会话打开是否卡顿”的真实判断。已清理本轮 `verify-gui-smoke` 派生的临时编译链，未清理用户已有的长期 `tauri:dev:headless` / 其他 Rust 校验进程。 +- 真实 Chrome 页签 E2E 续测暂未完成。当前前端 `http://127.0.0.1:1420/` 可访问，但 `http://127.0.0.1:3030/health` 未监听；需先恢复 DevBridge 后再复测多历史会话切换。 + +下一步： + +1. 恢复 DevBridge 3030 后，用现有 Chrome 页签复测：新建对话、打开两个历史会话、来回切换、观察控制台 error 和 `switchTopic.*` 日志。 +2. 若旧会话仍有体感卡顿，继续 P1：把 tab/shell 层的 running/queued/needs_input 胶囊化，并进一步降低非当前过程面板的更新频率。 + +### 2026-04-30：P1 第二刀，覆盖 history window hydrate 后的重投影卡顿 + +已完成： + +- 扩展 `useWorkspaceConversationSceneRuntime.tsx` 的旧会话运行轨迹延迟投影触发条件： + - 不再只依赖 `isAutoRestoringSession`。 + - 当 `sessionHistoryWindow.totalMessages > sessionHistoryWindow.loadedMessages` 时，也按旧会话窗口处理，覆盖 cached snapshot 已显示、detail hydrate 后恢复大量 `turns/threadItems` 的场景。 + - 同一个 session 尾部追加新消息时不重新延迟投影，避免首字和流式输出被 700ms 延迟误伤。 + - 继续保持发送中、timeline 聚焦、存在 pending A2UI 表单时不延迟，保护实时反馈和用户待处理动作。 + - 延迟投影状态写入改为“值未变化则返回当前 state”，避免 ready/pending 状态重复写入造成额外 render。 +- 补齐 `useWorkspaceConversationSceneRuntime.test.ts` 回归： + - `isAutoRestoringSession=false` 但存在截断历史窗口时，首帧仍先显示消息、延迟运行轨迹。 + - 发送中会话不延迟运行轨迹。 + - 聚焦 timeline 或存在 A2UI 表单时不延迟运行轨迹。 +- 顺手补齐语音快捷键相关测试 fixture 的 `fn_*` 字段，修复当前工作区 `typecheck` 阻塞；该修复只闭合测试 mock 类型，不扩展语音功能面。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts" +npm exec -- vitest run "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts" "src/components/agent/chat/workspace/WorkspaceConversationScene.test.tsx" "src/components/agent/chat/components/MessageList.test.tsx" +npm exec -- vitest run "src/components/onboarding/steps/VoiceShortcutTestStep.test.tsx" "src/components/settings-v2/general/hotkeys/hotkeyCatalog.test.ts" +npm run typecheck +curl -fsS "http://127.0.0.1:3030/health" +``` + +结果： + +- Workspace 单测：通过，`13` 个测试通过。 +- Workspace / MessageList 组合回归：通过，`99` 个测试通过。 +- 语音 fixture 定向回归：通过，`4` 个测试通过。 +- TypeScript：通过。 +- DevBridge 健康检查：失败，`127.0.0.1:3030` 未监听；当前只确认 `127.0.0.1:1420` 前端 dev server 在监听。第二轮 render 收紧后复查结果一致。 + +未完成验证： + +- 真实 Chrome 页签 E2E 暂未执行。原因是 DevBridge 3030 未就绪，强行启动 `verify:gui-smoke` 会触发新的 Rust 编译链并污染旧会话卡顿判断。 + +下一步： + +1. 恢复 DevBridge 3030 后，复用现有 Chrome/Lime 页签做多历史会话切换 E2E。 +2. 若体感卡顿仍明显，下一刀进入 tab/shell 层：冻结非活跃 tab 的重运行时投影，只保留标题、状态、未读和最后预览。 + +### 2026-04-30：P1 第三刀，收紧 tab/shell 同步小开销与验证阻塞 + +已完成： + +- `AgentChatWorkspace.tsx` 为 `topics` 建立单次 `topicById` Map： + - 初始会话切换策略、任务中心打开旧会话、detached 判断和顶部 tab item 生成复用 Map 查询。 + - 减少旧会话切换路径里多处 `topics.find/some` 扫描；当前非活跃 tab 本身只渲染标题/状态/未读，不渲染正文或 timeline。 +- `useWorkspaceConversationSceneRuntime.tsx` 收紧延迟投影状态更新： + - 空投影数组改为模块级稳定常量，避免下游 `useMemo` 因每次新建 `[]` 失效。 + - 延迟投影 ready/pending 写入保持“值未变则返回当前 state”，避免无意义二次 render。 +- 修复当前工作区两个验证阻塞： + - 语音快捷键测试 fixture 补齐 `fn_*` 字段。 + - 媒体任务测试 fixture 与浏览器 mock 输出补齐 transcript 汇总字段，保持 `MediaTaskModalityRuntimeContractIndex` 契约一致。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/index.test.tsx" -t "任务中心初始会话标签" +npm exec -- vitest run "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts" "src/components/agent/chat/workspace/WorkspaceConversationScene.test.tsx" "src/components/agent/chat/components/MessageList.test.tsx" +npm exec -- vitest run "src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.test.tsx" "src/components/agent/chat/workspace/useWorkspaceImageTaskPreviewRuntime.test.tsx" "src/lib/tauri-mock/core.test.ts" +npx eslint "src/components/agent/chat/AgentChatWorkspace.tsx" "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.tsx" "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts" "src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.test.tsx" "src/components/agent/chat/workspace/useWorkspaceImageTaskPreviewRuntime.test.tsx" "src/lib/tauri-mock/core.ts" "src/components/onboarding/steps/VoiceShortcutTestStep.test.tsx" "src/components/settings-v2/general/hotkeys/hotkeyCatalog.test.ts" --max-warnings 0 +npm run typecheck +npm run test:contracts +curl -fsS "http://127.0.0.1:3030/health" +``` + +结果： + +- 任务中心定向回归：通过，`11` 个测试通过。 +- Workspace / MessageList 组合回归：通过，`99` 个测试通过。 +- 媒体任务 / mock 回归：通过，`54` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：通过。 +- Contract：通过。 +- DevBridge 健康检查：失败，`127.0.0.1:3030` 未监听；`127.0.0.1:1420` 前端 dev server 仍在监听。 + +未完成验证： + +- 真实 Chrome 页签 E2E 仍未执行。原因同上一刀：DevBridge 3030 未就绪，当前不应启动新的 Playwright profile 或强行触发重编译链。 + +下一步： + +1. 待 DevBridge 恢复后，复用现有 Chrome/Lime 页签采样：打开旧会话、打开第二个旧会话、来回切换、发送 DeepSeek 短句，看首字前占位、真实 first text delta 和 long task。 +2. 若旧会话仍慢，下一刀不再继续做 tab 微调，优先做 MessageList 动态高度虚拟化或 `agent_runtime_get_session` 分块返回方案评估。 + +### 2026-04-30：P1 第四刀，旧会话首帧推迟 Prompt Cache 配置扫描 + +已完成： + +- `MessageList.tsx` 推迟旧会话恢复首帧的 Provider 配置自动加载： + - 旧会话恢复或历史窗口 hydrate 阶段，`useConfiguredProviders` 先以 `autoLoad: false` 运行。 + - 首帧消息可见后再在 idle 阶段允许 Provider 配置加载，避免 Prompt Cache 提示为了历史消息扫描抢占旧会话打开主链。 + - 保留实时发送、当前会话和普通新会话的 Provider 能力提示，不影响新 token 到达后的说明能力。 +- `MessageList.tsx` 同步收紧最新助手消息定位： + - `lastAssistantMessageId` 与当前 timeline 映射不再复制数组反向查找。 + - 减少旧会话窗口里消息列表重算时的短期内存峰值。 +- `MessageList.test.tsx` 补回归： + - 旧会话恢复首帧应关闭 Provider 配置自动加载。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" +npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" --max-warnings 0 +npm exec -- vitest run "src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts" "src/components/agent/chat/workspace/WorkspaceConversationScene.test.tsx" "src/components/agent/chat/components/MessageList.test.tsx" +npm run typecheck +npm run test:contracts +curl -fsS "http://127.0.0.1:3030/health" +``` + +结果： + +- MessageList 定向回归：通过，`85` 个测试通过。 +- MessageList ESLint：通过。 +- Workspace / MessageList 组合回归：通过，`100` 个测试通过。 +- TypeScript：通过。 +- Contract：通过。 +- DevBridge 健康检查：通过，返回 `status=ok`。 + +未完成验证： + +- 真实 Chrome 页签 E2E 仍需复用现有 Lime 页签继续；当前 Chrome DevTools MCP 被已有 `chrome-profile` 占用，不能按工具提示启动 isolated 新实例，也不应启用 `--no-sandbox` 的临时 Playwright profile。 + +下一步： + +1. 复用现有 Lime 页签或恢复 Chrome DevTools MCP 会话后，执行旧会话 A / 旧会话 B / 新建对话切换采样。 +2. 如果旧会话仍有明显鼠标 loading、CPU 或内存峰值，优先评估 MessageList 动态高度虚拟化，避免继续在 tab shell 做边际优化。 + +### 2026-04-30：P1 第五刀，点击旧会话不再即时预取抢占切换链路 + +已完成： + +- `AppSidebarConversationShelf.tsx` 取消点击路径上的即时预取： + - 移除 `onPointerDown` 的立即 prefetch。 + - `onFocus` 改为延迟预取，并在 `onClick` / `onBlur` / `onPointerLeave` 时取消尚未触发的预取。 + - 保留真正悬停或键盘聚焦停留时的旧会话预热，但避免“鼠标按下 -> 预取 hydrate -> 正式切换复用同一 promise”造成点击后主链多做一次旧会话 hydrate。 +- `MessageList.tsx` 继续减少热路径小分配： + - `renderGroups` 从 `assistantMessages.map(...).find(...)` 改为单次循环，避免每个消息组创建临时数组。 + +主线收益： + +- 用户点击旧会话时，正式切换链路直接进入 `switchTopic.fetchDetail`，不再先被侧栏 prefetch 的 snapshot hydrate 抢占。 +- 只有用户悬停停留足够久时才做预热，符合“后台预取不能拖慢明确点击”的优先级。 + +已验证： + +```bash +npm exec -- vitest run "src/components/AppSidebar.test.tsx" -t "旧会话预取|已有会话" +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" -t "旧会话恢复首帧|复杂任务完成后" +npx eslint "src/components/app-sidebar/AppSidebarConversationShelf.tsx" "src/components/AppSidebar.test.tsx" "src/components/agent/chat/components/MessageList.tsx" --max-warnings 0 +npm run typecheck +git diff --check -- "docs/exec-plans/agentui-implementation-progress.md" "src/components/app-sidebar/AppSidebarConversationShelf.tsx" "src/components/AppSidebar.test.tsx" "src/components/agent/chat/components/MessageList.tsx" +npm exec -- vitest run "src/components/AppSidebar.test.tsx" +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" +``` + +结果： + +- AppSidebar 定向回归：通过，`4` 个测试通过。 +- MessageList 定向回归：通过，`2` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：通过。 +- diff whitespace：通过。 +- AppSidebar 全量回归：通过，`38` 个测试通过。 +- MessageList 全量回归：通过，`85` 个测试通过。 + +未完成验证： + +- `npm run verify:gui-smoke` 本轮执行到 `smoke:agent-runtime-tool-surface-page` 时失败：`launch_browser_session` 对 DevBridge 的请求连续返回 `fetch failed`，随后 `3030` DevBridge 端口不可用。当前 `tauri dev` 正在重新编译 Rust 侧，暂不能继续真实 E2E。 +- Chrome DevTools MCP 仍被既有 `chrome-profile` 占用；不能按工具提示启动 isolated 新实例，也不应使用带 `--no-sandbox` 的临时 Playwright profile。 + +下一步： + +1. 等 DevBridge 3030 恢复后，先只做真实页签 E2E，不再重复触发会重启/重编 Rust 的 GUI smoke。 +2. E2E 重点采样：点击旧会话是否还出现鼠标 loading、打开第二个旧会话是否并发卡住、新建对话 tab 是否立即可打开。 +3. 若仍慢，下一刀进入真正的 MessageList 虚拟化或后端 `getSession` 分块返回，不继续做 prefetch / tab shell 边际优化。 + +### 2026-04-30：P1 第六刀，补 AgentUI 性能数值采集 + +已完成： + +- 新增 `src/lib/agentUiPerformanceMetrics.ts`： + - 浏览器内维护 `AgentUI` 性能 ring buffer，最多保留 `500` 条。 + - 自动暴露 `window.__LIME_AGENTUI_PERF__`，E2E 可直接读取： + - `window.__LIME_AGENTUI_PERF__.entries()` + - `window.__LIME_AGENTUI_PERF__.summary()` + - `window.__LIME_AGENTUI_PERF__.clear()` + - `summary()` 按 `sessionId` 汇总旧会话打开关键耗时： + - `clickToSwitchStartMs` + - `clickToCachedSnapshotMs` + - `clickToPendingShellMs` + - `clickToFetchStartMs` + - `fetchDetailDurationMs` + - `runtimeGetSessionDurationMs` + - `clickToSwitchSuccessMs` + - `clickToMessageListPaintMs` + - `finalRenderedMessagesCount / hiddenHistoryCount / persistedHiddenHistoryCount` + - 可用时同步采集 `usedJSHeapSize / totalJSHeapSize` +- 采样点已接入： + - `AppSidebarConversationShelf.tsx`：旧会话 hover/focus 预取 schedule / cancel / fire、点击旧会话。 + - `sessionClient.ts`：`agentRuntime.listSessions`、`agentRuntime.getSession` start / success / error 及 duration。 + - `useAgentSession.ts`：session prefetch、switch start、cached snapshot、pending shell、fetch detail、switch success。 + - `MessageList.tsx`：旧会话恢复 / 分页历史窗口 commit 与 post-paint 消息数、turn 数、timeline defer 状态。 +- E2E 读取建议： + +```ts +await page.evaluate(() => window.__LIME_AGENTUI_PERF__?.clear()); +// 执行：点击旧会话 A -> 点击旧会话 B -> 新建对话 -> 切回旧会话 +const snapshot = await page.evaluate(() => window.__LIME_AGENTUI_PERF__?.summary()); +console.table(snapshot?.sessions ?? []); +``` + +已验证： + +```bash +npm exec -- vitest run "src/lib/agentUiPerformanceMetrics.test.ts" "src/components/agent/chat/components/MessageList.test.tsx" -t "agentUiPerformanceMetrics|旧会话首帧应记录|旧会话恢复首帧" +npm exec -- vitest run "src/components/AppSidebar.test.tsx" -t "旧会话预取|已有会话" +npm exec -- vitest run "src/components/agent/chat/hooks/useAsterAgentChat.test.tsx" -t "thread timeline|hydrate|switchTopic" +npm exec -- vitest run "src/lib/api/agent.test.ts" -t "getSession|listSessions|runtime" +npx eslint "src/lib/agentUiPerformanceMetrics.ts" "src/lib/agentUiPerformanceMetrics.test.ts" "src/lib/api/agentRuntime/sessionClient.ts" "src/components/agent/chat/hooks/useAgentSession.ts" "src/components/app-sidebar/AppSidebarConversationShelf.tsx" "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" --max-warnings 0 +npm run typecheck +``` + +结果： + +- 性能采集器 / MessageList 定向回归：通过，`4` 个测试通过。 +- AppSidebar 旧会话点击 / 预取回归：通过，`4` 个测试通过。 +- useAsterAgentChat 定向回归：通过，`9` 个测试通过。 +- Agent Runtime API 定向回归：通过，`15` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：通过。 + +下一步： + +1. 复用现有 Lime 页签时先执行 `window.__LIME_AGENTUI_PERF__.clear()`。 +2. 完成多旧会话切换后导出 `summary().sessions`，用 `clickToMessageListPaintMs`、`runtimeGetSessionDurationMs`、`finalRenderedMessagesCount` 判断瓶颈是在桥接、后端查询还是前端渲染。 +3. 如果 `runtimeGetSessionDurationMs` 低但 `clickToMessageListPaintMs` 高，下一刀优先 MessageList 虚拟化；如果两者都高，优先后端 `getSession` 分块/缓存。 + +## 风险记录 + +- 当前工作区已有多处未提交改动，本计划只记录本轮 AgentUI 主线增量；合并前需继续保护用户已有改动，不做回滚。 +- 本轮没有新增 Tauri command，不需要同步 command catalog / mock；后续若新增 session snapshot 或 timeline page command，必须按命令边界四侧同步。 + +### 2026-04-30：P1 第七刀，旧会话列表分页降载与路由追平去重 + +采集事实： + +- 首轮 Playwright 性能探针显示，点击新建 / 旧会话附近仍有侧边栏 `agent_runtime_list_sessions(limit=37)` 在 DevBridge 通道内运行：浏览器侧记录约 `10.2s`，同时 `workspace_get` 出现 `timeout after 5000ms`，导致旧会话列表和新建任务入口体感卡顿。 +- 后端日志对比显示，降载前异常环境下 `limit=37` / `limit=60` 最高可到数秒级；降载后同一工作区常规请求收敛到 `limit=11` / `limit=21`，最近采样约 `47-218ms`。 +- 二次 Playwright 探针发现“侧栏打开旧会话 -> 路由追平”会让同一历史会话出现 `switchStartCount=2` / `runtimeGetSessionStartCount=2`，即同一次用户意图触发两次 `getSession`。 + +已完成： + +- `AppSidebar.tsx`：最近 / 归档会话列表改成可见数量 `+1` 的哨兵分页： + - 最近会话首屏请求从强制 `37` 降到 `11`。 + - 归档首屏请求从 `17` 降到 `9`。 + - 点击“查看更多”时再按当前可见数量继续增加请求窗口，避免首页一次性预取两页以上历史。 +- `useAgentSession.ts`：任务中心内部 topics 初始列表请求从 `60` 降到 `21`，保留 `topicsListMayBeTruncatedRef` 作为旧会话不在首屏列表时的 detached 恢复兜底。 +- `AppSidebarConversationShelf.tsx`：hover/focus 预取延迟从 `140ms` 提高到 `900ms`，点击路径仍会取消未触发预取，减少鼠标扫过列表时的隐性 `getSession`。 +- `useWorkspaceInitialSessionNavigation.ts` / `AgentChatWorkspace.tsx`：新增“外部任务打开已启动”去重标记；侧栏/任务中心事件已经触发 `switchTopic` 后，2s 内路由 `initialSessionId` 追平不再重复打开同一会话。 +- `.tmp/agentui-perf-probe.mjs` 临时探针增强：只从侧栏会话 shelf 取目标、记录 invoke command、采集 CDP CPU/heap delta、long task 与 detail 阶段 summary。 + +Playwright 复测摘要： + +- 页面：`http://127.0.0.1:1420/`，Chrome 持久化 profile `.tmp/lime-agentui-e2e-chrome-profile`，DevBridge `3030` 健康检查通过。 +- 旧会话 A `AI网关MVP规划`：`switchStartCount=1`，`runtimeGetSessionStartCount=1`，`runtimeGetSessionDurationMs≈82ms`，`clickToSwitchSuccessMs≈685ms`，`clickToMessageListPaintMs≈626ms`，`longTask=0`。 +- 旧会话 B `PPT大纲规划`：去重后目标会话 `switchStartCount=1`，`runtimeGetSessionStartCount=1`；`agent_runtime_get_session` 最慢约 `157ms`。 +- 切回旧会话 A：目标会话 `switchStartCount=1`，`runtimeGetSessionStartCount=1`，`clickToMessageListPaintMs≈18ms`，`longTask=0`。 +- 控制台：`0 error / 1 warning`；warning 为 `useAgentTopicSnapshot.skipWithoutActiveTopic`，不阻塞主链，但提示后续 topics 截断后仍可继续优化 topic snapshot 噪音。 +- 网络：最慢 `/invoke` 从上一轮 `agent_runtime_get_session≈6.1s` / `agent_runtime_list_sessions≈10.2s`，恢复到本轮 `agent_runtime_get_session≤157ms`、`agent_runtime_list_sessions≤218ms`；上一轮 6s 主要由探针未等待 detail、连续发起多个旧会话请求叠加当时 Rust 编译负载污染，已用直接 curl 与等待 detail 的探针复核。 + +已验证： + +```bash +npx eslint "src/components/AppSidebar.tsx" "src/components/AppSidebar.test.tsx" "src/components/app-sidebar/AppSidebarConversationShelf.tsx" "src/components/agent/chat/hooks/useAgentSession.ts" "src/components/agent/chat/hooks/useAsterAgentChat.test.tsx" --max-warnings 0 +npm exec -- vitest run "src/components/AppSidebar.test.tsx" "src/components/agent/chat/hooks/useAsterAgentChat.test.tsx" -t "最近对话|窗口重新聚焦|打开已有会话|归档动作|加载话题时应后台预热" +npx eslint "src/components/agent/chat/AgentChatWorkspace.tsx" "src/components/agent/chat/workspace/useWorkspaceInitialSessionNavigation.ts" "src/components/agent/chat/index.test.tsx" --max-warnings 0 +npm exec -- vitest run "src/components/agent/chat/index.test.tsx" "src/components/agent/chat/workspace/useWorkspaceInitialSessionNavigation.test.tsx" -t "外层侧边栏通知打开历史会话|初始会话|dedupe|连续打开历史会话|任务中心初始会话标签" +npm run typecheck +node ".tmp/agentui-perf-probe.mjs" > ".tmp/agentui-perf-probe-latest.json" +``` + +结果： + +- AppSidebar / useAsterAgentChat 定向回归：通过，`8` 个测试通过。 +- AgentChatWorkspace / initial session navigation 定向回归：通过，`16` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：通过。 +- Playwright 性能探针：完成，旧会话切换不再出现同一目标重复 `switchTopic` / 重复 `getSession`；列表请求窗口已降到 `11/21`。 + +下一步： + +1. 修复或降噪 `useAgentTopicSnapshot.skipWithoutActiveTopic`：在 topics 初始窗口被截断时，active session 已由 detail 恢复，不应持续警告。 +2. 新建任务探针当前用性能事件等待会超时，应改成 DOM 空态断言（`home-start-surface` / `青柠一下，灵感即来`）后再纳入数值汇总，避免把探针等待超时误判为产品卡顿。 +3. 如果用户继续反馈“打开后消息正文仍慢”，下一刀不要再扩 AppSidebar；优先进入 `agent_runtime_get_session` 首包/分块返回或 MessageList 真虚拟化。 + +### 2026-04-30：P1 第八刀，新建页旧会话本地打开与探针降噪 + +采集事实： + +- 上一轮 Playwright 探针只剩 `0 error / 1 warning`，warning 来自旧会话 pending shell 阶段：`sessionId` 已切到目标会话，但 topics 还没等 `getSession` detail upsert，`useAgentTopicSnapshot` 误判为 active topic 缺失。 +- 新建任务按钮探针此前仍等待 `session.switch.*` / `messageList.paint` 性能事件；新建首页本身不会触发旧会话切换事件，导致 14s 超时被误算成产品卡顿。 +- 从新建页点击旧会话时，侧栏原先只在 `agentEntry=claw` 时走任务中心本地事件；在 `new-task` 首页会回到路由跳转路径，容易触发页面切换、路由追平和首刀延迟。 + +已完成： + +- `useAsterAgentChat.ts`：`useAgentTopicSnapshot` 在 `isSessionHydrating=true` 时抑制 active topic 暂缺 warning；真实缺失仍会在非 hydrating 阶段继续暴露。 +- `AppSidebar.tsx`：新建任务首页点击已有会话改走 `TASK_CENTER_OPEN_TASK_EVENT`，交给当前 Agent workspace 本地新增/切换标签，不再跳出当前页面做 claw 路由导航。 +- `AppSidebar.test.tsx`：更新新建任务首页点击历史会话的回归，断言发出本地 open event 且不调用 `onNavigate`。 +- `.tmp/agentui-perf-probe.mjs`：新建任务改为等待 `home-start-surface` / `青柠一下，灵感即来` DOM 就绪；会话选择避开已标记 active 的侧栏项，避免把 no-op 点击计入旧会话恢复耗时。 + +Playwright 复测摘要： + +- 页面：`http://127.0.0.1:1420/`，Chrome 持久化 profile `.tmp/lime-agentui-e2e-chrome-profile`，DevBridge `3030` 健康检查通过。 +- 旧会话 B `PPT大纲规划`：`clickToSwitchStartMs≈3ms`，`runtimeGetSessionDurationMs≈71ms`，`clickToSwitchSuccessMs≈81ms`，`switchStartCount=1`，`runtimeGetSessionStartCount=1`，`longTask=0`。 +- 切回旧会话 A `AI网关MVP规划`：`clickToSwitchStartMs≈1ms`，`runtimeGetSessionDurationMs≈108ms`，`clickToSwitchSuccessMs≈113ms`，`switchStartCount=1`，`runtimeGetSessionStartCount=1`，`longTaskMax≈51ms`。 +- 新建任务：`clickToHomeMs≈83ms`，`home-start-surface=true`，`0` 次 `getSession`，`longTask=0`。 +- 控制台：`0 error / 0 warning`；上一轮 `useAgentTopicSnapshot.skipWithoutActiveTopic` 已消失。 +- 网络：最慢有效 `/invoke` 为 `agent_runtime_list_sessions≈413ms`，`agent_runtime_get_session≤113ms`；长连接类 `/events` 在浏览器关闭时出现 `ERR_ABORTED`，为探针关闭上下文产生的非阻塞噪音。 + +已验证： + +```bash +node --check ".tmp/agentui-perf-probe.mjs" +npx eslint "src/components/agent/chat/hooks/useAsterAgentChat.ts" --max-warnings 0 +npm exec -- vitest run "src/components/agent/chat/hooks/useAgentTopicSnapshot.test.tsx" "src/components/agent/chat/hooks/useAsterAgentChat.test.tsx" -t "detached 会话缺少活动话题|切换到无本地快照的话题" +npx eslint "src/components/AppSidebar.tsx" "src/components/AppSidebar.test.tsx" "src/components/agent/chat/hooks/useAsterAgentChat.ts" --max-warnings 0 +npm exec -- vitest run "src/components/AppSidebar.test.tsx" "src/components/agent/chat/index.test.tsx" -t "新建任务首页点击已有会话|new-task 首页收到外层侧栏打开历史会话|任务中心内点击已有会话|打开已有会话时若导航已有缓存任务" +npm run typecheck +npm run bridge:health -- --timeout-ms 120000 +node ".tmp/agentui-perf-probe.mjs" > ".tmp/agentui-perf-probe-latest.json" +``` + +结果： + +- Hook 定向回归：通过，`2` 个测试通过。 +- AppSidebar / AgentChatPage 定向回归：通过，`4` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：通过。 +- Playwright 性能探针：完成，旧会话点击到 switch start 已收敛到 `1-3ms`；新建任务首页首帧收敛到 `~83ms`；控制台 warning 清零。 + +下一步： + +1. 如果用户继续感知“旧会话正文慢”，优先看真实重历史会话的 `messageList.paint` 是否稳定产出；当前小会话 detail 很快，但首个 no-op 样本说明探针还需要进一步区分“已在内存中的会话”与“真实冷打开”。 +2. 对大历史会话继续采 `renderedMessagesCount / threadItemsCount / timelineGroupsCount / longTaskMaxMs`，若 `getSession≤150ms` 但 paint 或 long task 高，下一刀进入 MessageList 虚拟化 / timeline worker。 +3. 当前磁盘空间不足时不跑 `npm run tauri:dev:headless` / `verify:gui-smoke` 这类会触发 Rust 编译的重验证；本轮以已就绪 DevBridge + 浏览器实测覆盖主路径。 + +### 2026-04-30：P1 第九刀，旧会话 pending shell 直接进入会话布局 + +采集事实： + +- 上一轮探针中，旧会话 `switch.success` 与 `agent_runtime_get_session` 已较快返回，但部分样本 `messageList.paintCount=0`，探针等待到超时，体感上等同“打开旧会话后仍停在新建首页 / 空态”。 +- 代码复核发现 `chatLayoutVisibility` 已支持 `isSessionHydrating`，但 `AgentChatWorkspace` 还没有把该状态传入 `shouldShowChatLayout`；同时 `effectiveShowChatPanel` 的 `new-task` 分支也没有把 hydrating 算作会话活动。 +- 这会导致旧会话 pending shell 阶段虽然已进入 session switch 主链，但布局层仍按空白新建页处理，MessageList 无法及时挂载并产生 paint 指标。 + +已完成： + +- `AgentChatWorkspace.tsx`： + - `shouldShowChatLayout(...)` 调用补入 `isSessionHydrating`。 + - `effectiveShowChatPanel` 在 `agentEntry="new-task"` 时把 `isSessionHydrating` 计入会话活动，旧会话恢复的 pending shell 不再被空白首页分支吞掉。 +- `chatLayoutVisibility.ts` / `chatLayoutVisibility.test.ts`：保留第八刀新增的 hydrating 可见性参数与回归，确保旧会话恢复 pending shell 阶段直接进入会话布局。 + +已验证： + +```bash +npx eslint "src/components/agent/chat/AgentChatWorkspace.tsx" "src/components/agent/chat/utils/chatLayoutVisibility.ts" "src/components/agent/chat/utils/chatLayoutVisibility.test.ts" "src/components/agent/chat/index.test.tsx" "src/components/AppSidebar.tsx" "src/components/AppSidebar.test.tsx" "src/components/agent/chat/taskCenterDraftTaskEvents.ts" --max-warnings 0 +npm exec -- vitest run "src/components/agent/chat/utils/chatLayoutVisibility.test.ts" "src/components/agent/chat/index.test.tsx" "src/components/AppSidebar.test.tsx" -t "旧会话恢复 pending shell|new-task 首页收到外层侧栏打开历史会话|新建任务首页点击已有会话|任务中心内点击已有会话|空白新建任务首页应保留浏览器式工作区顶栏" +npm run typecheck +``` + +结果： + +- ESLint touched files：通过。 +- AgentUI 布局 / 新建页 / 侧栏打开历史会话定向回归：通过，`5` 个测试通过。 +- TypeScript：通过。 + +Playwright / DevBridge 状态： + +- 下一次真实复测必须继续按仓库规则使用 Chrome 持久化 profile：`.tmp/lime-agentui-e2e-chrome-profile`，不使用 isolated 临时 profile，不传 `--no-sandbox`。 +- 本轮真实 E2E 暂未完成：`npm run bridge:health -- --timeout-ms 5000` 多次失败，`http://127.0.0.1:3030/health` 未监听。 +- 尝试拉起 `npm run tauri:dev:headless` 时，当前机器已有其它 Rust / GUI smoke / cargo test 编译链占用 Cargo lock，输出持续停在 `Blocking waiting for file lock on package cache / artifact directory`；同时磁盘可用空间约 `8.8-9.3GiB`，继续叠加编译会污染旧会话卡顿采样。 +- 已停止本轮自己拉起的 headless 链路，未清理用户已有的长期 Tauri / cargo 校验进程；待 DevBridge 恢复后再复跑真实旧会话多标签性能探针。 + +下一步： + +1. DevBridge 3030 恢复后，复测：新建任务首页 -> 打开旧会话 A -> 打开旧会话 B -> 切回旧会话 A -> 新建空白任务标签。 +2. 采集并对比 `clickToSwitchStartMs`、`runtimeGetSessionDurationMs`、`clickToSwitchSuccessMs`、`messageListPaintCount`、`clickToMessageListPaintMs`、`longTaskMaxMs`。 +3. 如果 `messageList.paintCount` 仍为 `0`，下一刀继续查 `WorkspaceShellScene` / `MessageList` 条件渲染；如果 paint 正常但 `longTaskMaxMs` 高，转入 MessageList 虚拟化或 timeline worker。 + +### 2026-04-30：P1 第十刀，旧会话首帧延后底部运行状态行重计算 + +采集事实： + +- 第九刀修复布局后，旧会话 pending shell 能进入会话布局；下一处首帧风险集中在 `MessageList` 首帧仍会为了最后一条 assistant 的底部运行状态行同步扫描 `threadItems`。 +- 已分页旧会话或历史窗口里，timeline 本体虽然已经延后，但 `buildInputbarRuntimeStatusLineModel` / `buildAgentTaskRuntimeCardModel` 仍可能在首帧扫描并过滤大量历史 `threadItems`，造成 CPU 峰值和短期数组分配。 +- 这些底部完成态状态行不是旧会话首帧正文可见的必要条件；等待 historical timeline idle 后再补上，不影响发送中、等待输入、排队或 active turn 的实时反馈。 + +已完成： + +- `MessageList.tsx`： + - 对旧会话恢复窗口增加 `shouldDeferTailRuntimeStatusLine`。 + - 当旧会话历史 timeline 正在 idle 延后、且没有发送中 / active turn / pending action / queued turn / pending request 时，首帧不再构建底部运行状态行。 + - `messageList.commit/paint` 指标增加 `tailRuntimeStatusDeferred`，后续 E2E 可直接判断首帧是否成功避开这段同步计算。 +- `MessageList.test.tsx`：旧会话消息少但执行过程多的场景，新增断言：首帧不渲染 `inputbar-runtime-status-line`，idle 后再恢复。 +- `inputbarRuntimeStatusLine.ts` / `agentTaskRuntime.ts`：把多处 `[...array].reverse()` 与 `filter(...).find(...)` 改为从尾部循环或单次扫描，减少旧会话打开时的短期数组复制。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" -t "旧会话消息较少但执行过程很多时也应延后构建 timeline|旧会话首帧应记录可汇总的渲染采样数值|复杂任务完成后应把运行状态" +npm exec -- vitest run "src/components/agent/chat/components/Inputbar/components/InputbarRuntimeStatusLine.test.tsx" "src/components/agent/chat/components/MessageList.test.tsx" -t "旧会话消息较少但执行过程很多时也应延后构建 timeline|复杂任务完成后应把运行状态|InputbarRuntimeStatusLine" +npm exec -- vitest run "src/components/agent/chat/utils/agentTaskRuntime.test.ts" +npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" "src/components/agent/chat/utils/agentTaskRuntime.ts" "src/components/agent/chat/utils/inputbarRuntimeStatusLine.ts" --max-warnings 0 +npm run typecheck +npm run bridge:health -- --timeout-ms 5000 +``` + +结果： + +- MessageList 定向回归：通过，`3` 个测试通过。 +- Inputbar / MessageList 组合回归：通过，`4` 个测试通过。 +- Agent task runtime 单测：通过，`4` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：通过。 +- DevBridge 健康检查：失败，`http://127.0.0.1:3030/health` 仍未监听；因此本轮仍未做真实 Playwright 多旧会话采样。 + +下一步： + +1. DevBridge 恢复后，优先复测旧会话 A/B 切换，确认 `messageList.paint` 中 `tailRuntimeStatusDeferred=true` 的首帧是否降低 `longTaskMaxMs`。 +2. 如果 E2E 仍显示 `longTaskMaxMs` 高，下一刀继续把 historical `threadItems` 的摘要计算改成按 `latestTurnId` 建索引或移入 idle/worker。 +3. 如果 E2E 显示 `runtimeGetSessionDurationMs` 高于渲染耗时，回到后端 `getSession` 分块/缓存，不再继续前端微调。 + +### 2026-04-30：P1 第十一刀，旧会话按可见 turns 精确裁剪 threadItems + +采集事实： + +- 第十刀已把旧会话首帧的底部运行状态行延后，但 `MessageList` 仍会在历史窗口首帧按消息数量粗略截取尾部 `threadItems`。 +- 对“消息少、工具过程多”的旧会话，粗略按 `messageCount * factor` 截取容易把与当前可见消息无关的 turn item 也带入 `buildMessageTurnTimeline` / runtime status 计算，继续造成短期 CPU 与数组分配峰值。 +- 旧会话分页窗口的首帧只需要渲染尾部可见 assistant 关联的 turns；其它历史 turns 可以等待“加载完整历史”或后续展开再参与计算。 + +已完成： + +- `MessageList.tsx`： + - 移除按消息数倍数裁剪 `threadItems` 的粗略常量。 + - 根据当前实际渲染的 assistant 消息数推导旧会话恢复窗口的 `renderedTurns` tail window，并额外保留 `currentTurnId`，避免进行中 turn 被裁掉。 + - 在分页旧会话 / hidden history 场景下构建 `renderedTurnIdSet`，`renderedThreadItems` 只保留这些 turns 对应的 `threadItems`。 +- `MessageList.test.tsx`：新增“已分页旧会话首帧应只把尾部相关 turns 的 threadItems 纳入计算”回归，覆盖 8 个 turns、40 个 threadItems、首帧只渲染 2 条消息的场景；断言首帧只纳入尾部 2 个 turns / 10 个 threadItems。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" -t "已分页旧会话首帧应只把尾部相关 turns 的 threadItems 纳入计算|旧会话消息较少但执行过程很多时也应延后构建 timeline|已分页旧会话的完成执行过程应先折叠为轻量摘要" +npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" --max-warnings 0 +python3 - <<'PY' +# 120s wrapper for npm run typecheck +PY +``` + +结果： + +- MessageList 定向回归：通过，`3` 个测试通过。 +- ESLint touched files：通过。 +- TypeScript：本轮 wrapper 在 `120s` 超时后终止；未看到类型错误输出。当前机器仍有其它 Rust / cargo 编译链与编辑器 tsserver 占用 CPU，先记录为环境性未完成，不继续叠加新的全量 typecheck。 +- DevBridge / Playwright：尚未复测；需先确认 `3030` 恢复，避免在 bridge 未就绪时把前端优化误判为 GUI 卡顿。 + +下一步： + +1. DevBridge 恢复后，复测多旧会话切换并采集 `messageList.commit/paint`，重点看 `threadItemsCount` 是否随可见 turn 数下降，而不是随全量历史增长。 +2. 如果 `getSession≤150ms` 但 `clickToMessageListPaintMs` 或 `longTaskMaxMs` 仍高，下一刀继续把 `buildMessageTurnTimeline` 的历史 mapping / sort 移到 idle 或建立 `turn_id -> items` 索引。 +3. 如果 `runtimeGetSessionDurationMs` 高于渲染耗时，转回后端 `getSession` 分块 / 缓存，不再继续前端微调。 + +### 2026-04-30：P1 第十二刀，旧会话首帧完全跳过 threadItems 扫描 + +采集事实： + +- 第十一刀已经把旧会话恢复窗口的 `threadItems` 从“按消息数粗略截尾”改成“按可见 turns 精确裁剪”，但首帧仍需要遍历全量 `threadItems` 才能筛出尾部 turns。 +- 对工具轨迹特别多的旧会话，哪怕最终只渲染 10 条相关 items，首帧的全量数组扫描仍会造成鼠标 loading、CPU 峰值和短期内存分配。 +- 旧会话首帧的核心目标是先让消息文本和吸顶布局可见；完成态历史 timeline / 底部运行状态行可以继续等 idle 后补齐。 + +已完成： + +- `MessageList.tsx`： + - `shouldDeferHistoricalTimeline` 改为基于原始 `threadItems.length` 与恢复窗口判断，避免为了判断是否延后而先扫描 / 裁剪 `threadItems`。 + - 新增 `shouldDeferThreadItemsScan`：旧会话历史 timeline 尚未 idle-ready、且没有 active turn 时，`renderedThreadItems` 首帧直接返回空数组，不再遍历全量历史 items。 + - `timelineHydrationKey` 改用原始 `threadItems.length + lastItemId`，确保后台 item 变化仍能触发 idle hydrate。 + - `messageList.commit/paint` 指标增加 `threadItemsScanDeferred`，后续 Playwright 能直接看到首帧是否跳过了全量扫描。 +- `MessageList.test.tsx`：更新旧会话窗口测试，首帧断言 `threadItemsScanDeferred=true` 且 `threadItemsCount=0`；idle 后再断言精确裁剪到尾部 2 个 turns / 10 个 `threadItems`。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" -t "已分页旧会话首帧应只把尾部相关 turns 的 threadItems 纳入计算|旧会话消息较少但执行过程很多时也应延后构建 timeline|已分页旧会话的完成执行过程应先折叠为轻量摘要" +npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" --max-warnings 0 +npm run bridge:health -- --timeout-ms 5000 +python3 - <<'PY' +# 180s wrapper for npm run typecheck +PY +``` + +结果： + +- MessageList 定向回归：通过，`3` 个测试通过。 +- ESLint touched files：通过。 +- DevBridge 健康检查：失败，`http://127.0.0.1:3030/health` 未监听；本轮没有进入 Playwright 真实旧会话采样。 +- TypeScript：本轮 wrapper 未超时，但 `tsc --noEmit` 约 `40s` 后以 code `241` 退出且无错误诊断；检查后未发现残留 `tsc` 进程。当前仍有其它 `run-vitest-smart` / Vite 进程占用 CPU，先记录为环境性未闭环，避免继续叠加重校验影响用户本机性能。 + +下一步： + +1. DevBridge 3030 恢复后复测旧会话 A/B 切换，重点读取 `messageList.commit/paint` 的 `threadItemsScanDeferred`、`threadItemsCount`、`clickToMessageListPaintMs` 和 `longTaskMaxMs`。 +2. 若首帧 `threadItemsScanDeferred=true` 后仍卡顿，下一刀优先检查 `StreamingRenderer` / Markdown light render 的长文本成本，而不是继续优化 timeline。 +3. 若 idle 后 timeline hydrate 才出现长任务，则把 `buildMessageTurnTimeline` 的 grouping / sort 继续放到 worker 或分片 idle 队列。 + +### 2026-04-30：P1 第十三刀，旧会话首帧延后 contentParts 细节扫描 + 流式首字立即可见 + +采集事实： + +- DevBridge `3030` 仍未监听，当前无法用 Playwright 复测真实旧会话 A/B 切换；同时本机已有多条 Rust 编译链占用 CPU，继续启动 `tauri:dev:headless` 会污染“卡顿”采样。 +- 第十二刀已经让旧会话首帧跳过全量 `threadItems` 扫描，但 `MessageList` 在渲染每条历史 assistant 时仍会同步 `sanitizeContentPartsForDisplay`、过滤工具/思考片段、计算 inline process coverage。 +- 对已完成的旧会话，这些 `contentParts` 细节不是首帧文本可见的必要条件；可以等 historical timeline idle 后再恢复。 +- 流式纯文本首次挂载时，`StreamingText` 之前以空字符串作为首帧，必须等下一次 `requestAnimationFrame` 才开始吐字；在模型已有首个分片时，这会放大“首字慢”的体感。 + +已完成： + +- `MessageList.tsx`： + - 增加 `shouldDeferHistoricalAssistantMessageDetails`，旧会话恢复窗口、historical timeline 尚未 ready、没有 active turn / sending / action request / toolCalls / thinking 时，首帧不再处理历史 assistant 的 `contentParts` 细节。 + - 增加 `historicalContentPartsDeferredCount` 指标，后续 E2E 可判断是否命中旧会话首帧轻量路径。 + - 保留文本正文优先渲染；idle 后再恢复 contentParts / timeline 细节，避免首帧被旧工具过程拖慢。 +- `MessageList.test.tsx`：新增“旧会话首帧应延后历史助手 contentParts 细节扫描”回归，断言首帧 `data-content-parts=0`、`historicalContentPartsDeferredCount=1`，idle 后指标恢复为 `0`。 +- `StreamingRenderer.tsx`： + - 增加 `STREAMING_TEXT_INITIAL_VISIBLE_CHARS` 和 `resolveInitialStreamingDisplayText`。 + - 流式纯文本首次挂载时立即显示前 `12` 个字符；结构化内容仍保持空首帧，避免把未完整的 A2UI / write_file 协议直接露出。 + - 当流式文本发生非 append 替换时，也立即种下同样的纯文本前缀，减少重置后的空白等待。 +- `StreamingRenderer.test.tsx`：新增“流式纯文本首帧应立即显示前缀”回归，并保留大 backlog 追赶测试。 + +已验证： + +```bash +npm exec -- vitest run "src/components/agent/chat/components/MessageList.test.tsx" -t "旧会话首帧应延后历史助手 contentParts 细节扫描|已分页旧会话首帧应只把尾部相关 turns 的 threadItems 纳入计算|旧会话消息较少但执行过程很多时也应延后构建 timeline" +npm exec -- vitest run "src/components/agent/chat/components/StreamingRenderer.test.tsx" -t "流式纯文本首帧应立即显示前缀|流式正文积压较多时应快速追上最新目标文本|纯文本内容应短路跳过结构化解析" +npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" "src/components/agent/chat/components/StreamingRenderer.tsx" "src/components/agent/chat/components/StreamingRenderer.test.tsx" --max-warnings 0 +npm run bridge:health -- --timeout-ms 5000 +``` + +结果： + +- MessageList 定向回归：通过，`3` 个测试通过。 +- StreamingRenderer 定向回归：通过，`3` 个测试通过。 +- ESLint touched files：通过。 +- DevBridge 健康检查：失败，`http://127.0.0.1:3030/health` 未监听。 +- TypeScript：本轮暂不追加全量 typecheck；当前机器已有多条 Rust 编译链高负载运行，继续叠加全量 TS 校验会影响用户正在反馈的 CPU/鼠标 loading 问题。待编译链空闲后补跑。 + +下一步： + +1. DevBridge 恢复后复测真实旧会话：读取 `threadItemsScanDeferred`、`historicalContentPartsDeferredCount`、`clickToMessageListPaintMs`、`longTaskMaxMs`。 +2. 如果旧会话首帧仍慢，下一刀只看 `MarkdownRenderer` 本体：把 restored history 的短正文也先走纯文本预览 / idle 后 Markdown hydrate，避免 ReactMarkdown 在首帧解析多条历史短回复。 +3. 如果流式仍首字慢，下一步看后端事件到达时间：区分“前端首帧空白”与“runtime 第一段 delta 到达慢”。 + +第十三刀补充： + +- `agentUiPerformanceMetrics.ts`：summary 增加 `historicalContentPartsDeferredMax` 与 `threadItemsScanDeferredCount`，让 Playwright 不只读 raw entries，也能在 session summary 里直接看到旧会话首帧轻量路径命中次数。 +- `agentUiPerformanceMetrics.test.ts`：补充 summary 断言，覆盖 contentParts 延后最大值与 threadItems 扫描延后次数。 +- 追加验证： + +```bash +npm exec -- vitest run "src/lib/agentUiPerformanceMetrics.test.ts" +npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" "src/components/agent/chat/components/StreamingRenderer.tsx" "src/components/agent/chat/components/StreamingRenderer.test.tsx" "src/lib/agentUiPerformanceMetrics.ts" "src/lib/agentUiPerformanceMetrics.test.ts" --max-warnings 0 +``` + +结果：性能指标汇总单测通过，ESLint touched files 继续通过。 diff --git a/docs/exec-plans/multimodal-runtime-contract-plan.md b/docs/exec-plans/multimodal-runtime-contract-plan.md index b3be01aba..35758ebe0 100644 --- a/docs/exec-plans/multimodal-runtime-contract-plan.md +++ b/docs/exec-plans/multimodal-runtime-contract-plan.md @@ -62,6 +62,32 @@ runtime identity 2. `src/lib/governance/modalityCapabilityMatrix.json` 3. contract 守卫检查 `required_capabilities` 与 `routing_slot` 是否引用已登记能力和模型角色 +### Phase 3：ModalityExecutionProfile + +状态：进行中。 + +本阶段输出： + +1. `docs/roadmap/warp/execution-profile.md` +2. `src/lib/governance/modalityExecutionProfiles.json` +3. `image_generation`、`browser_control`、`pdf_extract`、`voice_generation`、`audio_transcription`、`web_research`、`text_transform` 的最小 execution profile。 +4. contract 守卫检查 current contract 必须被 profile 覆盖，且 profile 的模型角色、权限、LimeCore policy、artifact policy 必须覆盖 contract。 +5. `src/lib/governance/modalityExecutionProfiles.ts` 把 profile / adapter registry 解析成 launch metadata 快照。 +6. 后续仍需把 profile registry 接入 Rust runtime policy merge、thread read、evidence 和 GUI 可视化。 + +### Phase 5：Executor Adapter registry + +状态：进行中。 + +本阶段输出： + +1. `src/lib/governance/modalityExecutionProfiles.json` 的 `executor_adapters` +2. `skill:image_generate`、`browser:browser_assist`、`skill:pdf_read`、`service_skill:voice_runtime`、`skill:transcription_generate`、`skill:research`、`skill:text_transform` 的最小 adapter 声明。 +3. contract 守卫检查 current contract 的 `executor_binding` 必须能解析到 `executor_kind:binding_key` adapter。 +4. adapter 的 progress / cancel / resume / artifact 支持位、artifact output、permission requirements 与 failure mapping 必须覆盖 contract。 +5. 前端 `runtime_contract` snapshot 已携带 `executor_adapter` 摘要，避免上层入口继续只传 executor binding。 +6. 后续仍需把 adapter registry 接入真实执行前检查、统一 task index 与 LimeCore policy snapshot。 + ### Phase 7：上层入口绑定 状态：进行中。 @@ -80,10 +106,11 @@ runtime identity 1. `browser_control` 先作为底层 Browser Assist 运行合同进入 `harness.browser_assist`，不新增独立 browser task 协议。 2. `@浏览器 / @Browser Agent / @Mini Tester` 仍保留原始用户消息，只把 `browser_requirement`、launch URL 与 contract snapshot 交给后端 runtime 决策。 -3. `browser_control.bound_entries` 暂不扩展；`@` 命令只是入口层，不能成为底层 contract 事实源。 +3. `browser_control.bound_entries` 只登记 `@浏览器`、`@Browser Agent`、`@Mini Tester`、`@Web Scheduler`、`@Web Manage` 的 entry metadata；`@` 命令仍只是入口层，不能写 task / artifact / viewer 事实源。 4. 后端 `BrowserAssistRuntimeHint` 必须解析并保留 `browser_control` 合同快照，browser tool result metadata 需要携带同一组 contract 字段。 5. Evidence Pack 从 browser tool timeline metadata 导出 `browser_control` 的 `modalityRuntimeContracts`，使浏览器动作也进入 replay 可用的事实源。 -6. Replay / grader 必须把 `browser_control` 识别成 Browser Assist 回归样本，检查 browser action trace、禁止 WebSearch 替代真实浏览器动作。 +6. Evidence Pack 的 `modalityRuntimeContracts.snapshotIndex.browserActionIndex` 必须汇总 action、session、URL、observation 与 screenshot，使 Browser Assist 不再只能靠 raw snapshots 人工查找。 +7. Replay / grader 必须把 `browser_control` 识别成 Browser Assist 回归样本，检查 browser action trace / browserActionIndex、禁止 WebSearch 替代真实浏览器动作。 ### Phase 9：PDF Extract 底层合同注入 @@ -112,7 +139,7 @@ runtime identity 4. 服务型场景 prompt 必须显式说明当前底层合同，禁止退回 `legacy_tts_test_command` 或伪造“云端已提交”。 5. Evidence Pack 必须能从 voice runtime / service scene timeline args 或 metadata 导出 `voice_generation` 的 `modalityRuntimeContracts`，使配音合同进入 replay 可评分事实源。 6. Replay / grader 必须把 `voice_generation` 识别成本地 ServiceSkill/voice runtime 回归样本，检查 `service_scene_launch(scene_key=voice_runtime)` 或后续 audio_task/audio_output 证据，并禁止 `legacy_tts_test_command`、伪造云端提交、普通聊天文本或通用文件卡替代。 -7. 本阶段新增最小 `audio_generate` task artifact 协议，把 `voice_generation` 写入标准 `audio_task/audio_output` 产物；`audio viewer` 与真实音频 worker 继续后置，避免提前拆出第二套音频执行事实源。 +7. 本阶段新增最小 `audio_generate` task artifact 协议，把 `voice_generation` 写入标准 `audio_task/audio_output` 产物；聊天区提供最小音频任务卡与运行时文档 viewer，并允许执行器通过同一 task artifact 回写 `audio_output.completed`。当前 `lime-audio-worker` 已接入 OpenAI-compatible speech provider seam：有 current API Key Provider 凭证时写入 `.lime/runtime/audio/*.mp3` 并回写 `payload.audio_output.completed`；未配置 provider / model / client 时必须显式写回 `audio_provider_*` 错误，聊天任务卡与运行时文档 viewer 都从同一 task artifact 展示完成/失败事实，不回退 legacy TTS，也不伪造音频路径。 ### Phase 11：Web Research 合同注入 @@ -143,6 +170,20 @@ runtime identity 6. Replay / grader 必须把 `text_transform` 识别成文本转换回归样本，检查文本转换 Skill trace 与必要文件读取证据，并禁止 `frontend_direct_text_transform`、ToolSearch、WebSearch 或普通聊天替代。 7. 本阶段不拆 `summary_generation`、`translation`、`analysis`、`publish_compliance` 或 `logo_decomposition` 平行 contract；只有当独立 artifact、执行器或 viewer 形成稳定事实源时才拆子合同。 +### Phase 13：Audio Transcription 合同注入 + +状态：进行中。 + +本阶段输出： + +1. `audio_transcription` 作为音频转写底层运行合同进入 `harness.transcription_skill_launch.transcription_task`，`@转写 / @transcribe / @Audio Extractor` 仅保留为 entry source。 +2. 前端 `@转写` 发送时保留原始消息，并注入 `modality_contract_key=audio_transcription`、`required_capabilities`、`routing_slot=audio_transcription_model` 与 `runtime_contract` 快照。 +3. 后端 `prepare_transcription_skill_launch_request_metadata` 需要为旧客户端或缺省 metadata 补齐同一组合同字段，避免回退成前端直连 ASR、普通文件读取或 generic file transcript。 +4. `Skill(transcription_generate)` prompt 必须显式说明当前底层合同，继续压制 ToolSearch / WebSearch / Read / Glob / Grep 首刀偏航。 +5. 标准 `transcription_generate` task file 必须写入同一 `audio_transcription` 合同与 `transcript.pending` 子产物，CLI / Agent tool 回退入口不能另写平行协议。 +6. 媒体任务索引、Evidence Pack 与 Replay / grader 必须能定位 transcript 状态、来源、语言、输出格式与失败码，使 `audio_transcription` 不再只停留在入口 metadata。 +7. 本阶段接入最小 `lime-transcription-worker` 与用户可见恢复层：OpenAI-compatible provider 成功时写入 `.lime/runtime/transcripts/*` 并回写 `transcript.completed`，provider/source/contract 失败时回写 `transcript.failed` 与明确错误码；聊天任务卡和 `.lime/runtime/transcription-generate/*.md` 运行时文档优先消费统一媒体任务索引的 `transcript_*` snapshot，并在完成态读取 `transcript_path` 文本内容供复制校对；当前 viewer 已能从 JSON / SRT / VTT transcript 中解析时间轴与说话人，生成可逐段编辑校对的段落表；保存校对稿时复用 ArtifactDocument 版本链，写入 `transcriptCorrection*` / `transcriptSegmentsCorrected` / `transcriptCorrectionDiffSummary` metadata，不改写原始 ASR 输出，并在 viewer 中展示“校对稿已保存”状态；更多 ASR adapter 与本地离线 ASR 后续继续消费同一 task artifact 与 `transcriptIndex`。 + ## 本轮最小闭环 第一轮已完成治理底座；当前纠偏后的最小闭环必须至少贯通一个真实运行时 vertical slice。 @@ -177,16 +218,40 @@ runtime identity 18. `web_research` 第二十刀把 `@研报 / @竞品` 收为报告型子入口：入口仍走 `report_skill_launch -> Skill(report_generate)`，底层合同、evidence、replay 统一归入 `web_research`。 19. `text_transform` 第二十一刀把 `@读文件 / @总结 / @翻译 / @分析 / @发布合规 / @Logo拆解` 收为文本/文档转换子入口：入口仍分别走 `summary_skill_launch`、`translation_skill_launch`、`analysis_skill_launch`，底层合同、evidence、replay 统一归入 `text_transform`。 20. `voice_generation` 第二十三刀新增最小 `audio_generate` artifact protocol：`create_audio_generation_task_artifact` / `lime_create_audio_generation_task` 写入标准 `audio_task/audio_output` payload，前端 API、DevBridge mock、task index、evidence pack 与 replay grader 都消费同一份 `voice_generation` 合同快照。 +21. `voice_generation` 第二十四刀把 `audio_generate` tool result 恢复为聊天区任务卡与 `.lime/runtime/audio-generate/*.md` 运行时文档；打开任务卡不再尝试展示隐藏 `.lime/tasks/audio_generate/*.json`，而是进入可读的音频任务 viewer，同时继续把 task JSON 作为 primary source。 +22. `voice_generation` 第二十五刀新增最小 `audio_output` 完成态回写：`complete_audio_generation_task_artifact` 只更新标准 `audio_generate` task file，把 `audio_path / mime_type / duration_ms` 同步写入 payload、`payload.audio_output` 与 `record.result`，并通过同一媒体任务事件刷新前端，不新增云端执行协议。 +23. `voice_generation` 第二十六刀把完成态回流接回聊天任务卡：`useWorkspaceAudioTaskPreviewRuntime` 监听同一媒体任务事件并轮询已追踪的 `audio_generate` task file，从 `payload.audio_output` / `record.result.audio_output` 恢复 `audioUrl / mimeType / durationMs`，让执行器回写后能直接在现有轻卡中出现可播放结果。 +24. `voice_generation` 第二十七刀新增最小音频执行器骨架：`create_audio_generation_task_artifact` 创建标准 `audio_generate` 后会进入同一媒体任务执行链；在真实音频 provider 尚未接通时，worker 明确把 task 写成 `failed`、`last_error.code=audio_worker_unavailable`、`payload.audio_output.status=failed`，不伪造 `audio_path`，也不回流 `legacy_tts_test_command`。 +25. `voice_generation` 第二十八刀把音频执行器从纯骨架推进到 OpenAI-compatible provider seam：worker 通过 current API Key Provider 解析 `provider_id/model/api_key/base_url`，调用 `/audio/speech` 获取真实音频 bytes，保存到 `.lime/runtime/audio/.mp3`，并用同一 task artifact 回写 `payload.audio_output.completed`、`record.result.audio_output` 与 `worker_id=lime-audio-worker`；provider 未配置、resolver 缺失或非 OpenAI-compatible client 时分别写回 `audio_provider_unconfigured` / `audio_provider_resolver_unavailable` / `audio_provider_client_missing`，不回 legacy。 +26. `voice_generation` 第二十九刀把 worker 失败态回流到聊天音频任务卡：`useWorkspaceAudioTaskPreviewRuntime` 从 `payload.audio_output` / `record.last_error` 解析 `audio_provider_*` 错误码、错误原因与 retryable，不保留失败 artifact 里不存在的 `audio_path`；`TaskMessagePreview` 显示 Provider 错误码和原因，让“未配置 provider / resolver 不可用 / adapter 缺失”成为用户可见事实，而不是泛化成普通失败。 +27. `voice_generation` 第三十刀把完成/失败态同步到同一个运行时文档 viewer：音频任务轮询更新 `taskPreview` 的同时，会重建 `.lime/runtime/audio-generate/*.md` 对应的 ArtifactDocument metadata；打开任务卡时看到的是 `audio_output.completed` 的音频路径或 `audio_provider_*` 失败原因，而不是创建任务时的旧“等待执行器”文档。 +28. `voice_generation` 第三十一刀把 `audio_output` 细节接入 evidence / replay：`audio_task.modality_runtime_contract` 快照现在携带 `audioOutput.status/audioPath/providerId/model/workerId/errorCode`，`snapshotIndex.audioOutputIndex` 可检索完成/失败态；replay tags、failure modes、expected 与 grader 会区分 `audio_output.completed` 和 Provider 失败，禁止把已完成音频降级成 Markdown 文本、普通聊天或通用文件卡。 +29. `voice_generation` 第三十二刀把 `audio_output` 细节接入统一媒体任务索引：`list_media_task_artifacts` 的 `modality_runtime_contracts` 现在输出 `audio_output_count`、`audio_output_statuses`、`audio_output_error_codes` 与每个 snapshot 的 `audio_output_*` 字段；前端类型与浏览器 mock 同步，任务列表/恢复逻辑不必重新打开 task JSON 才能知道音频已完成、失败或缺少 Provider。 +30. `voice_generation` 第三十三刀让聊天音频任务恢复优先消费统一媒体任务索引：`useWorkspaceAudioTaskPreviewRuntime` 会先读取 `list_media_task_artifacts(modalityContractKey=voice_generation, taskType=audio_generate)` 的 `audio_output_*` snapshot，把完成态音频路径或 Provider 失败直接同步到任务卡与同一条运行时文档；只有索引缺失该 task 或没有 `audio_output_status` 时才回退读取单个 task artifact。 +31. `browser_control` 第三十八刀把同一份 `browserActionIndex` 接入最小 `browser_replay_viewer`：Harness evidence 面板的“打开复盘”会构造 `browser_assist` Artifact，并由 `BrowserAssistRenderer` 消费 `browserActionIndex` 展示 action/session/URL/observation/screenshot 与最近 `browser_session` / `browser_snapshot` 项；本刀不新增 browser task 协议，viewer 仍消费 Evidence Pack 的同一事实源。 +32. `audio_transcription` 第三十九刀把 `@转写 / @transcribe / @Audio Extractor` 收为 registry entry binding：前端 `transcription_skill_launch.transcription_task` 与 Rust prompt prepare 都会保留 `audio_transcription` 合同快照，`transcript` artifact graph 也反向登记该 current contract；本刀只建立底层合同，不新增真实转写 worker、完整 transcript viewer 或 replay 索引。 +33. `audio_transcription` 第四十刀把 `transcription_generate` task artifact 接到底层合同事实源：Agent tool 与 CLI 回退入口现在写入标准 `.lime/tasks/transcription_generate/*.json`，payload 固化 `audio_transcription` contract、`requested_target=transcript` 与 `transcript.pending`；`list_media_task_artifacts` 输出 transcript 状态/来源/语言/格式，Evidence Pack 输出 `snapshotIndex.transcriptIndex`，Replay / grader 能识别缺失 task trace、缺失 transcript index 与 `transcript.failed`。 +34. `audio_transcription` 第四十一刀接入最小 `lime-transcription-worker`：Agent tool 创建 task 后会启动转写执行链，OpenAI-compatible provider 成功时把响应写入 `.lime/runtime/transcripts/.json|txt|srt|vtt` 并回写 `transcript.completed` / `record.result.transcript`；缺少 AppHandle、provider/model、API Key、source 文件、非 OpenAI-compatible adapter 或 provider 请求失败时回写 `transcript.failed` 与 `transcription_*` 错误码，不回退 frontend ASR、普通文件读取或 generic_file transcript。 +35. `audio_transcription` 第四十二刀把 transcript 恢复层接回聊天区：`lime_create_transcription_task` 工具结果现在生成 `.lime/runtime/transcription-generate/*.md` 运行时文档；`useWorkspaceTranscriptionTaskPreviewRuntime` 监听同一媒体任务事件并优先按 `audio_transcription + transcription_generate` 查询 `list_media_task_artifacts`，从 `transcript_status/path/source/language/output_format/error_code` snapshot 恢复任务卡和 viewer，索引缺失时才回退读取单个 task artifact。 +36. `audio_transcription` 第四十三刀把 transcript 文本接入最小校对 viewer：完成态恢复时读取 `transcript_path` 指向的 `.lime/runtime/transcripts/*` 文本，并把内容嵌入 `.lime/runtime/transcription-generate/*.md` ArtifactDocument 的 `code_block`，打开任务卡即可复制和校对转写文本；失败态仍只展示 `transcription_*` 错误码，不伪造 transcript 内容。 +37. `audio_transcription` 第四十四刀把 transcript viewer 从纯文本推进到时间轴/说话人段落层：恢复层会解析 `.json` verbose transcript、SRT 与 VTT 的 segment / speaker / start-end 时间码，写回 `taskPreview.transcriptSegments`、聊天轻卡时间轴预览和 `.lime/runtime/transcription-generate/*.md` 的“转写时间轴（可逐段编辑校对）”表格；仍不读取隐藏 task JSON 作为主路径，也不新增前端 ASR 或 `generic_file` 旁路。 +38. `audio_transcription` 第四十五刀把 transcript viewer 从只读校对推进到可保存校对稿：`.lime/runtime/transcription-generate/*.md` ArtifactDocument 会标记 `transcriptCorrectionEnabled`、提示保存不改写原始 ASR 输出；用户编辑“转写文本”或“转写时间轴”并保存时，Workbench 复用同一 ArtifactDocument 版本链写入 `transcriptCorrectionStatus=saved`、`transcriptCorrectionEditedBlockId`、`transcriptCorrectionSourceTranscriptPath`、`transcriptCorrectionTextLength`、`transcriptCorrectionSegmentCount`、`transcriptCorrectionSpeakerCount` 与 `transcriptSegmentsCorrected`，不新增 Tauri command、frontend ASR 或并行 patch 协议。 +39. `audio_transcription` 第四十六刀把已保存校对稿显性化：保存后会插入/更新 `transcript-correction-status` callout，显示原始 ASR 输出保持不可变、原文/校对稿文本长度差异、段落差异和说话人数差异；同时写入 `transcriptCorrectionDiffSummary`，让后续专用 transcript viewer、diff 面板或导出器不必重新扫描 block 才能识别校对变更。 +40. `Phase 3 / Phase 5` 第四十七刀新增最小 `ModalityExecutionProfile` 与 `ExecutorAdapter` 事实源：`modalityExecutionProfiles.json` 覆盖 7 个 current contracts 与 7 个 executor adapters，`governance:modality-contracts` 会校验 profile 覆盖、adapter 绑定、支持位、产物、权限、LimeCore policy 与 failure mapping；本刀只建立 current 治理事实源，不新增命令、bridge、mock 或运行时执行分支。 +41. `Phase 3 / Phase 5` 第四十八刀把 profile / adapter 事实源接入前端 runtime contract resolver：`resolveModalityRuntimeContractBinding()` 现在会从 `modalityExecutionProfiles.json` 解析 `execution_profile` 与 `executor_adapter` 快照，并随 `runtime_contract` 进入所有 current launch metadata；本刀仍不新增 Tauri command、bridge、mock 或 Rust executor 分支，为下一步 Rust runtime preflight / evidence 可视化提供统一输入。 +42. `Phase 3 / Phase 5` 第四十九刀把 profile / adapter 快照接入 Evidence Pack：`runtime_evidence_pack_service` 会从 `runtime_contract.execution_profile.profile_key` 与 `runtime_contract.executor_adapter.adapter_key` 提取 `executionProfileKey` / `executorAdapterKey`，并写入 `modalityRuntimeContracts.snapshots[]`、`snapshotIndex.executionProfileKeys`、`snapshotIndex.executorAdapterKeys` 与 `toolTraceIndex.items[]`；本刀仍不改变真实 executor 行为，只让 evidence/replay 主链能看见 Phase 3/5 决策输入。 暂不做： -1. 暂不把所有 `@`、按钮、Scene 批量登记到 `bound_entries`；本轮只绑定 `image_generation` 第一条 vertical slice。 +1. 暂不把所有 `@`、按钮、Scene 批量登记到 `bound_entries`；已进入主链的 `image_generation`、`browser_control`、`pdf_extract`、`voice_generation`、`web_research`、`text_transform`、`audio_transcription` 继续按合同逐条收口，未进入 current 事实源的入口不批量扩张。 2. 暂不把 LimeCore 写成默认执行器；LimeCore 仍只作为 catalog / policy / offer / audit 控制面。 -3. 暂不批量改造视频、语音、PDF、搜索等其他 viewer；先让 `image_generation` vertical slice 收口。 +3. 暂不批量改造视频、PDF、搜索、研报等其他 viewer；已补到最小 viewer / index 的 browser 与 voice 继续沿同一事实源演进，未形成稳定 artifact graph 的能力先不另开 viewer 协议。 4. 暂不新增 PDF task artifact / reader viewer；PDF 本轮只收 executor trace、evidence 和 replay，避免在底层事实源未完全稳定前再开一套产物协议。 -5. 暂不新增 audio viewer / 真实音频 worker；voice 本轮只落最小 `audio_generate` artifact protocol，不伪造云端提交，也不把普通聊天文本或通用文件卡当作音频产物。 +5. 暂不新增完整音频工作台，也暂不把 Gemini / Azure 专有语音 API、LimeCore 云 run/poll 或语音后处理工作流并入主链；voice 目前只提供最小 `audio_generate` 任务卡、运行时文档 viewer、OpenAI-compatible speech provider seam 与完成/失败态回流。未配置可用 provider 时仍必须显式失败，不伪造云端提交，也不把普通聊天文本或通用文件卡当作音频产物。 6. 暂不新增独立 `report_generation` 合同；`@研报 / @竞品` 继续走 `report_skill_launch -> Skill(report_generate)` 主链，但其底层能力归属先收敛到 `web_research`，避免把 report artifact 协议提前扩张成第二套事实源。 7. 暂不新增独立 `summary_generation`、`translation`、`analysis`、`publish_compliance` 或 `logo_decomposition` 合同；这组轻量文本/文档转换入口先统一收敛到 `text_transform`，避免把上层 `@` 命令提前扩张成平行底层事实源。 +8. 暂不新增非 OpenAI-compatible ASR adapter 或本地离线 ASR 执行器；`audio_transcription` 当前交付标准 `transcription_generate` task writer、`lime-transcription-worker`、`transcript.completed/failed` 回写、统一媒体任务索引、聊天任务卡、可编辑校对运行时文档 viewer、JSON/SRT/VTT 时间轴与说话人段落展示、ArtifactDocument 版本化校对稿保存、校对稿状态/差异摘要、Evidence `transcriptIndex` 与 Replay 检查。 +9. 暂不在本刀实现 Rust 侧真实 profile merge、tenant policy snapshot、adapter runtime preflight 或 GUI/evidence 可视化；当前先把 Phase 3 / Phase 5 的底层合同关系落成可检查 registry，并接入前端 launch metadata 与 Evidence Pack 快照，后续继续让 Rust runtime 消费同一事实源。 ## 分类 @@ -259,9 +324,35 @@ runtime identity - 2026-04-29：继续第十五刀 `pdf_extract` entry binding：`modalityRuntimeContracts.json` 现在登记 `at_pdf_read_command`，覆盖 `@读PDF / @pdf / @read_pdf`，并把 launch path 固定为 `harness.pdf_read_skill_launch.pdf_read_request`；`useWorkspaceSendActions` 从 `resolvePdfExtractRuntimeContractBinding().boundEntrySources` 读取 `entry_source`，只保留 `PDF_EXTRACT_DEFAULT_ENTRY_SOURCE` 作为 registry 缺省 fallback。 - 2026-04-29：继续第十六刀 `voice_generation`：新增前端 registry resolver `resolveVoiceGenerationRuntimeContractBinding()`，`voice_generation.bound_entries` 登记 `at_voice_command`；`@配音` 发送时在 `harness.service_scene_launch.service_scene_run` 注入 `voice_generation` contract 快照，后端 `prepare_service_scene_launch_request_metadata` 也会为 `voice_runtime` 旧 metadata 补齐合同字段，服务型场景 prompt 会输出底层合同、所需能力与 `runtime_contract(JSON)`，并禁止退回 `legacy_tts_test_command`。 - 2026-04-29：继续第十七刀 `web_research`：新增前端 registry resolver `resolveWebResearchRuntimeContractBinding()`，`web_research.bound_entries` 登记 `at_search_command`、`at_deep_search_command` 与 `at_site_search_command`；`@搜索 / @深搜 / @站点搜索` 发送时分别在 research / deep search / site search request 中注入 `web_research` contract 快照，Rust 侧 prepare 函数也会把缺省合同字段补齐到 launch 与 request，prompt 显式输出底层合同、所需能力与 `runtime_contract(JSON)`，并禁止退回模型记忆、本地文件搜索、通用 WebSearch 或 ToolSearch 偏航。 -- 2026-04-30：继续第十八刀 `web_research` evidence/replay：`runtime_evidence_pack_service` 现在会从 `Skill(research)` / `Skill(site_search)` tool timeline 的 args 或 metadata 导出 `web_research` 的 `modalityRuntimeContracts`，`source=web_research_skill_trace.modality_runtime_contract`、`routingEvent=executor_invoked`、`routingOutcome=accepted/failed`、`expectedRoutingSlot=report_generation_model`；`runtime_replay_case_service` 同步增加 `web-research`、`research-skill`、`web-research-trace` tags，并在 expected / grader 中要求继续走 `Skill(research)` / `Skill(site_search)` 与真实 search_query / `lime_site_*` 证据，禁止 `model_memory_only_answer`、`local_file_search_before_research_skill`、ToolSearch 或普通聊天替代。 +- 2026-04-30：继续第十八刀 `web_research` evidence/replay：`runtime_evidence_pack_service` 现在会从 `Skill(research)` / `Skill(site_search)` tool timeline 的 args 或 metadata 导出 `web_research` 的 `modalityRuntimeContracts`，`source=web_research_skill_trace.modality_runtime_contract`、`routingEvent=executor_invoked`、`routingOutcome=accepted/failed`、`expectedRoutingSlot=report_generation_model`；`runtime_replay_case_service` 同步增加 `web-research`、`research-skill`、`web-research-trace` tags，并在 expected / grader 中要求继续走 `Skill(research)` / `Skill(site_search)` 与真实 search*query / `lime_site*\*`证据，禁止`model_memory_only_answer`、`local_file_search_before_research_skill`、ToolSearch 或普通聊天替代。 - 2026-04-30：继续第十九刀 `web_research` 索引化：`runtime_evidence_pack_service` 的 `modalityRuntimeContracts` 现在输出 `snapshotIndex`，汇总 `contractKeys`、`sourceCounts`、`routingOutcomeCounts`、`expectedRoutingSlots` 与 `toolTraceIndex.items`；`web_research` replay input 会携带同一份索引，使回归样本能直接定位 `executorBindingKey=research`、`entrySource=at_search_command` 与 `web_research_skill_trace`，不再只能扫描 raw snapshots。 - 2026-04-30：继续第二十刀 `web_research` 报告型子入口：`web_research.bound_entries` 纳入 `at_report_command` 与 `at_competitor_command`，前端 `@研报 / @竞品` 发送会在 `harness.report_skill_launch.report_request` 注入同一份 `web_research` contract；Rust `report_skill_launch` prepare / prompt 也补齐合同字段，同时保留首刀 `Skill(report_generate)`，并把 evidence / replay 扩展到 `report_generate` trace，避免另开 `report_generation` 平行事实源。 - 2026-04-30：继续第二十一刀 `text_transform`：新增 `text_transform` contract 并登记 `at_file_read_command`、`at_summary_command`、`at_translation_command`、`at_analysis_command`、`at_publish_compliance_command`、`at_logo_decomposition_command`；前端 `@读文件 / @总结 / @翻译 / @分析 / @发布合规 / @Logo拆解`、Rust summary/translation/analysis prepare 与 prompt、evidence `text_transform_skill_trace`、replay tags / expected / grader 已统一收敛到同一份底层合同，入口层不再新增 summary / translation / analysis 平行事实源。 - 2026-04-30：继续第二十二刀 `voice_generation` evidence/replay：`runtime_evidence_pack_service` 现在会从 voice runtime / service scene timeline args 或 metadata 导出 `voice_generation` 的 `modalityRuntimeContracts`，`source=voice_generation_service_scene_trace.modality_runtime_contract`、`routingEvent=executor_invoked`、`routingOutcome=accepted/failed`、`expectedRoutingSlot=voice_generation_model`；`runtime_replay_case_service` 同步增加 `voice-generation`、`voice-runtime`、`voice-generation-trace` tags，并在 expected / grader 中要求继续走 `service_scene_launch(scene_key=voice_runtime)` / 本地 ServiceSkill runtime 或后续 audio_task/audio_output 证据，禁止 `legacy_tts_test_command`、伪造云端提交、普通聊天文本或通用文件卡替代。 - 2026-04-30：继续第二十三刀 `voice_generation` audio artifact protocol：新增 `audio_generate` task type、`create_audio_generation_task_artifact` Tauri 命令与 `lime_create_audio_generation_task` Agent tool，只写入标准 `audio_task/audio_output` 产物，不生成真实音频、不提交云端；前端 API、DevBridge mock、`list_media_task_artifacts` contract index、evidence `audio_task.modality_runtime_contract` 与 replay grader 已同步消费同一份 `voice_generation` 合同快照，下一刀再补 audio viewer / 真实 worker。 +- 2026-04-30：继续第二十四刀 `voice_generation` audio preview/viewer：`lime_create_audio_generation_task` 工具结果现在会带回 `source_text / voice / audio_path / duration_ms` 等任务元数据，前端 `taskPreviewFromToolResult` 可生成 `audio_generate` 任务卡，并同步生成 `.lime/runtime/audio-generate/*.md` 运行时文档 viewer；聊天区任务卡采用 Lime 现有轻卡视觉，打开后进入可读文档而不是暴露隐藏 task JSON，真实音频 worker 与完整音频工作台继续后置。 +- 2026-04-30：继续第二十五刀 `voice_generation` audio_output 回写闭环：新增 `complete_audio_generation_task_artifact` 命令与前端 API / DevBridge mock / governance catalog / mockPriority 同步，执行器可把标准 `audio_generate` task 从 pending 更新为 `succeeded`，并把完成态 `audio_output` 同时写入 payload 与 result；本刀仍不引入真实音频 worker、LimeCore 云执行 run/poll 或 legacy TTS。 +- 2026-04-30：继续第二十六刀 `voice_generation` audio preview runtime：新增 `useWorkspaceAudioTaskPreviewRuntime`，聊天工作区会监听 `lime://creation_task_submitted` 并轮询仍在 running 或缺少音频 URL 的 `audio_generate` 任务，从标准 task artifact 恢复完成态 `audio_output` 到任务卡；这一步把第二十五刀写回口真正接回用户可见轻卡，但仍不新增完整音频工作台。 +- 2026-04-30：继续第二十七刀 `voice_generation` audio worker skeleton：`create_audio_generation_task_artifact` 现在会在 emit 标准媒体任务事件后启动 `lime-audio-worker` 骨架；骨架先把任务推进到 `running`，随后在真实 provider 未接通时写回 `audio_worker_unavailable` 可重试失败与 `payload.audio_output.status=failed`，不生成假 `audio_path`、不调用 legacy TTS，也不新增 LimeCore 云 run/poll 协议。 +- 2026-04-30：继续第二十八刀 `voice_generation` OpenAI-compatible audio provider seam：`lime-audio-worker` 不再停在纯骨架失败，而是从标准 `audio_generate` task payload / 全局 `media_defaults.voice` 解析 provider 与 model，通过 current API Key Provider 主链拿到 OpenAI-compatible `api_key/base_url`，调用 `/audio/speech` 返回真实音频 bytes 后保存到 `.lime/runtime/audio/.mp3`，再回写 `payload.audio_output.completed` 与 `record.result.audio_output`；无 AppHandle、DbConnection、ApiKeyProviderService、provider/model/API Key 或非 OpenAI-compatible client 时写回明确 `audio_provider_*` 失败码，不回退 `legacy_tts_test_command`、`test_tts`、伪云端提交或通用文件卡。 +- 2026-04-30：继续第二十九刀 `voice_generation` audio failure preview：聊天区 `useWorkspaceAudioTaskPreviewRuntime` 现在会从标准 `audio_generate` task artifact 的 `payload.audio_output` / `record.last_error` 读取 `audio_provider_*` 错误码、错误原因与 retryable，并在失败 artifact 没有 `audio_path` 时清空任务卡音频路径；`TaskMessagePreview` 同步展示 Provider 错误码和原因，让未配置 provider、resolver 不可用或 adapter 缺失直接回到用户可见任务卡，不再泛化成普通失败，也不伪造音频结果。 +- 2026-04-30：继续第三十刀 `voice_generation` audio viewer sync：`useWorkspaceAudioTaskPreviewRuntime` 在从标准 `audio_generate` task artifact 恢复完成/失败态时，会同步重建同一条 `.lime/runtime/audio-generate/*.md` 运行时文档的 ArtifactDocument metadata；打开聊天任务卡时，viewer 会展示 `audio_output.completed` 的真实音频路径，或展示 `audio_provider_*` 错误码与原因，不再停留在创建任务时的“等待音频执行器”文档。 +- 2026-04-30：继续第三十一刀 `voice_generation` audio evidence/replay details：`runtime_evidence_pack_service` 现在会从 `audio_generate` task artifact 导出 `audioOutput` 摘要，并在 `snapshotIndex.audioOutputIndex` 汇总 audio output 状态、Provider 错误码与 worker；`runtime_replay_case_service` 同步把 `audio-output-completed` / `audio-output-failed` 纳入 suite tags / failure modes，并在 expected / grader 中要求 replay 保留 `audio_output.completed` 与真实 `audio_path`，或显式处理 Provider 失败，不允许回退 legacy TTS、伪造音频路径、普通聊天文本或通用文件卡。 +- 2026-04-30：继续第三十二刀 `voice_generation` audio task index details：`media_task_cmd` 的 `list_media_task_artifacts` 索引现在把 `audio_output` 完成/失败事实直接纳入 `modality_runtime_contracts`，包含按状态汇总、Provider 错误码汇总，以及每个 task snapshot 的 `audio_output_status/path/mime_type/duration_ms/error_code/retryable`；`src/lib/api/agentRuntime/types.ts` 与浏览器 fallback mock 同步更新，避免浏览器链路或前端恢复逻辑只能靠重新读取隐藏 task JSON 才能判断音频产物状态。 +- 2026-04-30：继续第三十三刀 `voice_generation` audio index restore：聊天工作区的 `useWorkspaceAudioTaskPreviewRuntime` 现在会先按 `voice_generation + audio_generate` 查询统一媒体任务索引，从 snapshot 的 `audio_output_status/path/mime_type/duration_ms/error_code/retryable` 直接恢复任务卡和 `.lime/runtime/audio-generate/*.md` viewer；索引缺失或旧 task 没有 `audio_output_status` 时才保留原来的单 task artifact 读取兜底，避免主路径继续依赖隐藏 task JSON。 +- 2026-04-30：继续第三十四刀 `Phase 4 artifact graph`：新增 `docs/roadmap/warp/artifact-graph.md` 与 `src/lib/governance/modalityArtifactGraph.json`，把 `image/audio/browser/pdf/report/presentation/webpage/generic_file` 的 artifact kind、truth source、viewer、evidence 与 task index 字段落成可检查事实源；`governance:modality-contracts` 现在会校验 contract 的 `artifact_kinds` 必须能在 artifact graph 中找到 truth source / viewer / evidence 交集，防止新增多模态能力继续退回未知 viewer 或通用文件卡旁路。 +- 2026-04-30：继续第三十五刀 `browser_control entry binding`：`browser_control.bound_entries` 现在登记 `@浏览器 / @browser / @browse`、`@Browser Agent`、`@Mini Tester`、`@Web Scheduler`、`@Web Manage`，所有入口只声明 `harness.browser_assist` launch metadata、默认输入映射与可见性策略；前端 Browser Assist launch 会按触发词从 registry 派生 `entry_source`，不再把所有浏览器别名压成同一个入口来源，也不让入口直接写 task / artifact / viewer。 +- 2026-04-30：继续第三十六刀 `browser_control browserActionIndex`：`runtime_evidence_pack_service` 现在会从 Browser Assist tool metadata 提取 `browserAction` 摘要，并在 `modalityRuntimeContracts.snapshotIndex.browserActionIndex` 汇总 actionCount、sessionCount、lastUrl、observationCount、screenshotCount、backend/status/action 计数与逐项 `browser_session` / `browser_snapshot` artifactKind；`runtime_replay_case_service` 同步把 `browser-action-index` 纳入 suite tags / expected / grader，要求 replay 继续导出可查询 browser session/snapshot 索引，而不是只靠 raw snapshots 或 WebSearch 替代。 +- 2026-04-30：继续第三十七刀 `browser_control Harness visible index`：Evidence Pack 的 `observabilitySummary.modalityRuntimeContracts.snapshotIndex.browserActionIndex` 现在会携带裁剪后的 Browser Assist 摘要，前端 `AgentRuntimeEvidencePack` 类型与 normalizer 同步解析该索引；`HarnessStatusPanel` 在导出问题证据包后展示 Browser Assist 索引卡，直接暴露 action/session/URL/observation/screenshot 与最近 `browser_session` / `browser_snapshot` 项，证明浏览器复盘已从 runtime.json 人工查找前进到用户可见 evidence 面板。 +- 2026-04-30：继续第三十八刀 `browser_control browser_replay_viewer`：`HarnessStatusPanel` 的 Browser Assist 索引卡新增“打开复盘”，以 evidence pack + `browserActionIndex` 构造临时 `browser_assist` Artifact；`BrowserAssistRenderer` 现在可从 artifact meta 或 JSON content 读取 `browserActionIndex`，渲染最小 `browser_replay_viewer`，展示动作统计、最近 URL、会话/target/entry、截图可用性与最近 `browser_session` / `browser_snapshot`。这一步把 evidence 可见摘要推进到 Lime 内部复盘面，但仍不新增 browser task 文件协议；完整交互回放、权限 profile 可视化与截图/DOM/network 深层展开继续后置。 +- 2026-04-30：继续第三十九刀 `audio_transcription contract`：新增 current `audio_transcription` contract，绑定 `@转写 / @transcribe / @Audio Extractor` 到 `harness.transcription_skill_launch.transcription_task`；前端 `@转写` metadata 现在携带 `modality_contract_key=audio_transcription`、`required_capabilities=[text_generation,audio_transcription]`、`routing_slot=audio_transcription_model` 与 `runtime_contract`，Rust `transcription_skill_launch` 也会为旧 metadata 补齐同一合同并在 prompt 中禁止 `frontend_direct_asr`、`generic_file_transcript`、转写前 ToolSearch/WebSearch 偏航。`transcript` artifact graph 已反向登记该合同；真实转写 worker、transcript viewer 与 evidence/replay 索引继续后置。 +- 2026-04-30：继续第四十刀 `audio_transcription transcript task/index/evidence/replay`：`lime_create_transcription_task`、`create_transcription_task_artifact_inner` 与 CLI `lime task create transcription` / `lime media transcription generate` 现在写入同一标准 `transcription_generate` task file，payload 固化 `audio_transcription` 合同、`requested_target=transcript` 与 `transcript.pending`；`list_media_task_artifacts` 增加 `transcript_count/statuses/error_codes` 与每个 task 的 `transcript_*` snapshot，Evidence Pack 增加 `snapshotIndex.transcriptIndex`，Replay / grader 增加 `audio-transcription`、`transcription-task-trace`、`transcript-index` 标签与缺失/失败阻断检查。真实 ASR worker、完成态 transcript 回写和专属 transcript viewer 仍作为下一刀。 +- 2026-04-30：继续第四十一刀 `audio_transcription transcription worker`：`lime_create_transcription_task` 现在在创建标准 `transcription_generate` task 后启动 `lime-transcription-worker`；worker 校验 `audio_transcription` contract、读取本地 source file 或下载 `source_url`，通过 current API Key Provider 解析 OpenAI-compatible credential，调用 `/audio/transcriptions`，成功时写入 `.lime/runtime/transcripts/*` 并回写 `transcript.completed`、`transcript_path`、`provider_id/model` 与 `record.result.transcript`；失败时回写 `transcript.failed` 和 `transcription_provider_*` / `transcription_source_*` / contract 错误码，不回退 frontend ASR、普通文件读取或 generic_file transcript。最小 GUI 恢复层已在第四十二刀接回，文本校对已在第四十三刀接回，时间轴/说话人段落展示已在第四十四刀接回；编辑保存型 transcript viewer、更多 ASR adapter 与本地离线 ASR 继续后置。 +- 2026-04-30：继续第四十二刀 `audio_transcription transcript visible restore`：聊天工作区新增 `useWorkspaceTranscriptionTaskPreviewRuntime`，会优先从 `list_media_task_artifacts(modalityContractKey=audio_transcription, taskType=transcription_generate)` 的 `transcript_*` snapshot 恢复完成/失败态，把 `transcript_path`、source、language、outputFormat、Provider 错误码同步到任务卡与 `.lime/runtime/transcription-generate/*.md` 运行时文档；只有索引缺失时才回退读取单个 task artifact，避免用户可见路径继续依赖隐藏 task JSON 或 generic_file transcript。 +- 2026-04-30：继续第四十三刀 `audio_transcription transcript proofreading viewer`：`useWorkspaceTranscriptionTaskPreviewRuntime` 在完成态恢复时读取 `transcript_path` 对应文本，并把 `transcriptText` 写回任务预览和 `.lime/runtime/transcription-generate/*.md` 运行时文档；`buildTranscriptionTaskArtifactDocument` 现在输出“转写文本（可编辑校对）”代码块，让打开任务卡即可在 Lime 内部读取、复制与校对 transcript，而不是只看到路径或退回 generic_file。 +- 2026-04-30：继续第四十四刀 `audio_transcription transcript timeline viewer`：新增 transcript segment 解析器，支持从 OpenAI verbose JSON、SRT 与 VTT 读取 `start/end/speaker/text`，并把结果回写到聊天任务卡时间轴预览与 `.lime/runtime/transcription-generate/*.md` ArtifactDocument 表格；本刀只消费同一 `transcription_generate` task file 与 `transcript_path` 输出，不新增协议、Tauri command、frontend ASR 或 generic_file 旁路。编辑保存型 transcript viewer、更多 ASR adapter 与本地离线 ASR 继续后置。 +- 2026-04-30：继续第四十五刀 `audio_transcription transcript correction save`：复用现有 ArtifactDocument Workbench 编辑保存链路，让转写运行时文档的文本与时间轴从“可复制校对”推进到“可保存校对稿”；生成文档会标记 `transcriptCorrectionEnabled` / `transcriptCorrectionSource=artifact_document_version` 并提示保存不改写原始 ASR 输出，保存时补 `transcriptCorrectionStatus=saved`、编辑 block、原始 transcript 路径、文本长度、段落/说话人数与 `transcriptSegmentsCorrected`。本刀没有新增 Tauri command、Rust worker 分支、frontend ASR 或 generic_file patch 旁路。 +- 2026-04-30：继续第四十六刀 `audio_transcription transcript correction status`：保存校对稿时会在同一 ArtifactDocument 中插入/更新 `transcript-correction-status` 成功提示块，明确“校对稿已保存为当前运行时文档的新版本；原始 ASR 输出文件保持不变”，并写入 `transcriptCorrectionDiffSummary`，汇总原文/校对稿文本长度、段落数量、变更段落和说话人数差异。该状态仍随同一 `.lime/runtime/transcription-generate/*.md` 运行时文档版本保存，不新增命令、worker 分支或并行 patch artifact。 +- 2026-04-30：继续第四十七刀 `Phase 3 / Phase 5 execution profile registry`：新增 `docs/roadmap/warp/execution-profile.md` 与 `src/lib/governance/modalityExecutionProfiles.json`，把 7 个 current contracts 的 profile、artifact policy、LimeCore policy refs 与 executor adapters 落成机器事实源；`check-modality-runtime-contracts.mjs` 现在会读取 profile registry，校验每个 current contract 都被 profile 覆盖、每个 `executor_binding` 都有 adapter、adapter 支持位/产物/权限/failure mapping 与 contract 对齐。该刀不改 Tauri command、bridge、mock 或真实 executor 行为，只把 Phase 3/5 的主线底座从文档要求推进成可阻断错误配置的 current 守卫。 +- 2026-04-30：继续第四十八刀 `Phase 3 / Phase 5 profile resolver`：新增 `src/lib/governance/modalityExecutionProfiles.ts` 与定向测试，`resolveModalityRuntimeContractBinding()` 会把 current contract 对应的 `execution_profile`、`executor_adapter`、`executionProfileKey`、`executorAdapterKey` 注入同一 runtime contract binding；所有现有上层入口继续只调用 runtime contract resolver，即可随 launch metadata 携带 profile / adapter 快照。本刀没有新增命令、bridge、mock、Rust executor 或 GUI surface，只把上一刀的机器事实源推进到前端主路径输入。 +- 2026-04-30：继续第四十九刀 `Phase 3 / Phase 5 evidence snapshot`：`runtime_evidence_pack_service` 现在会从 runtime contract 中提取 `executionProfileKey` 与 `executorAdapterKey`，并写入 `modalityRuntimeContracts.snapshots[]`、`snapshotIndex.executionProfileKeys`、`snapshotIndex.executorAdapterKeys` 与 `toolTraceIndex.items[]`；图片 contract preflight 失败样本与 web_research Skill trace 样本都增加断言，证明 profile / adapter 已进入 evidence 主链，而不是只停留在前端 metadata 或治理 JSON。本刀未改命令、bridge、mock、GUI 或真实 executor 行为。 diff --git a/docs/exec-plans/old-session-open-performance-plan.md b/docs/exec-plans/old-session-open-performance-plan.md index bf6e55a3e..47ac418be 100644 --- a/docs/exec-plans/old-session-open-performance-plan.md +++ b/docs/exec-plans/old-session-open-performance-plan.md @@ -112,6 +112,8 @@ - 同轮收口：任务中心内部 `handleOpenSidebarTaskTopic` / `handleResumeSidebarTask` / `handleResumeRecentSession` / fallback restore 默认走追加/激活语义，只有外部直达会话仍保留 replace 语义，避免旧缓存污染首开深链。 - `2026-04-30` 追加旧会话卡顿收口：MessageList 对恢复中或已分页旧会话使用更小的首屏批次（20 条）和 idle 小批量补齐；已完成的旧会话最后一个 turn 不再按 current turn 处理，避免历史 timeline 在首帧被当作活跃执行过程同步展开。 - `2026-04-30` 追加 invoke 降噪：`workspace_get` 增加 1s 短 TTL 与同 id in-flight 合并，避免旧会话切换期间工作区详情重复抢占 bridge；打开旧会话时自动回填的 `recent_preferences` / `recent_team_selection` 改为同 session 后台合并队列，12s 后 idle 再写，若期间已切走则直接丢弃，手动偏好修改仍保持即时低延迟同步。 +- `2026-04-30` 追加旧会话预取：侧边栏会话按钮在 focus / pointer enter / pointer down 时发送 task-center prefetch 事件，当前 Agent 工作区复用 `runtime.getSession(historyLimit: 40)` 预取结果；若用户随后点击同一会话，`switchTopic` 直接等待同一个 in-flight 详情请求，不再重复打 `agent_runtime_get_session`，并把预取结果写入本地 tail snapshot。话题列表就绪后还会在测试环境外 idle 预取最近 3 条非当前会话，降低无本地 snapshot 时的首次点击等待。 +- `2026-04-30` 追加路由恢复收口：从首页/侧栏通过 `initialSessionId` 进入旧会话时不再默认 `forceRefresh`，仅 `workspace_error` 会话继续强制刷新；命中 fresh tail snapshot 时先渲染缓存，远端详情刷新延后到 1.2s + idle，避免点击主链马上再打 `agent_runtime_get_session`。 - `npx vitest run "src/components/agent/chat/index.test.tsx" -t "任务中心初始会话标签"` 通过（11 tests），新增覆盖任务中心内部连续打开旧会话、外层侧栏事件 + 路由追平后保留两个历史 tab。 - `npx vitest run "src/components/AppSidebar.test.tsx"` 通过（35 tests），新增覆盖任务中心内点击已有会话会先通知本地标签栏，再继续导航。 - `npx eslint "src/components/AppSidebar.tsx" "src/components/AppSidebar.test.tsx" "src/components/agent/chat/AgentChatWorkspace.tsx" "src/components/agent/chat/index.test.tsx" "src/components/agent/chat/taskCenterDraftTaskEvents.ts" --max-warnings 0` 通过。 @@ -131,3 +133,47 @@ - `npx vitest run "src/components/agent/chat/hooks/useSelectedTeamPreference.test.tsx"` 通过（13 tests），覆盖自动 fallback Team 回填走后台同步、手动切换仍即时回写。 - `npx eslint "src/lib/api/project.ts" "src/lib/api/project.test.ts" "src/components/agent/chat/AgentChatWorkspace.tsx" "src/components/agent/chat/hooks/useSelectedTeamPreference.ts" "src/components/agent/chat/hooks/useSelectedTeamPreference.test.tsx" --max-warnings 0` 通过。 - GUI smoke / Playwright 后置复测暂被本地并发 Cargo/Tauri 构建阻塞：`tauri:dev:headless` 首次暴露非本轮 `creation_tools.rs` 大型 `serde_json::json!` 递归限制，已在 `src-tauri/src/lib.rs` 补 `recursion_limit = "256"` 解除；随后仍有多个非本轮 `cargo test` / `cargo check` 并发进程占用 target/package lock，需环境空闲后重跑 `npm run bridge:health -- --timeout-ms 120000`、`npm run verify:gui-smoke` 与 Playwright 旧会话性能采样。 +- `2026-04-30` Playwright 预取前采样（`http://127.0.0.1:1420/`）：刷新后点击 `Slow typing E2E`，早期 invoke 仅 `workspace_get_default` / `workspace_list` / `agent_runtime_get_session(historyLimit:40)`，其中 `get_session` 约 `1261ms`；`localStorage/sessionStorage` 未命中 `aster_session_snapshots*`，说明当前慢点主要是无本地 tail snapshot 时必须等后端详情返回。 +- `2026-04-30` Playwright 预取后采样（`http://127.0.0.1:1420/`）：focus `AI Trends Task` 后预取 `agent_runtime_get_session(historyLimit:40)` 约 `116ms` 并写入 `aster_session_snapshots_*`；随后点击同会话，前 `500ms` 内不再出现 `agent_runtime_get_session`，仅剩 route/workspace 初始化类 `workspace_get/default/list`，消息组已可见，`body/html cursor=auto`。 +- `npx vitest run "src/components/agent/chat/hooks/useAsterAgentChat.test.tsx" "src/components/AppSidebar.test.tsx"` 通过（2 files / 191 tests），覆盖旧会话预取 in-flight 复用与侧边栏 focus 预取事件。 +- `npx eslint "src/components/agent/chat/hooks/useAgentSession.ts" "src/components/agent/chat/hooks/useAsterAgentChat.ts" "src/components/agent/chat/hooks/useAsterAgentChat.test.tsx" "src/components/agent/chat/AgentChatWorkspace.tsx" "src/components/agent/chat/taskCenterDraftTaskEvents.ts" "src/components/AppSidebar.tsx" "src/components/AppSidebar.test.tsx" "src/components/app-sidebar/AppSidebarConversationShelf.tsx" --max-warnings 0` 通过。 +- `npx tsc --noEmit --pretty false` 通过。 +- `2026-04-30` 追加旧会话首帧渲染降载：历史恢复路径的助手正文新增 `MarkdownRenderer renderMode="light"`，旧会话首屏跳过高成本 `rehypeRaw` / `rehypeKatex` / Prism 代码高亮；实时流式与普通新消息继续走标准模式。已分页旧会话首屏批次从 `20` 缩小到 `10`，idle 补齐批次从 `12` 缩小到 `6`，且恢复路径 idle 延迟从 `120ms` 提高到 `600ms`，避免点击后立即连续吃主线程。 +- `2026-04-30` 追加后端历史 payload 裁剪：`agent_thread_items` 的 tail / cursor 历史查询在 `file_artifact.content` 之外，继续裁剪超大的 `tool_call.output`、`command_execution.aggregated_output` 与 `web_search.output` 到约 `16KB` 并附截断提示；完整 `list_items_by_thread` 不裁剪，保留全量诊断能力。这针对本地 runtime.db 中数 MB 级 `social_generate_cover_image` tool output 导致旧会话恢复 CPU/内存尖峰的问题。 +- Context7 / WebSearch 校准结论：React 官方文档说明 `useDeferredValue` / `startTransition` 只能降低优先级，不能让慢列表本身变快；TanStack Virtual 对动态高度长列表建议 `useVirtualizer + measureElement + overscan`；Tauri 文档建议大 payload 走 async command 与 Channel/chunk。当前先落地低风险的“减少首帧同步工作 + 裁剪历史 payload”，后续若 40 条历史仍卡，下一刀应评估 MessageList 动态高度虚拟化或 `agent_runtime_get_session` 分块返回。 +- Playwright E2E（`http://127.0.0.1:1420/`）：刷新后点击 `Slow typing E2E`，`agent_runtime_get_session(historyLimit:40)` 约 `322ms`，点击到首屏可见约 `605ms`，DOM 约 `491`，`body/html cursor=auto`，存在一次约 `113ms` long task；进一步缩小历史首屏批次后，等待页面稳定再切 `AI Trends Task -> Slow typing E2E`，可见旧会话 DOM 保持约 `480-626`，`cursor=auto`。当前 dev 环境 heap 读数约 `1.4GB`，更像 Vite/WebView 长会话累计值，不能单独当作本轮回归证据。 +- `npm test -- "src/components/agent/chat/components/StreamingRenderer.test.tsx"` 通过（29 tests），覆盖 `markdownRenderMode="light"` 透传到 `MarkdownRenderer`。 +- `npm test -- "src/components/agent/chat/components/MessageList.test.tsx"` 通过（82 tests），覆盖旧会话轻量 Markdown、长历史正文预览、历史 timeline 折叠与更小首屏批次。 +- `npm run lint -- "src/components/agent/chat/components/MarkdownRenderer.tsx" "src/components/agent/chat/components/StreamingRenderer.tsx" "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/StreamingRenderer.test.tsx" "src/components/agent/chat/components/MessageList.test.tsx"` 通过。 +- `cd src-tauri && CARGO_TARGET_DIR="/tmp/lime-core-test-target" cargo test -p lime-core item_tail_query_should` 通过（2 tests），覆盖历史 tail 查询裁剪大 file artifact 与大 tool output，完整查询不裁剪。 +- `npm run test:contracts` 通过；本轮未新增命令，仅收窄 `agent_runtime_get_session` 历史窗口返回中的超大 inline payload。 +- `npm run verify:gui-smoke` 通过；复用现有 headless Tauri 与 DevBridge，workspace-ready、browser-runtime、site-adapters、service-skill-entry、runtime tool-surface 与 runtime tool-surface page smoke 均通过。 +- `npm run typecheck` 通过。 +- `2026-04-30` 追加旧会话首帧二次降载：已分页/恢复旧会话中的长助手回复不再首帧挂载 `StreamingRenderer` / Markdown 解析，而是先展示纯文本预览，用户点击“展开完整内容”后才按轻量 Markdown 物化；超长历史回复的预览长度从约 `6000` 字收窄到约 `2000` 字，降低点击旧会话后的主线程解析与 DOM 压力。 +- 同轮将历史 timeline 的 idle 物化从旧会话首帧后约 `80ms` 延后到约 `900ms`，避免消息刚可见后立刻构建完整 `buildMessageTurnTimeline` 抢占主线程；非旧会话仍保留原 `80ms` 行为。 +- `npm test -- "src/components/agent/chat/components/MessageList.test.tsx"` 通过（83 tests），新增覆盖长助手回复纯文本预览与展开后再物化轻量 Markdown。 +- Playwright E2E 复测（`http://127.0.0.1:1420/`）：刷新后切 `AI Trends Task -> Slow typing E2E`，旧会话首屏约 `77ms` 可见，DOM 节点约 `403`，历史助手正文纯文本预览 `1` 个，历史 timeline 预览 `1` 个、真实 timeline 挂载 `0` 个，`longTasks=[]`，`body/html cursor=auto`，控制台 `0 error / 0 warning`。invoke trace 中两次 `agent_runtime_get_session` 均携带 `historyLimit: 40`，无 `/invoke` 错误。 +- `npm run verify:gui-smoke` 通过；复用现有 headless Tauri 与 DevBridge，workspace-ready、browser-runtime、site-adapters、service-skill-entry、runtime tool-surface 与 runtime tool-surface page smoke 均通过。 +- `2026-04-30` 追加旧会话后台渲染节流：恢复/已分页旧会话不再自动 idle 补齐当前窗口里更早的消息，首屏只保留尾部批次，用户点击“立即展开更早消息”时才继续物化，避免打开后数秒内后台持续挂载 Markdown / timeline 造成 CPU 与内存尖峰；普通非旧会话仍保留空闲自动补齐。 +- 同轮进一步扩大历史 timeline 延后范围：即使旧会话只有少量消息，只要 execution/thread items 很多，也先显示正文，约 `900ms` 后再生成 timeline 摘要预览；这覆盖 `Slow typing E2E` 这类“消息少但工具轨迹多”的慢会话。 +- `npm test -- "src/components/agent/chat/components/MessageList.test.tsx"` 通过（84 tests），新增覆盖旧会话不自动补齐历史窗口、少消息多执行过程延后构建 timeline。 +- `npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" --max-warnings 0` 通过。 +- `npm run typecheck` 通过。 +- 后置 `npm run verify:gui-smoke` 已尝试两次但未完成：当前本机已有非本轮 `cargo test --manifest-path src-tauri/Cargo.toml audio_generation_task` 与旧 `tauri:dev:headless` 长时间占用 Cargo artifact lock，DevBridge 无法就绪；为避免继续拉高 CPU，已停止本轮新拉起的 smoke 进程，未终止非本轮 cargo/tauri 进程。待该锁释放后需重跑 GUI smoke / Playwright。 +- `2026-04-30` 追加首字前反馈优化：Playwright MCP 当前 transport closed，改用本地 Playwright + Chrome 复测；真实 DevBridge 恢复后发送 `首字延迟 E2E ...`，点击到用户消息可见约 `295ms`，新的 `assistant-first-token-placeholder` 同步约 `295ms` 可见。后端随后因本机缺少 `claude` 凭证在约 `5.3s` 返回失败，因此本轮无法测真实模型首 token，但已消除“首个模型事件前 assistant 气泡为空/像没响应”的 UI 体感慢点。 +- 同轮代码收口：`MessageList` 在 assistant `isThinking` 且正文/parts/timeline 均为空但存在启动态 `runtimeStatus` 时，渲染轻量首字前占位；不把 runtimeStatus 塞回 `StreamingRenderer` 正文，也不渲染底部重复运行态，首个真实文本到来后自动切回正常流式渲染。 +- `npm test -- "src/components/agent/chat/components/MessageList.test.tsx"` 通过（84 tests），新增覆盖启动态 runtimeStatus 只渲染轻量首字前占位、已有正文时不重复占位。 +- `npx eslint "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx" --max-warnings 0` 通过。 +- `npm run typecheck` 通过。 +- `git diff --check -- "src/components/agent/chat/components/MessageList.tsx" "src/components/agent/chat/components/MessageList.test.tsx"` 通过。 +- `npm run verify:gui-smoke` 通过；复用现有 headless Tauri 与 DevBridge，workspace-ready、browser-runtime、site-adapters、service-skill-entry、runtime tool-surface 与 runtime tool-surface page smoke 均通过。 +- `2026-04-30` 追加 DeepSeek 首字续测：Playwright MCP 仍为 `Transport closed`，继续用本地 Chrome + Playwright；模型选择入口需先展开“高级设置”，再点击 `button[role="combobox"]`，可切到 `DeepSeek / deepseek-v4-flash`。 +- 同轮复现输出排版问题：DeepSeek 实际流里会返回 `thinking` 分片再返回最终 `text` 分片，例如 `llm_request.2.jsonl` 中先有 `thinking: "我们…好。"`，再有 `text: "好"`；旧前端在 thinking 关闭时仍渲染 thinking block，导致 UI 显示类似 `思考中…好。好`。 +- 同轮代码收口：`agentStreamTurnEventBinding` 将本轮 `thinking` 偏好透传给 `agentStreamRuntimeHandler`；当 thinking 关闭时，`thinking_delta` 只激活流状态、不再写入 assistant `thinkingContent/contentParts`，后续 `text_delta` 会清理已有 thinking part，避免 DeepSeek 隐式 reasoning 混入最终正文。 +- DeepSeek E2E 成功样本（`/tmp/lime-deepseek-first-token-e2e-retry.json`）：`agent_runtime_create_session` 约 `73ms`，`agent_runtime_submit_turn` 约 `136ms`，首字前占位约 `117ms` 可见，最终 assistant 文本为 `好已完成·00:06`，不再出现 `思考中` / reasoning 重复吐字；过程中仍有一次非阻塞 `workspace_get` 5s timeout 噪音。 +- DeepSeek E2E 失败样本（`/tmp/lime-deepseek-first-token-e2e-final.json`）：同样选择 `DeepSeek / deepseek-v4-flash` 后，`agent_runtime_submit_turn` 偶发在 `60s` 前端超时并返回 DevBridge timeout，且本地 `aster/state/logs` 没有对应 LLM request 新日志，说明失败发生在提交链路/bridge 阻塞或运行时排队阶段，未进入 provider 首 token；这是后续首字慢优化的下一刀证据。 +- `npm test -- "src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts"` 通过（10 tests），新增覆盖 thinking 关闭时不渲染 reasoning_delta。 +- `npm test -- "src/components/agent/chat/hooks/agentStreamTurnEventBinding.test.ts" "src/components/agent/chat/hooks/agentStreamSubmitExecution.test.ts"` 通过（7 tests）。 +- `npx eslint "src/components/agent/chat/hooks/agentStreamRuntimeHandler.ts" "src/components/agent/chat/hooks/agentStreamTurnEventBinding.ts" "src/components/agent/chat/hooks/agentStreamSubmitExecution.ts" "src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts" --max-warnings 0` 通过。 +- `npm run typecheck` 通过。 +- `npm run verify:gui-smoke` 通过；本轮临时 target 首次编译耗时较长，随后 DevBridge / workspace-ready / browser-runtime / site-adapters / service-skill-entry / runtime tool-surface page smoke 均通过。 diff --git a/docs/roadmap/agentui/README.md b/docs/roadmap/agentui/README.md new file mode 100644 index 000000000..c5b1e44aa --- /dev/null +++ b/docs/roadmap/agentui/README.md @@ -0,0 +1,55 @@ +# Lime AgentUI 路线图文档 + +> 状态：路线图与架构设计 +> 更新时间：2026-04-30 +> 范围：Lime 对话工作区下一阶段 AgentUI，包括 UI 架构、代码层级、事件流程、时序图、后端协作与落地顺序。 + +## 目标 + +AgentUI 不是再做一个聊天页面，而是把 Lime 已有的 runtime、timeline、artifact、task、team、harness、evidence 能力收束成一个可观察、可控制、可交付的工作台。 + +本目录回答四类问题： + +1. **产品结构**：Lime 的 AgentUI 应该由哪些层组成，哪些信息应该出现在首屏，哪些应该进入展开详情。 +2. **代码结构**：现有前端、协议、Tauri command、Rust runtime、service、持久化层分别负责什么。 +3. **运行流程**：发送消息、打开旧会话、排队输入、权限确认、产物生成、证据导出如何流动。 +4. **落地顺序**：哪些改动先解决体感慢、卡顿、重复吐字和多任务管理，哪些进入中长期演进。 + +## 阅读顺序 + +| 文档 | 作用 | +| --- | --- | +| [agent-ui-research-and-lime-direction.md](agent-ui-research-and-lime-direction.md) | 竞品与本地参考调研，说明为什么 Lime 要走“对话 + 过程 + 任务 + 产物 + 证据”路线。 | +| [lime-agentui-target-architecture.md](lime-agentui-target-architecture.md) | 目标 UI 架构图与五层模型，是后续 UI 改造的总图。 | +| [lime-agentui-code-map.md](lime-agentui-code-map.md) | Lime 当前代码层级地图，标出前端、协议、后端、服务和测试入口。 | +| [lime-agentui-event-flow.md](lime-agentui-event-flow.md) | 关键流程图，包括发送消息、旧会话恢复、queue/steer、权限、artifact、evidence。 | +| [lime-agentui-sequence-diagrams.md](lime-agentui-sequence-diagrams.md) | 端到端时序图，适合实现和排查首字慢、恢复慢、流式错乱。 | +| [lime-agentui-backend-coordination.md](lime-agentui-backend-coordination.md) | 后端配合代码架构，定义 UI 需要后端继续补齐的投影、分页、指标与批量接口。 | +| [lime-agentui-implementation-roadmap.md](lime-agentui-implementation-roadmap.md) | P0/P1/P2/P3 落地顺序、验收标准和验证命令。 | + +## 当前结论 + +Lime AgentUI 的主线应保持一个事实源，不新增第二套事件系统： + +```text +Agent runtime event + -> session / timeline / thread_read / artifact / evidence projection + -> frontend state + -> Conversation / Process / Task / Artifact / Evidence UI +``` + +下一阶段 UI 的关键词不是“更像某个竞品”，而是： + +- **首屏轻**：旧会话打开先展示 shell、缓存快照、最近消息，再渐进补 timeline / tool / artifact。 +- **流式稳**：text、thinking、tool、artifact、runtime status 分型渲染，防止重复吐字和正文污染。 +- **任务可压缩**：运行中、排队、needs input、plan ready、failed 统一进入 capsule / task strip。 +- **产物离开正文**：最终交付进入 Artifact / Canvas / Workbench，聊天正文负责解释和协作。 +- **证据可追溯**：harness、evidence、review、replay 消费同一条 runtime/timeline 事实链。 + +## 设计约束 + +1. 不复制 Claude Code、Warp、CodexMonitor 或 Codex TUI 的表面视觉，只借鉴结构模式。 +2. 不把过程日志塞回最终回答正文。 +3. 不让旧会话恢复阻塞 UI 挂载。 +4. 不让 sidebar list、session detail、timeline build、artifact preview 在同一时刻抢主线程和 invoke 通道。 +5. 不新增 parallel runtime/event 协议；新增 UI 只消费现有 AgentEvent、timeline、thread_read、artifact、evidence 投影。 diff --git a/docs/roadmap/agentui/agent-ui-research-and-lime-direction.md b/docs/roadmap/agentui/agent-ui-research-and-lime-direction.md new file mode 100644 index 000000000..411650f0b --- /dev/null +++ b/docs/roadmap/agentui/agent-ui-research-and-lime-direction.md @@ -0,0 +1,645 @@ +# AI Agent UI 调研与 Lime 对话体验路线图 + +> 状态：调研与路线图 +> 更新时间：2026-04-30 +> 目标：把“AI Agent 最佳 UI 是什么”从竞品印象、局部控件和聊天气泡，收敛成 Lime 可执行的对话体验路线图。 +> 结论：Lime 不应做纯聊天界面，也不应复制 Claude Code 或 Warp；Lime 应走“对话 + 过程 + 任务 + 产物 + 证据”的可执行工作区 Agent UI。 + +## 1. 核心判断 + +当前 AI Agent UI 已经从“输入框 + 消息列表”演进成复合工作台。最好的产品不再只回答“模型说了什么”，而是同时回答： + +1. 用户现在在做什么目标？ +2. Agent 正在什么阶段？ +3. 哪些工具、权限、成本、文件、网页和子代理被使用了？ +4. 哪些内容是最终产物，哪些只是推理过程？ +5. 结果是否可追溯、可重放、可继续编辑？ + +这和 Lime 的方向高度一致。Lime 已经有 workspace、artifact、timeline、task、runtime、browser、service skill、team/subagent、evidence pack 等底座。真正要补的不是一个更像 ChatGPT 的聊天页，而是把这些底座组织成清晰、低负担、可长时间使用的 Agent 工作台。 + +本文档的产品结论是： + +- **Chat 负责意图、推理与协作。** +- **Capsule 负责执行状态、后台任务和多 Agent 队列。** +- **Timeline 负责过程证据。** +- **Artifact 负责最终交付。** +- **Evidence 负责复盘、验证和治理。** + +一句话：Lime 应当让 Agent 像一个可观察、可控、可交付的工作系统，而不是一个不断吐字的文本框。 + +## 2. 调研范围 + +### 2.1 商业产品与官方资料 + +| 产品 / 资料 | 观察重点 | 对 Lime 的启发 | +| --- | --- | --- | +| OpenAI Agents / AgentKit / ChatKit | Agent 构建、工具、会话组件、任务链路 | Chat UI 应该消费结构化 agent event，而不是只消费文本流 | +| ChatGPT Canvas | 对话之外的编辑画布 | 产物应离开聊天正文，进入可编辑的 Artifact / Canvas | +| ChatGPT Projects / Tasks | 项目上下文、长期任务、跨会话组织 | Lime 的 workspace/session/tab 需要成为一等导航对象 | +| Claude Artifacts | 回复旁路展示代码、文档、图表等产物 | “正文解释 + 右侧产物”比把全部内容塞进消息更稳定 | +| Claude Extended Thinking | 思考内容可控展示 | thinking 应折叠、摘要化、可关闭，不能污染最终答案 | +| Claude Code | 胶囊式后台任务、工具结果压缩、计划审批 | Lime 可借鉴 pill / plan / tool rich output，但不复制终端 UI | +| Gemini Deep Research | 多步研究进度、来源、最终报告 | 研究型任务需要进度、来源和报告产物分离 | +| Gemini Canvas | 对话与创作空间组合 | 编辑型任务应进入持续画布，而不是每次生成一段 Markdown | +| Cursor / Windsurf | IDE 内 agent、工具调用、diff、上下文 | 代码任务 UI 的核心是可审查变更和可恢复控制 | +| Warp Agent Mode | 终端 block、AI block、上下文命令 | 命令和 AI 过程适合 block 化，而不是混成纯聊天流 | + +### 2.2 Context7 开源与 SDK 资料 + +| 资料 | 关键模式 | 对 Lime 的取舍 | +| --- | --- | --- | +| `assistant-ui` | Thread、ThreadList、Composer、Message parts、ActionBar、branch、copy、reload、feedback、attachment、lazy list | 可参考 primitives 分层和 lazy list 思路，不直接引入整套视觉 | +| `Vercel AI SDK` | UIMessage parts、reasoning part、tool part、data part、custom stream、tool progress | Lime 应继续把 text / thinking / tool / status / artifact 映射为结构化 part | +| `CopilotKit` | CopilotChat、Generative UI、Human-in-the-loop、frontend action、shared agent state | Lime 的 action_required、权限确认、工作区状态更新应作为 HITL UI，而不是普通文本 | + +### 2.3 本地参考项目 + +主要参考 `/Users/coso/Documents/dev/js/claudecode`，辅助参考 `/Users/coso/Documents/dev/rust/warp`、`/Users/coso/Documents/dev/rust/CodexMonitor` 与 `/Users/coso/Documents/dev/rust/codex`。 + +Claude Code 本地源码显示三类尤其值得借鉴： + +1. `src/tasks/pillLabel.ts` 把后台任务压缩为短标签，例如 shell、monitor、team、local agent、cloud session、ultraplan、dreaming。只有 `needs_input` 和 `plan_ready` 这种注意力状态才提示 CTA。 +2. `src/tools/MCPTool/UI.tsx` 对工具输入输出做分层：非 verbose 截断输入、progress bar、大输出 token warning、图片占位、空输出提示、小 JSON 扁平展示、dominant text payload 解包。 +3. `src/tools/ExitPlanModeTool/UI.tsx` 把计划流做成明确状态：退出计划、提交审批、用户批准、保存路径、拒绝计划，而不是只渲染一段 Markdown。 + +Warp 本地源码与文档显示三类可借鉴方向： + +1. Terminal block / AI block 把命令、输出、AI 解释组织为可选择、可复制、可恢复的块。 +2. Markdown visuals、Mermaid、本地图片渲染说明 AI 输出不应只是纯文本。 +3. Agent Mode 的价值在于贴近工作上下文，而不是开一个独立聊天网页。 + +CodexMonitor 更接近 Lime 当前桌面 GUI 形态，尤其值得补充： + +1. 多 workspace / thread 管理：支持 recent agent activity、unread/running 状态、pin/rename/archive/copy、resume thread 和 remote backend。对 Lime 的启发是 session/tab 不只是列表，而是多工作区执行态索引。 +2. Composer 控制层：支持 Queue vs Steer、模型选择、collaboration mode、reasoning effort、access mode、context usage ring、图片附件、dictation、skills / prompts / review / file path autocomplete。对 Lime 的启发是输入区应显式承载“执行意图 + 模型/权限/上下文预算”，不是只有 textarea。 +3. 消息渲染层：reasoning、tool、diff、review、userInput、plan follow-up 分成不同 row，tool output 做窗口化末尾展示，图片用 lazy thumbnail + lightbox。对 Lime 的启发是 process item 必须分型渲染。 +4. App-server event reference 把支持/缺失事件列成清单，并记录 `item/reasoning/*`、`item/commandExecution/*`、`turn/diff/updated`、`thread/status/changed` 等事件。对 Lime 的启发是 Agent UI 需要协议覆盖表，避免 UI 靠猜事件。 + +Codex TUI 则提供了更底层的交互与性能经验： + +1. `docs/tui-chat-composer.md` 把 composer 当成状态机：popup、slash、file、skill、paste burst、history search、local/persistent history、image rows、draft recovery 分开处理。 +2. `docs/tui-request-user-input.md` 把用户输入请求做成 overlay：一题一页、option + freeform notes、焦点路由、紧凑布局降级。对 Lime 的 `action_required` / elicitation UI 很直接。 +3. `docs/tui-stream-chunking-*.md` 把流式展示拆成 Smooth / CatchUp 模式，并用 queue depth、oldest age、mode transition、rapid re-entry 等指标验证。对 Lime 的首字慢、流式卡顿、CPU 飙高有直接借鉴价值。 +4. `status_indicator_widget.rs`、`unified_exec_footer.rs`、`pending_input_preview.rs` 展示了状态行、后台 terminal 摘要、pending steer / queued messages 预览的压缩方式。对 Lime 的 capsule / queue UI 有参考意义。 + +### 2.4 Lime 当前事实源 + +Lime 不是从零开始。当前仓库已有基础： + +- `src/components/agent/chat/types.ts` 已有 `ContentPart`：`text`、`thinking`、`tool_use`、`action_required`。 +- `src/components/agent/chat/components/StreamingRenderer.tsx` 已有 thinking block、content parts、runtime status、流式渲染与历史轻量渲染入口。 +- `src/components/agent/chat/components/MessageList.tsx` 已有首 token placeholder、runtime status pill、timeline 分离、历史 timeline 延迟构建、长历史窗口。 +- `docs/roadmap/task/event-chain.md` 已定义事件链方向：不要新造第二条事件系统，应在现有 `AgentEvent -> timeline -> thread_read -> evidence / RequestLog` 主链上补齐任务、路由、经济事件。 +- `docs/roadmap/artifacts/roadmap.md` 已明确 Artifact 方向，并包含核心原则：`Chat for reasoning, Artifact for delivery`。 +- `docs/roadmap/harness-engine/README.md` 已把 evidence pack、replay、analysis、review、UI 展示纳入同源治理。 +- `docs/roadmap/warp/README.md` 已明确 Agent loop、tool protocol、permission、slash、SkillTool、subagent 参考 Claude Code。 + +这意味着 Agent UI 的下一步不是再加一个 parallel UI，而是把现有结构化事实源的展示层收束起来。 + +## 3. Agent UI 模式全景 + +### 3.1 输入区 Composer + +| 模式 | 代表形态 | 适用场景 | Lime 建议 | +| --- | --- | --- | --- | +| 底部吸附输入框 | ChatGPT、Claude、assistant-ui | 长对话、持续问答 | 保留，作为默认主入口 | +| 浮动胶囊输入框 | 新建任务、命令模式、轻量 prompt | 首页、空态、新任务 | 适合 Lime 首页和新建对话，但不应覆盖工作区信息 | +| Slash command | Claude Code、Cursor、Warp | 精确触发模式、工具、模板 | 必须支持，且映射到 skill / command / task contract | +| Mention / context chip | Cursor、Windsurf、Copilot 类产品 | 引用文件、网页、任务、会话 | 适合 Lime workspace、artifact、browser、session 引用 | +| 附件托盘 | ChatGPT、Claude、Gemini | 多模态输入、文档、图片 | 应继续做成输入前结构化 context，不只显示文件名 | +| 模型 / Provider 选择 | 多模型产品 | 高级用户、成本控制 | 可以是紧凑 selector，不要干扰普通输入 | +| 权限 / 模式 chip | plan、act、safe、browser、write | 高风险执行前约束 | 应与 execution profile / permission policy 绑定 | +| Queue / Steer 切换 | CodexMonitor、Codex TUI | 当前 turn 运行中继续补充输入 | Lime 应明确区分“排队下一条”和“转向当前 turn” | +| Context usage ring | CodexMonitor | 长会话、上下文预算、压缩前提示 | Lime 应把上下文预算做成低调但常驻的预算信号 | +| History search / draft recovery | Codex TUI | 长期高频输入、误清空恢复 | Lime 可做 P1，避免新建/切换/取消时丢 draft | +| 语音 / 截图 / 拖拽 | 通用 AI 助手 | 快速输入、视觉任务 | 可作为 P2，不应阻塞主链 | + +输入区最重要的不是“做成胶囊”本身，而是让用户在发出任务前就能看见：当前上下文、执行模式、目标 workspace、模型/成本策略和权限边界。 + +Codex TUI 的 composer 状态机对 Lime 很有价值：多行粘贴、IME、slash command、附件 placeholder、历史召回、外部编辑器回填都不应靠 textarea 事件临时判断。Lime 如果继续强化 composer，应把这些行为沉到可测试的状态机，而不是在组件里继续堆条件分支。 + +### 3.2 消息展示 Message + +| 模式 | 优点 | 风险 | Lime 建议 | +| --- | --- | --- | --- | +| 用户气泡 + Assistant 正文 | 清晰区分双方 | 长输出易拥挤 | 默认采用 | +| 双侧气泡 | 聊天感强 | 对桌面工作台效率低 | 不作为主体验 | +| Assistant 无气泡正文 | 文档阅读舒服 | 状态和工具容易混入正文 | 适合最终回答 | +| 分段消息 parts | 可结构化渲染 text/tool/thinking/source | 需要事件协议稳定 | Lime 已有基础，应强化 | +| Branch / 版本切换 | 支持重试和多答案 | 增加复杂度 | 对关键回合 P1 引入 | +| Message action bar | copy/edit/retry/feedback | 操作过多会压视觉 | 保留低调操作栏 | +| Inline citation | 研究、网页、证据 | 无来源时容易假装严谨 | 只对真实 source/evidence 显示 | + +Lime 的消息层应做到：最终答案干净、过程信息可见但不抢位、操作入口稳定、历史恢复轻量。 + +### 3.3 Thinking / Reasoning UI + +| 模式 | 表现 | 优点 | 风险 | Lime 建议 | +| --- | --- | --- | --- | --- | +| 完全隐藏 | 只显示最终答复 | 简洁 | 用户不知道是否卡住 | 不作为唯一模式 | +| “正在思考”状态 | 文本或 spinner | 首字前有反馈 | 解释力弱 | 必须有，用于首 token 前 | +| 折叠 thinking block | 默认显示摘要/耗时，可展开 | 平衡透明与清爽 | 摘要质量要稳定 | P0 推荐 | +| 实时 reasoning step | 一步步显示 | 强透明 | 易泄漏、噪声大、卡顿 | 只对支持模型和调试模式 | +| 完成后摘要 | “思考了 N 秒 / 检查了 X 项” | 历史阅读友好 | 可能过度拟人化 | 适合默认态 | +| 原文可展开 | 展示模型 reasoning 内容 | 方便调试 | 不同模型政策不同 | 仅在 provider 明确支持且用户开启 | +| Sources-first reasoning | 先展示来源/证据，再展示结论 | 研究型任务可信 | 对普通任务太重 | 用于 research / browser / evidence 场景 | + +Lime 的原则：thinking 是过程层，不是正文层。它可以解释“正在发生什么”，但不能与最终答案重复吐字，也不能把 ``、`thinking_delta` 或工具日志混进 Markdown 正文。 + +### 3.4 Runtime Status UI + +| 状态 | UI 表现 | 说明 | +| --- | --- | --- | +| submitted | 首 token placeholder | 明确“请求已提交，等待首个响应” | +| routing | 阶段 pill | 显示模型/路由/队列决策，不刷屏 | +| streaming | 轻量流式光标 | 不用大 spinner 打断阅读 | +| tool_running | 工具 pill + 可展开详情 | 正文不直接塞工具输出 | +| action_required | 高亮确认卡 | 必须有清晰 CTA | +| retrying | 小型重试状态 | 显示次数和原因 | +| failed | 可恢复错误卡 | retry / copy diagnostic / switch provider | +| cancelled | 安静的取消状态 | 不应显示成失败 | +| completed | 状态收起 | 保留摘要，避免历史噪声 | + +首字慢的体感往往来自“用户完全看不到系统是否活着”。Runtime Status UI 的目标不是假装更快，而是在真实首 token 前给出低成本、可信的阶段反馈。 + +Codex TUI 的 `StatusIndicatorWidget` 说明状态行应把动画、耗时、interrupt hint 和短详情放在同一条稳定表面中，避免布局跳动。Lime 的 runtime status 可以吸收这个结构：一行显示阶段、耗时和可取消动作，详情最多 2 到 3 行，超出进入 timeline 或诊断抽屉。 + +### 3.5 Tool Call UI + +| 模式 | 适用 | Lime 建议 | +| --- | --- | --- | +| 行内工具卡 | 单个小工具、读文件、短命令 | 用于当前 turn 的关键步骤 | +| 批量摘要 | 多个工具、重复搜索、长列表 | 默认压缩，避免刷屏 | +| 详情抽屉 | 大输出、调试、审计 | 与 timeline/evidence 连接 | +| 输入截断 | 参数很长 | 默认显示关键字段，提供展开 | +| 输出预览 | 小 JSON、短文本、图片 | rich output 解析后展示 | +| 大输出警告 | token 很大 | 借鉴 Claude Code，提示上下文影响 | +| 空输出提示 | 工具无内容 | 显示“无输出”，不要留白 | +| 可重跑 | 幂等工具、失败工具 | 高风险工具需要确认 | +| 来源链接 | browser/search/doc | 与 citation/evidence 同源 | + +Claude Code 的 MCP UI 给 Lime 一个清晰信号：tool UI 的好坏不在于“展示全部原始数据”，而在于分层、截断、摘要、异常提示和可追溯。 + +### 3.6 Capsule / Pill UI + +胶囊式 UI 是本次调研里最适合 Lime 形成特色的模式。它不应只是装饰，而应成为任务状态入口。 + +| 胶囊类型 | 示例 | Lime 映射 | +| --- | --- | --- | +| 本地命令 | `1 shell`、`1 monitor` | terminal / command runtime | +| 后台 terminal | `1 background terminal running` | command / terminal dock | +| 后台 workflow | `1 background workflow` | task runtime | +| 本地 Agent | `1 local agent` | local runtime / subagent | +| 云端 Agent | `◇ 1 cloud session` | remote runtime / cloud scene | +| Team | `1 team` | team/subagent orchestration | +| Plan ready | `◆ plan ready` | plan approval / ExitPlanMode | +| Needs input | `◇ needs your input` | action_required / elicitation | +| Dreaming / background idea | `dreaming` | 可选 P2，不作为主链 | + +胶囊应该遵循三条规则： + +1. **普通 running 只显示事实，不打扰。** +2. **needs input / plan ready / failed 才抢注意力。** +3. **点击胶囊展开任务中心或 timeline，不跳出当前上下文。** + +这也能解决多 tab、多会话和后台 agent 带来的卡顿：用户看到的是压缩后的任务索引，而不是所有会话全量渲染。 + +CodexMonitor 与 Codex TUI 都支持“当前 turn 运行中继续输入”：一个偏 GUI 队列，一个偏 TUI pending input preview。Lime 可以把这类输入合并进胶囊/队列层：pending steer 显示为“将注入当前 turn”，queued follow-up 显示为“本轮后发送”，并允许编辑/删除。 + +### 3.7 Plan / Human-in-the-loop UI + +AI Agent 越能执行，计划和审批 UI 越重要。最佳实践不是让模型说“我准备这么做”，而是把计划变成可批准、可拒绝、可编辑、可追踪的对象。 + +| 场景 | UI 形态 | Lime 建议 | +| --- | --- | --- | +| 普通计划 | Markdown plan + steps | 可读即可 | +| 高风险计划 | approval card | 必须有批准 / 拒绝 | +| 团队审批 | waiting for lead | 显示审批人/状态 | +| 计划被拒绝 | rejected plan message | 保留理由并可继续编辑 | +| 计划保存 | saved path / artifact | 不只留在聊天正文 | +| 权限升级 | danger confirmation | 与 command policy 统一 | +| 成本确认 | cost / model estimate | 与 task/model routing 统一 | + +Lime 已有 `action_required` 和 timeline，可以把 HITL 从“文本解释”升级为“状态明确的交互卡”。 + +### 3.8 Artifact / Canvas UI + +Agent UI 的关键分水岭是：最终产物是否离开聊天正文，进入可继续工作的表面。 + +| 模式 | 代表 | Lime 建议 | +| --- | --- | --- | +| Inline preview | ChatGPT 小预览、tool result preview | 适合短结果 | +| 右侧 Canvas | Claude Artifacts、ChatGPT Canvas、Gemini Canvas | Lime 应强化为主交付面 | +| 文件树 / 产物列表 | IDE agent、workspace 产品 | 适合多文件、多版本 | +| Diff view | Cursor、Windsurf | 代码/文档修改必须支持 | +| Version history | Artifacts / docs | Artifact Workbench P1/P2 | +| Source drawer | 研究报告、网页任务 | 与 timeline/evidence 同源 | +| Export / share | 文档、图片、报告 | 使用桌面保存与 artifact export 主链 | + +Lime 的既有路线图已经说清楚：`Chat for reasoning, Artifact for delivery`。Agent UI 文档应把这条原则提升为所有交互设计的第一约束。 + +### 3.9 Timeline / Evidence UI + +Timeline 和 Evidence 是 Lime 可以走出自己特色的地方。多数竞品只显示“看起来可信”的过程，Lime 可以显示真正可复查的运行事实。 + +| 层级 | 目的 | 展示方式 | +| --- | --- | --- | +| Inline process | 当前 turn 发生了什么 | 少量关键状态 | +| Turn timeline | 工具、thinking、artifact、action | 折叠时间线 | +| Session timeline | 多回合运行历史 | 默认懒加载 | +| Evidence pack | 可导出事实包 | Harness / review 面板 | +| Replay | 复现问题 | 调试/验证入口 | +| Request log | 模型、provider、路由、成本 | 诊断面 | + +重要边界：不是所有事件都要直接展示给用户。事件可以进入事实源，UI 只投影必要摘要。否则 timeline 会变成另一种刷屏。 + +### 3.10 Multi-agent / Team UI + +多 Agent UI 不应一开始做成复杂组织图。先从任务状态和责任边界做起。 + +| 模式 | 使用时机 | Lime 建议 | +| --- | --- | --- | +| 成员胶囊 | 多个子代理并行 | 顶部/底部状态压缩 | +| 子任务列表 | 可拆解任务 | 任务中心展示 | +| Handoff 卡 | 子代理交接结果 | 与 evidence / artifact 绑定 | +| Leader approval | 需要审批 | 计划 UI 的扩展 | +| Queue view | 多任务排队 | 避免全量 tab 同时渲染 | +| Per-agent trace | 调试失败 | 默认折叠 | + +Lime 的 team/subagent 特色应该是“可观察、可交接、可验证”，而不是多几个头像。 + +### 3.11 Browser / Research UI + +研究类 Agent 需要的 UI 与普通问答不同。 + +| 模式 | 目的 | Lime 建议 | +| --- | --- | --- | +| 查询计划 | 解释将查什么 | 可折叠 | +| 来源列表 | 可信度和复查 | 与 citation/evidence 同源 | +| 浏览器快照 | 证明页面状态 | browser runtime artifact | +| 进度步骤 | 多轮搜索 | capsule + timeline | +| 最终报告 | 交付物 | artifact document | +| 引用回跳 | 从报告到来源 | artifact source drawer | + +Gemini Deep Research 的核心不是“搜得多”，而是把研究过程、来源和最终报告分层。Lime 如果结合 browser runtime 和 evidence pack，可以做得更扎实。 + +### 3.12 Tab / Session UI + +用户已经多次反馈：多会话、多 tab 和旧会话恢复会导致卡顿。这里的 UI 不能只学聊天产品，要学习浏览器。 + +| 浏览器式概念 | Agent 会话映射 | 目的 | +| --- | --- | --- | +| Active tab | 当前可交互会话 | 全量渲染 | +| Recent tab | 最近使用会话 | 轻量快照 | +| Pinned tab | 固定重要任务 | 不被自动丢弃 | +| Suspended tab | 暂停渲染但保留状态 | 降低 CPU/内存 | +| Discarded tab | 释放重资源，保留恢复入口 | 控制长时间使用成本 | +| Restore | 重新加载窗口历史 | 渐进恢复 | +| Tab group | 项目/任务分组 | workspace 维度组织 | +| Remote backend thread | 远端机器上的 Codex session | 显示连接、运行、未读、恢复状态 | +| Worktree / clone agent | 隔离工作区任务 | 作为 session 派生关系，而不是普通平铺 tab | + +Lime 的多会话体验应避免“打开一个历史对话就把所有东西恢复出来”。旧会话默认显示窗口化消息、轻量 timeline 摘要和 artifact 索引；只有用户展开时才加载完整历史和细节。 + +CodexMonitor 的 thread 管理提醒我们：session UI 还要表达 pin、archive、copy、resume、running、unread、remote backend 和 worktree 派生关系。Lime 不宜只做浏览器标签外观，而应把 tab 当成“可恢复执行单元”的索引。 + +### 3.13 Error / Recovery UI + +Agent 产品一定会失败，好的 UI 不是隐藏失败，而是让失败可恢复。 + +| 错误类型 | UI 方式 | +| --- | --- | +| Provider credential | 指向设置页，保留当前输入 | +| Rate limit | 显示等待/换模型/重试 | +| Tool permission denied | 展示被拒绝动作和替代路径 | +| Tool timeout | 可重跑、可取消、可导出日志 | +| Empty output | 明确无输出，不留白 | +| Bad formatting | 提供重新渲染/原文查看 | +| Stream interrupted | 保留已生成内容，允许继续 | +| Old session corrupt | 轻量恢复 + 诊断导出 | + +错误态必须和 runtime/evidence 绑定，不能只在 toast 里一闪而过。 + +### 3.14 Performance UI + +性能不是纯工程问题，也是一种 UI 设计。 + +| 模式 | 解决问题 | Lime 建议 | +| --- | --- | --- | +| Skeleton | 首屏等待 | 打开旧会话先挂 UI | +| First token placeholder | 首字前无反馈 | 已有基础，继续强化 | +| Progressive hydration | 旧会话重 | 先文本，后 timeline/tool | +| Virtual list | 消息多 | 长历史必须使用窗口化 | +| Lazy timeline | threadItems 重 | 默认摘要，展开再构建 | +| Suspended tab | 多会话卡顿 | 浏览器式 session 管理 | +| Output cap | 工具输出巨大 | 预览 + warning + 详情 | +| Worker parsing | Markdown/timeline 重 | P2 可引入 | +| Adaptive stream chunking | 流式 backlog 和 UI 卡顿 | Smooth / CatchUp 策略，按队列压力切换 | +| Queue pressure metrics | 流式调优无证据 | 记录 queue depth、oldest age、mode transition、rapid re-entry | +| Stable status row | 运行态布局抖动 | 状态行固定高度，详情有最大行数 | + +“看起来慢”经常来自三个源头：首 token 前无反馈、历史恢复同步计算、后台会话全量渲染。Agent UI 路线图必须把性能策略写进交互，而不是只靠后端优化。 + +Codex TUI 的流式分块文档给 Lime 一个可执行验证方向：不要只观察“首字时间”，还要记录 UI 消费流的 backlog。至少应能看到每次渲染提交的队列深度、最老未渲染 delta 年龄、是否进入 catch-up、是否频繁抖动切换。这样才能解释“模型已经吐了但 UI 还慢”。 + +## 4. Lime 应该形成的 UI 架构 + +### 4.1 五层模型 + +Lime Agent UI 建议按五层组织： + +| 层 | 负责 | 不负责 | +| --- | --- | --- | +| 对话层 | 用户意图、最终回答、少量协作文本 | 展示全部工具日志 | +| 过程层 | thinking、runtime status、tool summary、timeline 摘要 | 承载最终产物 | +| 任务层 | 胶囊、队列、后台任务、子代理、plan approval | 替代消息列表 | +| 产物层 | Artifact、Canvas、文件、diff、版本、导出 | 解释所有推理 | +| 证据层 | evidence、replay、request log、review、diagnostic | 日常默认展开 | + +这五层对应 Lime 的现有能力，不需要新造第二套 runtime。关键是让 UI 只消费同一条事实链的不同投影。 + +### 4.2 信息优先级 + +默认屏幕优先级应为： + +1. 当前用户目标和输入区。 +2. 当前 active turn 的可读状态。 +3. 最终回答或 artifact 交付物。 +4. 需要用户介入的 action。 +5. 后台任务胶囊。 +6. 可展开过程和证据。 + +不应默认抢占注意力的内容： + +- 全量 thinking。 +- 全量 tool JSON。 +- 完整 request log。 +- 历史 turn 的所有 timeline 细节。 +- 已完成后台任务的长日志。 + +### 4.3 Lime 视觉语言约束 + +根据 `docs/aiprompts/design-language.md`，Agent UI 应保持： + +- 轻盈、清晰、专业，信息优先。 +- 桌面应用感强，避免网页营销风格。 +- 主表面使用实体底色，不依赖半透明和磨砂。 +- 胶囊可以使用小圆角状态标签，但不要把所有内容都做成卡片。 +- 状态色保持语义稳定：成功 emerald，提醒 amber，信息 sky/slate，错误 rose/red。 +- 中文排版优先，不使用过大的英文 tracking 破坏阅读。 + +这意味着 Lime 可以使用胶囊式 UI，但胶囊必须是状态语义，不是视觉噱头。 + +## 5. 适合 Lime 的优先级 + +### P0：对话主链 UI 收敛 + +目标：解决旧会话卡顿、首字慢、重复吐字、排版污染这些用户已经感知到的问题。 + +建议动作： + +1. `thinking`、`tool_use`、`action_required` 与最终正文严格分层。 +2. 首 token 前始终显示可信 runtime status，不让用户面对空白等待。 +3. tool 输出默认摘要化，正文只保留必要解释。 +4. 历史消息默认轻量渲染：文本优先，timeline/tool 详情延后。 +5. 长输出和大 JSON 必须有 preview、warning 和详情入口。 + +验收： + +- DeepSeek / reasoning 模型不会把思考内容重复吐到正文。 +- 打开旧会话不会触发长时间鼠标 loading 和明显 CPU 飙高。 +- 用户能在 1 秒内看到“请求已提交/正在路由/正在等待模型”等阶段反馈。 +- 工具执行不会把正文排版挤乱。 + +### P1：执行胶囊层 + +目标：让后台任务、多 Agent、计划审批和 needs input 有统一入口。 + +建议动作： + +1. 引入任务胶囊状态模型：running、queued、needs_input、plan_ready、failed、completed。 +2. 普通 running 低调显示；needs_input / plan_ready / failed 明确提示。 +3. 胶囊点击展开任务中心或当前 session timeline。 +4. 胶囊与 tab/session 管理联动，避免所有会话全量恢复。 + +验收： + +- 多个后台任务不会刷屏。 +- 用户能一眼知道哪个任务需要介入。 +- 打开多个历史对话时，非 active 会话不持续吃 CPU。 + +### P1：Artifact-first 交付路径 + +目标：让最终产物离开聊天正文，进入可编辑、可版本化、可导出的工作台。 + +建议动作： + +1. 对文档、代码、报告、图片、网页研究等结果默认创建 artifact。 +2. 聊天正文只解释意图、摘要和下一步。 +3. Artifact 与 timeline item 双向跳转。 +4. 工具来源、引用和 evidence 能回挂到 artifact source drawer。 + +验收： + +- 长报告不再只是一大段聊天 Markdown。 +- 用户可从 artifact 找到生成它的 turn、tool、source。 +- 修改、重写、导出走 Artifact Workbench，而不是重发一段聊天。 + +### P2：浏览器式 Session / Tab 管理 + +目标：解决多会话长期运行的性能和导航问题。 + +建议动作： + +1. Session tab 引入 active、recent、pinned、suspended、discarded 状态。 +2. 非 active 历史会话只保留标题、摘要、任务胶囊和 artifact 索引。 +3. 恢复旧会话采用 progressive hydration。 +4. pinned session 不被自动丢弃，普通 session 可按内存压力降级。 + +验收： + +- 同时打开多个历史对话不会线性增加主线程压力。 +- 切回旧会话先显示可读快照，再渐进恢复细节。 +- 新建任务不会被旧会话恢复阻塞。 + +### P2：Team / Evidence 工作台 + +目标：让 Lime 的团队代理和证据链成为差异化能力。 + +建议动作： + +1. Team/subagent 先以胶囊和任务列表呈现，不急于做复杂图谱。 +2. Handoff 结果进入 timeline 和 artifact。 +3. Evidence pack 和 replay 入口只在需要诊断、审核、验证时展开。 +4. Harness 面板与对话工作台共享同一份 verification facts。 + +验收： + +- 多 Agent 执行能看出责任边界。 +- 失败时能导出证据，而不是让用户截图聊天。 +- Review / replay / UI 不出现事实漂移。 + +## 6. 推荐路线图 + +### Phase 1：对话主链稳定化 + +交付内容： + +- thinking / final text / tool / action_required 的展示边界收紧。 +- 首 token placeholder 与 runtime status 统一。 +- 历史恢复默认轻量渲染。 +- tool rich output 支持截断、预览、大输出警告、空输出提示。 +- 流式输出引入 UI backlog 指标，区分模型慢、invoke 慢和前端渲染慢。 +- E2E 覆盖新建对话、旧会话、流式、thinking、tool 输出、错误恢复。 + +这是最先做的一刀，因为它直接对应当前用户痛点：旧会话慢、首字慢、重复吐字、排版不稳。 + +### Phase 2：胶囊任务层 + +交付内容： + +- 任务胶囊状态模型。 +- 胶囊栏或状态区。 +- needs input / plan ready / failed 的 attention CTA。 +- 胶囊点击进入任务中心或 timeline。 +- 后台任务与 session/tab 状态关联。 + +这一刀会把 Claude Code 的 pill 思路转化为 Lime 自己的桌面工作台语言。 + +### Phase 3：Artifact 与 Timeline 双向增强 + +交付内容： + +- artifact 默认承载长产物。 +- timeline item 可打开 artifact 定位。 +- artifact source drawer 可回跳 source / tool / evidence。 +- report / code / image / browser snapshot 等 domain artifact 的 viewer 收敛。 + +这一刀让 Lime 从“聊天里有产物”变成“围绕产物协作”。 + +### Phase 4：浏览器式会话管理 + +交付内容： + +- active / recent / pinned / suspended / discarded session 状态。 +- 非 active session 的资源降级。 +- 旧会话 progressive hydration。 +- 多 tab 下的 CPU、内存、主线程指标回归。 + +这一刀专门解决“对话多了以后卡”的长期问题。 + +### Phase 5：Team / 多 Agent 工作台 + +交付内容： + +- team/subagent 胶囊。 +- 子任务列表和 handoff 卡。 +- leader approval / plan approval。 +- evidence / replay / verification 与 UI 联动。 + +这一刀把 Lime 的长期特色显性化：不是单个聊天助手，而是可治理的 Agent 工作系统。 + +## 7. 验收标准 + +### 7.1 用户体验验收 + +1. 新建对话可以快速打开，不被旧会话恢复阻塞。 +2. 打开旧会话先出现可读内容，再渐进恢复细节。 +3. 首 token 前有可信状态，不出现长时间空白。 +4. thinking 不与最终答案重复。 +5. tool 输出不会破坏正文排版。 +6. action_required 有明确 CTA。 +7. 后台任务有胶囊入口，且注意力状态清晰。 +8. 长报告、代码、图片、研究结果进入 artifact。 +9. artifact 能回跳来源、timeline 和 evidence。 +10. 多个 tab/session 长时间存在时 CPU 和内存可控。 + +### 7.2 工程验收 + +1. 不新增第二套事件系统。 +2. UI 消费 `AgentEvent -> timeline -> thread_read -> evidence` 主链投影。 +3. `ContentPart` 继续作为消息结构化 part 的前端边界。 +4. tool / thinking / action / artifact 的渲染有稳定测试。 +5. GUI 主路径补 Playwright / smoke 验证。 +6. 旧会话性能改动有 E2E 或性能日志证明。 +7. 文档、代码和测试都不扩展 deprecated / compat 路径。 + +## 8. 明确不做 + +1. 不直接复制 Claude、ChatGPT、Gemini、Cursor、Warp 的视觉。 +2. 不把所有 tool 输出塞进聊天正文。 +3. 不把 thinking 当作最终答案的一部分。 +4. 不新增与 timeline/evidence 平行的第二套事实系统。 +5. 不为了“胶囊好看”把所有状态都做成 pill。 +6. 不默认全量恢复每个历史会话。 +7. 不为了展示透明度牺牲旧会话性能。 +8. 不用营销式 hero 或大面积装饰背景包装 Agent 工作台。 + +## 9. 参考资料 + +### 9.1 官方与公开资料 + +- OpenAI Agents 指南：`https://platform.openai.com/docs/guides/agents` +- OpenAI Agent Builder：`https://platform.openai.com/docs/guides/agent-builder` +- OpenAI ChatKit：`https://platform.openai.com/docs/guides/chatkit` +- OpenAI Canvas 发布文：`https://openai.com/index/introducing-canvas/` +- ChatGPT Projects 帮助文档：`https://help.openai.com/en/articles/10169521-projects-in-chatgpt` +- Claude Artifacts 帮助文档：`https://support.claude.com/en/articles/9487310-what-are-artifacts-and-how-do-i-use-them` +- Claude Extended Thinking 文档：`https://platform.claude.com/docs/en/docs/build-with-claude/extended-thinking` +- Anthropic Computer Use 发布文：`https://www.anthropic.com/news/3-5-models-and-computer-use` +- Gemini Canvas：`https://gemini.google/overview/canvas/` +- Gemini Deep Research 帮助文档：`https://support.google.com/gemini/answer/15719111` +- Warp Agent Mode 文档：`https://docs.warp.dev/agents/warp-ai/agent-mode` +- Cursor Docs：`https://docs.cursor.com/` +- Windsurf Docs：`https://docs.windsurf.com/` +- Microsoft Human-AI Interaction Guidelines：`https://www.microsoft.com/en-us/research/project/guidelines-for-human-ai-interaction/` +- Google People + AI Guidebook：`https://pair.withgoogle.com/guidebook/` + +### 9.2 Context7 查询源 + +- `assistant-ui`：`/assistant-ui/assistant-ui` +- `Vercel AI SDK`：`/vercel/ai` +- `CopilotKit`：`/copilotkit/copilotkit` + +### 9.3 本地参考源 + +- Claude Code pill：`/Users/coso/Documents/dev/js/claudecode/src/tasks/pillLabel.ts` +- Claude Code MCP tool UI：`/Users/coso/Documents/dev/js/claudecode/src/tools/MCPTool/UI.tsx` +- Claude Code plan UI：`/Users/coso/Documents/dev/js/claudecode/src/tools/ExitPlanModeTool/UI.tsx` +- Warp 项目说明：`/Users/coso/Documents/dev/rust/warp/WARP.md` +- Warp Agent Mode 测试：`/Users/coso/Documents/dev/rust/warp/crates/integration/src/test/agent_mode.rs` +- CodexMonitor README：`/Users/coso/Documents/dev/rust/CodexMonitor/README.md` +- CodexMonitor app-server event reference：`/Users/coso/Documents/dev/rust/CodexMonitor/docs/app-server-events.md` +- CodexMonitor composer：`/Users/coso/Documents/dev/rust/CodexMonitor/src/features/composer/components/Composer.tsx` +- CodexMonitor queue：`/Users/coso/Documents/dev/rust/CodexMonitor/src/features/composer/components/ComposerQueue.tsx` +- CodexMonitor messages：`/Users/coso/Documents/dev/rust/CodexMonitor/src/features/messages/components/Messages.tsx` +- Codex TUI composer state machine：`/Users/coso/Documents/dev/rust/codex/docs/tui-chat-composer.md` +- Codex TUI request-user-input overlay：`/Users/coso/Documents/dev/rust/codex/docs/tui-request-user-input.md` +- Codex TUI stream chunking：`/Users/coso/Documents/dev/rust/codex/docs/tui-stream-chunking-tuning.md` +- Codex TUI stream validation：`/Users/coso/Documents/dev/rust/codex/docs/tui-stream-chunking-validation.md` +- Codex TUI status / queue UI：`/Users/coso/Documents/dev/rust/codex/codex-rs/tui/src/status_indicator_widget.rs`、`/Users/coso/Documents/dev/rust/codex/codex-rs/tui/src/bottom_pane/pending_input_preview.rs` + +### 9.4 Lime 事实源 + +- `src/components/agent/chat/types.ts` +- `src/components/agent/chat/components/StreamingRenderer.tsx` +- `src/components/agent/chat/components/MessageList.tsx` +- `docs/roadmap/task/event-chain.md` +- `docs/roadmap/artifacts/roadmap.md` +- `docs/roadmap/harness-engine/README.md` +- `docs/roadmap/warp/README.md` + +## 10. 最终建议 + +Lime 的 Agent UI 不应把“胶囊式 UI”“思考 UI”“工具 UI”“Artifact UI”当作互相独立的组件库任务。它们应该围绕同一个问题收束： + +> 用户如何在一个桌面工作区里，快速知道 Agent 正在做什么、什么时候需要自己介入、产物在哪里、过程是否可信、失败后如何恢复？ + +因此下一阶段最务实的顺序是： + +1. 先修对话主链：thinking、tool、首 token、旧会话性能。 +2. 再建胶囊任务层：后台任务、needs input、plan ready、多 session 压缩。 +3. 然后强化 Artifact：最终产物离开正文，进入 Workbench。 +4. 最后把 session/tab、team/subagent、evidence/replay 组织成 Lime 独有的 Agent 工作系统。 + +这条路既吸收了 Claude Code 的执行状态、Warp 的 block 化工作上下文、ChatGPT/Claude/Gemini 的 Canvas/Artifact 经验，也保留 Lime 自己最重要的差异化：可执行、可观察、可交付、可验证。 diff --git a/docs/roadmap/agentui/lime-agentui-backend-coordination.md b/docs/roadmap/agentui/lime-agentui-backend-coordination.md new file mode 100644 index 000000000..6d228d5f1 --- /dev/null +++ b/docs/roadmap/agentui/lime-agentui-backend-coordination.md @@ -0,0 +1,361 @@ +# Lime AgentUI 后端协作架构 + +> 状态：后端协作设计 +> 更新时间：2026-04-30 +> 目标：明确 AgentUI 下一阶段需要后端稳定提供哪些投影、分页、指标和批量接口，避免前端靠全量 detail 和字符串推断撑 UI。 + +## 1. 后端职责总览 + +AgentUI 的后端不只是“把模型输出传给前端”。它需要提供五类能力： + +| 能力 | 后端职责 | UI 收益 | +| --- | --- | --- | +| 事件流 | 尽早发 `runtime_status`，分型发 text/thinking/tool/artifact/action/queue | 首字前有反馈，流式不串音 | +| 会话分页 | tail window、cursor、older page、summary snapshot | 旧会话秒开，长历史不卡 | +| 过程持久化 | runtime event -> timeline turn/item | tool/thinking/process 可追溯 | +| 产物持久化 | artifact document、版本、metadata、diff | 交付物离开正文进入 workbench | +| 证据导出 | evidence、review、replay、handoff | 结果可审计、可复盘、可继续 | + +后端协作的核心原则： + +- 前端首屏只拿必要 summary，不拿全量历史。 +- 运行态由事件流驱动，历史态由 session detail 和 timeline 驱动。 +- process、artifact、evidence 必须有稳定 id，不能只靠显示文本。 +- 慢点日志要前后端同名可拼接。 + +## 2. 后端模块分层 + +```mermaid +flowchart TB + FE[Frontend agentRuntime clients] --> Cmd[command_api/runtime_api.rs] + Cmd --> Turn[runtime_turn.rs] + Cmd --> Session[session_runtime.rs] + Cmd --> Action[action_runtime.rs] + Cmd --> Subagent[subagent_runtime.rs] + + Turn --> Queue[runtime_queue.rs] + Queue --> Aster[Aster Runtime] + Aster --> Converter[event_converter.rs] + Converter --> Events[AgentEvent stream] + Converter --> Timeline[AgentTimelineRecorder] + + Cmd --> Store[session_store.rs] + Store --> Db[(SQLite)] + Timeline --> Db + + Turn --> Artifact[artifact services] + Artifact --> ArtifactFiles[.lime/artifacts] + + Cmd --> Evidence[evidence/review/replay services] + Evidence --> HarnessFiles[.lime/harness] + + Store --> ThreadRead[thread_read projection] + ThreadRead --> FE + Events --> FE +``` + +## 3. Command API 合同 + +### 3.1 Session 读取 + +当前主入口： + +- 前端：`getAgentRuntimeSession(sessionId, options)` +- 后端：`agent_runtime_get_session(...)` + +当前已支持： + +- `historyLimit` +- `historyOffset` +- `historyBeforeMessageId` +- `resumeSessionStartHooks` +- `history_cursor` +- `history_truncated` +- `messages_count` +- `thread_read` +- `queued_turns` + +下一阶段建议： + +| 需求 | 接口建议 | 说明 | +| --- | --- | --- | +| tab 快速恢复 | `agent_runtime_get_session_snapshot` 或 list summary 扩展 | 只返回 title、last message preview、status、unread/running、capsule summary | +| 旧会话分段 hydrate | 保持 `getSession(historyLimit=40)`，补 `timelineDetailLimit` | 首屏 messages 先到，timeline detail 可更少 | +| 更早历史 | `historyBeforeMessageId` 优先于 offset | offset 对持续插入不稳定，cursor 更适合分页 | +| 非活跃 tab 刷新 | summary refresh | 避免后台 tab 拉 detail | +| list 与 detail 抢通道 | list low priority / debounce | 点击历史会话时 detail 优先 | + +### 3.2 Turn 提交 + +当前主入口： + +- 前端：`submitAgentRuntimeTurn(request)` +- 后端：`agent_runtime_submit_turn` + +必须保持： + +- event listener 先注册。 +- command 尽早接受请求。 +- runtime 尽早发 `runtime_status`。 +- busy 时明确 queue 或 steer。 + +下一阶段建议： + +| 需求 | 后端配合 | +| --- | --- | +| 首字慢观测 | 在 submit accepted、runtime queued、runtime starting、provider request started、first model event 处发 status 或 tracing | +| Queue / Steer 区分 | request 和 event 均带 `submission_mode` / `queue_policy` / `target_turn_id` | +| 取消与中断 | interrupt marker 进入 thread_read，事件流发 cancelled/aborted 状态 | +| provider 慢 | runtime status 包含 provider/model/routing stage，但不暴露敏感配置 | + +### 3.3 Action Required + +当前主入口： + +- 前端：`respondAgentRuntimeAction(request)` +- 后端：`agent_runtime_respond_action` +- 事件：`action_required` +- 投影：`thread_read.pending_requests` + +下一阶段建议： + +| 需求 | 后端配合 | +| --- | --- | +| needs input capsule | thread_read 输出 pending request count、highest severity、oldest age | +| 权限确认可审计 | approval request item 写入 timeline，并记录 response summary | +| replay pending request | pending request 带 replay metadata 和稳定 request id | +| 表单型请求 | action data 使用稳定 schema，而不是 Markdown 文本 | + +### 3.4 Queue + +当前主入口： + +- 前端：`removeAgentRuntimeQueuedTurn`、`promoteAgentRuntimeQueuedTurn` +- 后端：runtime queue service +- 事件：`queue_added`、`queue_removed`、`queue_started`、`queue_cleared` + +下一阶段建议： + +| 需求 | 后端配合 | +| --- | --- | +| task capsule | queue snapshot 返回 `queued_turn_id`、preview、created_at、mode、target turn | +| 多 tab 快 | list session summary 带 queue/running counts，不需要 detail | +| 队列操作稳定 | queue mutation 以 id 为键，返回最新 queue snapshot | +| 后台恢复 | `resume_runtime_queue_if_needed` 不由普通 getSession 顺手触发 | + +## 4. Session Store 与历史分页 + +当前关键入口： + +- `get_runtime_session_detail_with_history_page` +- `get_session_sync_with_history_page` +- `AgentDao::get_message_window_info` +- `count_session_messages_sync` + +当前已做到： + +- tail page 读取 messages。 +- 对 turns/items 使用同窗口限制。 +- limited history 下跳过部分重型 runtime overlay。 +- full history 时启用 tool IO eviction。 + +下一阶段建议后端继续提供三种粒度： + +| 粒度 | 用途 | 内容 | +| --- | --- | --- | +| summary | sidebar/tab/capsule | session id、title、preview、status、running/queued/pending counts、last activity | +| window detail | active conversation 首屏 | 最近 N 条 messages、少量 turns/items、thread_read、history_cursor | +| full detail page | 用户主动加载更早历史或展开 timeline | older messages、对应 turns/items、artifact refs | + +不要把“打开会话”和“加载所有证据”绑成同一个接口。 + +## 5. Timeline 与 Process 投影 + +当前关键入口： + +- `AgentTimelineRecorder` +- `AgentTimelineDao::list_turns_by_thread_tail_page` +- `AgentTimelineDao::list_items_by_thread_tail_page` +- `thread_reliability_projection_service` + +Timeline 后端应对 UI 提供两层数据： + +| 层 | 内容 | UI 使用 | +| --- | --- | --- | +| compact process summary | turn status、tool count、artifact count、last tool、errors、duration | 消息旁 process summary、capsule、tab | +| detailed timeline items | reasoning、tool call、command、web search、approval、artifact | Timeline drawer / evidence | + +下一阶段建议： + +1. 给 `getSession` 增加可选 `timelineDetailLimit` 或 `includeTimelineDetails=false`。 +2. 提供 `getThreadTimelinePage(sessionId, cursor, limit)`，用于展开过程详情时再加载。 +3. 对 tool 大输出默认返回 summary + offload ref，详情按需读取。 +4. timeline item 需要稳定 type 和 status，前端不要解析中文过程文本。 + +## 6. Artifact 协作 + +当前关键入口： + +- `artifact_document_service.rs` +- `artifact_ops_service.rs` +- `artifact_protocol.rs` +- `write_artifact_events.rs` +- `AgentThreadTimelineArtifactCard` +- `useWorkspaceArtifactPreviewActions` + +后端需要保证： + +- artifact id、request id、path key 归一化一致。 +- `artifact_snapshot` 能映射到 timeline item 和 ArtifactDocument。 +- 大 artifact 内容可以按 preview 读取，不把完整内容塞进 message。 +- artifact ops envelope 可增量展示： + - `artifact.begin` + - `artifact.meta.patch` + - `artifact.source.upsert` + - `artifact.block.upsert` + - `artifact.block.remove` + - `artifact.complete` + - `artifact.fail` + +下一阶段建议： + +| 需求 | 后端配合 | +| --- | --- | +| workbench 自动打开 | artifact snapshot 返回 display title、mime/kind、preview mode | +| artifact 列表快 | session summary 带 latest artifacts summary | +| artifact diff | file checkpoint / artifact diff service 打通 | +| artifact 失败恢复 | artifact fail event 进入 thread_read incident | + +## 7. Evidence / Review / Replay 协作 + +当前关键入口： + +- `runtime_evidence_pack_service.rs` +- `runtime_review_decision_service.rs` +- `runtime_replay_case_service.rs` +- `runtime_analysis_handoff_service.rs` +- `runtime_handoff_artifact_service.rs` + +证据层应保持： + +```text +session detail + thread_read + timeline + artifacts + verification + -> evidence pack + -> review decision / replay / handoff +``` + +下一阶段建议： + +| 需求 | 后端配合 | +| --- | --- | +| evidence 后台导出 | command 返回 job/snapshot 状态，UI 用 capsule 展示 | +| review 状态 | session summary 带 latest review status / pending review | +| replay 可见 | replay case 生成后进入 evidence panel | +| 验证摘要 | thread_read 或 evidence summary 输出 pass/fail/needs_review，不由前端推断 | + +## 8. 日志与性能指标合同 + +### 8.1 旧会话恢复 + +后端已经输出： + +- `total_ms` +- `resume_queue_ms` +- `detail_ms` +- `hooks_ms` +- `queue_snapshots_ms` +- `projection_ms` +- `interrupt_marker_ms` +- `dto_ms` +- `history_limit` +- `history_offset` +- `messages` +- `turns` +- `items` +- `queued_turns` + +前端应对齐输出： + +- `switchTopic.start` +- `snapshot.apply.ms` +- `runtimeGetSession.start/success/slow` +- `hydrate.messages.ms` +- `messageList.timelineBuild.ms` +- `messageList.firstStablePaint.ms` +- `historicalTimeline.idleComplete.ms` + +### 8.2 首字慢 + +建议后端补齐： + +- `submit.accepted.ms` +- `queue.wait.ms` +- `turn.start.ms` +- `provider.request.start.ms` +- `provider.first_event.ms` +- `provider.first_text_delta.ms` +- `tool.first_start.ms` + +建议前端补齐： + +- `listener.bound.ms` +- `submit.invoke.ms` +- `first_event.ms` +- `first_runtime_status.ms` +- `first_text_delta.ms` +- `first_text_paint.ms` +- `text_delta.queue_depth` +- `text_delta.oldest_unrendered_age_ms` +- `stream.catch_up_mode` + +### 8.3 CPU / 内存 + +建议补： + +- active tab count。 +- hydrated detail tab count。 +- mounted MessageList count。 +- mounted timeline item count。 +- streaming buffer chars。 +- deferred timeline pending count。 +- artifact preview loaded bytes。 + +这些指标可以先通过 debug log 实现，不必一开始接入完整遥测系统。 + +## 9. UI 需要的后端新增能力清单 + +| 优先级 | 能力 | 建议位置 | 验收 | +| --- | --- | --- | --- | +| P0 | session summary snapshot | `session_runtime.rs` / DTO | 打开 tab 不拉全量 detail | +| P0 | first event / first text tracing | `runtime_turn.rs` / event converter | 首字慢可分段定位 | +| P0 | getSession 可关闭 timeline detail | `runtime_api.rs` / `session_store.rs` | 旧会话 messages 先渲染 | +| P1 | timeline page API | runtime command + DAO | 展开过程时分页加载 | +| P1 | batch session patch | session update command | access mode/provider/strategy 回填合并 | +| P1 | queue/capsule summary in list | session list DTO | sidebar/tab 不抢 detail | +| P1 | tool detail on-demand | tool IO offload + command | 大工具输出不进正文 | +| P2 | background evidence job | evidence service | 证据导出不阻塞 UI | +| P2 | artifact preview manifest | artifact service | workbench 列表快速加载 | +| P3 | worker-friendly projection DTO | protocol projection | 前端 worker 可直接消费 | + +## 10. 命令边界注意事项 + +任何新增或修改 Tauri command 必须同步四侧： + +1. 前端 API client。 +2. Rust command 注册。 +3. command catalog。 +4. mock / browser fallback。 + +最低校验： + +```bash +npm run test:contracts +``` + +如果改动影响 GUI 主路径，再补： + +```bash +npm run verify:gui-smoke +``` + +本轮文档只定义目标，不新增 command。后续实现时必须按命令边界治理执行。 diff --git a/docs/roadmap/agentui/lime-agentui-code-map.md b/docs/roadmap/agentui/lime-agentui-code-map.md new file mode 100644 index 000000000..007babf0a --- /dev/null +++ b/docs/roadmap/agentui/lime-agentui-code-map.md @@ -0,0 +1,165 @@ +# Lime AgentUI 代码层级地图 + +> 状态：代码事实地图 +> 更新时间：2026-04-30 +> 目标：把 AgentUI 相关前端、协议、Tauri command、Rust runtime、service、持久化入口分层，作为后续拆分和性能优化的定位图。 + +## 1. 总览 + +```mermaid +flowchart TB + UI[src/components/agent/chat UI] + Hooks[src/components/agent/chat/hooks state + stream] + Api[src/lib/api/agentRuntime + agentProtocol] + Tauri[src-tauri commands/aster_agent_cmd] + AgentCrate[src-tauri/crates/agent runtime crate] + Services[src-tauri/src/services] + Store[(SQLite + .lime files)] + + UI --> Hooks + Hooks --> Api + Api --> Tauri + Tauri --> AgentCrate + Tauri --> Services + AgentCrate --> Services + Services --> Store + AgentCrate --> Store +``` + +当前 AgentUI 不是缺少底座，而是底座集中在少数大入口里。下一步代码架构的重点是把聚合入口拆成稳定子层，而不是复制一条新链。 + +## 2. 前端 UI 层 + +| 区域 | 关键文件 | 当前职责 | 后续架构方向 | +| --- | --- | --- | --- | +| 工作区总入口 | `src/components/agent/chat/AgentChatWorkspace.tsx` | 组合 chat、inputbar、workspace、artifact、timeline、team、harness、task center | 保留 shell 入口，逐步拆出 `SessionChrome`、`ConversationPane`、`WorkbenchPane`、`ProcessDrawer` | +| 消息列表 | `src/components/agent/chat/components/MessageList.tsx` | 渲染 messages、timeline、queued、pending action、历史窗口、progressive render | 继续做首屏轻量投影；timeline/tool/artifact detail 延迟或虚拟化 | +| 流式渲染 | `src/components/agent/chat/components/StreamingRenderer.tsx` | 渲染 text/thinking/tool/action/runtime status、A2UI、计划块 | 保持 content part 分型；补 backlog/catch-up 指标和重复吐字防线 | +| 输入区 | `src/components/agent/chat/components/Inputbar/index.tsx` | 输入容器与状态接线 | 收敛为 composer shell | +| 输入核心 | `src/components/agent/chat/components/Inputbar/components/InputbarCore.tsx` | 文本、图片、provider、model、execution strategy、access mode、team、workflow、hint popup | 抽出 composer state machine，明确 send / queue / steer | +| 运行状态行 | `src/components/agent/chat/components/Inputbar/components/InputbarRuntimeStatusLine.tsx` | 输入区状态提示 | 升级为首字前可信 runtime strip，可和 task capsule 共享投影 | +| 队列面板 | `src/components/agent/chat/components/Inputbar/components/QueuedTurnsPanel.tsx` | 展示 queued turns | 作为 task capsule 的详情面板 | +| 时间线 | `src/components/agent/chat/components/AgentThreadTimeline.tsx` | 展示 turn/item/tool/artifact 过程 | 从 MessageList 正文中继续抽离，默认延迟渲染历史项 | +| 工具步骤 | `src/components/agent/chat/components/InlineToolProcessStep.tsx`、`ToolCallDisplay.tsx` | 工具调用、工具结果展示 | 大输出进入详情抽屉，正文只保留摘要 | +| 计划与决策 | `AgentPlanBlock.tsx`、`DecisionPanel.tsx`、`ActionRequestA2UIPreviewCard.tsx` | 计划、审批、action_required | 统一为 HITL pattern：批准、拒绝、编辑、重放 | +| 任务中心 | `TaskCenterTabStrip.tsx`、`A2UITaskCard.tsx` | task center / A2UI 任务卡 | 进入 task layer，和 tab/capsule 统一 | +| Team 工作台 | `components/team-workspace-board/*` | 子代理、团队、画布式协作 | 成为 task layer 的 team 视图，不和普通 chat 首屏抢渲染 | +| Harness | `HarnessStatusPanel.tsx`、`RuntimeReviewDecisionDialog.tsx` | evidence、review、verification 展示 | 成为 evidence layer，默认由 task/evidence 胶囊进入 | +| Workbench | `CanvasWorkbenchLayout.tsx`、`GeneralWorkbench*`、`useWorkspaceArtifactPreviewActions.ts` | artifact/canvas/general workbench | 成为 artifact layer 的首要承载面 | + +## 3. 前端状态与 Hook 层 + +| 文件 | 当前职责 | 关键事实 | 后续方向 | +| --- | --- | --- | --- | +| `hooks/useAgentSession.ts` | 会话主状态、topic、messages、threadTurns、threadItems、queuedTurns、threadRead、executionRuntime | 已有 `SESSION_HISTORY_LOAD_PAGE_SIZE = 50`、`SESSION_DETAIL_PREFETCH_HISTORY_LIMIT = 40`、deferred hydration、prefetch、cached snapshot | 逐步拆为 session controller、history controller、topic/tab controller、runtime projection selector | +| `hooks/agentStreamSubmitExecution.ts` | 提交流式 turn | 先注册 event binding，再 `submitOp` | 保持顺序，避免首事件丢失 | +| `hooks/agentStreamTurnEventBinding.ts` | 注册事件 listener、watchdog、silent recovery | `STREAM_FIRST_EVENT_TIMEOUT_MS = 12_000`、`STREAM_INACTIVITY_TIMEOUT_MS = 120_000` | 首字慢排查应采集 first event / first text / inactivity 指标 | +| `hooks/agentStreamRuntimeHandler.ts` | `handleTurnStreamEvent` 分发事件到前端状态 | `TEXT_DELTA_RENDER_FLUSH_MS = 32`，含 `reconcileFinalContentParts` | 抽成 reducer，补 backlog depth / oldest age / catch-up mode | +| `utils/threadTimelineView.ts` | messages + turns + items -> timeline 投影 | MessageList 同步计算热点之一 | 大会话改为 idle/worker/懒加载 | +| `utils/messageTurnGrouping.ts` | timeline grouping | 与渲染批次绑定 | 与 virtualization 结合 | +| `utils/processDisplayText.ts` | 过程文本规范化 | 防 `` / 工具日志污染正文 | 继续作为 text/thinking/tool 分型防线 | + +## 4. 前端协议 / API 层 + +| 文件 | 当前职责 | 关键能力 | +| --- | --- | --- | +| `src/lib/api/agentRuntime/index.ts` | agent runtime API 统一导出 | session、thread、export、subagent、site、media client | +| `src/lib/api/agentRuntime/sessionClient.ts` | create/list/get/update session | `getAgentRuntimeSession(sessionId, { historyLimit, historyOffset, historyBeforeMessageId })`；有 `runtimeGetSession.slow` 日志 | +| `src/lib/api/agentRuntime/threadClient.ts` | submit、interrupt、compact、resume、respond action、queue 操作、thread read | `submitAgentRuntimeTurn`、`respondAgentRuntimeAction`、`getAgentRuntimeThreadRead` | +| `src/lib/api/agentRuntime/types.ts` | DTO 类型 | `AsterSessionDetail`、`AgentRuntimeThreadReadModel`、`AgentRuntimeSubmitTurnRequest`、history cursor | +| `src/lib/api/agentProtocol.ts` | AgentEvent 类型和兼容 normalizer | `turn_started`、`text_delta`、`thinking_delta`、`tool_start/end`、`artifact_snapshot`、`runtime_status`、queue、subagent、`done/final_done` | +| `src/lib/api/agentTextNormalization.ts` | legacy 文本与 item normalizer | 兼容旧事件/旧 item | 继续收口 compat，不新增旧形态 | + +协议层的架构判断： + +- UI 只能消费结构化事件和 DTO，不应通过字符串猜测 Agent 状态。 +- `parseAgentEvent` 是兼容边界，不是鼓励继续新增 legacy 事件。 +- `historyLimit/historyOffset/historyBeforeMessageId` 是旧会话性能主接口，后续 pagination/virtualization 要继续沿用。 + +## 5. Tauri Command 层 + +| 文件 | 当前职责 | AgentUI 相关能力 | +| --- | --- | --- | +| `src-tauri/src/commands/aster_agent_cmd/command_api/runtime_api.rs` | runtime Tauri command 主入口 | submit、interrupt、compact、resume、get session、thread read、file checkpoint、evidence/review export | +| `src-tauri/src/commands/aster_agent_cmd/runtime_turn.rs` | turn 执行主链 | 提前发 `runtime_status`、执行 provider/runtime/tool/hook/memory/artifact/auto compact | +| `src-tauri/src/commands/aster_agent_cmd/session_runtime.rs` | session create/list/update/recent runtime context | sidebar/session summary 和 tab 管理基础 | +| `src-tauri/src/commands/aster_agent_cmd/subagent_runtime.rs` | subagent runtime | team/capsule 子代理状态基础 | +| `src-tauri/src/commands/aster_agent_cmd/action_runtime.rs` | action required / response | HITL、权限确认、用户输入请求 | +| `src-tauri/src/commands/aster_agent_cmd/tool_runtime/*` | tool bridge | browser、workspace、service skill、media、mcp、subagent、site | +| `src-tauri/src/commands/aster_agent_cmd/dto.rs` | 后端 DTO | `AgentRuntimeSessionDetail`、history cursor、thread_read 投影 | + +`agent_runtime_get_session` 当前关键行为： + +- 默认 `RUNTIME_SESSION_OPEN_HISTORY_LIMIT = 40`。 +- 最大 `RUNTIME_SESSION_MAX_HISTORY_LIMIT = 2000`。 +- `historyLimit = 0` 表示不裁剪。 +- 返回 detail、queue snapshots、interrupt marker、thread_read、history cursor。 +- tracing 已拆出 `detail_ms`、`hooks_ms`、`queue_snapshots_ms`、`projection_ms`、`dto_ms`。 + +这意味着旧会话恢复慢的排查不应只看前端感受，必须把前端 `runtimeGetSession.slow` 与后端 `agent_runtime_get_session` 分段日志对齐。 + +## 6. Rust Agent Crate 层 + +| 文件 | 当前职责 | AgentUI 意义 | +| --- | --- | --- | +| `src-tauri/crates/agent/src/runtime_queue.rs` | turn queue、resume、queue event | queue/capsule/task layer 的事实源 | +| `src-tauri/crates/agent/src/session_store.rs` | session detail、messages、turns、items、todo、runtime overlay、subagent context | old session restore、history pagination、thread_read 的底层事实源 | +| `src-tauri/crates/agent/src/session_query.rs` | parent/child/cascade 查询 | team/subagent 视图 | +| `src-tauri/crates/agent/src/event_converter.rs` | Aster event -> Lime runtime event | text/thinking/tool/artifact/status 的协议生成点 | +| `src-tauri/crates/agent/src/protocol_projection.rs` | current projection 入口 | 新 UI 应优先跟 current projection 对齐 | +| `src-tauri/crates/agent/src/runtime_projection_snapshot.rs` | runtime summary snapshot | tab/capsule/sidebar summary 可复用 | +| `src-tauri/crates/agent/src/session_execution_runtime.rs` | execution runtime/cost/limit/routing | provider/model/cost/context usage UI | +| `src-tauri/crates/agent/src/queued_turn.rs` | queue snapshot 数据结构 | queue panel、task capsule | +| `src-tauri/crates/agent/src/tool_io_offload.rs` | tool 大输出 offload | tool UI 大输出不卡顿的后端基础 | +| `src-tauri/crates/agent/src/write_artifact_events.rs` | artifact write event | artifact snapshot / workbench 联动 | + +## 7. Service 层 + +| 文件 | 当前职责 | AgentUI 使用方式 | +| --- | --- | --- | +| `src-tauri/src/services/agent_timeline_service.rs` | `AgentTimelineRecorder` 持久化 runtime event -> turn/item | Process layer 与 Evidence layer 的过程事实 | +| `src-tauri/src/services/artifact_document_service.rs` | ArtifactDocument 持久化、版本、`.lime/artifacts` | Artifact layer 主事实源 | +| `src-tauri/src/services/artifact_ops_service.rs` | incremental artifact envelope | artifact.begin/meta/source/block/complete/fail | +| `src-tauri/src/services/runtime_evidence_pack_service.rs` | evidence pack 导出 | Evidence layer 输出 `summary.md/runtime.json/timeline.json/artifacts.json` | +| `src-tauri/src/services/runtime_review_decision_service.rs` | review decision 模板与保存 | Human review 闭环 | +| `src-tauri/src/services/runtime_replay_case_service.rs` | replay case | 失败复现与验证 | +| `src-tauri/src/services/runtime_handoff_artifact_service.rs` | handoff bundle | 跨 agent/人工交接 | +| `src-tauri/src/services/runtime_file_checkpoint_service.rs` | file checkpoint / diff | 代码任务可审查变更 | +| `src-tauri/src/services/thread_reliability_projection_service.rs` | reliability projection | thread_read、incident、pending request | + +## 8. 存储与文件事实源 + +| 事实源 | 内容 | UI 映射 | +| --- | --- | --- | +| SQLite session tables | sessions、messages、metadata | sidebar、tabs、Conversation layer | +| SQLite timeline tables | turns、items、tool、artifact item | Process layer、Evidence layer | +| `.lime/artifacts` | artifact documents、版本、snapshot | Artifact layer | +| `.lime/harness/sessions//evidence` | evidence pack | Evidence panel | +| `.lime/harness/sessions//review` | review decision | Review dialog/panel | +| runtime in-memory state | running turns、interrupt marker、queue | Runtime strip、capsule、queue | + +## 9. 测试入口 + +| 范围 | 现有测试入口 | +| --- | --- | +| MessageList | `src/components/agent/chat/components/MessageList.test.tsx` | +| StreamingRenderer | `src/components/agent/chat/components/StreamingRenderer.test.tsx` | +| Stream event binding | `src/components/agent/chat/hooks/agentStreamTurnEventBinding.test.ts` | +| Stream runtime reducer | `src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts` | +| Inputbar | `InputbarCore.test.tsx`、`InputbarRuntimeStatusLine.test.tsx`、`QueuedTurnsPanel.test.tsx` | +| Timeline | `AgentThreadTimeline.test.tsx`、`AgentThreadTimelineArtifactCard.test.tsx` | +| Task center / team | `TaskCenterTabStrip.test.tsx`、`TeamWorkspaceBoard*.test.tsx` | +| Harness | `HarnessStatusPanel.test.tsx` | +| Backend history/session | `src-tauri/crates/agent/src/session_store.rs` unit tests | +| Command contract | `npm run test:contracts` | +| GUI smoke | `npm run verify:gui-smoke` | + +## 10. 后续拆分边界 + +建议优先拆这些边界，避免继续扩大主入口： + +1. `useAgentSession` 拆出 `useSessionHistoryWindow`、`useTopicTabs`、`useRuntimeProjection`。 +2. `agentStreamRuntimeHandler` 抽出纯 reducer，测试 first event、first text、thinking/final 去重、tool/action/artifact 合并。 +3. `MessageList` 把 timeline projection 和 rendering 分离，历史 timeline 默认 idle/worker。 +4. `AgentChatWorkspace` 把 workbench/harness/team 的状态接线移到各自 controller hook。 +5. `InputbarCore` 把 textarea、attachments、slash/hint、queue/steer 拆为 composer state machine。 diff --git a/docs/roadmap/agentui/lime-agentui-event-flow.md b/docs/roadmap/agentui/lime-agentui-event-flow.md new file mode 100644 index 000000000..031c01557 --- /dev/null +++ b/docs/roadmap/agentui/lime-agentui-event-flow.md @@ -0,0 +1,238 @@ +# Lime AgentUI 事件流程 + +> 状态：流程设计 +> 更新时间：2026-04-30 +> 目标：固定 AgentUI 的关键运行流，避免 UI 层用局部状态猜测 runtime 行为。 + +## 1. 事件事实源 + +AgentUI 的运行事实来自五类投影： + +| 投影 | 入口 | UI 消费 | +| --- | --- | --- | +| AgentEvent stream | `src/lib/api/agentProtocol.ts` | 流式文本、thinking、tool、artifact、runtime status、queue、done | +| Session detail | `getAgentRuntimeSession` / `agent_runtime_get_session` | 打开旧会话、历史窗口、thread items、queued turns、thread read | +| Thread read | `getAgentRuntimeThreadRead` / session detail 内联 | pending requests、last outcome、incident、interrupt、queue | +| Timeline | `AgentTimelineRecorder` 持久化的 turn/item | Process layer、Evidence layer | +| Artifact/Evidence files | `.lime/artifacts`、`.lime/harness` | Workbench、Harness、Review、Replay | + +事件流原则： + +1. 新 UI 只新增投影或 selector，不新增第二条 runtime event。 +2. 对话正文只消费 `text` 和最终答复相关 preview。 +3. thinking/tool/status/action/artifact 必须分型，不拼接进同一个 Markdown 字符串。 +4. 历史恢复消费 session detail，流式执行消费 AgentEvent，两者最终进入同一前端 state shape。 + +## 2. 发送消息流程 + +```mermaid +flowchart TB + Start([用户点击发送]) --> Validate[Inputbar 校验 prompt / attachments / model / access mode] + Validate --> Optimistic[前端创建乐观 user message 和 assistant placeholder] + Optimistic --> Bind[注册 AgentEvent listener] + Bind --> Submit[submitAgentRuntimeTurn] + Submit --> Command[agent_runtime_submit_turn] + Command --> Queue{runtime queue 是否忙} + Queue -->|空闲| StartTurn[启动 turn] + Queue -->|忙碌且允许排队| Enqueue[写入 queued turn 并发 queue_added] + Queue -->|忙碌且 steer| Steer[注入当前 turn 或 pending input] + + StartTurn --> EarlyStatus[尽早发 runtime_status preparing/routing] + EarlyStatus --> Runtime[Aster Agent Runtime] + Runtime --> Events[turn_started / thinking_delta / text_delta / tool / artifact / action_required] + Events --> FrontendReducer[handleTurnStreamEvent 更新前端状态] + FrontendReducer --> UI[Conversation / Process / Task / Artifact UI] + Events --> Recorder[AgentTimelineRecorder] + Recorder --> TimelineDb[(timeline DB)] + Runtime --> Done[done / final_done] + Done --> Reconcile[reconcileFinalContentParts 防重复吐字] + Reconcile --> Complete[停止 sending，收起 transient status] +``` + +关键约束： + +- listener 必须先于 submit 注册，避免首事件丢失。 +- `runtime_status` 应早于首个 `text_delta`，用于首字前反馈。 +- `text_delta` 只进入 text part；`thinking_delta` 只进入 thinking part。 +- `final_done` 只能 reconcile，不应把完整 final text 再追加一遍。 +- tool/artifact/action 事件进入 process/task/artifact 投影，不污染最终正文。 + +## 3. 打开旧会话流程 + +```mermaid +flowchart TB + Click([用户打开历史会话]) --> Tab[创建或激活 topic/tab] + Tab --> Snapshot{有 cached snapshot?} + Snapshot -->|有| ApplySnapshot[立即应用标题/最近消息/运行态快照] + Snapshot -->|无| Shell[渲染 workspace shell + skeleton] + ApplySnapshot --> Shell + Shell --> Detail[getSession historyLimit=40] + Detail --> Command[agent_runtime_get_session] + Command --> Store[SessionStore tail window + turns/items] + Store --> ThreadRead[构建 thread_read / queue snapshots / history cursor] + ThreadRead --> HydrateMessages[优先 hydrate messages] + HydrateMessages --> RenderLight[MessageList 轻量渲染最近消息] + RenderLight --> DeferTimeline[延迟 timeline/tool/artifact detail] + DeferTimeline --> Idle[空闲时补过程层] + RenderLight --> HistoryCursor{history_truncated?} + HistoryCursor -->|是| LoadMore[显示加载更早历史入口] + HistoryCursor -->|否| Done[完成恢复] +``` + +关键约束： + +- 不等待全量历史再挂载 UI。 +- 不在打开旧会话时同时触发无 `historyLimit` 的 `getSession`。 +- sidebar list 刷新应低优先级，不能抢 session detail 主链。 +- 非活跃 tab 只保留 snapshot，不构建完整 MessageList/timeline。 +- full history 只能分页，不回退到 `historyLimit: 0` 的默认路径。 + +## 4. Queue / Steer 流程 + +```mermaid +flowchart TB + Input([当前 turn 运行中用户继续输入]) --> Mode{用户选择模式} + Mode -->|Queue| QueueSubmit[submitTurn queueIfBusy=true] + Mode -->|Steer| SteerSubmit[submitTurn steer/current-turn intent] + QueueSubmit --> QueueAdded[queue_added] + QueueAdded --> TaskCapsule[Task capsule 显示排队数量和下一条摘要] + TaskCapsule --> QueuePanel[QueuedTurnsPanel 支持 promote/remove/edit] + SteerSubmit --> PendingPreview[pending steer preview] + PendingPreview --> Runtime[当前 runtime 消费 steer] + Runtime --> Status[queue_started / queue_removed / runtime_status] + Status --> UI[Task layer + Process layer 更新] +``` + +UI 规则： + +- Queue 表示“本轮结束后执行”，Steer 表示“影响当前执行”。 +- 两者必须有不同视觉和文案。 +- queue 入口不应该触发新 tab 全量恢复。 +- queue 操作应以 `queued_turn_id` 为稳定键，避免列表重排导致操作错位。 + +## 5. Action Required / Human-in-the-loop 流程 + +```mermaid +flowchart TB + Runtime[Agent Runtime] --> ActionEvent[action_required] + ActionEvent --> ThreadItem[approval_request / request_user_input item] + ActionEvent --> Frontend[前端 pendingActions] + Frontend --> Promote{是否提升到输入区 A2UI?} + Promote -->|是| InputA2UI[Inputbar / bottom pending form] + Promote -->|否| InlineCard[消息内 action card] + InputA2UI --> UserDecision[用户批准/拒绝/填写] + InlineCard --> UserDecision + UserDecision --> Respond[respondAgentRuntimeAction] + Respond --> Command[agent_runtime_respond_action] + Command --> RuntimeResume[Runtime 继续执行] + RuntimeResume --> Events[后续 text/tool/artifact/done] +``` + +UI 规则： + +- `needs_input` 和 `plan_ready` 是 task capsule 的高优先级状态。 +- 高风险动作必须显示明确操作对象、影响范围和确认按钮。 +- action card 完成后应收起为历史摘要，不继续占据首屏。 +- replay pending request 应从 `thread_read.pending_requests` 进入，不从文本猜测。 + +## 6. Artifact 流程 + +```mermaid +flowchart TB + Runtime[Agent Runtime / Tool] --> ArtifactEvent[artifact_snapshot 或 write_artifact event] + ArtifactEvent --> Timeline[AgentTimelineRecorder 写入 file_artifact item] + ArtifactEvent --> ArtifactService[ArtifactDocument / ArtifactOps] + ArtifactService --> Files[.lime/artifacts] + ArtifactEvent --> Frontend[前端 artifact signal] + Frontend --> MessagePreview[消息内简短 preview/card] + Frontend --> Workbench[Canvas / Artifact Workbench 自动选择或提示] + Workbench --> UserEdit[用户编辑 / diff / 导出] +``` + +UI 规则： + +- artifact 主体不长期留在正文。 +- 消息内 card 只做摘要和打开入口。 +- artifact 版本、路径、metadata 应来自 artifact service，不由前端拼路径。 +- `artifact_snapshot` 同时属于 process evidence 和 artifact delivery。 + +## 7. Evidence / Review 流程 + +```mermaid +flowchart TB + User[用户或系统触发导出] --> Export[agent_runtime_export_evidence_pack] + Export --> Detail[读取 session detail + thread_read] + Detail --> Timeline[读取 timeline] + Detail --> Artifacts[读取 artifact metadata] + Timeline --> EvidenceService[runtime_evidence_pack_service] + Artifacts --> EvidenceService + EvidenceService --> Files[.lime/harness/sessions//evidence] + Files --> HarnessPanel[HarnessStatusPanel 展示] + HarnessPanel --> Review[导出 review decision template] + Review --> ReviewService[runtime_review_decision_service] + ReviewService --> ReviewFiles[.lime/harness/sessions//review] +``` + +UI 规则： + +- evidence/review 是证据层，不应阻塞聊天流式输出。 +- 导出动作应显示后台任务状态，完成后给 capsule/harness 入口。 +- evidence 的 summary、timeline、artifact、verification 必须同源，不允许前端伪造通过状态。 + +## 8. 慢点排查流程 + +### 8.1 旧会话恢复慢 + +```mermaid +flowchart LR + UserClick[click old session] --> UIStart[frontend switchTopic start] + UIStart --> GetSession[runtimeGetSession.start] + GetSession --> BackendLog[agent_runtime_get_session total/detail/projection/dto] + BackendLog --> FrontendHydrate[finalize detail / hydrate messages] + FrontendHydrate --> MessageCompute[MessageList timeline/group memo] + MessageCompute --> Paint[first stable paint] +``` + +需要同时记录： + +- click -> shell rendered。 +- click -> `runtimeGetSession.start`。 +- `runtimeGetSession.start` -> success。 +- 后端 `detail_ms/projection_ms/dto_ms`。 +- detail success -> messages rendered。 +- messages rendered -> historical timeline idle completed。 + +### 8.2 首字慢 + +```mermaid +flowchart LR + Send[send click] --> Bind[event listener bound] + Bind --> Submit[submit invoke start] + Submit --> Accepted[submit accepted] + Accepted --> FirstEvent[first AgentEvent] + FirstEvent --> FirstStatus[first runtime_status] + FirstStatus --> FirstText[first text_delta] + FirstText --> FirstPaint[first text paint] +``` + +需要同时记录： + +- listener bound 时间。 +- submit invoke 耗时。 +- first event 时间。 +- first runtime status 时间。 +- first text delta 时间。 +- delta flush queue depth。 +- oldest unrendered delta age。 +- catch-up mode transition。 + +### 8.3 CPU / 内存飙高 + +优先检查： + +1. 是否多个 tab 同时全量恢复。 +2. 是否非活跃会话仍在构建 `MessageList` 和 timeline。 +3. 是否 `getSession(historyLimit: 0)` 或无 limit 请求被触发。 +4. 是否 sidebar list 与 session detail 抢同一 invoke 通道。 +5. 是否 streaming text 逐字动画在 backlog 很大时仍慢速推进。 +6. 是否 tool output / artifact preview 被一次性塞入正文渲染。 diff --git a/docs/roadmap/agentui/lime-agentui-implementation-roadmap.md b/docs/roadmap/agentui/lime-agentui-implementation-roadmap.md new file mode 100644 index 000000000..9c0c18d5e --- /dev/null +++ b/docs/roadmap/agentui/lime-agentui-implementation-roadmap.md @@ -0,0 +1,303 @@ +# Lime AgentUI 实施路线图 + +> 状态：实施路线图 +> 更新时间：2026-04-30 +> 目标：把 AgentUI 架构落成可分批交付的工程任务，优先解决旧会话恢复慢、首字慢、tab 卡顿、流式重复吐字和过程信息噪声。 + +## 1. 阶段目标 + +| 阶段 | 目标 | 用户体感 | +| --- | --- | --- | +| P0 | 稳住当前对话主链和性能热点 | 旧会话能快速显示，首字前有可信反馈，流式不重复吐字 | +| P1 | 建立 task capsule 与 tab 管理 | 多会话、多后台任务不拖慢主页面 | +| P2 | 强化 artifact/workbench/evidence 分层 | 最终产物和证据离开正文，进入可编辑/可审计工作台 | +| P3 | 抽象 AgentUI 子系统 | `AgentChatWorkspace` 从巨石入口变成稳定 shell | + +## 2. P0：体验止血 + +### 2.1 旧会话渐进恢复 + +目标： + +- 点击历史会话后立即显示 shell、tab、标题和 skeleton。 +- `getSession(historyLimit=40)` 返回后先 hydrate messages。 +- timeline/tool/artifact detail 延迟到 idle 或用户展开。 +- 禁止打开旧会话时触发无 limit detail。 + +代码入口： + +- `src/components/agent/chat/hooks/useAgentSession.ts` +- `src/components/agent/chat/components/MessageList.tsx` +- `src/lib/api/agentRuntime/sessionClient.ts` +- `src-tauri/src/commands/aster_agent_cmd/command_api/runtime_api.rs` + +验收： + +- 旧会话 click -> shell paint 不依赖 getSession 完成。 +- detail 请求带 `historyLimit`。 +- MessageList 首屏不等待完整 timeline。 +- Playwright E2E 能打开两个历史会话并来回切换。 + +### 2.2 首字慢分段日志 + +目标： + +- 发送后立刻显示 submitted/routing/preparing。 +- 记录 listener bound、submit invoke、first event、first status、first text delta、first paint。 +- 后端记录 submit accepted、runtime start、provider request start、first provider event。 + +代码入口： + +- `agentStreamSubmitExecution.ts` +- `agentStreamTurnEventBinding.ts` +- `agentStreamRuntimeHandler.ts` +- `runtime_turn.rs` +- `event_converter.rs` + +验收： + +- 首字慢时日志能判断慢在 invoke、queue、provider、event bridge、前端 flush 还是渲染。 +- Playwright E2E 发送真实 prompt 后能采集首字链路。 +- 首字前 UI 不再表现为鼠标 loading + 页面无反馈。 + +### 2.3 流式重复吐字防线 + +目标： + +- `thinking_delta`、`text_delta`、`final_done` 严格分型。 +- `final_done` 只 reconcile，不二次 append 完整正文。 +- ``、工具日志、过程 status 不进入最终 Markdown 正文。 + +代码入口： + +- `agentStreamRuntimeHandler.ts` +- `StreamingRenderer.tsx` +- `processDisplayText.ts` +- `StreamingRenderer.test.tsx` +- `agentStreamRuntimeHandler.test.ts` + +验收： + +- 深度思考模型不会把 thinking 和 final answer 混合重复显示。 +- 同一段最终答复不会出现“先流式一次，完成后又整段追加一次”。 +- 历史恢复不会把已完成 thinking 当正文重放。 + +### 2.4 Streaming backlog catch-up + +目标： + +- 大量 delta 到达时自动进入 catch-up，避免逐字动画拖慢。 +- 指标记录 queue depth、oldest unrendered age、flush count。 +- MessageList / Markdown 轻量模式对历史和 backlog 区分处理。 + +代码入口： + +- `StreamingRenderer.tsx` +- `agentStreamRuntimeHandler.ts` +- `MarkdownRenderer.tsx` +- `MessageList.tsx` + +验收： + +- 长回答中途不会出现 UI 越吐越慢。 +- CPU 不因逐字动画长期满载。 +- catch-up 触发和恢复可从日志看到。 + +## 3. P1：Tab 与 Task Capsule + +### 3.1 浏览器式 tab 管理 + +目标： + +- 新建对话是新增 tab，不跳出当前界面。 +- 打开历史会话是新增或激活 tab，不导致已有 tab 全量重渲染。 +- 非活跃 tab 只保留 snapshot。 +- 关闭 tab 释放重型渲染对象。 + +代码入口： + +- `AgentChatWorkspace.tsx` +- `useAgentSession.ts` +- `TaskCenterTabStrip.tsx` +- sidebar/topic 相关组件 + +验收： + +- 连续打开多个历史会话不再 CPU/内存飙高。 +- 可同时保留新建对话和多个旧会话。 +- 切回旧 tab 先显示 snapshot，再后台刷新。 + +### 3.2 Task capsule strip + +目标： + +- running、queued、needs_input、plan_ready、failed、team/subagent 统一为胶囊。 +- 普通 running 低调展示；needs_input/plan_ready/failed 抢注意力。 +- 点击胶囊打开 task center 或 process drawer，不跳离上下文。 + +代码入口： + +- `TaskCenterTabStrip.tsx` +- `QueuedTurnsPanel.tsx` +- `InputbarRuntimeStatusLine.tsx` +- `AgentRuntimeStrip.tsx` +- `thread_read` projection + +验收： + +- 当前 turn 状态不再只靠按钮 spinner。 +- pending action 可从胶囊快速定位。 +- queue 和 steer 的视觉、文案和行为区分清楚。 + +### 3.3 Sidebar list 降优先级 + +目标： + +- 点击历史会话时 session detail 优先。 +- recent list / archived list 刷新低优先级、可取消、可 debounce。 +- list summary 不携带重型 detail。 + +代码入口： + +- sidebar refresh 逻辑 +- `sessionClient.ts` +- `session_runtime.rs` + +验收： + +- 打开历史会话时不因 sidebar list 刷新阻塞 detail。 +- invoke 通道没有 list 与 getSession 串行抢占。 + +## 4. P2：Artifact / Evidence 分层 + +### 4.1 Artifact 从正文迁移到 Workbench + +目标： + +- `artifact_snapshot` 在正文只显示摘要卡。 +- Workbench 自动选择最新关键 artifact 或显示建议。 +- 大 artifact 按 preview 加载，不塞入 message。 + +代码入口： + +- `StreamingRenderer.tsx` +- `AgentThreadTimelineArtifactCard.tsx` +- `useWorkspaceArtifactPreviewActions.ts` +- artifact services + +验收： + +- 长文档/代码/报告不会让消息列表卡顿。 +- artifact 能从 timeline、message card、workbench 三处指向同一 id/path。 + +### 4.2 Evidence / Review 后台化 + +目标： + +- evidence export 显示为后台任务。 +- 完成后进入 harness panel，不阻塞当前聊天。 +- review decision 和 replay 与 evidence pack 同源。 + +代码入口： + +- `HarnessStatusPanel.tsx` +- `runtime_evidence_pack_service.rs` +- `runtime_review_decision_service.rs` +- `runtime_replay_case_service.rs` + +验收： + +- 导出期间仍可继续查看/发送消息。 +- evidence/review 路径和状态来自后端返回，不由前端猜。 + +## 5. P3：结构收敛 + +### 5.1 拆分 AgentChatWorkspace + +目标结构： + +```text +AgentChatWorkspace + AgentWorkspaceShell + SessionChrome + ConversationPane + WorkbenchPane + ProcessDrawer + TaskCapsuleStrip +``` + +拆分原则： + +- 每次只拆一个职责边界。 +- 不改变用户行为时先补 snapshot/interaction 测试。 +- 不把视觉重构和协议重构混在一刀。 + +### 5.2 拆分 useAgentSession + +建议目标： + +```text +useAgentSession + useTopicTabs + useSessionHistoryWindow + useRuntimeStream + useRuntimeProjection + useQueuedTurns + usePendingActions +``` + +收益： + +- 性能日志能按子系统归因。 +- 单元测试更容易覆盖 stream / history / queue。 +- 后续 worker 化 timeline 不需要改整个 hook。 + +### 5.3 Worker / Virtualization + +目标： + +- timeline 构建可移到 worker 或 idle task。 +- 历史 MessageList 可虚拟化。 +- 大 tool/artifact 详情按需加载。 + +触发条件： + +- 旧会话 messages > 200。 +- threadItems > 500。 +- 单条 assistant content > 24k。 +- tool output preview > 64k。 + +## 6. 验证矩阵 + +| 改动类型 | 最低验证 | +| --- | --- | +| 文档变更 | markdown 格式检查、`git status --short` | +| 前端 UI/Hook | 受影响 `*.test.tsx` / `*.test.ts`，默认 `npm run verify:local` | +| 流式事件逻辑 | `agentStreamRuntimeHandler.test.ts`、`agentStreamTurnEventBinding.test.ts` | +| Tauri command / bridge | `npm run verify:local` + `npm run test:contracts` | +| GUI 主路径 | `npm run verify:local` + `npm run verify:gui-smoke` | +| 真实交互 | Playwright E2E：新建对话、打开两个历史会话、发送消息、queue/steer、pending action | + +## 7. E2E 场景清单 + +优先补这些 Playwright 场景： + +1. 新建对话：点击新任务后新增 tab，输入区立即可用。 +2. 打开旧会话：历史消息快速显示，输入区吸顶/吸底行为符合目标设计。 +3. 多历史会话：连续打开两个旧会话，二者都能切换，不卡死。 +4. 发送消息：首字前出现 runtime status，首字后流式稳定。 +5. 深度思考模型：thinking 折叠，不重复吐字，不污染 final answer。 +6. Queue：当前 turn 运行中发送下一条，进入 queue capsule。 +7. Steer：当前 turn 运行中修正当前任务，显示 pending steer。 +8. Artifact：生成 artifact 后正文显示卡片，workbench 可打开。 +9. Evidence：导出 evidence 后 harness panel 能打开文件。 + +## 8. 完成定义 + +AgentUI 下一阶段不能用“组件做出来了”作为完成标准。完成标准应同时满足： + +1. 当前用户主路径更快：旧会话恢复、首字、tab 切换有日志和 E2E 证明。 +2. UI 分层更清楚：正文、过程、任务、产物、证据不混在一个 Markdown 流里。 +3. 后端事实源更稳定：summary、detail、timeline、artifact、evidence 各自有清晰接口。 +4. 代码边界更小：新增能力不继续扩大 `AgentChatWorkspace` 和 `useAgentSession` 的职责。 +5. 验证可重复：单元测试、GUI smoke、Playwright 覆盖核心流程。 diff --git a/docs/roadmap/agentui/lime-agentui-sequence-diagrams.md b/docs/roadmap/agentui/lime-agentui-sequence-diagrams.md new file mode 100644 index 000000000..a0a8aa19a --- /dev/null +++ b/docs/roadmap/agentui/lime-agentui-sequence-diagrams.md @@ -0,0 +1,288 @@ +# Lime AgentUI 时序图 + +> 状态：时序设计 +> 更新时间：2026-04-30 +> 目标：为实现、E2E、性能日志和故障排查提供端到端时序基线。 + +## 1. 新建会话并发送消息 + +```mermaid +sequenceDiagram + participant U as User + participant Home as HomeStartSurface / EmptyState + participant Shell as AgentChatWorkspace + participant Hook as useAgentSession + participant Bind as AgentEvent Binding + participant API as threadClient + participant Cmd as agent_runtime_submit_turn + participant Queue as runtime_queue + participant RT as Aster Runtime + participant State as Frontend State + participant UI as MessageList / StreamingRenderer + + U->>Home: 新建对话并输入 prompt + Home->>Shell: create topic/session + Shell->>Hook: handleSend(prompt) + Hook->>State: append user message + assistant placeholder + Hook->>Bind: registerAgentStreamTurnEventBinding(eventName) + Hook->>API: submitAgentRuntimeTurn(request) + API->>Cmd: invoke agent_runtime_submit_turn + Cmd->>Queue: submit_runtime_turn + Queue-->>Cmd: accepted / enqueued + Cmd-->>State: early runtime_status + Queue->>RT: start turn + RT-->>Bind: turn_started + Bind->>State: currentTurnId / threadTurns + RT-->>Bind: thinking_delta + Bind->>State: contentParts.thinking + RT-->>Bind: text_delta + Bind->>State: buffered text flush + State->>UI: render first text + RT-->>Bind: tool_start / tool_end + Bind->>State: threadItems + tool parts + RT-->>Bind: artifact_snapshot + Bind->>State: artifact signal + RT-->>Bind: done / final_done + Bind->>State: reconcileFinalContentParts + State->>UI: completed message +``` + +验收点： + +- `registerAgentStreamTurnEventBinding` 早于 submit invoke。 +- 首个 `runtime_status` 能在首个 `text_delta` 前显示。 +- `final_done` 不导致完整答案重复追加。 + +## 2. 打开旧会话渐进恢复 + +```mermaid +sequenceDiagram + participant U as User + participant Tabs as TopicTabs / Sidebar + participant Hook as useAgentSession + participant Cache as Session Snapshot Cache + participant API as sessionClient + participant Cmd as agent_runtime_get_session + participant Store as SessionStore + participant Projection as ThreadRead Projection + participant ML as MessageList + participant Timeline as AgentThreadTimeline + + U->>Tabs: click old session + Tabs->>Hook: switchTopic(sessionId) + Hook->>Cache: read cached snapshot + alt snapshot exists + Cache-->>Hook: title + last messages + status + Hook->>ML: render cached shell/light messages + else no snapshot + Hook->>ML: render shell + skeleton + end + Hook->>API: getSession(sessionId, historyLimit=40) + API->>Cmd: agent_runtime_get_session + Cmd->>Store: get_runtime_session_detail_with_history_page + Store-->>Cmd: messages tail + turns + items + Cmd->>Projection: build thread_read + queue snapshots + cursor + Projection-->>Cmd: session detail + Cmd-->>API: AsterSessionDetail + API-->>Hook: normalized detail + Hook->>ML: hydrate messages first + ML-->>U: stable recent history visible + Hook->>Timeline: defer historical timeline to idle + Timeline-->>U: process detail becomes available +``` + +验收点： + +- `switchTopic` 后 shell 立即出现。 +- `historyLimit=40` 是默认恢复路径。 +- timeline 的历史构建不阻塞最近消息可读。 +- 非活跃 tab 不触发完整 timeline 渲染。 + +## 3. 运行中 Queue 后续输入 + +```mermaid +sequenceDiagram + participant U as User + participant Input as Inputbar + participant Hook as useAgentSession + participant API as threadClient + participant Cmd as agent_runtime_submit_turn + participant Queue as runtime_queue + participant State as Frontend State + participant Task as Task Capsule / QueuedTurnsPanel + + U->>Input: 当前 turn 运行中输入 follow-up + Input->>Hook: send mode = queue + Hook->>API: submitAgentRuntimeTurn(queueIfBusy=true) + API->>Cmd: agent_runtime_submit_turn + Cmd->>Queue: enqueue turn + Queue-->>State: queue_added + State->>Task: show queued count + preview + U->>Task: promote/remove queued turn + Task->>API: promoteQueuedTurn / removeQueuedTurn + API->>Cmd: queue mutation command + Cmd->>Queue: mutate queue + Queue-->>State: queue_removed / queue_started + State->>Task: update capsule +``` + +验收点： + +- queue 不创建假消息正文。 +- queue 操作不会触发旧会话全量恢复。 +- 运行中任务和排队任务视觉不同。 + +## 4. 运行中 Steer 当前任务 + +```mermaid +sequenceDiagram + participant U as User + participant Input as Inputbar + participant Hook as useAgentSession + participant API as threadClient + participant Cmd as agent_runtime_submit_turn + participant RT as Aster Runtime + participant State as Frontend State + participant UI as Process / Capsule + + U->>Input: 当前 turn 运行中输入修正 + Input->>Hook: send mode = steer + Hook->>State: show pending steer preview + Hook->>API: submitAgentRuntimeTurn(steer intent) + API->>Cmd: agent_runtime_submit_turn + Cmd->>RT: deliver steer to running turn + RT-->>State: runtime_status steer_received + State->>UI: update current task status + RT-->>State: subsequent text/tool/action events +``` + +验收点： + +- steer 文案明确“影响当前任务”。 +- pending steer 可取消或至少可见。 +- steer 后状态进入当前 turn，而不是排队 turn。 + +## 5. Action Required / 权限确认 + +```mermaid +sequenceDiagram + participant RT as Aster Runtime + participant Bind as AgentEvent Binding + participant State as Frontend State + participant UI as Action Card / Input A2UI + participant User as User + participant API as threadClient + participant Cmd as agent_runtime_respond_action + + RT-->>Bind: action_required(requestId, actionType, scope, data) + Bind->>State: pendingActions + threadItems.approval_request + State->>UI: show needs input / approval CTA + User->>UI: approve / reject / fill form + UI->>API: respondAgentRuntimeAction + API->>Cmd: agent_runtime_respond_action + Cmd-->>RT: resolve request + RT-->>Bind: runtime_status / text_delta / tool / done + Bind->>State: mark action resolved + State->>UI: collapse card to summary +``` + +验收点： + +- `needs_input` 在 task capsule 中可见。 +- action 完成后不能继续以 pending 状态吸顶。 +- 高风险动作必须保留审批结果摘要。 + +## 6. Artifact Snapshot 到 Workbench + +```mermaid +sequenceDiagram + participant RT as Aster Runtime / Tool + participant Bind as AgentEvent Binding + participant Timeline as AgentTimelineRecorder + participant ArtifactSvc as Artifact Services + participant State as Frontend State + participant Msg as Message Preview + participant Workbench as Artifact Workbench + + RT-->>Bind: artifact_snapshot(artifactId, path, metadata) + RT-->>Timeline: record file_artifact item + RT-->>ArtifactSvc: persist document / snapshot + ArtifactSvc-->>State: artifact metadata available + Bind->>State: add artifact signal to current turn + State->>Msg: render compact artifact card + State->>Workbench: select or suggest artifact + Workbench->>ArtifactSvc: read preview / version / diff +``` + +验收点： + +- 聊天正文只显示 artifact 摘要卡。 +- Workbench 是 artifact 主编辑面。 +- timeline 和 evidence 能追到同一个 artifact id/path。 + +## 7. Evidence Pack 与 Review Decision + +```mermaid +sequenceDiagram + participant User as User + participant Harness as HarnessStatusPanel + participant API as export client + participant Cmd as agent_runtime_export_evidence_pack + participant Store as SessionStore + participant Timeline as Timeline DB + participant Evidence as runtime_evidence_pack_service + participant Review as runtime_review_decision_service + participant Files as .lime/harness + + User->>Harness: export evidence + Harness->>API: exportEvidencePack(sessionId) + API->>Cmd: agent_runtime_export_evidence_pack + Cmd->>Store: load session detail + Cmd->>Timeline: load turns/items + Cmd->>Evidence: build summary/runtime/timeline/artifacts + Evidence->>Files: write evidence pack + Files-->>Harness: show evidence files + User->>Harness: export review decision + Harness->>Review: build template from evidence/replay/handoff + Review->>Files: write review decision +``` + +验收点： + +- evidence 导出不阻塞当前流式 turn。 +- review decision 只保存人工审核，不自动批准或自动应用修复。 +- UI 展示的证据路径来自后端返回值。 + +## 8. 多 Tab / 多历史会话 + +```mermaid +sequenceDiagram + participant U as User + participant Tabs as Tab Manager + participant Snapshot as Tab Snapshot Store + participant Hook as useAgentSession + participant API as sessionClient + participant UI as Active Workspace + + U->>Tabs: open session A + Tabs->>Hook: activate A + Hook->>API: getSession(A, historyLimit=40) + API-->>Hook: detail A + Hook->>Snapshot: save A lightweight snapshot + U->>Tabs: open session B + Tabs->>Hook: freeze A render state + Hook->>Snapshot: keep A summary only + Tabs->>Hook: activate B + Hook->>API: getSession(B, historyLimit=40) + API-->>Hook: detail B + U->>Tabs: switch back A + Tabs->>Snapshot: restore A snapshot immediately + Snapshot-->>UI: show A shell + Hook->>API: optional refresh A detail in background +``` + +验收点： + +- 打开第二个历史会话不会让第一个历史会话继续全量渲染。 +- tab 切换先恢复 snapshot，再后台刷新。 +- 关闭 tab 释放 MessageList/timeline 重对象。 diff --git a/docs/roadmap/agentui/lime-agentui-target-architecture.md b/docs/roadmap/agentui/lime-agentui-target-architecture.md new file mode 100644 index 000000000..2e08d28a4 --- /dev/null +++ b/docs/roadmap/agentui/lime-agentui-target-architecture.md @@ -0,0 +1,194 @@ +# Lime AgentUI 目标架构 + +> 状态：目标架构 +> 更新时间：2026-04-30 +> 适用范围：`src/components/agent/chat` 主工作区、`src/lib/api/agentRuntime` 前端协议层、`src-tauri` agent runtime 主链。 + +## 1. 架构原则 + +Lime 的下一代 AgentUI 应是一个桌面工作台，而不是单列聊天页。它应当同时回答： + +1. 用户当前目标是什么。 +2. Agent 正在执行哪一步。 +3. 哪些工具、权限、文件、网页、子代理和队列参与了执行。 +4. 哪些内容是最终产物。 +5. 结果能否被验证、复盘、重放和继续编辑。 + +因此目标 UI 分为五层： + +| 层 | 用户问题 | 主要 UI | 主要事实源 | +| --- | --- | --- | --- | +| 对话层 Conversation | 我和 Agent 说了什么，最终答复是什么 | `MessageList`、`StreamingRenderer`、用户消息、助手正文 | `messages`、`contentParts`、`text_delta` | +| 过程层 Process | Agent 现在在做什么，卡在哪里 | runtime strip、thinking、tool step、timeline | `runtime_status`、`thinking_delta`、`threadItems`、`threadTurns` | +| 任务层 Task | 还有哪些后台任务、排队输入、子代理 | capsule、task center、queue panel、team board | `queuedTurns`、`childSubagentSessions`、`thread_read` | +| 产物层 Artifact | 最终交付物在哪里继续编辑 | canvas、artifact preview、workbench | `artifact_snapshot`、ArtifactDocument、`.lime/artifacts` | +| 证据层 Evidence | 这次执行是否可靠，如何复盘 | harness panel、evidence pack、review decision、replay | `thread_read`、timeline、evidence/review services | + +## 2. 总体架构图 + +```mermaid +flowchart TB + User[用户] --> Shell[AgentChatWorkspace 工作台壳] + + Shell --> Composer[输入层 Inputbar] + Shell --> Conversation[对话层 MessageList / StreamingRenderer] + Shell --> Process[过程层 RuntimeStrip / Timeline / Tool Step] + Shell --> Task[任务层 Capsule / Queue / Team / Task Center] + Shell --> Artifact[产物层 Canvas / Artifact Workbench] + Shell --> Evidence[证据层 Harness / Evidence / Review] + + Composer --> RuntimeApi[agentRuntime API clients] + RuntimeApi --> Protocol[AgentEvent / Session DTO / ThreadRead] + RuntimeApi --> Tauri[Tauri Commands agent_runtime_*] + + Tauri --> RuntimeQueue[Runtime Queue] + Tauri --> SessionStore[Session Store] + Tauri --> TimelineRecorder[AgentTimelineRecorder] + Tauri --> ArtifactServices[Artifact Services] + Tauri --> EvidenceServices[Evidence / Review Services] + + RuntimeQueue --> Aster[Aster Agent Runtime] + Aster --> Events[AgentEvent stream] + Events --> Protocol + Events --> TimelineRecorder + Events --> ConversationState[前端 messages / contentParts] + Events --> ProcessState[前端 threadItems / turns / status] + Events --> TaskState[前端 queuedTurns / subagents] + + SessionStore --> Db[(SQLite session / timeline)] + TimelineRecorder --> Db + ArtifactServices --> ArtifactFiles[.lime/artifacts] + EvidenceServices --> HarnessFiles[.lime/harness] + + ConversationState --> Conversation + ProcessState --> Process + TaskState --> Task + ArtifactFiles --> Artifact + HarnessFiles --> Evidence +``` + +## 3. UI 布局目标 + +下一阶段建议把 `AgentChatWorkspace` 从“组件巨石”逐步收敛为稳定布局骨架： + +```text +AgentChatWorkspace + AgentWorkspaceShell + SessionChrome + TopicTabs + TaskCapsuleStrip + RuntimeHealthIndicator + AgentMainLayout + ConversationPane + MessageList + StreamingRenderer + Inputbar + WorkbenchPane + ArtifactCanvas + TaskCenter + EvidencePanel + ProcessDrawer + AgentThreadTimeline + ToolDetail + RuntimeDiagnostics +``` + +这不是要求一次性重写，而是给后续拆分一个归宿。当前可以继续复用 `AgentChatWorkspace.tsx`，但新增能力应尽量下沉到更小的 selector、hook、panel，而不是继续把状态和渲染堆到主文件。 + +## 4. 五层 UI 的职责边界 + +### 4.1 对话层 + +对话层只负责高信噪比协作： + +- 用户消息。 +- 助手最终答复。 +- 必要的流式占位。 +- 与最终答复强相关的引用、预览和操作。 + +对话层不应承担： + +- 全量 tool output。 +- 全量 thinking 原文。 +- 后台队列管理。 +- 证据包文件树。 +- artifact 编辑器主体。 + +### 4.2 过程层 + +过程层负责把“Agent 是否活着、正在做什么”变成低噪声状态： + +- 首事件前显示可信阶段：submitted、routing、preparing、waiting provider。 +- 首字前显示 `runtime_status`，不要只给鼠标 loading。 +- thinking 默认折叠，完成后显示摘要。 +- tool 默认压缩为 step，点击进入详情。 +- 历史会话默认延迟渲染 timeline。 + +过程层的关键是分型，不是堆文本。 + +### 4.3 任务层 + +任务层负责多会话、多 turn、多子代理的压缩索引： + +- `running`：低调胶囊，不抢注意力。 +- `queued`：显示队列长度和下一条摘要。 +- `needs_input`：高优先级 CTA。 +- `plan_ready`：高优先级 CTA，进入计划审批。 +- `failed`：可恢复错误胶囊，点击打开诊断。 +- `team/subagent`：显示数量、状态和焦点会话。 + +任务层应学习浏览器 tab 的管理方式：可快速打开、切换、关闭、懒加载、后台冻结，避免多个旧会话同时全量恢复导致 CPU 和内存飙高。 + +### 4.4 产物层 + +产物层承接最终交付： + +- 文档、代码、报告、图片、视频、网页、表单等不应长期塞在聊天正文里。 +- `artifact_snapshot` 应生成 artifact card，并自动同步到 workbench。 +- artifact 的编辑、预览、diff、保存、导出由 workbench 负责。 +- 聊天消息只保留摘要、解释和打开入口。 + +### 4.5 证据层 + +证据层负责证明执行可信： + +- `thread_read` 负责待处理请求、last outcome、incident、queue、interrupt。 +- timeline 负责过程证据。 +- evidence pack 负责导出 `summary.md`、`runtime.json`、`timeline.json`、`artifacts.json`。 +- review decision 负责人工审核闭环。 +- replay 负责复现失败场景。 + +证据层默认不出现在首屏正文里，但必须能从任务、timeline 或 harness panel 快速进入。 + +## 5. 状态所有权 + +| 状态 | 所有权建议 | 当前关键入口 | +| --- | --- | --- | +| 当前会话、topic、messages | `useAgentSession` 继续作为临时聚合层，逐步拆 selector | `src/components/agent/chat/hooks/useAgentSession.ts` | +| 流式事件绑定 | 独立 stream controller | `agentStreamSubmitExecution.ts`、`agentStreamTurnEventBinding.ts` | +| 流式 reducer | 独立 runtime event reducer | `agentStreamRuntimeHandler.ts` | +| 消息渲染投影 | MessageList selector / memo | `MessageList.tsx`、`threadTimelineView.ts`、`messageTurnGrouping.ts` | +| 队列与 capsule | Task state selector | `queuedTurns`、`TaskCenterTabStrip.tsx`、`QueuedTurnsPanel.tsx` | +| artifact 工作台 | Workbench controller | `useWorkspaceArtifactPreviewActions.ts`、artifact services | +| evidence/review | Harness controller | `HarnessStatusPanel.tsx`、runtime evidence/review services | + +## 6. 性能目标 + +| 场景 | 目标体验 | 架构要求 | +| --- | --- | --- | +| 打开旧会话 | 立即显示 shell 和最近消息骨架，随后渐进 hydrate | `getSession(historyLimit: 40)`，timeline 延迟，full history 分页 | +| 首字前等待 | 用户看到明确阶段，而不是应用卡死 | 先发 `runtime_status`，前端显示 runtime strip | +| 大量 text_delta | 流式平滑，catch-up 时不逐字拖慢 | delta buffer、flush 指标、catch-up mode | +| 多 tab / 多历史会话 | 非活跃 tab 不全量渲染 | tab snapshot、懒加载、后台冻结、关闭释放 | +| 大 tool output | 正文不卡，详情可查 | tool output 截断、offload、timeline 详情 | +| evidence/review | 导出不阻塞聊天 | 后台任务 + 完成胶囊 + workbench 入口 | + +## 7. 非目标 + +短期不做这些事： + +- 不新增第二套 event bus。 +- 不把所有 timeline 迁到前端本地推断。 +- 不把 artifact 编辑器塞回 `MessageList`。 +- 不为了视觉新鲜感引入和 Lime 现有设计语言冲突的大面积渐变、半透明主表面或嵌套卡片。 +- 不一次性重写整个 `AgentChatWorkspace`；先用 selector、panel、controller 拆出稳定边界。 diff --git a/docs/roadmap/voice/README.md b/docs/roadmap/voice/README.md new file mode 100644 index 000000000..df1a42377 --- /dev/null +++ b/docs/roadmap/voice/README.md @@ -0,0 +1,96 @@ +# Lime 离线语音模型路线图 + +> 状态：current planning source +> 更新时间：2026-04-30 +> 目标：把截图参考里的“语音模型”能力收敛为 Lime 可实现的离线语音主线：limecore 下发 SenseVoice Small 下载地址、本地 ASR 转写、Fn 按住说话、测试转写与转写历史。 + +## 1. 本路线图回答什么 + +本目录统一回答下面几类问题： + +1. Lime 如何接入 SenseVoice Small 离线 ASR，而不是继续只依赖云端 ASR 凭证。 +2. 离线模型如何通过 limecore 模型清单与对象存储/CDN 按需下载，避免安装包内置大文件。 +3. Fn 按住录音、松开转写的快捷键体验如何落到 macOS 桌面主链。 +4. 语音设置页如何区分语音输入 ASR、语音润色 LLM 与配音/TTS 服务模型。 +5. 新能力如何接回现有 `voice_command_service -> voice_asr_service -> voice-core` 主链，而不是新造第二套语音系统。 + +## 2. 固定结论 + +### 2.1 模型不内置进安装包 + +SenseVoice Small、Silero VAD、Whisper、GGUF 等大模型文件都不进入 App 初始安装包。 + +安装包只包含： + +1. 模型清单读取 +2. 下载与校验能力 +3. 本地模型状态管理 +4. 已安装模型的推理运行时 +5. 缺模型时的明确引导 + +模型必须由用户显式触发下载。后台不能静默拉取几百 MB 的模型文件。 + +### 2.2 P0 只产品化 SenseVoice Small + +P0 只把截图参考中的 `SenseVoice Small` 做成主路径： + +1. 通过 `sherpa-onnx` 运行本地 ASR。 +2. 通过 limecore `voice-model-catalog` 获取对象存储/CDN 下载地址。 +3. 支持中、粤、英、日、韩自动识别。 +4. 默认启用 ITN，把语音识别结果归一成更适合输入的文本。 + +已有 `WhisperLocal` 能力作为现有代码资产保留，但不作为 P0 的产品化下载模型。 + +### 2.3 Fn 是专门输入模式，不是普通快捷键字符串 + +现有 `tauri_plugin_global_shortcut` 适合 `CommandOrControl+Shift+V` 这类组合键。Fn 在 macOS 上属于特殊硬件/系统层按键，不能简单塞进现有 shortcut 字符串解析。 + +P0 固定为： + +1. Apple 键盘上优先提供 Fn 按住说话体验。 +2. 无法捕获 Fn 或权限不足时，降级到现有自定义快捷键。 +3. 第三方键盘默认走自定义快捷键，不承诺 Fn。 + +## 3. 先读顺序 + +1. [sensevoice-small-integration.md](./sensevoice-small-integration.md) +2. [fn-dictation-shortcut.md](./fn-dictation-shortcut.md) + +## 4. 当前 Lime 事实源 + +当前仓库已经有语音主链基础： + +1. `src-tauri/crates/voice-core` 负责录音、转写核心类型、云端 ASR 客户端与可选 Whisper 本地识别。 +2. `src-tauri/crates/services/src/voice_asr_service.rs` 统一管理 ASR 服务。 +3. `src-tauri/crates/services/src/voice_command_service.rs` 封装转写、润色和输出流程。 +4. `src-tauri/src/voice/shortcut.rs` 负责普通全局快捷键 press/release。 +5. `src/components/settings-v2/agent/voice/index.tsx` 已有语音输入、语音处理、语音服务模型三段设置。 +6. `src/components/agent/chat/components/Inputbar/hooks/useInputbarDictation.ts` 已把输入栏语音听写接到现有转写主链。 + +后续实现必须复用这些入口，不新增平行 ASR 流程。 + +## 5. 当前实现进度 + +2026-04-30 已推进到可验证主链： + +1. `SenseVoice Small` 模型目录、安装状态、下载、删除、设为默认已接到设置页“语音模型”卡片。 +2. 模型文件仍按需下载到用户数据目录，不进入 App 安装包；下载地址优先来自 limecore `GET /api/v1/public/tenants/:tenantId/client/voice-model-catalog`，对象存储/CDN 域名由 `server.voiceModelAssetBaseUrl` 管理。 +3. `voice_asr_service` 已把 `SenseVoiceLocal` 接到 `voice_core::SenseVoiceTranscriber`，通过 `sherpa-onnx` 执行 non-streaming 本地转写。 +4. macOS Fn 按住录音已落第一刀：原生监听 Fn press/release，失败时保留普通快捷键 fallback。 +5. 已补“已安装模型后的 WAV 测试转写”入口：设置页可原生选择或手动输入本机 16-bit PCM WAV 路径后，通过 `voice_models_test_transcribe_file` 复用 `voice_asr_service` 真实本地推理链路。 +6. 仍未做 P1 能力：视频抽音、实时录音测试、VAD 分段、转写历史、`trigger_mode` / `fn_shortcut_enabled` 配置分流。 + +## 6. 当前必须避免的误区 + +1. 把 SenseVoice Small 打进安装包，导致桌面包体积膨胀。 +2. 把 Fn 当成普通 shortcut 字符串，导致配置看似成功但实际不可用。 +3. 把语音输入 ASR 和配音/TTS 的“语音服务模型”混在同一个设置语义里。 +4. 因为接入 `sherpa-onnx` 就绕过 `voice_asr_service` 主链。 +5. 缺模型时自动下载，造成不可预期流量和等待。 +6. 在 Tauri 客户端保存对象存储 Access Key 或让客户端生成签名 URL；签名或公开域名必须由 limecore 管理。 + +## 7. 这一步如何服务主线 + +这套文档的直接主线收益是： + +**把 Lime 语音输入从“云端 ASR 凭证 + 普通快捷键”推进到“按需下载本地模型 + Fn 按住说话 + 可测试可删除”的 current 语音主链。** diff --git a/docs/roadmap/voice/fn-dictation-shortcut.md b/docs/roadmap/voice/fn-dictation-shortcut.md new file mode 100644 index 000000000..7fc8f3ac3 --- /dev/null +++ b/docs/roadmap/voice/fn-dictation-shortcut.md @@ -0,0 +1,266 @@ +# Fn 语音输入快捷键方案 + +> 状态：current planning source +> 更新时间：2026-04-30 +> 目标：定义截图参考中的 Fn 按住录音、松开转写体验，并明确它与现有普通全局快捷键的边界。 + +## 1. 固定目标 + +P0 目标： + +1. Apple 键盘上支持 Fn 按住录音。 +2. 松开 Fn 后停止录音并触发转写。 +3. Fn 不可用时，稳定降级到自定义快捷键。 +4. 设置页明确提示 Fn 的系统限制和第三方键盘处理方式。 +5. 现有 `CommandOrControl+Shift+V` 类普通快捷键继续工作。 + +非目标： + +1. 不承诺 Windows 上支持 Fn。 +2. 不承诺所有第三方键盘支持 Fn。 +3. 不把 Fn 写入现有 shortcut 字符串解析器。 +4. 不做系统键盘设置自动修改。 + +## 2. 现有实现边界 + +当前语音快捷键在 `src-tauri/src/voice/shortcut.rs` 中实现： + +1. 使用 `tauri_plugin_global_shortcut`。 +2. 通过字符串解析 `Shortcut`。 +3. `Pressed` 时打开语音浮窗。 +4. `Released` 时发送停止录音事件。 + +这条路径适合普通组合键： + +```text +CommandOrControl+Shift+V +Alt+Space +Control+Option+V +``` + +Fn 不应进入这条路径。原因： + +1. Fn 在 macOS 上经常不是普通 keyboard accelerator。 +2. 系统可能把 Fn/地球仪键绑定到输入法、表情、听写或其他系统行为。 +3. 第三方键盘对 Fn 的上报方式不统一。 +4. 直接注册可能出现“配置成功但无事件”的假成功。 + +## 3. 配置模型 + +语音输入配置新增触发模式： + +```rust +pub enum VoiceShortcutTriggerMode { + FnHold, + CustomShortcut, +} +``` + +配置字段： + +```rust +pub struct VoiceInputConfig { + pub trigger_mode: VoiceShortcutTriggerMode, + pub shortcut: String, + pub fn_shortcut_enabled: bool, +} +``` + +默认策略： + +| 平台 / 设备 | 默认模式 | +| --- | --- | +| macOS + Apple 键盘 | `FnHold` | +| macOS + 未知键盘 | `CustomShortcut` | +| Windows | `CustomShortcut` | +| Linux | `CustomShortcut` | + +`shortcut` 字段继续保留，作为 fallback 与用户自定义快捷键。 + +## 4. macOS Fn 主路径 + +### 4.0 当前实现进度 + +2026-04-30 已落地第一刀： + +1. `src-tauri/src/voice/fn_shortcut.rs` 通过 macOS `NSEventMaskFlagsChanged` 监听 Fn 修饰键状态。 +2. Fn 按下复用 `open_floating_window_with_voice`，Fn 松开复用 `send_voice_stop_event`，不新增第二套语音窗口流程。 +3. 监听随语音输入启用/停用注册和注销；普通语音快捷键仍作为 fallback 保留。 +4. 运行时状态已接入 `get_voice_shortcut_runtime_status` 和设置页 Fn 状态展示。 +5. 尚未实现本方案里的 `trigger_mode` / `fn_shortcut_enabled` 配置分流；当前是“启用语音输入时同时注册普通快捷键与 macOS Fn 监听”的最小可用形态。 + +### 4.1 监听模块 + +新增 macOS-only 模块，例如： + +```text +src-tauri/src/voice/fn_shortcut_macos.rs +``` + +职责： + +1. 启动 Fn 事件监听。 +2. 判断 press / release。 +3. 去抖，避免重复打开语音窗。 +4. 把事件转发给现有语音窗口函数。 +5. 暴露状态给设置页。 + +事件行为： + +| 事件 | 行为 | +| --- | --- | +| Fn pressed | 打开语音浮窗并开始录音 | +| Fn repeated | 忽略 | +| Fn released | 发送停止录音事件 | +| 监听失败 | 标记 Fn 不可用，降级自定义快捷键 | + +### 4.2 权限与系统设置 + +Fn 监听可能需要系统辅助功能、输入监听或键盘相关权限。实现时必须提供可解释状态： + +| 状态 | 含义 | UI 提示 | +| --- | --- | --- | +| `available` | Fn 可监听 | 正常显示 Fn | +| `permission_required` | 缺少权限 | 引导到系统设置授权 | +| `system_reserved` | 系统占用 Fn/地球仪键 | 提示改系统“按地球仪键”为“什么都不做” | +| `unsupported_keyboard` | 当前键盘不支持 | 建议使用自定义快捷键 | +| `unknown_error` | 其他错误 | 保留诊断信息 | + +UI 文案沿用截图含义： + +> Fn 键仅支持 Apple 键盘（内置或妙控键盘），第三方键盘请点击左侧按钮自定义快捷键。建议在「系统设置 -> 键盘 -> 按地球仪键」中设为「什么都不做」以避免干扰。 + +### 4.3 降级策略 + +Fn 不可用时： + +1. 不阻断语音功能。 +2. 自动显示自定义快捷键配置。 +3. 保留现有 `shortcut` 注册路径。 +4. 运行时状态说明“Fn 当前不可用，已使用自定义快捷键”。 + +## 5. 普通快捷键兼容 + +`src-tauri/src/voice/shortcut.rs` 继续负责普通快捷键。 + +需要调整： + +1. 注册时根据 `trigger_mode` 分流。 +2. `FnHold` 模式只启动 macOS Fn 监听。 +3. `CustomShortcut` 模式继续注册 `shortcut` 字符串。 +4. 切换模式时先注销旧监听，再注册新监听。 +5. 如果 Fn 注册失败，尝试注册 `shortcut` fallback。 + +不改动： + +1. 现有 press 打开语音窗逻辑。 +2. 现有 release 停止录音事件。 +3. 翻译快捷键的普通 shortcut 路径。 + +翻译模式 P0 不绑定 Fn。Fn 只作为主语音输入触发键。 + +## 6. 前端设置页 + +设置页顶部新增 Fn 控制区： + +1. Fn 胶囊按钮。 +2. 启用开关。 +3. 当前运行状态。 +4. 系统限制提示。 +5. 自定义快捷键 fallback 入口。 + +状态展示： + +| 状态 | UI | +| --- | --- | +| Fn 可用 | 显示绿色可用状态 | +| 权限缺失 | 显示警告和打开系统设置按钮 | +| 系统占用 | 显示系统设置建议 | +| 第三方键盘 | 显示自定义快捷键入口 | +| fallback 生效 | 显示当前 fallback 快捷键 | + +交互规则： + +1. 用户关闭 Fn 开关后，直接进入 `CustomShortcut`。 +2. 用户打开 Fn 开关但不可用时，不覆盖现有可用 shortcut。 +3. 保存失败时恢复旧设置。 + +## 7. 命令与合同 + +若新增 Tauri command，必须同步四侧： + +1. 前端 `safeInvoke(...)` +2. Rust `tauri::generate_handler!` +3. `agentCommandCatalog` +4. `mockPriorityCommands` / `defaultMocks` + +建议命令： + +| 命令 | 作用 | +| --- | --- | +| `voice_shortcut_get_runtime_status` | 获取 Fn 与普通快捷键运行状态 | +| `voice_shortcut_set_trigger_mode` | 切换 Fn / 自定义快捷键模式 | +| `voice_shortcut_probe_fn` | 探测 Fn 是否可监听 | +| `voice_shortcut_open_keyboard_settings` | 打开系统键盘设置或权限设置 | + +运行时状态需要结构化返回： + +```json +{ + "triggerMode": "fn_hold", + "fnStatus": "permission_required", + "fallbackShortcut": "CommandOrControl+Shift+V", + "registeredShortcut": null, + "message": "需要授权输入监听权限" +} +``` + +## 8. 测试计划 + +Rust： + +1. `FnHold` 模式不调用普通 shortcut parser。 +2. `CustomShortcut` 模式仍注册普通快捷键。 +3. Fn 注册失败时 fallback 到 `shortcut`。 +4. 切换模式时旧监听被注销。 +5. repeated press 不重复打开窗口。 + +前端： + +1. Fn 可用状态展示。 +2. 权限缺失提示。 +3. 第三方键盘 fallback 提示。 +4. 用户关闭 Fn 后显示自定义快捷键。 +5. 保存失败恢复旧配置。 + +GUI： + +```bash +npm run verify:gui-smoke +``` + +合同： + +```bash +npm run test:contracts +``` + +收口： + +```bash +npm run verify:local +``` + +## 9. 验收标准 + +1. Apple 键盘上按住 Fn 能启动录音，松开后转写。 +2. Fn 不可用时，用户能清楚知道原因和 fallback。 +3. 第三方键盘用户仍能使用自定义快捷键完成同一语音输入流程。 +4. 现有输入栏听写、悬浮语音窗、翻译快捷键不被破坏。 +5. 设置页不会展示“Fn 已启用”但运行时实际没有监听的假状态。 + +## 10. 这一步如何服务主线 + +Fn 快捷键的主线收益是： + +**把语音输入从“需要记住组合键”推进到“按住说话”的低摩擦桌面交互，同时保留普通快捷键作为可靠兜底。** diff --git a/docs/roadmap/voice/sensevoice-small-integration.md b/docs/roadmap/voice/sensevoice-small-integration.md new file mode 100644 index 000000000..25c0c343e --- /dev/null +++ b/docs/roadmap/voice/sensevoice-small-integration.md @@ -0,0 +1,346 @@ +# SenseVoice Small 离线 ASR 接入方案 + +> 状态：current planning source +> 更新时间：2026-04-30 +> 目标：定义 SenseVoice Small 在 Lime 中的模型分发、配置、运行时接入、UI 状态和验证口径。 + +## 1. 固定目标 + +P0 交付一个可真实使用的本地语音输入模型： + +1. 用户在设置页看到 `SenseVoice Small` 本地模型卡。 +2. 未安装时，用户手动点击下载。 +3. 下载完成后，模型状态变为已安装。 +4. 用户可选择音频文件、视频文件或实时录音测试转写。 +5. 语音输入主链可把录音交给 SenseVoice Small 本地转写。 +6. 用户可以删除本地模型缓存。 + +非目标： + +1. 不在 P0 同时产品化多个离线 ASR 模型。 +2. 不把模型文件打包进 App。 +3. 不重写现有语音润色、输出和输入栏听写链路。 +4. 不在语音模块内实现独立 LLM 调用栈。 + +## 2. 技术选型 + +SenseVoice Small 通过 `sherpa-onnx` 接入。 + +调研结论： + +1. `sherpa-onnx` 支持本地离线 ASR、VAD、TTS 等语音任务。 +2. SenseVoice Small 有 ONNX INT8 模型包，适合桌面端按需下载。 +3. 模型包主文件为 `model.int8.onnx` 与 `tokens.txt`。 +4. 支持普通话、粤语、英语、日语、韩语。 +5. `use_itn = true` 可做逆文本归一化，更适合输入场景。 +6. `silero_vad.onnx` 可作为可选文件，用于长音频和实时录音分段。 + +参考源： + +1. Context7: `/k2-fsa/sherpa-onnx` +2. `sherpa-onnx` Tauri 示例：`tauri-examples/non-streaming-speech-recognition-from-file` +3. `sherpa` SenseVoice 预训练模型文档 +4. Context7: `/modelscope/modelscope`，用于确认 ModelScope `snapshot_download` / `modelscope download` 文件下载能力。 +5. ModelScope: `https://modelscope.cn/models/iic/SenseVoiceSmall-onnx`，用于确认阿里系 SenseVoice Small ONNX 源。 +6. 阿里云 OSS 文档：`https://help.aliyun.com/zh/oss/developer-reference/`，用于确认公开读对象和服务端签名 URL 的分发方式。 + +## 3. 模型分发 + +### 3.1 limecore / 对象存储模型清单 + +limecore control-plane-svc 提供公开客户端接口： + +```text +GET /api/v1/public/tenants/:tenantId/client/voice-model-catalog +``` + +固定结论： + +1. 模型对象放在 `/Users/coso/Documents/dev/ai/limecloud/limecore` 对应后端管理范围内，由 control-plane-svc 下发目录。 +2. 实际大文件放在对象存储或 CDN 公开域名后；P0 可用阿里云 OSS、Cloudflare R2 或等价静态文件服务，客户端只消费下发 URL。 +3. 对象存储 Access Key、Secret、签名逻辑不得进入 Tauri 客户端。 +4. 若需要私有桶，后续由 limecore 生成短期 presigned GET URL；P0 先支持公开域名 / 自定义域名。 +5. Lime 客户端保留同一组 `model_id`、必需文件与安装目录，不再把 GitHub Release URL 作为产品事实源。 + +首个模型条目： + +```json +{ + "version": 1, + "assetBaseURL": "https://models.example.com", + "items": [ + { + "id": "sensevoice-small-int8-2024-07-17", + "name": "SenseVoice Small INT8", + "provider": "FunAudioLLM / sherpa-onnx", + "description": "本地离线 ASR，支持中文、英文、日文、韩文和粤语；客户端按需下载到用户数据目录，不随安装包内置。", + "version": "2024-07-17", + "languages": ["zh", "en", "ja", "ko", "yue"], + "runtime": "sherpa-onnx", + "bundled": false, + "sizeBytes": 262144000, + "requiredFiles": ["model.int8.onnx", "tokens.txt", "silero_vad.onnx"], + "download": { + "archive": { + "downloadPath": "voice/sensevoice-small-int8-2024-07-17/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2", + "downloadUrl": "https://models.example.com/voice/sensevoice-small-int8-2024-07-17/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2", + "sha256": "" + }, + "vad": { + "modelId": "silero-vad-onnx", + "downloadPath": "voice/silero-vad-onnx/silero_vad.onnx", + "downloadUrl": "https://models.example.com/voice/silero-vad-onnx/silero_vad.onnx" + } + } + } + ] +} +``` + +字段语义： + +| 字段 | 含义 | +| --- | --- | +| `id` | 本地安装记录与配置引用的稳定 ID | +| `name` | UI 展示名 | +| `runtime` | 运行时类型，P0 为 `sherpa-onnx` | +| `download.archive.downloadUrl` | limecore 拼装后的模型归档下载地址 | +| `download.archive.sha256` | 可选压缩包校验值；存在时客户端必须校验 | +| `download.vad.downloadUrl` | Silero VAD 文件下载地址 | +| `bundled` | 是否内置到客户端；P0 固定为 `false` | +| `requiredFiles` | 解压后必须存在的文件 | + +limecore 配置： + +```yaml +server: + voiceModelAssetBaseUrl: "https://models.example.com" +``` + +可用环境变量： + +```bash +SERVER_VOICE_MODEL_ASSET_BASE_URL="https://models.example.com" +VOICE_MODEL_ASSET_BASE_URL="https://models.example.com" +``` + +阿里系来源调研结论： + +1. ModelScope 上存在 `iic/SenseVoiceSmall-onnx`，属于阿里系官方模型源，可通过 `modelscope download`、`snapshot_download` 或 `https://modelscope.cn/api/v1/models/iic/SenseVoiceSmall-onnx/resolve/master/` 下载文件。 +2. 该源文件是 `model_quant.onnx`、`tokens.json`、`config.yaml`，不是 sherpa-onnx release 的 `model.int8.onnx`、`tokens.txt` 归档形态。 +3. P0 为了不扩散安装器复杂度，推荐在 limecore 侧把可运行形态重新打包并托管到阿里云 OSS / CDN 或 R2；若要直连 ModelScope，需新增 direct-file 安装模式，并把 `tokens.json` 转换为 sherpa-onnx 可读的 `tokens.txt`。 + +### 3.2 本地存储 + +模型下载到 Lime 统一数据目录下： + +```text +/models/voice/sensevoice-small-int8-2024-07-17/ +``` + +实现要求： + +1. 目录必须通过统一目录封装获取，禁止硬编码 macOS 或 Windows 路径。 +2. 下载临时文件放在同一数据目录的临时子目录，成功校验后原子移动。 +3. 下载完成先校验压缩包 sha256，再校验解压后的必需文件。 +4. 校验失败标记为 `corrupted`，不能标记为 `installed`。 +5. 删除模型只删除本地模型目录和安装记录，不删除用户的语音配置。 + +### 3.3 安装状态 + +客户端至少维护下面状态： + +| 状态 | 含义 | UI 行为 | +| --- | --- | --- | +| `not_installed` | 未下载 | 显示下载模型 | +| `downloading` | 下载中 | 显示进度，禁用测试 | +| `verifying` | 校验中 | 禁用删除与测试 | +| `installed` | 已安装 | 可设为默认、测试、删除 | +| `corrupted` | 文件缺失或校验失败 | 显示重新下载 | +| `failed` | 下载失败 | 显示重试 | + +## 4. Rust 运行时接入 + +### 4.0 当前实现进度 + +2026-04-30 已落地到真实推理主链： + +1. `voice-core` 新增 `SenseVoiceTranscriber`，通过 `sherpa-onnx` offline recognizer 加载 `model.int8.onnx` 和 `tokens.txt`。 +2. `voice_asr_service` 的 `SenseVoiceLocal` 分支已从占位错误切换为本地转写，输入仍复用现有 `AudioData`。 +3. 录音采样率不是 16kHz 时，会通过 sherpa-onnx `LinearResampler` 转为 16kHz mono float samples。 +4. 模型文件仍只来自设置页显式下载目录，不进入应用安装包；运行时缺文件会提示回到“设置 -> 语音模型”下载。 +5. `voice_models_test_transcribe_file` 已提供已安装模型后的 WAV 文件测试转写入口，读取本机 16-bit PCM WAV 后复用 `AsrService::transcribe`，不新造第二套推理路径。 +6. `voice_models_download` 优先使用前端从 limecore `voice-model-catalog` 取得的对象存储/CDN 下载 URL；无前端目录时才读取 `LIME_VOICE_MODEL_CATALOG_URL` / `LIME_VOICE_MODEL_ASSET_BASE_URL` 这类运行时配置。 +7. P0 仍是 non-streaming decode；VAD 文件随模型状态校验保留，但本轮推理路径未做分段 VAD。 + +### 4.1 配置类型 + +配置层新增本地 ASR provider： + +```rust +pub enum AsrProviderType { + WhisperLocal, + SenseVoiceLocal, + Xunfei, + Baidu, + OpenAI, +} +``` + +新增 SenseVoice 配置： + +```rust +pub struct SenseVoiceLocalConfig { + pub model_id: String, + pub model_dir: Option, + pub use_itn: bool, + pub num_threads: u16, + pub vad_model_id: Option, +} +``` + +默认值： + +1. `model_id = "sensevoice-small-int8-2024-07-17"` +2. 识别语言沿用 `AsrCredentialEntry.language`，设为默认模型时写入 `auto` +3. `use_itn = true` +4. `num_threads = 4` +5. `vad_model_id = "silero-vad-onnx"` + +### 4.2 服务层 + +`voice_asr_service.rs` 增加 `SenseVoiceLocal` 分支： + +```text +AsrService::transcribe + -> credential.provider == SenseVoiceLocal + -> transcribe_sensevoice_local(...) + -> voice_core::SenseVoiceTranscriber +``` + +失败口径： + +1. 未安装模型：返回“本地 SenseVoice Small 尚未安装，请先在设置 -> 语音模型中下载”。 +2. 文件损坏：返回“模型文件校验失败，请重新下载”。 +3. 运行时依赖不可用：返回可诊断错误，并提示使用自定义快捷键或云端 ASR 不是解决方向。 + +### 4.3 voice-core + +`voice-core` 新增 `SenseVoiceTranscriber`： + +1. 输入接收现有 `AudioData`。 +2. 转换为 16kHz mono float samples。 +3. 创建 sherpa-onnx offline recognizer。 +4. `language = auto`。 +5. `use_itn = true`。 +6. 输出复用现有 `TranscribeResult`。 + +P0 只做 non-streaming decode。实时录音体验仍按当前录音完成后转写，不做流式字幕。 + +### 4.4 sherpa-onnx 集成策略 + +优先顺序： + +1. 优先验证 Rust crate / C API 直连方案。 +2. 若动态库打包风险高，使用 sidecar 进程封装 sherpa-onnx 调用。 +3. 不管采用哪种方式，对上都只暴露 `voice_core::SenseVoiceTranscriber`，避免 UI 或服务层感知运行时细节。 + +需要在实现前做最小 spike： + +1. macOS aarch64 本地加载模型并转写 wav。 +2. Windows x64 本地加载模型并转写 wav。 +3. Tauri 打包后动态库可找到。 +4. 无模型时错误稳定。 + +## 5. 前端设置页 + +设置页新增“语音模型”视图，参考截图组织： + +1. 顶部：语音输入快捷键，显示 Fn 模式开关与说明。 +2. 模型卡：`SenseVoice Small`、`本地`、简介、大小、安装状态。 +3. 操作：下载模型、删除模型、设为默认。 +4. 测试转写：P0 已提供原生选择或手动输入本机 WAV 路径；视频文件抽音和实时录音测试保留为 P1。 +5. 历史：所有转写历史入口。 + +文案边界： + +1. `语音模型` 指语音输入 ASR 模型。 +2. `语音处理` 指转写后的 LLM 润色与翻译。 +3. `语音服务模型` 指配音/TTS 生成模型。 + +这三个概念不能混用。 + +## 6. 命令与合同 + +若新增 Tauri command，必须同步四侧： + +1. 前端 `safeInvoke(...)` +2. Rust `tauri::generate_handler!` +3. `agentCommandCatalog` +4. `mockPriorityCommands` / `defaultMocks` + +建议命令： + +| 命令 | 作用 | +| --- | --- | +| `voice_models_list_catalog` | 获取服务端模型清单 | +| `voice_models_get_install_state` | 获取本地安装状态 | +| `voice_models_download` | 下载并校验模型；可接收前端透传的后端目录项 | +| `voice_models_delete` | 删除本地模型 | +| `voice_models_test_transcribe_file` | 文件测试转写 | + +下载与删除都必须返回结构化状态，不能只返回字符串。 + +## 7. 测试计划 + +Rust： + +1. 模型清单解析与平台过滤。 +2. sha256 校验失败。 +3. 必需文件缺失。 +4. 未安装模型时转写失败。 +5. 短音频沿用现有错误语义。 +6. 已安装模型路径解析。 + +前端： + +1. 模型卡显示未安装、下载中、已安装、损坏、失败。 +2. 未安装时测试按钮禁用。 +3. 已安装时选择或输入 WAV 路径并点击测试，会调用 `voice_models_test_transcribe_file` 并显示转写结果、采样率与时长。 +4. 下载进度展示。 +5. 删除模型需要确认。 +6. 设为默认后保存 ASR 配置。 + +合同： + +```bash +npm run test:contracts +``` + +GUI： + +```bash +npm run verify:gui-smoke +``` + +收口： + +```bash +npm run verify:local +``` + +## 8. 验收标准 + +1. 新安装 App 不包含 SenseVoice 模型文件。 +2. 用户能从设置页显式下载模型。 +3. 断网但已安装模型时，语音转写可工作。 +4. 删除模型后，转写不再假装可用，并提示重新下载。 +5. 输入栏听写和悬浮语音窗消费同一条 ASR 主链。 +6. 已安装模型后，可以用本机 16-bit PCM WAV 文件做测试转写，证明模型目录和本地推理链路可用。 +7. 日志能区分下载失败、模型损坏、运行时加载失败和识别失败。 + +## 9. 这一步如何服务主线 + +SenseVoice Small 接入的主线收益是： + +**让 Lime 的语音输入在无云端 ASR 凭证、无网络的情况下仍能完成转写，同时不牺牲安装包体积。** diff --git a/docs/roadmap/warp/README.md b/docs/roadmap/warp/README.md index 531af077b..fa8bd7f1f 100644 --- a/docs/roadmap/warp/README.md +++ b/docs/roadmap/warp/README.md @@ -1,7 +1,7 @@ # Warp 对照下的 Lime 多模态管理路线图 -> 状态：current planning source -> 更新时间：2026-04-29 +> 状态：current planning source +> 更新时间：2026-04-30 > 目标：吸收 Warp 开源客户端在 Agent Harness、Execution Profile、Artifact、Attachment、Task Index 与 Cloud/Local 分层上的可借鉴原则，把 Lime 的多模态能力收敛成统一运行合同，而不是继续按 `@` 命令和单点 viewer 分散扩张。 ## 1. 本路线图回答什么 @@ -143,6 +143,10 @@ Lime 的 artifact graph 不应只有 `document` / `file`。 通用文件只作为兜底，不作为多模态默认主结果。 +当前 `browser_session` / `browser_snapshot` 不新建平行 task 协议；Browser Assist tool timeline 先通过 evidence `snapshotIndex.browserActionIndex` 进入可查询索引层，并已在 Harness evidence panel 暴露摘要、通过最小 `browser_replay_viewer` 打开复盘。后续完整交互回放、权限 profile 与截图/DOM/network 深层展开继续消费同一事实源。 + +当前 `transcript` 已绑定到底层 `audio_transcription` contract；`@转写 / @transcribe / @Audio Extractor` 只是上层入口，前端、Rust metadata、`transcription_generate` task file、CLI 回退入口与 `lime-transcription-worker` 会保留同一份 `audio_transcription` runtime contract snapshot。当前闭环已经能写入 `.lime/tasks/transcription_generate/*.json`，在 payload 下生成 `transcript.pending`，通过 OpenAI-compatible transcription provider seam 回写 `transcript.completed/failed`，并把 transcript 状态/路径/来源/语言/格式/Provider 错误纳入 `list_media_task_artifacts`、聊天任务卡、`.lime/runtime/transcription-generate/*.md` 运行时文档、Evidence Pack `snapshotIndex.transcriptIndex` 与 Replay / grader。第四十三刀已让运行时文档读取 `.lime/runtime/transcripts/*` 文本内容，打开任务卡即可看到可复制校对的转写文本；第四十四刀继续解析 JSON / SRT / VTT transcript 的时间轴与说话人，并在聊天轻卡和运行时文档中展示可逐段编辑校对的段落表；第四十五刀复用 ArtifactDocument 保存链路，保存校对稿时写入 `transcriptCorrection*` / `transcriptSegmentsCorrected` metadata，并明确不改写原始 ASR 输出文件；第四十六刀补上 viewer 内“校对稿已保存”状态卡与 `transcriptCorrectionDiffSummary`，让原文/校对稿的文本长度、段落、说话人数差异可见。后续仍需要更专用的逐段 transcript viewer 交互、更多 ASR adapter 与本地离线 ASR 执行器。 + ## 4. 目录文档分工 1. [runtime-fact-map.md](./runtime-fact-map.md) @@ -151,40 +155,46 @@ Lime 的 artifact graph 不应只有 `document` / `file`。 - Phase 1 `ModalityRuntimeContract` 字段语义与机器校验入口。 3. [capability-matrix.md](./capability-matrix.md) - Phase 2 多模态能力矩阵、模型角色槽位与 capability gap 口径。 -4. [implementation-plan.md](./implementation-plan.md) +4. [execution-profile.md](./execution-profile.md) + - Phase 3 / Phase 5 `ModalityExecutionProfile`、executor adapter registry 与治理守卫。 +5. [artifact-graph.md](./artifact-graph.md) + - Phase 4 领域化产物图，明确 artifact kind、truth source、viewer、evidence 与 task index 映射。 +6. [implementation-plan.md](./implementation-plan.md) - 分阶段开发计划、改动面、验收输出和验证入口。 -5. [evolution-guide.md](./evolution-guide.md) +7. [evolution-guide.md](./evolution-guide.md) - Lime 自下而上的演进总图、泳道图、阶段门禁和每轮收口模板。 -6. [acceptance.md](./acceptance.md) +8. [acceptance.md](./acceptance.md) - 关键场景验收标准，防止路线图停留在抽象层。 当前机器可检查事实源： 1. `src/lib/governance/modalityRuntimeContracts.json` 2. `src/lib/governance/modalityCapabilityMatrix.json` -3. `scripts/check-modality-runtime-contracts.mjs` -4. `npm run governance:modality-contracts` +3. `src/lib/governance/modalityArtifactGraph.json` +4. `src/lib/governance/modalityExecutionProfiles.json` +5. `src/lib/governance/modalityExecutionProfiles.ts` +6. `scripts/check-modality-runtime-contracts.mjs` +7. `npm run governance:modality-contracts` 后续如果继续推进，再按需新增： -1. `artifact-graph.md` -2. `limecore-integration.md` -3. `browser-computer-use.md` -4. `migration-map.md` +1. `limecore-integration.md` +2. `browser-computer-use.md` +3. `migration-map.md` ## 5. 分阶段总览 -| 阶段 | 目标 | 主产物 | -| --- | --- | --- | -| Phase 0 | 盘点底层运行事实源 | runtime fact map | -| Phase 1 | 建底层运行合同 schema | `ModalityRuntimeContract` + governance check | -| Phase 2 | 扩展模型能力矩阵 | modality capability matrix + routing evidence | -| Phase 3 | 建统一 execution profile | model roles + permission profile + tenant override | -| Phase 4 | 领域化 artifact graph | domain artifact kinds + viewer mapping | -| Phase 5 | 建 executor / Browser typed action 边界 | executor adapter + browser evidence | -| Phase 6 | LimeCore 目录与策略接线 | cloud catalog + model offer + Gateway/Scene policy | -| Phase 7 | 绑定上层入口 | `@` / button / scene launch mapping | -| Phase 8 | 任务索引与复盘 | modality task index + audit + replay hooks | +| 阶段 | 目标 | 主产物 | +| ------- | --------------------------------------- | -------------------------------------------------- | +| Phase 0 | 盘点底层运行事实源 | runtime fact map | +| Phase 1 | 建底层运行合同 schema | `ModalityRuntimeContract` + governance check | +| Phase 2 | 扩展模型能力矩阵 | modality capability matrix + routing evidence | +| Phase 3 | 建统一 execution profile | `modalityExecutionProfiles.json` + profile / policy guard | +| Phase 4 | 领域化 artifact graph | domain artifact kinds + viewer mapping | +| Phase 5 | 建 executor / Browser typed action 边界 | executor adapter registry + browser evidence | +| Phase 6 | LimeCore 目录与策略接线 | cloud catalog + model offer + Gateway/Scene policy | +| Phase 7 | 绑定上层入口 | `@` / button / scene launch mapping | +| Phase 8 | 任务索引与复盘 | modality task index + audit + replay hooks | ## 6. 当前必须避免的误区 diff --git a/docs/roadmap/warp/acceptance.md b/docs/roadmap/warp/acceptance.md index 85fc6cf7d..231bac84c 100644 --- a/docs/roadmap/warp/acceptance.md +++ b/docs/roadmap/warp/acceptance.md @@ -1,7 +1,7 @@ # Warp 对照多模态管理验收标准 -> 状态：current planning source -> 更新时间：2026-04-29 +> 状态：current planning source +> 更新时间：2026-04-30 > 目标：为 [implementation-plan.md](./implementation-plan.md) 提供可验证场景，避免“参考 Warp”停留在架构口号。 ## 1. 总体验收口径 @@ -32,6 +32,7 @@ 1. 上层入口不得直接创建任务、写 artifact 或决定 viewer。 2. 上层入口不得直接决定模型和权限。 3. 上层入口只能补 launch metadata，并绑定到底层 contract。 +4. 找不到 execution profile 或 executor adapter 的 current contract 不能继续绑定入口。 ## 2. `@配图` 验收 @@ -62,7 +63,7 @@ 4. 执行动作是 typed browser/computer action。 5. 每次关键动作产生 observation：screenshot、DOM、network 或 URL state。 6. timeline 展示真实 browser tool 过程。 -7. evidence pack 导出 browser trace。 +7. evidence pack 导出 browser trace，并在 `snapshotIndex.browserActionIndex` 中提供 action/session/URL/observation 可查询摘要；Harness evidence panel 能展示该摘要，并能打开最小 `browser_replay_viewer` 复盘最近浏览器动作。 8. 禁用 browser_control 时给出阻断或询问，不回退 WebSearch 假装完成。 禁止： @@ -93,17 +94,19 @@ 必须证明： 1. `@配音` 走 `service_scene_launch(scene_key=voice_runtime)` 或后续同构 contract。 -2. `@转写` 走 transcription contract，不和普通文件读取混淆。 +2. `@转写` 走 `audio_transcription` contract，不和普通文件读取混淆；前端与 Rust metadata 都保留同一份 runtime contract snapshot。 3. 模型路由分别识别 `voice_generation` / `audio_transcription`。 4. 媒体上传、读取和生成权限进入 profile。 5. 输出区分 `audio_task`、`audio_output`、`transcript`。 6. 音频 artifact 带时长、mime、来源、任务状态。 -7. evidence pack 能导出媒体任务与产物。 +7. evidence pack 能导出媒体任务与产物；`audio_transcription` 必须把 `transcription_generate` task 与 `transcript` 子产物纳入同一 `snapshotIndex.transcriptIndex`，Replay / grader 能检查 transcript 状态、来源、语言、格式与失败码。 +8. 当前转写 worker 必须通过同一 task artifact 回写 `transcript.completed` 或 `transcript.failed`；成功时 transcript 文件落在 `.lime/runtime/transcripts/*`，失败时保留 provider/source/contract 错误码，聊天任务卡与 `.lime/runtime/transcription-generate/*.md` 运行时文档必须优先消费 `list_media_task_artifacts` 的 `transcript_*` snapshot，并在完成态读取 `transcript_path` 指向的文本内容用于内部校对；如果 transcript 文件是 JSON / SRT / VTT，还必须解析时间轴与说话人并在同一运行时文档里展示可逐段编辑校对的段落表。用户保存校对稿时必须落回同一 ArtifactDocument 新版本，并写入 `transcriptCorrection*` / `transcriptSegmentsCorrected` / `transcriptCorrectionDiffSummary` metadata，同时在 viewer 中展示“校对稿已保存”状态；原始 ASR 输出文件保持不可变，不能另写普通文件卡。 禁止： 1. 回流旧本地 TTS 测试命令作为 current。 2. 把音频输出只当通用文件卡。 +3. 前端直连 ASR、普通文件读取或 generic file transcript 绕过 `audio_transcription` 合同。 ## 6. `@搜索` / `@深搜` / `@研报` 验收 @@ -179,10 +182,11 @@ 每次推进本路线图，至少选择相关命令： 1. 文档/contract 改动：`npm run harness:doc-freshness` 或等价文档检查。 -2. command/runtime contract 改动：`npm run test:contracts`。 -3. UI 可见改动：相关 `*.test.tsx` 与 `npm run verify:gui-smoke`。 -4. 模型路由改动：相关 Rust / TS 路由测试与 thread read 断言。 -5. LimeCore 接口改动：同步改 LimeCore OpenAPI、SDK、类型与客户端消费测试。 +2. contract / profile / artifact graph 改动：`npm run governance:modality-contracts`。 +3. command/runtime contract 改动：`npm run test:contracts`。 +4. UI 可见改动：相关 `*.test.tsx` 与 `npm run verify:gui-smoke`。 +5. 模型路由改动：相关 Rust / TS 路由测试与 thread read 断言。 +6. LimeCore 接口改动：同步改 LimeCore OpenAPI、SDK、类型与客户端消费测试。 最终收口前，至少跑： diff --git a/docs/roadmap/warp/artifact-graph.md b/docs/roadmap/warp/artifact-graph.md new file mode 100644 index 000000000..f6bbb960f --- /dev/null +++ b/docs/roadmap/warp/artifact-graph.md @@ -0,0 +1,212 @@ +# 领域化 Artifact Graph + +> 状态：current planning source +> 更新时间：2026-04-30 +> 目标：把 Warp 路线图 Phase 4 从“artifact kind 列表”推进成可检查事实源，约束多模态结果必须落到领域化产物、viewer、evidence 与 task index，而不是继续退回普通文件卡或聊天文本。 + +## 1. 事实源 + +当前机器可检查事实源： + +1. Graph：`src/lib/governance/modalityArtifactGraph.json` +2. Contract：`src/lib/governance/modalityRuntimeContracts.json` +3. Check：`scripts/check-modality-runtime-contracts.mjs` +4. npm 入口：`npm run governance:modality-contracts` + +本文件解释字段语义；JSON graph 是校验输入。 + +## 2. 固定原则 + +1. Artifact kind 描述底层产物，不描述 `@` 命令。 +2. Contract 引用的每个 `artifact_kinds` 必须存在于 artifact graph。 +3. Contract 的 `truth_source`、`viewer_surface`、`evidence_events` 必须能与对应 artifact kind 对上。 +4. `generic_file` 只允许作为兜底，不允许成为图片、音频、浏览器、PDF、报告、PPT、网页等多模态主结果。 +5. Viewer 只消费 artifact graph 或 runtime truth source，不从 UI 状态反推执行事实。 +6. Evidence / replay / task index 消费同一组 artifact kind，不另建第二套恢复事实源。 + +## 3. Graph 字段 + +| 字段 | 说明 | +| ----------------------- | -------------------------------------------------------------------------------- | +| `kind` | artifact kind 主键，例如 `audio_output` | +| `lifecycle` | `current` / `compat` / `deprecated` / `dead` | +| `modality` | `text` / `image` / `audio` / `video` / `browser` / `document` / `code` / `mixed` | +| `implementation_status` | `current` / `partial` / `planned`，描述实现成熟度，不改变生命周期 | +| `truth_sources` | 产物事实源，例如 `audio_task_artifact`、`runtime_timeline_event` | +| `viewer_surfaces` | 允许消费该产物的 viewer surface | +| `evidence_events` | Evidence pack 至少能解释到的 runtime event | +| `task_index_fields` | 进入任务索引 / 复盘 / 诊断的最小字段 | +| `current_contracts` | 当前引用该 artifact kind 的 contract | +| `notes` | 当前缺口或边界说明 | + +## 4. 当前 Artifact Graph + +| artifact kind | 实现状态 | 当前合同 | viewer | 主缺口 | +| ----------------------- | -------------- | ------------------------------------------------- | -------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `image_task` | current | `image_generation` | `image_workbench` | 继续把更多图片子入口收敛到同一 task graph | +| `image_output` | current | `image_generation` | `image_workbench` | 防止普通文件卡重复展示 | +| `audio_task` | current | `voice_generation` | `audio_player` | 后续接 execution profile / provider policy | +| `audio_output` | current | `voice_generation` | `audio_player` | 后续接 Provider 设置修复入口与 LimeCore offer | +| `transcript` | partial | `audio_transcription` | `document_viewer` / 后续 `transcript_viewer` | 已有 `transcription_generate` task writer、`lime-transcription-worker`、`transcript.completed/failed` 回写、媒体任务索引、聊天任务卡、可编辑校对的运行时文档 viewer、JSON/SRT/VTT 时间轴与说话人段落展示、ArtifactDocument 版本化校对稿保存、校对稿状态/差异摘要、Evidence `transcriptIndex` 与 Replay 检查；还缺更专用的逐段 transcript viewer 交互与更多 ASR adapter | +| `browser_session` | partial | `browser_control` | `browser_replay_viewer` | entry binding 已回挂 registry；`snapshotIndex.browserActionIndex` 已进入 Harness evidence 面板与最小复盘 viewer，还缺权限 profile 与完整交互回放 | +| `browser_snapshot` | partial | `browser_control` | `browser_replay_viewer` | observation / screenshot 已进 evidence 索引、Harness 摘要与最小复盘 viewer；截图/DOM/network 深层展开仍需补齐 | +| `pdf_extract` | partial | `pdf_extract` | `document_viewer` | 还缺页码/引用恢复与专属 PDF artifact viewer | +| `report_document` | partial | `pdf_extract` / `web_research` / `text_transform` | `document_viewer` / `report_viewer` | 还缺 report viewer 与来源索引闭环 | +| `presentation_document` | planned | 无 | `presentation_viewer` / `document_viewer` | 还缺 presentation contract | +| `webpage_artifact` | planned | `web_research` | `webpage_viewer` / `document_viewer` | 还缺 webpage artifact / viewer 分离 | +| `generic_file` | current compat | `text_transform` | `generic_file_viewer` / `document_viewer` | 只能兜底，不能继续承载多模态主结果 | + +## 5. 运行关系图 + +```mermaid +flowchart TB + Contract["ModalityRuntimeContract"] --> ArtifactKind["artifact_kinds"] + Contract --> Truth["truth_source"] + Contract --> Viewer["viewer_surface"] + Contract --> Evidence["evidence_events"] + + ArtifactKind --> Graph["modalityArtifactGraph.json"] + Graph --> Domain["domain artifact"] + Graph --> Index["task index fields"] + Graph --> Replay["evidence / replay"] + Graph --> Surface["viewer mapping"] + + Domain --> Surface + Domain --> Index + Domain --> Replay +``` + +读法： + +1. Contract 声明“会产出什么 kind”。 +2. Artifact graph 声明“这个 kind 的事实源、viewer、evidence 和索引字段是什么”。 +3. Viewer、task index、replay 都从 artifact graph 消费同一事实，不允许各自猜。 + +## 6. `voice_generation` 样板 + +当前最接近闭环的样板是： + +```mermaid +sequenceDiagram + participant Entry as @配音 entry + participant Contract as voice_generation contract + participant Task as audio_task artifact + participant Worker as lime-audio-worker + participant Output as audio_output + participant Index as media task index + participant Viewer as audio task viewer + participant Evidence as evidence / replay + + Entry->>Contract: harness.service_scene_launch + contract snapshot + Contract->>Task: create audio_generate task + Task->>Worker: standard audio task payload + Worker-->>Task: running / failed / completed + Worker-->>Output: audio_path / mime / duration or provider error + Task-->>Index: audio_output_* snapshot + Index-->>Viewer: restore task card and runtime document + Task-->>Evidence: audioOutput snapshot + Evidence-->>Viewer: replay / review can explain output +``` + +这条样板已经证明： + +1. `audio_task` 与 `audio_output` 不再是普通文件卡。 +2. 失败态保留 `audio_provider_*` 错误码，不伪造音频路径。 +3. 媒体任务索引可直接恢复任务卡与 viewer。 +4. Evidence / replay 能区分 `audio_output.completed` 和 provider 失败。 + +## 7. `browser_control` 索引闭环 + +本轮补齐的最小 browser index 不新增 browser task 协议，只消费已有 Browser Assist tool timeline metadata： + +```mermaid +sequenceDiagram + participant Entry as @浏览器 entry + participant Contract as browser_control contract + participant Tool as mcp__lime-browser__* + participant Trace as browser_action_trace + participant Index as snapshotIndex.browserActionIndex + participant Harness as Harness evidence panel + participant Viewer as browser_replay_viewer + participant Replay as replay / grader + + Entry->>Contract: harness.browser_assist + entry_source + Contract->>Tool: Browser Assist 决策 typed action + Tool-->>Trace: action / backend / session / target / result + Trace-->>Index: browser_session / browser_snapshot summary + Index-->>Harness: action / session / URL / observation 摘要 + Harness-->>Viewer: browser_assist Artifact + browserActionIndex + Index-->>Viewer: 最近动作 / URL / session / screenshot + Index-->>Replay: actionCount / sessionCount / lastUrl / observationCount / screenshotCount +``` + +索引字段先固定为： + +1. `actionCount`、`statusCounts`、`actionCounts`：判断是否真的产生 browser action。 +2. `sessionIds`、`targetIds`、`profileKeys`、`backendCounts`：定位执行环境和 backend。 +3. `lastUrl`：快速恢复最近页面位置。 +4. `observationCount`、`screenshotCount`：区分普通 session action 与 snapshot/observation。 +5. `items[].artifactKind`：在同一索引内区分 `browser_session` 与 `browser_snapshot`。 + +这一步把 `browser_control` 从 raw trace 人工扫描推进到可检索诊断层，并让 Harness evidence panel 能直接展示 Browser Assist 摘要、打开最小 `browser_replay_viewer`；后续完整交互回放、权限 profile 可视化与截图/DOM/network 深层展开仍消费同一份 index，不能另起 browser task 文件协议。 + +## 8. `audio_transcription` Transcript 索引闭环 + +本轮把 `audio_transcription` 从入口合同推进到最小 transcript task / index / evidence / replay 闭环： + +```mermaid +sequenceDiagram + participant Entry as @转写 entry + participant Contract as audio_transcription contract + participant Skill as Skill(transcription_generate) + participant Task as transcription_generate task + participant Transcript as transcript.pending + participant Index as media task index + participant Evidence as snapshotIndex.transcriptIndex + participant Replay as replay / grader + participant Viewer as document_viewer / transcript_viewer + + Entry->>Contract: harness.transcription_skill_launch + contract snapshot + Contract->>Skill: Agent 首刀执行转写技能 + Skill->>Task: lime_create_transcription_task / CLI fallback + Task-->>Transcript: payload.transcript(status=pending) + Task->>Transcript: lime-transcription-worker -> completed/failed + Task-->>Index: transcript status/path/source/language/outputFormat/errorCode + Task-->>Evidence: transcription_task.modality_runtime_contract + Evidence-->>Replay: transcriptIndex + failure modes + Index-->>Viewer: transcript snapshot + transcript file text/segments -> 任务卡 / 可编辑校对文档 / 时间轴段落表 + Viewer-->>Viewer: 保存校对稿 -> ArtifactDocument 新版本 + transcriptCorrection metadata + Viewer-->>Viewer: 校对稿状态卡 + transcriptCorrectionDiffSummary +``` + +当前已经固定的索引字段： + +1. `transcript_count`、`transcript_statuses`、`transcript_error_codes`：判断转写产物是否已进入统一媒体任务索引。 +2. `items[].transcript_status`、`transcript_path`、`transcript_source_url`、`transcript_source_path`：定位 transcript 状态、输出路径与来源。 +3. `items[].transcript_language`、`transcript_output_format`：保留用户请求或 Provider 返回的语言和输出格式。 +4. `snapshotIndex.transcriptIndex.items[]`：在 evidence / replay 中携带 provider、model、worker、errorCode 与 retryable。 + +这一步把 transcript 从 pending 事实推进到可执行 worker 与用户可见恢复闭环：OpenAI-compatible provider 成功时写入 `.lime/runtime/transcripts/*` 并回写 `transcript.completed`；Provider / source / contract 失败时写入 `transcript.failed` 与明确错误码；聊天区恢复层优先消费 `list_media_task_artifacts` 的 `transcript_*` snapshot，同步任务卡与 `.lime/runtime/transcription-generate/*.md` 运行时文档，只有索引缺失时才回退读取单个 task artifact。第四十三刀继续读取 `transcript_path` 指向的文本文件，把 transcript 内容嵌入运行时文档的 `code_block`，使用户能在 Lime 内部复制和校对文本。第四十四刀继续从 JSON verbose transcript、SRT 与 VTT 解析 `start/end/speaker/text` 段落，把时间轴预览展示到聊天轻卡，并把“转写时间轴（可逐段编辑校对）”表格写入同一个运行时文档。第四十五刀复用 ArtifactDocument 编辑保存链路：用户保存转写文本或时间轴校对结果时，只写回同一运行时文档的新版本和 `transcriptCorrection*` / `transcriptSegmentsCorrected` metadata，不改写原始 ASR 输出文件。第四十六刀继续把保存状态显性化：保存后运行时文档会插入/更新“校对稿已保存”状态卡，并写入 `transcriptCorrectionDiffSummary`，用于展示原文与校对稿的文本长度、段落和说话人数差异。更专用的逐段 transcript viewer、更多 ASR adapter 与本地离线 ASR 后续继续消费同一份 `transcription_generate` task file 和 `transcriptIndex`，不能另起 `generic_file` 或前端直连 ASR 旁路。 + +## 9. 未完成主线 + +下一批必须继续补： + +1. `browser_session` / `browser_snapshot`：Harness evidence panel 与最小 `browser_replay_viewer` 已能展示摘要；继续把权限 profile、截图/DOM/network 深层展开与完整交互回放接到 `snapshotIndex.browserActionIndex`，形成可操作复盘闭环。 +2. `pdf_extract`：页码、引用、来源文件与 viewer 恢复还未形成稳定 artifact graph。 +3. `report_document`：研报 / 竞品 / 分析仍缺 report viewer 与来源索引。 +4. `transcript`：task writer、OpenAI-compatible ASR worker、完成/失败态回写、媒体任务索引、聊天任务卡、可编辑校对运行时文档 viewer、JSON/SRT/VTT 时间轴与说话人段落展示、ArtifactDocument 版本化校对稿保存、校对稿状态/差异摘要、Evidence `transcriptIndex` 与 Replay 检查已补齐；继续补更专用的逐段 transcript viewer、更多 ASR adapter 与本地离线 ASR，不允许由 `generic_file` 冒充。 +5. `presentation_document` / `webpage_artifact`：还缺对应 contract、artifact writer 与 viewer mapping。 +6. `generic_file`：继续收口为 compat fallback，不能作为新增多模态能力的默认输出。 + +## 10. 机器守卫 + +`npm run governance:modality-contracts` 现在必须检查： + +1. 每个已声明 artifact kind 都在 graph 中有定义。 +2. 每个 contract 引用的 artifact kind 都能找到 viewer intersection。 +3. 每个 contract 引用的 artifact kind 都能找到 truth source intersection。 +4. 每个 contract 引用的 artifact kind 都能找到 evidence event intersection。 +5. Graph 中的 current / partial artifact 至少声明 `task_id`、`contract_key`、`artifact_kind`、`status` 这组 task index 字段。 + +这样 Phase 4 不再只停留在文档表格，而是会阻止新 contract 继续把多模态结果丢给未知 viewer、未知 evidence 或 `generic_file` 旁路。 diff --git a/docs/roadmap/warp/contract-schema.md b/docs/roadmap/warp/contract-schema.md index 166a533f9..d108b1e59 100644 --- a/docs/roadmap/warp/contract-schema.md +++ b/docs/roadmap/warp/contract-schema.md @@ -10,6 +10,8 @@ - Registry：`src/lib/governance/modalityRuntimeContracts.json` - Capability Matrix：`src/lib/governance/modalityCapabilityMatrix.json` +- Artifact Graph：`src/lib/governance/modalityArtifactGraph.json` +- Execution Profiles：`src/lib/governance/modalityExecutionProfiles.json` - Check：`scripts/check-modality-runtime-contracts.mjs` - npm 入口：`npm run governance:modality-contracts` @@ -24,6 +26,8 @@ 5. `required_capabilities` 必须全部出现在 [capability-matrix.md](./capability-matrix.md)。 6. `routing_slot` 必须出现在 capability matrix 的 `model_roles`。 7. contract 不允许引用不存在的 artifact kind、viewer surface、permission key 或 evidence event。 +8. contract 引用的 artifact kind 必须能在 artifact graph 中找到 truth source / viewer / evidence 交集。 +9. current contract 必须能在 [execution-profile.md](./execution-profile.md) 中找到 profile 与 executor adapter 覆盖。 ## 3. 顶层结构 @@ -107,13 +111,15 @@ Phase 7 才允许大量补 entry binding。 ## 7. 首批 contract -首批 registry 先覆盖底层能力；`image_generation` 已作为第一条 vertical slice 进入 Phase 7 entry binding： +首批 registry 先覆盖底层能力；进入 current 的 contract 必须继续被 capability matrix、artifact graph、execution profile 与 executor adapter registry 同步覆盖： 1. `image_generation` 2. `browser_control` 3. `pdf_extract` 4. `voice_generation` -5. `web_research` +5. `audio_transcription` +6. `web_research` +7. `text_transform` 这些 contract 用于验证 schema 和治理守卫；后续 vertical slice 继续按“先底层、后 entry binding”的顺序推进。 @@ -132,3 +138,5 @@ npm run governance:modality-contracts 5. executor binding 声明能力与 failure mapping。 6. entry binding 不携带底层事实源字段。 7. entry binding 必须声明 `entry_source`，且 `launch_metadata_path` 必须留在 `harness.*`。 +8. `artifact_kinds` 必须存在于 artifact graph，并与 contract 的 truth source、viewer、evidence 至少各有一个交集。 +9. current contract 必须被 execution profile 覆盖，且 `executor_binding` 必须能解析到已声明 executor adapter。 diff --git a/docs/roadmap/warp/evolution-guide.md b/docs/roadmap/warp/evolution-guide.md index 6cacac5d1..df8c5a51a 100644 --- a/docs/roadmap/warp/evolution-guide.md +++ b/docs/roadmap/warp/evolution-guide.md @@ -1,7 +1,7 @@ # Lime 多模态演进迭代指南 -> 状态：current planning source -> 更新时间：2026-04-29 +> 状态：current planning source +> 更新时间：2026-04-30 > 目标：把 Warp / ClaudeCode 参考转成 Lime 可持续演进的工程节奏，明确每一阶段先交付什么、如何验收、何时允许接上层入口。 ## 1. 演进原则 @@ -148,6 +148,8 @@ sequenceDiagram participant Browser as Browser Executor participant Obs as Observation participant Evidence as Evidence + participant Index as browserActionIndex + participant Harness as Harness evidence panel participant Viewer as Browser replay viewer participant Entry as @浏览器 binding @@ -156,7 +158,10 @@ sequenceDiagram Runtime->>Browser: typed action Browser-->>Obs: screenshot / DOM / URL / network Obs-->>Evidence: browser trace - Evidence-->>Viewer: replay material + Evidence-->>Index: action / session / URL / observation summary + Index-->>Harness: visible evidence summary + Harness-->>Viewer: browser_assist artifact preview + Index-->>Viewer: replay material Runtime-->>Entry: 允许绑定 @浏览器 ``` @@ -164,17 +169,17 @@ sequenceDiagram ## 6. 阶段门禁 -| 阶段 | 允许进入下一阶段的条件 | 不允许的捷径 | -| --- | --- | --- | -| P0 Fact Map | 每个底层事实源有 owner、读写方、持久化和 evidence 关系 | 先从 `@` 命令盘点开始 | -| P1 Contract | Contract 以底层能力为主键，引用真实 truth source / artifact / viewer | 用入口名当 contract 主键 | -| P2 Capability | 路由能输出候选、唯一候选、候选为空和能力缺口 | 只看 provider/model id | -| P3 Profile | 模型、权限、租户策略、用户锁定能合并解释 | skill 内部临时判断权限 | -| P4 Artifact | domain artifact 能被 viewer 和 evidence 共同消费 | 所有结果写成 generic file | -| P5 Executor | executor 声明 progress/cancel/resume/artifact/failure mapping | 自由 Bash 或裸 CLI | -| P6 LimeCore | catalog/policy/offer/audit 能约束本地执行 | LimeCore 默认代跑所有入口 | -| P7 Entry | 入口只提交 metadata 并绑定 contract | 入口直建 task / artifact / viewer | -| P8 Index | task index 能按 contract、entry、modality、artifact 查询 | 只靠聊天线程恢复 | +| 阶段 | 允许进入下一阶段的条件 | 不允许的捷径 | +| ------------- | -------------------------------------------------------------------- | --------------------------------- | +| P0 Fact Map | 每个底层事实源有 owner、读写方、持久化和 evidence 关系 | 先从 `@` 命令盘点开始 | +| P1 Contract | Contract 以底层能力为主键，引用真实 truth source / artifact / viewer | 用入口名当 contract 主键 | +| P2 Capability | 路由能输出候选、唯一候选、候选为空和能力缺口 | 只看 provider/model id | +| P3 Profile | `modalityExecutionProfiles.json` 覆盖 current contracts；模型、权限、租户策略、用户锁定能合并解释 | skill 内部临时判断权限 | +| P4 Artifact | domain artifact 能被 viewer 和 evidence 共同消费 | 所有结果写成 generic file | +| P5 Executor | executor adapter registry 对齐 contract 绑定、支持位、产物、权限与 failure mapping | 自由 Bash 或裸 CLI | +| P6 LimeCore | catalog/policy/offer/audit 能约束本地执行 | LimeCore 默认代跑所有入口 | +| P7 Entry | 入口只提交 metadata 并绑定 contract | 入口直建 task / artifact / viewer | +| P8 Index | task index 能按 contract、entry、modality、artifact 查询 | 只靠聊天线程恢复 | ## 7. 迭代选择流程 @@ -206,17 +211,17 @@ flowchart TB ## 8. 文档产物演进表 -| 阶段 | 文档产物 | 代码产物 | 验证入口 | -| --- | --- | --- | --- | -| P0 | `runtime-fact-map.md` | 无或只读脚本 | 链接/owner 一致性检查 | -| P1 | `contract-schema.md` | schema / governance check | contract check | -| P2 | capability matrix doc | routing 类型和证据 | 路由单测 / thread read 断言 | -| P3 | execution profile doc | profile merge / policy source | 权限与降级测试 | -| P4 | `artifact-graph.md` | artifact kind / viewer mapping | artifact/viewer 回归 | -| P5 | executor adapter doc | executor registry / browser action | executor / browser trace 测试 | -| P6 | `limecore-integration.md` | catalog/policy SDK 消费 | Lime + LimeCore contract test | -| P7 | entry binding inventory | `@` / button / scene binding | acceptance 场景 | -| P8 | task index doc | index / replay / audit query | evidence pack / replay 测试 | +| 阶段 | 文档产物 | 代码产物 | 验证入口 | +| ---- | -------------------------------------------------- | ------------------------------------------------- | ------------------------------------------ | +| P0 | `runtime-fact-map.md` | 无或只读脚本 | 链接/owner 一致性检查 | +| P1 | `contract-schema.md` | schema / governance check | contract check | +| P2 | capability matrix doc | routing 类型和证据 | 路由单测 / thread read 断言 | +| P3 | `execution-profile.md` | `modalityExecutionProfiles.json` / profile merge / policy source | profile registry 守卫 + 权限与降级测试 | +| P4 | `artifact-graph.md` + `modalityArtifactGraph.json` | artifact kind / viewer / evidence / index mapping | artifact graph 守卫 + artifact/viewer 回归 | +| P5 | `execution-profile.md` 的 executor adapter 章节 | executor adapter registry / browser action | adapter registry 守卫 + executor / browser trace 测试 | +| P6 | `limecore-integration.md` | catalog/policy SDK 消费 | Lime + LimeCore contract test | +| P7 | entry binding inventory | `@` / button / scene binding | acceptance 场景 | +| P8 | task index doc | index / replay / audit query | evidence pack / replay 测试 | ## 9. 版本演进目标 @@ -228,7 +233,8 @@ flowchart TB 1. `runtime-fact-map.md` 存在。 2. `contract-schema.md` 存在。 -3. 至少一个 contract 完成 capability/profile/artifact/evidence 字段。 +3. `capability-matrix.md`、`execution-profile.md`、`artifact-graph.md` 存在。 +4. 至少一个 contract 完成 capability/profile/artifact/evidence 字段。 ### V1：单能力闭环 diff --git a/docs/roadmap/warp/execution-profile.md b/docs/roadmap/warp/execution-profile.md new file mode 100644 index 000000000..c05bde9b2 --- /dev/null +++ b/docs/roadmap/warp/execution-profile.md @@ -0,0 +1,154 @@ +# ModalityExecutionProfile 与 Executor Adapter + +> 状态：current planning source +> 更新时间：2026-04-30 +> 目标：把 Warp 路线图 Phase 3 / Phase 5 从散文约束推进成可机器检查的 profile 与 executor adapter registry，确保每个 current 多模态合同都能解释模型角色、权限、执行器、产物策略、LimeCore 策略引用和失败映射。 + +## 1. 事实源 + +当前机器可检查事实源： + +1. Profile registry：`src/lib/governance/modalityExecutionProfiles.json` +2. Contract registry：`src/lib/governance/modalityRuntimeContracts.json` +3. Capability matrix：`src/lib/governance/modalityCapabilityMatrix.json` +4. Artifact graph：`src/lib/governance/modalityArtifactGraph.json` +5. TS resolver：`src/lib/governance/modalityExecutionProfiles.ts` +6. Check：`scripts/check-modality-runtime-contracts.mjs` +7. npm 入口：`npm run governance:modality-contracts` + +本文件解释字段语义；JSON registry 是校验输入。当前已建立治理事实源与前端 TS resolver，所有 current contract 的 launch metadata 可以携带 `execution_profile` 与 `executor_adapter` 快照；仍不新增 Tauri command、bridge、mock 或具体 Skill / ServiceSkill / Browser executor 分支。 + +## 2. 固定原则 + +1. Execution profile 描述底层运行策略，不描述 `@` 命令。 +2. 每个 `current` contract 必须被至少一个 execution profile 覆盖。 +3. Contract 的 `routing_slot`、`permission_profile_keys`、`limecore_policy_refs`、`artifact_kinds` 与 `executor_binding` 必须能在 profile / adapter registry 中找到对应声明。 +4. Executor adapter 的 `adapter_key` 必须等于 `executor_kind:binding_key`，并与 contract 的 `executor_binding` 对齐。 +5. Adapter 的 progress / cancel / resume / artifact 支持位必须和 contract 一致，不能伪造能力。 +6. `generic_file` 只能作为 compat fallback；新增多模态主结果必须优先落到领域化 artifact kind。 +7. LimeCore 在本阶段只作为 catalog / policy / offer / audit 引用，不是默认执行器。 + +## 3. Profile 字段 + +| 字段 | 说明 | +| --- | --- | +| `profile_key` | profile 主键，例如 `audio_transcription_profile` | +| `lifecycle` | `current` / `compat` / `deprecated` / `dead` | +| `supported_contracts` | 该 profile 覆盖的底层合同 | +| `model_role_slots` | 对应 capability matrix 的模型角色槽位 | +| `permission_profile_keys` | 运行前必须合并解释的权限面 | +| `executor_adapter_keys` | 允许调用的 executor adapter | +| `artifact_policy.write_mode` | 产物写入模式，例如 `domain_task_artifact` | +| `artifact_policy.artifact_kinds` | 允许写出的领域产物 | +| `artifact_policy.viewer_surfaces` | 允许消费该产物的 viewer surface | +| `limecore_policy_refs` | LimeCore 控制面引用，例如 `model_catalog`、`provider_offer`、`tenant_feature_flags` | +| `user_lock_policy` | 用户显式模型锁定的处理规则 | +| `fallback_behavior` | 权限、能力、执行器或来源失败时的降级 / 阻断口径 | +| `evidence_events` | profile 决策至少需要解释到的 evidence event | +| `audit_fields` | 后续 thread read / audit / evidence 需要携带的字段 | +| `notes` | 当前边界与后续缺口 | + +## 4. Executor Adapter 字段 + +| 字段 | 说明 | +| --- | --- | +| `adapter_key` | `executor_kind:binding_key`，例如 `skill:transcription_generate` | +| `lifecycle` | 生命周期分类 | +| `executor_kind` | `skill` / `tool` / `service_skill` / `browser` / `gateway` / `scene_cloud` / `local_cli` | +| `binding_key` | 执行器在 runtime 中的绑定名 | +| `supported_contracts` | 该 adapter 允许服务的底层合同 | +| `supports_progress` | 是否能报告进度 | +| `supports_cancel` | 是否能取消 | +| `supports_resume` | 是否能恢复 | +| `supports_artifact` | 是否能写标准 artifact | +| `artifact_output_kinds` | adapter 可以写出的 artifact kind | +| `permission_requirements` | adapter 所需权限 | +| `credential_requirements` | adapter 所需凭证或云控制面引用 | +| `failure_mapping` | 失败必须映射到的标准原因 | +| `evidence_events` | adapter 执行至少需要解释到的 evidence event | +| `notes` | 当前实现边界 | + +## 5. 当前覆盖 + +| contract | profile | executor adapter | artifact policy | 当前说明 | +| --- | --- | --- | --- | --- | +| `image_generation` | `image_generation_profile` | `skill:image_generate` | `image_task` / `image_output` | 图片生成只能写标准 image task/output，不回退 legacy CLI | +| `browser_control` | `browser_control_profile` | `browser:browser_assist` | `browser_session` / `browser_snapshot` | 浏览器动作必须保留 typed action 与 observation trace，不降级 WebSearch | +| `pdf_extract` | `pdf_extract_profile` | `skill:pdf_read` | `pdf_extract` / `report_document` | PDF 读取必须保留文件读取证据，页码/引用 viewer 后续补齐 | +| `voice_generation` | `voice_generation_profile` | `service_skill:voice_runtime` | `audio_task` / `audio_output` | 本地 ServiceSkill/worker 写音频任务，不把 LimeCore 当默认执行器 | +| `audio_transcription` | `audio_transcription_profile` | `skill:transcription_generate` | `transcript` | 转写固定走 transcription task / transcriptIndex，不走 frontend ASR 或 generic_file | +| `web_research` | `web_research_profile` | `skill:research` | `report_document` / `webpage_artifact` | 联网研究保留搜索来源与报告型产物，来源索引后续继续补 | +| `text_transform` | `text_transform_profile` | `skill:text_transform` | `report_document` / `generic_file` | `generic_file` 只保留为 compat fallback，主结果继续向 document viewer 收敛 | + +## 6. 决策流程 + +```mermaid +flowchart TB + Contract["ModalityRuntimeContract"] --> Profile["ModalityExecutionProfile"] + Matrix["Capability Matrix\nmodel_roles"] --> Profile + Policy["LimeCore policy refs\nmodel_catalog / offer / tenant flags"] --> Profile + UserLock["User explicit model lock"] --> Profile + Profile --> Decision{"allow / ask / deny"} + Decision -->|allow| Adapter["Executor Adapter"] + Decision -->|ask| Ask["ask_user_question"] + Decision -->|deny| Gap["permission / capability / policy gap"] + Adapter --> Artifact["Domain artifact"] + Artifact --> Evidence["evidence / replay / task index"] + Evidence --> Viewer["viewer surface"] +``` + +读法： + +1. Contract 声明底层能力需求。 +2. Profile 合并模型角色、权限、LimeCore 策略、用户锁定与 fallback。 +3. 前端 launch metadata 先携带 profile / adapter 快照，后续 Rust runtime preflight 再消费同一事实源执行 allow / ask / deny。 +4. Adapter 只在 profile 允许后执行，并按声明写 domain artifact。 +5. Evidence / task index / viewer 消费同一执行与产物事实。 + +## 7. Browser adapter 时序 + +```mermaid +sequenceDiagram + participant Contract as browser_control contract + participant Profile as browser_control_profile + participant Adapter as browser:browser_assist + participant Action as Typed browser action + participant Observation as screenshot / DOM / URL / network + participant Evidence as browserActionIndex + participant Viewer as browser_replay_viewer + + Contract->>Profile: routing_slot + browser_control permission + gateway_policy + Profile-->>Contract: allow / ask / deny + Contract->>Adapter: executor_binding browser:browser_assist + Adapter->>Action: plan typed action + Action-->>Observation: collect observation + Observation-->>Evidence: browser_session / browser_snapshot + Evidence-->>Viewer: replay summary and latest artifacts +``` + +固定约束：如果 profile 未允许 `browser_control`，不得把浏览器任务改写成普通 WebSearch；如果 adapter 没有 observation，就不能写 `browser_snapshot`。 + +## 8. 机器守卫 + +`npm run governance:modality-contracts` 现在必须检查： + +1. `modalityExecutionProfiles.json` 存在且 `version/status/owner` 正确。 +2. Profile / adapter 主键唯一。 +3. Profile 引用的 contract、model role、permission、adapter、artifact、viewer、LimeCore policy 与 evidence event 必须存在。 +4. 每个 current contract 必须被 profile 覆盖。 +5. 每个 current contract 的 `executor_binding` 必须存在于 `executor_adapters`。 +6. Adapter 的支持位、artifact output、permission requirements、failure mapping 必须覆盖 contract 声明。 +7. Profile 的 model role、permission、LimeCore policy 与 artifact policy 必须覆盖 contract 声明。 + +这一步把 Phase 3 / Phase 5 从“应该有 profile / executor adapter”推进成会阻断错误 contract 的机器事实源。 + +## 9. 未完成主线 + +后续继续补： + +1. Rust / Agent 运行时真实 `ExecutionProfile` merge：把 `modalityExecutionProfiles.json` 或其生成快照接入 `TaskProfile`、权限判断、用户模型锁定和 thread read。 +2. LimeCore policy snapshot：把 `model_catalog`、`provider_offer`、`tenant_feature_flags`、`gateway_policy` 的实际命中值写回 profile decision。 +3. GUI / evidence 可视化：在 Harness evidence、任务卡或 viewer 中展示 profile allow / ask / deny、adapter key 和 policy gap。 +4. Executor registry 运行时化：让 Skill / ServiceSkill / Browser / Gateway 的 adapter 能从同一事实源生成执行前检查。 +5. Task index 统一层：把 `executor_kind`、`adapter_key`、`profile_key`、`policy_snapshot` 纳入统一查询字段。 +6. LimeCore Phase 6：继续接目录、offer、Gateway/Scene policy 与 audit，不把 LimeCore 扩张成默认 executor。 diff --git a/docs/roadmap/warp/implementation-plan.md b/docs/roadmap/warp/implementation-plan.md index 21a577f89..1147e3f1e 100644 --- a/docs/roadmap/warp/implementation-plan.md +++ b/docs/roadmap/warp/implementation-plan.md @@ -1,7 +1,7 @@ # Warp 对照多模态管理实施计划 -> 状态：current planning source -> 更新时间：2026-04-29 +> 状态：current planning source +> 更新时间：2026-04-30 > 目标：把 [README.md](./README.md) 的路线图拆成自下而上的执行阶段，先建设底层多模态运行合同，再把 `@` 命令、按钮和 Scene 这类上层入口绑定上来。 ## 0. 排序修正 @@ -168,6 +168,8 @@ ## Phase 3：ModalityExecutionProfile +当前落点：见 [execution-profile.md](./execution-profile.md)、`src/lib/governance/modalityExecutionProfiles.json` 与 `src/lib/governance/modalityExecutionProfiles.ts`；最小 profile / executor adapter registry 与前端 launch metadata 快照已落地，真实 Rust runtime policy merge、tenant policy snapshot 与 GUI/evidence 可视化仍待继续。 + ### 目标 建立 Warp `AIExecutionProfile` 式的 Lime profile，把模型角色、权限策略、执行器策略合并到一个解释层。 @@ -220,9 +222,12 @@ 2. 租户禁止媒体上传时，图片/音频 contract 给出可解释降级。 3. 本地 explicit model lock 优先于自动优化，但能力不匹配时必须给出阻断原因。 4. Profile 决策写入 thread read 与 evidence，不只停留在设置页。 +5. 每个 current contract 都必须被 `modalityExecutionProfiles.json` 覆盖，且 profile 的模型角色、权限、LimeCore policy、artifact policy 与 contract 对齐。 ## Phase 4：领域化 Artifact Graph +当前落点：见 [artifact-graph.md](./artifact-graph.md) 与 `src/lib/governance/modalityArtifactGraph.json`。 + ### 目标 让多模态结果拥有领域 kind、恢复路径和 viewer 映射；上层入口只消费 artifact graph。 @@ -247,7 +252,7 @@ 1. `generic_file` 只做兜底。 2. 二进制图片输出不自动镜像成普通文件卡。 3. viewer 只消费 artifact graph，不自己猜类型。 -4. evidence pack 导出 artifact kind 和关联键。 +4. evidence pack 导出 artifact kind、关联键与可查询索引；`browser_control` 当前索引事实源是 `snapshotIndex.browserActionIndex`。 ### 验收 @@ -259,6 +264,8 @@ ## Phase 5：Executor Adapter 与 Browser typed action +当前落点：见 [execution-profile.md](./execution-profile.md) 与 `src/lib/governance/modalityExecutionProfiles.json` 的 `executor_adapters`；最小 adapter registry 已覆盖 `image_generation`、`browser_control`、`pdf_extract`、`voice_generation`、`audio_transcription`、`web_research`、`text_transform`，前端 runtime contract snapshot 已携带 `executor_adapter`，Rust runtime adapter preflight 仍待继续。 + ### 目标 先定义执行器边界，再允许上层入口调用它们；借鉴 Warp harness 和 computer use，但不把外部 CLI 作为 current 捷径。 @@ -293,8 +300,9 @@ Browser Assist 必须收成： 1. 浏览器操作不会退回普通 WebSearch。 2. 每次浏览器动作都有 action summary 与 observation。 3. 高风险动作按 profile 决定自动、询问或阻断。 -4. evidence pack 能导出 browser trace。 +4. evidence pack 能导出 browser trace，并在 `browserActionIndex` 汇总 action/session/URL/observation/screenshot；Harness evidence panel 能展示同一索引摘要，并能打开最小 `browser_replay_viewer`。 5. `local_cli` adapter 不支持 progress/resume/artifact 时必须显式标注，不可伪造 current 能力。 +6. 每个 current contract 的 `executor_binding` 必须能解析到 `executor_kind:binding_key` adapter，且 adapter 的 progress/cancel/resume/artifact 支持位、产物、权限与失败映射必须覆盖 contract。 ## Phase 6：LimeCore 目录与策略接线 diff --git a/package-lock.json b/package-lock.json index cd39f47f0..24c7feff9 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "lime", - "version": "1.24.0", + "version": "1.25.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "lime", - "version": "1.24.0", + "version": "1.25.0", "dependencies": { "@babel/standalone": "^7.29.0", "@fabianlars/tauri-plugin-oauth": "^2", diff --git a/package.json b/package.json index f51c96079..67d8b9400 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "lime", "private": true, - "version": "1.24.0", + "version": "1.25.0", "type": "module", "engines": { "node": ">=22.0.0" diff --git a/packages/lime-cli-npm/README.md b/packages/lime-cli-npm/README.md index f95903b48..a315fbf78 100644 --- a/packages/lime-cli-npm/README.md +++ b/packages/lime-cli-npm/README.md @@ -112,7 +112,7 @@ npm run build:release -- \ ```bash npm run build:release -- \ --target-triple "aarch64-apple-darwin" \ - --version "1.24.0" \ + --version "1.25.0" \ --out-dir "./dist" ``` diff --git a/packages/lime-cli-npm/package.json b/packages/lime-cli-npm/package.json index 14e2eda9d..0e0a59a3b 100644 --- a/packages/lime-cli-npm/package.json +++ b/packages/lime-cli-npm/package.json @@ -1,6 +1,6 @@ { "name": "@limecloud/lime-cli", - "version": "1.24.0", + "version": "1.25.0", "description": "Lime 官方任务 CLI", "bin": { "lime": "scripts/run.js" diff --git a/scripts/check-modality-runtime-contracts.mjs b/scripts/check-modality-runtime-contracts.mjs index 4303881b9..536a1072c 100755 --- a/scripts/check-modality-runtime-contracts.mjs +++ b/scripts/check-modality-runtime-contracts.mjs @@ -7,10 +7,15 @@ import process from "node:process"; const CONTRACT_PATH = "src/lib/governance/modalityRuntimeContracts.json"; const CAPABILITY_MATRIX_PATH = "src/lib/governance/modalityCapabilityMatrix.json"; +const ARTIFACT_GRAPH_PATH = "src/lib/governance/modalityArtifactGraph.json"; +const EXECUTION_PROFILE_PATH = + "src/lib/governance/modalityExecutionProfiles.json"; const REQUIRED_DOCS = [ "docs/roadmap/warp/runtime-fact-map.md", "docs/roadmap/warp/contract-schema.md", "docs/roadmap/warp/capability-matrix.md", + "docs/roadmap/warp/execution-profile.md", + "docs/roadmap/warp/artifact-graph.md", "docs/roadmap/warp/evolution-guide.md", ]; @@ -105,6 +110,24 @@ const FAILURE_REASONS = new Set([ "file_unavailable", "source_unavailable", ]); +const PROFILE_WRITE_MODES = new Set([ + "domain_task_artifact", + "domain_document_artifact", + "runtime_observation_trace", + "timeline_backed_document_artifact", + "document_or_compat_file_artifact", +]); +const ARTIFACT_IMPLEMENTATION_STATUSES = new Set([ + "current", + "partial", + "planned", +]); +const REQUIRED_ARTIFACT_INDEX_FIELDS = new Set([ + "task_id", + "contract_key", + "artifact_kind", + "status", +]); const ENTRY_KINDS = new Set([ "command", "button_action", @@ -431,7 +454,515 @@ function validateCapabilityMatrix(matrix) { }; } -function validateContractRegistry(registry, matrixRefs) { +function validateArtifactGraph(graph) { + const errors = []; + pushIf(errors, graph.version !== 1, "artifactGraph.version must be 1"); + pushIf( + errors, + graph.status !== "current", + "artifactGraph.status must be current", + ); + pushIf( + errors, + !isNonEmptyString(graph.owner), + "artifactGraph.owner must be set", + ); + + const artifactMap = collectUniqueObjects( + errors, + graph.artifact_kinds, + "kind", + "artifactGraph.artifact_kinds", + ); + + for (const kind of ARTIFACT_KINDS) { + pushIf( + errors, + !artifactMap.has(kind), + `artifactGraph.artifact_kinds is missing known artifact kind: ${kind}`, + ); + } + + for (const [kind, artifact] of artifactMap.entries()) { + pushIf( + errors, + !ARTIFACT_KINDS.has(kind), + `artifact ${kind}.kind is unknown: ${kind}`, + ); + pushIf( + errors, + !LIFECYCLES.has(artifact.lifecycle), + `artifact ${kind}.lifecycle is unknown: ${String(artifact.lifecycle)}`, + ); + pushIf( + errors, + !MODALITIES.has(artifact.modality), + `artifact ${kind}.modality is unknown: ${String(artifact.modality)}`, + ); + pushIf( + errors, + !ARTIFACT_IMPLEMENTATION_STATUSES.has(artifact.implementation_status), + `artifact ${kind}.implementation_status is unknown: ${String(artifact.implementation_status)}`, + ); + validateStringArray(errors, kind, "truth_sources", artifact.truth_sources); + validateEnumArray( + errors, + kind, + "viewer_surfaces", + artifact.viewer_surfaces, + VIEWER_SURFACES, + ); + validateEnumArray( + errors, + kind, + "evidence_events", + artifact.evidence_events, + EVIDENCE_EVENTS, + ); + validateStringArray( + errors, + kind, + "task_index_fields", + artifact.task_index_fields, + ); + validateStringArray( + errors, + kind, + "current_contracts", + artifact.current_contracts, + { allowEmpty: true }, + ); + pushIf( + errors, + !isNonEmptyString(artifact.notes), + `artifact ${kind}.notes must be set`, + ); + + if ( + artifact.implementation_status === "current" || + artifact.implementation_status === "partial" + ) { + const indexFields = new Set( + Array.isArray(artifact.task_index_fields) + ? artifact.task_index_fields + : [], + ); + for (const requiredField of REQUIRED_ARTIFACT_INDEX_FIELDS) { + pushIf( + errors, + !indexFields.has(requiredField), + `artifact ${kind}.task_index_fields must include ${requiredField}`, + ); + } + } + } + + return { + errors, + artifactMap, + }; +} + +function hasIntersection(left, right) { + if (!Array.isArray(left) || !Array.isArray(right)) { + return false; + } + const rightSet = new Set(right); + return left.some((value) => rightSet.has(value)); +} + +function validateContractArtifactGraph(errors, contract, artifactGraphRefs) { + const contractKey = contract.contract_key; + if (!Array.isArray(contract.artifact_kinds)) { + return; + } + + for (const artifactKind of contract.artifact_kinds) { + const artifact = artifactGraphRefs.artifactMap.get(artifactKind); + if (!artifact) { + errors.push(`${contractKey}.artifact_kinds references artifact not defined in graph: ${artifactKind}`); + continue; + } + + pushIf( + errors, + Array.isArray(artifact.current_contracts) && + artifact.current_contracts.length > 0 && + !artifact.current_contracts.includes(contractKey), + `${contractKey}.artifact_kinds contains ${artifactKind}, but artifactGraph.${artifactKind}.current_contracts does not include ${contractKey}`, + ); + pushIf( + errors, + !hasIntersection(contract.truth_source, artifact.truth_sources), + `${contractKey}.artifact_kinds ${artifactKind} has no truth_source intersection with artifact graph`, + ); + pushIf( + errors, + !hasIntersection(contract.viewer_surface, artifact.viewer_surfaces), + `${contractKey}.artifact_kinds ${artifactKind} has no viewer_surface intersection with artifact graph`, + ); + pushIf( + errors, + !hasIntersection(contract.evidence_events, artifact.evidence_events), + `${contractKey}.artifact_kinds ${artifactKind} has no evidence_events intersection with artifact graph`, + ); + } +} + +function resolveExecutorAdapterKey(executor) { + if (!isPlainObject(executor)) { + return null; + } + if (!isNonEmptyString(executor.executor_kind) || !isNonEmptyString(executor.binding_key)) { + return null; + } + return `${executor.executor_kind}:${executor.binding_key}`; +} + +function validateSubset(errors, label, actualValues, requiredValues) { + const actualSet = new Set(Array.isArray(actualValues) ? actualValues : []); + for (const requiredValue of requiredValues || []) { + pushIf( + errors, + !actualSet.has(requiredValue), + `${label} must include ${requiredValue}`, + ); + } +} + +function validateArtifactPolicy(errors, label, policy, artifactGraphRefs) { + pushIf(errors, !isPlainObject(policy), `${label}.artifact_policy must be an object`); + if (!isPlainObject(policy)) { + return; + } + pushIf( + errors, + !PROFILE_WRITE_MODES.has(policy.write_mode), + `${label}.artifact_policy.write_mode is unknown: ${String(policy.write_mode)}`, + ); + validateEnumArray( + errors, + label, + "artifact_policy.artifact_kinds", + policy.artifact_kinds, + ARTIFACT_KINDS, + ); + if (Array.isArray(policy.artifact_kinds)) { + for (const artifactKind of policy.artifact_kinds) { + pushIf( + errors, + !artifactGraphRefs.artifactMap.has(artifactKind), + `${label}.artifact_policy.artifact_kinds references artifact not defined in graph: ${artifactKind}`, + ); + } + } + validateEnumArray( + errors, + label, + "artifact_policy.viewer_surfaces", + policy.viewer_surfaces, + VIEWER_SURFACES, + ); +} + +function validateExecutionProfiles(registry, profiles, matrixRefs, artifactGraphRefs) { + const errors = []; + pushIf(errors, profiles.version !== 1, "executionProfiles.version must be 1"); + pushIf( + errors, + profiles.status !== "current", + "executionProfiles.status must be current", + ); + pushIf( + errors, + !isNonEmptyString(profiles.owner), + "executionProfiles.owner must be set", + ); + + const contractMap = collectUniqueObjects( + errors, + registry.contracts, + "contract_key", + "runtimeContracts.contracts", + ); + const profileMap = collectUniqueObjects( + errors, + profiles.profiles, + "profile_key", + "executionProfiles.profiles", + ); + const adapterMap = collectUniqueObjects( + errors, + profiles.executor_adapters, + "adapter_key", + "executionProfiles.executor_adapters", + ); + + for (const [adapterKey, adapter] of adapterMap.entries()) { + pushIf( + errors, + !LIFECYCLES.has(adapter.lifecycle), + `executor adapter ${adapterKey}.lifecycle is unknown: ${String(adapter.lifecycle)}`, + ); + pushIf( + errors, + !EXECUTOR_KINDS.has(adapter.executor_kind), + `executor adapter ${adapterKey}.executor_kind is unknown: ${String(adapter.executor_kind)}`, + ); + pushIf( + errors, + !isNonEmptyString(adapter.binding_key), + `executor adapter ${adapterKey}.binding_key must be set`, + ); + pushIf( + errors, + isNonEmptyString(adapter.executor_kind) && + isNonEmptyString(adapter.binding_key) && + adapterKey !== `${adapter.executor_kind}:${adapter.binding_key}`, + `executor adapter ${adapterKey}.adapter_key must equal executor_kind:binding_key`, + ); + validateEnumArray( + errors, + adapterKey, + "supported_contracts", + adapter.supported_contracts, + new Set(contractMap.keys()), + ); + for (const fieldName of [ + "supports_progress", + "supports_cancel", + "supports_resume", + "supports_artifact", + ]) { + pushIf( + errors, + typeof adapter[fieldName] !== "boolean", + `executor adapter ${adapterKey}.${fieldName} must be boolean`, + ); + } + validateEnumArray( + errors, + adapterKey, + "artifact_output_kinds", + adapter.artifact_output_kinds, + ARTIFACT_KINDS, + ); + validateEnumArray( + errors, + adapterKey, + "permission_requirements", + adapter.permission_requirements, + PERMISSIONS, + ); + validateStringArray( + errors, + adapterKey, + "credential_requirements", + adapter.credential_requirements, + { allowEmpty: true }, + ); + validateEnumArray( + errors, + adapterKey, + "failure_mapping", + adapter.failure_mapping, + FAILURE_REASONS, + ); + validateEnumArray( + errors, + adapterKey, + "evidence_events", + adapter.evidence_events, + EVIDENCE_EVENTS, + ); + pushIf( + errors, + !isNonEmptyString(adapter.notes), + `executor adapter ${adapterKey}.notes must be set`, + ); + } + + const supportedContractsByProfile = new Set(); + for (const [profileKey, profile] of profileMap.entries()) { + pushIf( + errors, + !LIFECYCLES.has(profile.lifecycle), + `execution profile ${profileKey}.lifecycle is unknown: ${String(profile.lifecycle)}`, + ); + validateEnumArray( + errors, + profileKey, + "supported_contracts", + profile.supported_contracts, + new Set(contractMap.keys()), + ); + validateEnumArray( + errors, + profileKey, + "model_role_slots", + profile.model_role_slots, + matrixRefs.modelRoleSlots, + ); + validateEnumArray( + errors, + profileKey, + "permission_profile_keys", + profile.permission_profile_keys, + PERMISSIONS, + ); + validateEnumArray( + errors, + profileKey, + "executor_adapter_keys", + profile.executor_adapter_keys, + new Set(adapterMap.keys()), + ); + validateArtifactPolicy(errors, profileKey, profile.artifact_policy, artifactGraphRefs); + validateEnumArray( + errors, + profileKey, + "limecore_policy_refs", + profile.limecore_policy_refs, + LIMECORE_POLICY_REFS, + ); + pushIf( + errors, + !isNonEmptyString(profile.user_lock_policy), + `execution profile ${profileKey}.user_lock_policy must be set`, + ); + validateStringArray( + errors, + profileKey, + "fallback_behavior", + profile.fallback_behavior, + ); + validateEnumArray( + errors, + profileKey, + "evidence_events", + profile.evidence_events, + EVIDENCE_EVENTS, + ); + validateStringArray(errors, profileKey, "audit_fields", profile.audit_fields); + pushIf( + errors, + !isNonEmptyString(profile.notes), + `execution profile ${profileKey}.notes must be set`, + ); + + for (const contractKey of profile.supported_contracts || []) { + const contract = contractMap.get(contractKey); + if (!contract) { + continue; + } + supportedContractsByProfile.add(contractKey); + const adapterKey = resolveExecutorAdapterKey(contract.executor_binding); + validateSubset( + errors, + `execution profile ${profileKey}.model_role_slots`, + profile.model_role_slots, + [contract.routing_slot], + ); + validateSubset( + errors, + `execution profile ${profileKey}.permission_profile_keys`, + profile.permission_profile_keys, + contract.permission_profile_keys, + ); + validateSubset( + errors, + `execution profile ${profileKey}.limecore_policy_refs`, + profile.limecore_policy_refs, + contract.limecore_policy_refs, + ); + validateSubset( + errors, + `execution profile ${profileKey}.artifact_policy.artifact_kinds`, + profile.artifact_policy?.artifact_kinds, + contract.artifact_kinds, + ); + validateSubset( + errors, + `execution profile ${profileKey}.artifact_policy.viewer_surfaces`, + profile.artifact_policy?.viewer_surfaces, + contract.viewer_surface, + ); + if (adapterKey) { + validateSubset( + errors, + `execution profile ${profileKey}.executor_adapter_keys`, + profile.executor_adapter_keys, + [adapterKey], + ); + } + } + } + + for (const [contractKey, contract] of contractMap.entries()) { + if (contract.lifecycle !== "current") { + continue; + } + pushIf( + errors, + !supportedContractsByProfile.has(contractKey), + `current contract ${contractKey} must be covered by an execution profile`, + ); + const adapterKey = resolveExecutorAdapterKey(contract.executor_binding); + const adapter = adapterKey ? adapterMap.get(adapterKey) : null; + pushIf( + errors, + !adapter, + `current contract ${contractKey}.executor_binding must be defined in executionProfiles.executor_adapters: ${String(adapterKey)}`, + ); + if (!adapter) { + continue; + } + validateSubset( + errors, + `executor adapter ${adapterKey}.supported_contracts`, + adapter.supported_contracts, + [contractKey], + ); + for (const fieldName of [ + "supports_progress", + "supports_cancel", + "supports_resume", + "supports_artifact", + ]) { + pushIf( + errors, + adapter[fieldName] !== contract.executor_binding[fieldName], + `current contract ${contractKey}.executor_binding.${fieldName} must match executor adapter ${adapterKey}`, + ); + } + validateSubset( + errors, + `executor adapter ${adapterKey}.artifact_output_kinds`, + adapter.artifact_output_kinds, + contract.artifact_kinds, + ); + validateSubset( + errors, + `executor adapter ${adapterKey}.permission_requirements`, + adapter.permission_requirements, + contract.permission_profile_keys, + ); + validateSubset( + errors, + `executor adapter ${adapterKey}.failure_mapping`, + adapter.failure_mapping, + contract.executor_binding.failure_mapping, + ); + } + + return { + errors, + profileCount: profileMap.size, + adapterCount: adapterMap.size, + }; +} + +function validateContractRegistry(registry, matrixRefs, artifactGraphRefs) { const errors = []; pushIf(errors, registry.version !== 1, "registry.version must be 1"); pushIf(errors, registry.status !== "current", "registry.status must be current"); @@ -508,6 +1039,7 @@ function validateContractRegistry(registry, matrixRefs) { } pushIf(errors, !isNonEmptyString(contract.owner_surface), `${contractKey}.owner_surface must be set`); validateEntryBindings(errors, contract); + validateContractArtifactGraph(errors, contract, artifactGraphRefs); } return errors; @@ -523,7 +1055,7 @@ function validateRequiredDocs() { }); } -function renderSuccess(registry, matrix) { +function renderSuccess(registry, matrix, graph, profileReport) { const currentCount = registry.contracts.filter( (contract) => contract.lifecycle === "current", ).length; @@ -533,19 +1065,35 @@ function renderSuccess(registry, matrix) { ` current: ${currentCount}`, ` capabilities: ${matrix.capabilities.length}`, ` model roles: ${matrix.model_roles.length}`, + ` artifact kinds: ${graph.artifact_kinds.length}`, + ` execution profiles: ${profileReport.profileCount}`, + ` executor adapters: ${profileReport.adapterCount}`, ` registry: ${CONTRACT_PATH}`, ` matrix: ${CAPABILITY_MATRIX_PATH}`, + ` graph: ${ARTIFACT_GRAPH_PATH}`, + ` profiles: ${EXECUTION_PROFILE_PATH}`, ].join("\n"); } function main() { const registry = readJson(CONTRACT_PATH); const matrix = readJson(CAPABILITY_MATRIX_PATH); + const graph = readJson(ARTIFACT_GRAPH_PATH); + const profiles = readJson(EXECUTION_PROFILE_PATH); const matrixReport = validateCapabilityMatrix(matrix); + const graphReport = validateArtifactGraph(graph); + const profileReport = validateExecutionProfiles( + registry, + profiles, + matrixReport, + graphReport, + ); const errors = [ ...validateRequiredDocs(), ...matrixReport.errors, - ...validateContractRegistry(registry, matrixReport), + ...graphReport.errors, + ...validateContractRegistry(registry, matrixReport, graphReport), + ...profileReport.errors, ]; if (errors.length > 0) { @@ -556,7 +1104,7 @@ function main() { process.exit(1); } - console.log(renderSuccess(registry, matrix)); + console.log(renderSuccess(registry, matrix, graph, profileReport)); } main(); diff --git a/scripts/lib/harness-eval-history-record.test.ts b/scripts/lib/harness-eval-history-record.test.ts index 37af9490a..20db4a630 100644 --- a/scripts/lib/harness-eval-history-record.test.ts +++ b/scripts/lib/harness-eval-history-record.test.ts @@ -287,7 +287,7 @@ describe("harness-eval-history-record", () => { }, ), ).not.toThrow(); - }, 30_000); + }, 60_000); it("应记录本地 summary 历史，并生成非 seed 的 trend / cleanup", () => { const tempRoot = createTempRoot(); @@ -442,7 +442,7 @@ describe("harness-eval-history-record", () => { ), ), ).toBe(true); - }, 30_000); + }, 90_000); it("并发记录 history 时不应覆盖已有样本", async () => { const tempRoot = createTempRoot(); @@ -478,5 +478,5 @@ describe("harness-eval-history-record", () => { expect(historyFiles).toHaveLength(2); expect(firstResult.recordedSummaryPath).not.toBe(secondResult.recordedSummaryPath); - }, 30_000); + }, 60_000); }); diff --git a/scripts/release-updater-manifest.test.mjs b/scripts/release-updater-manifest.test.mjs index 7d6b6259f..017927e20 100644 --- a/scripts/release-updater-manifest.test.mjs +++ b/scripts/release-updater-manifest.test.mjs @@ -303,7 +303,7 @@ describe("GitHub release asset staging", () => { "arm-sig", ); writeFile( - path.join(assetsDir, "aarch64-apple-darwin", "Lime_1.24.0_aarch64.dmg"), + path.join(assetsDir, "aarch64-apple-darwin", "Lime_1.25.0_aarch64.dmg"), ); writeFile(path.join(assetsDir, "x86_64-apple-darwin", "Lime.app.tar.gz")); writeFile( @@ -311,7 +311,7 @@ describe("GitHub release asset staging", () => { "x64-sig", ); writeFile( - path.join(assetsDir, "x86_64-apple-darwin", "Lime_1.24.0_x64.dmg"), + path.join(assetsDir, "x86_64-apple-darwin", "Lime_1.25.0_x64.dmg"), ); writeFile(latestPath, "{}"); @@ -319,29 +319,29 @@ describe("GitHub release asset staging", () => { assetsDir, extraAssets: [latestPath], outDir, - version: "v1.24.0", + version: "v1.25.0", }); expect(copied.map((item) => item.name).sort()).toEqual( [ - "Lime_1.24.0_aarch64.app.tar.gz", - "Lime_1.24.0_aarch64.app.tar.gz.sig", - "Lime_1.24.0_aarch64.dmg", - "Lime_1.24.0_x64.app.tar.gz", - "Lime_1.24.0_x64.app.tar.gz.sig", - "Lime_1.24.0_x64.dmg", + "Lime_1.25.0_aarch64.app.tar.gz", + "Lime_1.25.0_aarch64.app.tar.gz.sig", + "Lime_1.25.0_aarch64.dmg", + "Lime_1.25.0_x64.app.tar.gz", + "Lime_1.25.0_x64.app.tar.gz.sig", + "Lime_1.25.0_x64.dmg", "latest.json", ].sort(), ); expect( fs.readFileSync( - path.join(outDir, "Lime_1.24.0_aarch64.app.tar.gz.sig"), + path.join(outDir, "Lime_1.25.0_aarch64.app.tar.gz.sig"), "utf8", ), ).toBe("arm-sig"); expect( fs.readFileSync( - path.join(outDir, "Lime_1.24.0_x64.app.tar.gz.sig"), + path.join(outDir, "Lime_1.25.0_x64.app.tar.gz.sig"), "utf8", ), ).toBe("x64-sig"); diff --git a/src-tauri/Cargo.lock b/src-tauri/Cargo.lock index dc8352660..e0556e6fa 100644 --- a/src-tauri/Cargo.lock +++ b/src-tauri/Cargo.lock @@ -2381,7 +2381,7 @@ dependencies = [ "dtoa-short", "itoa", "matches", - "phf 0.8.0", + "phf 0.10.1", "proc-macro2", "quote", "smallvec", @@ -2397,7 +2397,7 @@ dependencies = [ "cssparser-macros", "dtoa-short", "itoa", - "phf 0.8.0", + "phf 0.11.3", "smallvec", ] @@ -4338,7 +4338,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.56.0", + "windows-core 0.62.2", ] [[package]] @@ -5065,7 +5065,7 @@ dependencies = [ [[package]] name = "lime" -version = "1.24.0" +version = "1.25.0" dependencies = [ "anyhow", "arboard", @@ -5075,7 +5075,9 @@ dependencies = [ "axum 0.7.9", "axum-server", "base64 0.22.1", + "block", "bytes", + "bzip2", "chrono", "chrono-tz", "cocoa", @@ -5168,7 +5170,7 @@ dependencies = [ [[package]] name = "lime-agent" -version = "1.24.0" +version = "1.25.0" dependencies = [ "anyhow", "aster-core", @@ -5197,7 +5199,7 @@ dependencies = [ [[package]] name = "lime-browser-runtime" -version = "1.24.0" +version = "1.25.0" dependencies = [ "chrono", "futures", @@ -5214,7 +5216,7 @@ dependencies = [ [[package]] name = "lime-cli" -version = "1.24.0" +version = "1.25.0" dependencies = [ "clap", "lime-core", @@ -5226,7 +5228,7 @@ dependencies = [ [[package]] name = "lime-config" -version = "1.24.0" +version = "1.25.0" dependencies = [ "async-trait", "lime-core", @@ -5242,7 +5244,7 @@ dependencies = [ [[package]] name = "lime-core" -version = "1.24.0" +version = "1.25.0" dependencies = [ "aster-models", "async-trait", @@ -5295,7 +5297,7 @@ dependencies = [ [[package]] name = "lime-gateway" -version = "1.24.0" +version = "1.25.0" dependencies = [ "aes", "axum 0.7.9", @@ -5325,7 +5327,7 @@ dependencies = [ [[package]] name = "lime-infra" -version = "1.24.0" +version = "1.25.0" dependencies = [ "chrono", "dashmap 5.5.3", @@ -5345,7 +5347,7 @@ dependencies = [ [[package]] name = "lime-mcp" -version = "1.24.0" +version = "1.25.0" dependencies = [ "async-trait", "dirs 5.0.1", @@ -5361,7 +5363,7 @@ dependencies = [ [[package]] name = "lime-media-runtime" -version = "1.24.0" +version = "1.25.0" dependencies = [ "axum 0.7.9", "chrono", @@ -5392,7 +5394,7 @@ dependencies = [ [[package]] name = "lime-processor" -version = "1.24.0" +version = "1.25.0" dependencies = [ "async-trait", "lime-core", @@ -5411,7 +5413,7 @@ dependencies = [ [[package]] name = "lime-providers" -version = "1.24.0" +version = "1.25.0" dependencies = [ "anyhow", "async-stream", @@ -5466,7 +5468,7 @@ dependencies = [ [[package]] name = "lime-server" -version = "1.24.0" +version = "1.25.0" dependencies = [ "aster-core", "async-stream", @@ -5510,7 +5512,7 @@ dependencies = [ [[package]] name = "lime-server-utils" -version = "1.24.0" +version = "1.25.0" dependencies = [ "axum 0.7.9", "futures", @@ -5525,7 +5527,7 @@ dependencies = [ [[package]] name = "lime-services" -version = "1.24.0" +version = "1.25.0" dependencies = [ "anyhow", "aster-core", @@ -5568,7 +5570,7 @@ dependencies = [ [[package]] name = "lime-skills" -version = "1.24.0" +version = "1.25.0" dependencies = [ "async-trait", "dirs 5.0.1", @@ -5586,7 +5588,7 @@ dependencies = [ [[package]] name = "lime-websocket" -version = "1.24.0" +version = "1.25.0" dependencies = [ "axum 0.7.9", "chrono", @@ -6265,7 +6267,7 @@ version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.4.0", "proc-macro2", "quote", "syn 2.0.117", @@ -7012,9 +7014,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" dependencies = [ - "phf_macros 0.8.0", "phf_shared 0.8.0", - "proc-macro-hack", ] [[package]] @@ -7023,7 +7023,9 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" dependencies = [ + "phf_macros 0.10.0", "phf_shared 0.10.0", + "proc-macro-hack", ] [[package]] @@ -7127,12 +7129,12 @@ dependencies = [ [[package]] name = "phf_macros" -version = "0.8.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" dependencies = [ - "phf_generator 0.8.0", - "phf_shared 0.8.0", + "phf_generator 0.10.0", + "phf_shared 0.10.0", "proc-macro-hack", "proc-macro2", "quote", @@ -7533,7 +7535,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d" dependencies = [ "anyhow", - "itertools 0.12.1", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -8256,6 +8258,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c665f33d38cea657d9614f766881e4d510e0eda4239891eea56b4cadcf01801b" dependencies = [ "aws-lc-rs", + "log", "once_cell", "ring", "rustls-pki-types", @@ -8619,7 +8622,7 @@ dependencies = [ "sentry-panic", "sentry-tracing", "tokio", - "ureq", + "ureq 3.2.0", ] [[package]] @@ -9032,7 +9035,29 @@ version = "3.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b1fdf65dd6331831494dd616b30351c38e96e45921a27745cf98490458b90bb" dependencies = [ - "dirs 4.0.0", + "dirs 6.0.0", +] + +[[package]] +name = "sherpa-onnx" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071b3daa7bf126fe47caf8f1f7fcd5af6a4ff40e67f410503c81de55b7608b20" +dependencies = [ + "serde", + "serde_json", + "sherpa-onnx-sys", +] + +[[package]] +name = "sherpa-onnx-sys" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c231bdd33c5b31cbf42996e426d186d453b3fc0574f194ecc661bfae7aea1114" +dependencies = [ + "bzip2", + "tar", + "ureq 2.12.1", ] [[package]] @@ -10951,6 +10976,22 @@ version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "once_cell", + "rustls 0.23.36", + "rustls-pki-types", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "ureq" version = "3.2.0" @@ -11124,6 +11165,7 @@ dependencies = [ "serde", "serde_json", "sha2", + "sherpa-onnx", "thiserror 1.0.69", "tokio", "tokio-tungstenite 0.24.0", @@ -11436,6 +11478,15 @@ version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.6", +] + [[package]] name = "webpki-roots" version = "1.0.6" diff --git a/src-tauri/Cargo.toml b/src-tauri/Cargo.toml index adc65dac6..05d53794d 100644 --- a/src-tauri/Cargo.toml +++ b/src-tauri/Cargo.toml @@ -9,7 +9,7 @@ exclude = [ resolver = "2" [workspace.package] -version = "1.24.0" +version = "1.25.0" edition = "2021" authors = ["coso"] repository = "https://github.com/aiclientproxy/lime" @@ -95,6 +95,7 @@ tar = "0.4" fs2 = "0.4" indexmap = { version = "2", features = ["serde"] } zip = "0.6" +bzip2 = "0.4" dashmap = "5" notify = { version = "6", default-features = false, features = ["macos_fsevent"] } parking_lot = "0.12" @@ -182,6 +183,9 @@ features = [ version = "0.52" # macOS 平台依赖 +[workspace.dependencies.block] +version = "0.1" + [workspace.dependencies.cocoa] version = "0.26" @@ -193,7 +197,7 @@ version = "2.4" [package] name = "lime" -version = "1.24.0" +version = "1.25.0" description = "AI API Proxy Desktop App" authors = ["you"] edition = "2021" @@ -295,6 +299,7 @@ tar.workspace = true fs2.workspace = true indexmap.workspace = true zip.workspace = true +bzip2.workspace = true dashmap.workspace = true notify.workspace = true parking_lot.workspace = true @@ -343,6 +348,7 @@ winreg.workspace = true # macOS specific dependencies for browser interceptor [target.'cfg(target_os = "macos")'.dependencies] +block.workspace = true cocoa.workspace = true objc.workspace = true @@ -351,8 +357,10 @@ proptest.workspace = true tempfile.workspace = true [features] -default = ["custom-protocol"] +default = ["custom-protocol", "local-sensevoice"] custom-protocol = ["tauri/custom-protocol"] # 本地 Whisper 语音识别（编译很慢，CI 默认不启用） local-whisper = ["lime-services/local-whisper"] +# 本地 SenseVoice Small 语音识别（模型按需下载，不进入安装包） +local-sensevoice = ["lime-services/local-sensevoice"] notification = [] # 预留特性：系统通知功能 diff --git a/src-tauri/capabilities/default.json b/src-tauri/capabilities/default.json index 03bcf1650..6102aa93f 100644 --- a/src-tauri/capabilities/default.json +++ b/src-tauri/capabilities/default.json @@ -21,6 +21,7 @@ "core:window:allow-set-title", "core:window:allow-inner-size", "core:window:allow-outer-size", + "core:window:allow-start-dragging", "core:webview:default", "core:app:default", "core:resources:default", @@ -40,4 +41,4 @@ "global-shortcut:allow-is-registered", "autostart:default" ] -} \ No newline at end of file +} diff --git a/src-tauri/crates/agent/src/credential_bridge.rs b/src-tauri/crates/agent/src/credential_bridge.rs index c1201b96c..fb867e4c1 100644 --- a/src-tauri/crates/agent/src/credential_bridge.rs +++ b/src-tauri/crates/agent/src/credential_bridge.rs @@ -408,11 +408,12 @@ fn should_disable_provider_default_fast_model(config: &AsterProviderConfig) -> b /// - `https://open.bigmodel.cn/api/paas/v4` -> (`https://open.bigmodel.cn`, `api/paas/v4`) /// - `https://localhost:8080/v1` -> (`https://localhost:8080`, `v1`) fn split_url_host_and_path(url: &str) -> (String, String) { + let url = strip_url_query_fragment(url); // 找到 scheme 之后的 authority 部分 let after_scheme = if let Some(pos) = url.find("://") { pos + 3 } else { - return (url.to_string(), String::new()); + return (url, String::new()); }; // 找到 authority 之后的第一个 /（即路径开始） @@ -450,6 +451,95 @@ fn resolve_anthropic_env_key(config: &AsterProviderConfig) -> &'static str { } } +const LIME_TENANT_HEADER: &str = "X-Lime-Tenant-ID"; +const LIME_TENANT_PARAM: &str = "lime_tenant_id"; +const OPENAI_CUSTOM_HEADERS_ENV: &str = "OPENAI_CUSTOM_HEADERS"; + +fn normalize_lime_tenant_id(value: &str) -> Option { + let tenant_id = value.trim(); + if tenant_id.is_empty() { + return None; + } + + tenant_id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + .then(|| tenant_id.to_string()) +} + +fn parse_lime_tenant_id_from_pairs(value: &str) -> Option { + value.split('&').find_map(|pair| { + let mut parts = pair.splitn(2, '='); + let key = parts.next()?.trim(); + let value = parts.next().unwrap_or_default(); + (key == LIME_TENANT_PARAM) + .then(|| normalize_lime_tenant_id(value)) + .flatten() + }) +} + +fn lime_tenant_id_from_api_host(api_host: &str) -> Option { + let trimmed = api_host.trim(); + let query = trimmed + .find('?') + .map(|pos| trimmed[pos + 1..].split('#').next().unwrap_or_default()); + let fragment = trimmed.find('#').map(|pos| &trimmed[pos + 1..]); + + query + .and_then(parse_lime_tenant_id_from_pairs) + .or_else(|| fragment.and_then(parse_lime_tenant_id_from_pairs)) +} + +fn strip_url_query_fragment(value: &str) -> String { + let trimmed = value.trim(); + let end = [trimmed.find('?'), trimmed.find('#')] + .into_iter() + .flatten() + .min() + .unwrap_or(trimmed.len()); + + trimmed[..end].trim_end_matches('/').to_string() +} + +fn parse_openai_custom_headers_env(value: &str) -> Vec<(String, String)> { + value + .split(',') + .filter_map(|header| { + let mut parts = header.splitn(2, '='); + let key = parts.next()?.trim(); + if key.is_empty() { + return None; + } + let value = parts.next().unwrap_or_default().trim(); + Some((key.to_string(), value.to_string())) + }) + .collect() +} + +fn update_openai_lime_tenant_custom_header(tenant_id: Option<&str>) { + let existing = std::env::var(OPENAI_CUSTOM_HEADERS_ENV).unwrap_or_default(); + let mut headers = parse_openai_custom_headers_env(&existing) + .into_iter() + .filter(|(key, _)| !key.eq_ignore_ascii_case(LIME_TENANT_HEADER)) + .collect::>(); + + if let Some(tenant_id) = tenant_id.and_then(normalize_lime_tenant_id) { + headers.push((LIME_TENANT_HEADER.to_string(), tenant_id)); + } + + if headers.is_empty() { + std::env::remove_var(OPENAI_CUSTOM_HEADERS_ENV); + return; + } + + let value = headers + .into_iter() + .map(|(key, value)| format!("{key}={value}")) + .collect::>() + .join(","); + std::env::set_var(OPENAI_CUSTOM_HEADERS_ENV, value); +} + fn set_provider_env_vars(config: &AsterProviderConfig) { tracing::info!( "[CredentialBridge] set_provider_env_vars: provider_name={}, has_api_key={}, base_url={:?}", @@ -491,6 +581,9 @@ fn set_provider_env_vars(config: &AsterProviderConfig) { } else { std::env::remove_var("OPENAI_FORCE_RESPONSES_API"); } + if config.base_url.is_none() { + update_openai_lime_tenant_custom_header(None); + } } // 设置 base_url @@ -499,18 +592,22 @@ fn set_provider_env_vars(config: &AsterProviderConfig) { if let Some(base_url) = &config.base_url { match config.provider_name.as_str() { "openai" => { + let tenant_id = lime_tenant_id_from_api_host(base_url); + update_openai_lime_tenant_custom_header(tenant_id.as_deref()); + // 解析 base_url，将路径部分拆分到 OPENAI_BASE_PATH。 // Responses 模式也复用这条前缀推导，避免出现 /v1/v1/responses。 // 例如 https://open.bigmodel.cn/api/paas/v4 // -> OPENAI_HOST = https://open.bigmodel.cn // -> OPENAI_BASE_PATH = api/paas/v4/chat/completions - let (host_part, path_part) = split_url_host_and_path(base_url); + let sanitized_base_url = strip_url_query_fragment(base_url); + let (host_part, path_part) = split_url_host_and_path(&sanitized_base_url); if path_part.is_empty() { // 无路径部分（如 https://api.openai.com），直接设置 - std::env::set_var("OPENAI_HOST", base_url); + std::env::set_var("OPENAI_HOST", &sanitized_base_url); // 清除可能残留的 OPENAI_BASE_PATH，使用 Aster 默认值 std::env::remove_var("OPENAI_BASE_PATH"); - tracing::info!("[CredentialBridge] 设置 OPENAI_HOST={}", base_url); + tracing::info!("[CredentialBridge] 设置 OPENAI_HOST={}", sanitized_base_url); } else { // base_url 包含路径，需要拆分 let base_path = format!("{}/chat/completions", path_part); @@ -606,6 +703,12 @@ fn map_provider_type_to_aster_with_api_type( #[cfg(test)] mod tests { use super::*; + use std::sync::{Mutex, MutexGuard, OnceLock}; + + fn env_lock() -> MutexGuard<'static, ()> { + static LOCK: OnceLock> = OnceLock::new(); + LOCK.get_or_init(|| Mutex::new(())).lock().unwrap() + } #[test] fn test_map_provider_type_to_aster_with_api_type() { @@ -631,6 +734,7 @@ mod tests { #[test] fn test_set_provider_env_vars_openai_codex_responses_splits_prefixed_base_url() { + let _env_guard = env_lock(); std::env::remove_var("OPENAI_HOST"); std::env::remove_var("OPENAI_BASE_PATH"); std::env::remove_var("OPENAI_FORCE_RESPONSES_API"); @@ -665,6 +769,7 @@ mod tests { #[test] fn test_set_provider_env_vars_openai_codex_responses_normalizes_v1_base_url() { + let _env_guard = env_lock(); std::env::remove_var("OPENAI_HOST"); std::env::remove_var("OPENAI_BASE_PATH"); std::env::remove_var("OPENAI_FORCE_RESPONSES_API"); @@ -814,10 +919,84 @@ mod tests { let (host, path) = split_url_host_and_path("https://api.openai.com/"); assert_eq!(host, "https://api.openai.com"); assert_eq!(path, ""); + + // 查询参数和 fragment 只用于附加元数据，不应进入真实请求 URL + let (host, path) = + split_url_host_and_path("https://llm.limeai.run/openai?lime_tenant_id=tenant-0001#x=1"); + assert_eq!(host, "https://llm.limeai.run"); + assert_eq!(path, "openai"); + } + + #[test] + fn test_set_provider_env_vars_openai_lime_hub_adds_tenant_header_from_fragment() { + let _env_guard = env_lock(); + std::env::remove_var("OPENAI_HOST"); + std::env::remove_var("OPENAI_BASE_PATH"); + std::env::set_var( + OPENAI_CUSTOM_HEADERS_ENV, + "X-Other=1,X-Lime-Tenant-ID=stale", + ); + + let config = AsterProviderConfig { + provider_name: "openai".to_string(), + provider_selector: Some("lime-hub".to_string()), + model_name: "gpt-5.5".to_string(), + api_key: Some("test-key".to_string()), + base_url: Some("https://llm.limeai.run#lime_tenant_id=tenant-0001".to_string()), + credential_uuid: "test-uuid".to_string(), + force_responses_api: false, + toolshim: false, + toolshim_model: None, + }; + + set_provider_env_vars(&config); + + assert_eq!( + std::env::var("OPENAI_HOST").ok().as_deref(), + Some("https://llm.limeai.run") + ); + assert!(std::env::var("OPENAI_BASE_PATH").is_err()); + assert_eq!( + std::env::var(OPENAI_CUSTOM_HEADERS_ENV).ok().as_deref(), + Some("X-Other=1,X-Lime-Tenant-ID=tenant-0001") + ); + + std::env::remove_var(OPENAI_CUSTOM_HEADERS_ENV); + } + + #[test] + fn test_set_provider_env_vars_openai_without_lime_tenant_clears_stale_header() { + let _env_guard = env_lock(); + std::env::set_var( + OPENAI_CUSTOM_HEADERS_ENV, + "X-Lime-Tenant-ID=stale,X-Other=1", + ); + + let config = AsterProviderConfig { + provider_name: "openai".to_string(), + provider_selector: Some("openai".to_string()), + model_name: "gpt-4o".to_string(), + api_key: Some("test-key".to_string()), + base_url: Some("https://api.openai.com/v1".to_string()), + credential_uuid: "test-uuid".to_string(), + force_responses_api: false, + toolshim: false, + toolshim_model: None, + }; + + set_provider_env_vars(&config); + + assert_eq!( + std::env::var(OPENAI_CUSTOM_HEADERS_ENV).ok().as_deref(), + Some("X-Other=1") + ); + + std::env::remove_var(OPENAI_CUSTOM_HEADERS_ENV); } #[test] fn test_set_provider_env_vars_anthropic_sets_host_and_base_url() { + let _env_guard = env_lock(); std::env::remove_var("ANTHROPIC_API_KEY"); std::env::remove_var("ANTHROPIC_AUTH_TOKEN"); @@ -852,6 +1031,7 @@ mod tests { #[test] fn test_set_provider_env_vars_official_anthropic_keeps_api_key_env() { + let _env_guard = env_lock(); std::env::remove_var("ANTHROPIC_API_KEY"); std::env::remove_var("ANTHROPIC_AUTH_TOKEN"); diff --git a/src-tauri/crates/aster-rust/crates/aster/src/providers/anthropic.rs b/src-tauri/crates/aster-rust/crates/aster/src/providers/anthropic.rs index 64fee99df..0c2c09cd6 100644 --- a/src-tauri/crates/aster-rust/crates/aster/src/providers/anthropic.rs +++ b/src-tauri/crates/aster-rust/crates/aster/src/providers/anthropic.rs @@ -7,6 +7,7 @@ use serde_json::Value; use std::io; use tokio::pin; use tokio_util::io::StreamReader; +use url::{form_urlencoded, Url}; use super::api_client::{ApiClient, ApiResponse, AuthMethod}; use super::base::{ConfigKey, MessageStream, ModelInfo, Provider, ProviderMetadata, ProviderUsage}; @@ -36,6 +37,8 @@ const ANTHROPIC_KNOWN_MODELS: &[&str] = &[ const ANTHROPIC_DOC_URL: &str = "https://docs.anthropic.com/en/docs/about-claude/models"; const ANTHROPIC_API_VERSION: &str = "2023-06-01"; +const LIME_TENANT_HEADER: &str = "X-Lime-Tenant-ID"; +const LIME_TENANT_PARAM: &str = "lime_tenant_id"; fn normalize_anthropic_host(host: &str) -> String { host.trim().trim_end_matches('/').to_ascii_lowercase() @@ -62,6 +65,61 @@ fn should_use_bearer_auth_for_anthropic_host(host: &str) -> bool { .any(|needle| normalized_host.contains(needle)) } +fn parse_config_url(host: &str) -> Option { + let trimmed = host.trim(); + if trimmed.is_empty() { + return None; + } + + Url::parse(trimmed) + .or_else(|_| Url::parse(&format!("https://{trimmed}"))) + .ok() +} + +fn strip_host_url_metadata(host: &str) -> String { + if let Some(mut url) = parse_config_url(host) { + url.set_query(None); + url.set_fragment(None); + return url.to_string().trim_end_matches('/').to_string(); + } + + let trimmed = host.trim(); + let end = [trimmed.find('?'), trimmed.find('#')] + .into_iter() + .flatten() + .min() + .unwrap_or(trimmed.len()); + trimmed[..end].trim_end_matches('/').to_string() +} + +fn normalize_lime_tenant_id(value: &str) -> Option { + let tenant_id = value.trim(); + if tenant_id.is_empty() { + return None; + } + + tenant_id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + .then(|| tenant_id.to_string()) +} + +fn parse_lime_tenant_id_from_pairs(value: &str) -> Option { + form_urlencoded::parse(value.as_bytes()).find_map(|(key, value)| { + (key == LIME_TENANT_PARAM) + .then(|| normalize_lime_tenant_id(&value)) + .flatten() + }) +} + +fn lime_tenant_id_from_host(host: &str) -> Option { + let url = parse_config_url(host)?; + + url.query() + .and_then(parse_lime_tenant_id_from_pairs) + .or_else(|| url.fragment().and_then(parse_lime_tenant_id_from_pairs)) +} + fn resolve_anthropic_auth(host: &str, secret: String) -> AuthMethod { if should_use_bearer_auth_for_anthropic_host(host) { AuthMethod::BearerToken(secret) @@ -74,16 +132,21 @@ fn resolve_anthropic_auth(host: &str, secret: String) -> AuthMethod { } fn build_anthropic_api_client(host: &str, auth_secret: String) -> Result { - let should_add_x_api_key_header = should_use_bearer_auth_for_anthropic_host(host); - let auth = resolve_anthropic_auth(host, auth_secret.clone()); + let sanitized_host = strip_host_url_metadata(host); + let should_add_x_api_key_header = should_use_bearer_auth_for_anthropic_host(&sanitized_host); + let auth = resolve_anthropic_auth(&sanitized_host, auth_secret.clone()); - let mut api_client = ApiClient::new(host.to_string(), auth)? + let mut api_client = ApiClient::new(sanitized_host, auth)? .with_header("anthropic-version", ANTHROPIC_API_VERSION)?; if should_add_x_api_key_header { api_client = api_client.with_header("x-api-key", &auth_secret)?; } + if let Some(tenant_id) = lime_tenant_id_from_host(host) { + api_client = api_client.with_header(LIME_TENANT_HEADER, &tenant_id)?; + } + Ok(api_client) } @@ -106,7 +169,8 @@ impl AnthropicProvider { .get_param("ANTHROPIC_HOST") .or_else(|_| config.get_param("ANTHROPIC_BASE_URL")) .unwrap_or_else(|_| "https://api.anthropic.com".to_string()); - let api_key: String = if should_use_bearer_auth_for_anthropic_host(&host) { + let sanitized_host = strip_host_url_metadata(&host); + let api_key: String = if should_use_bearer_auth_for_anthropic_host(&sanitized_host) { config .get_secret("ANTHROPIC_AUTH_TOKEN") .or_else(|_| config.get_secret("ANTHROPIC_API_KEY"))? @@ -116,7 +180,7 @@ impl AnthropicProvider { .or_else(|_| config.get_secret("ANTHROPIC_AUTH_TOKEN"))? }; - let automatic_prompt_cache = supports_automatic_prompt_cache_for_host(&host); + let automatic_prompt_cache = supports_automatic_prompt_cache_for_host(&sanitized_host); let api_client = build_anthropic_api_client(&host, api_key)?; Ok(Self { @@ -137,7 +201,8 @@ impl AnthropicProvider { .get_secret(&config.api_key_env) .map_err(|_| anyhow::anyhow!("Missing API key: {}", config.api_key_env))?; - let automatic_prompt_cache = supports_automatic_prompt_cache_for_host(&config.base_url); + let sanitized_host = strip_host_url_metadata(&config.base_url); + let automatic_prompt_cache = supports_automatic_prompt_cache_for_host(&sanitized_host); let api_client = build_anthropic_api_client(&config.base_url, api_key)?; Ok(Self { @@ -469,6 +534,23 @@ mod tests { ); } + #[test] + fn test_lime_tenant_fragment_adds_custom_header_to_api_client() { + let api_client = build_anthropic_api_client( + "https://llm.limeai.run#lime_tenant_id=tenant-0001", + "k1".to_string(), + ) + .expect("api client should build"); + let headers = api_client.default_headers_for_test(); + + assert_eq!( + headers + .get(LIME_TENANT_HEADER) + .and_then(|value| value.to_str().ok()), + Some("tenant-0001") + ); + } + #[test] fn test_official_host_keeps_single_auth_header_mode() { let api_client = build_anthropic_api_client("https://api.anthropic.com", "k2".to_string()) diff --git a/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai.rs b/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai.rs index a1d5dbb24..5578bc1c1 100644 --- a/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai.rs +++ b/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai.rs @@ -429,8 +429,16 @@ pub fn validate_tool_schemas(tools: &mut [Value]) { } /// Ensures that the given JSON value follows the expected JSON Schema structure. -fn ensure_valid_json_schema(schema: &mut Value) { +pub(crate) fn ensure_valid_json_schema(schema: &mut Value) { if let Some(params_obj) = schema.as_object_mut() { + if params_obj + .get("type") + .and_then(|t| t.as_str()) + .is_some_and(|t| t == "array") + { + params_obj.entry("items").or_insert_with(|| json!({})); + } + // Check if this is meant to be an object type schema let is_object_type = params_obj .get("type") @@ -448,15 +456,31 @@ fn ensure_valid_json_schema(schema: &mut Value) { if let Some(properties) = params_obj.get_mut("properties") { if let Some(properties_obj) = properties.as_object_mut() { for (_key, prop) in properties_obj.iter_mut() { - if prop.is_object() - && prop.get("type").and_then(|t| t.as_str()) == Some("object") - { + if prop.is_object() { ensure_valid_json_schema(prop); } } } } } + + if let Some(items) = params_obj.get_mut("items") { + ensure_valid_json_schema(items); + } + + for keyword in ["oneOf", "anyOf", "allOf"] { + if let Some(variants) = params_obj.get_mut(keyword).and_then(Value::as_array_mut) { + for variant in variants.iter_mut() { + ensure_valid_json_schema(variant); + } + } + } + + if let Some(additional_properties) = params_obj.get_mut("additionalProperties") { + if additional_properties.is_object() { + ensure_valid_json_schema(additional_properties); + } + } } } @@ -857,6 +881,33 @@ mod tests { let mut tools = vec![original_schema.clone()]; validate_tool_schemas(&mut tools); assert_eq!(tools[0], original_schema); + + // Test case 4: Nested oneOf array schema should get items for OpenAI-compatible APIs. + let mut tools = vec![json!({ + "type": "function", + "function": { + "name": "test_func", + "description": "test description", + "parameters": { + "type": "object", + "properties": { + "message": { + "oneOf": [ + { "type": "string" }, + { "type": "array" } + ] + } + }, + "required": ["message"] + } + } + })]; + + validate_tool_schemas(&mut tools); + assert_eq!( + tools[0]["function"]["parameters"]["properties"]["message"]["oneOf"][1]["items"], + json!({}) + ); } const OPENAI_TOOL_USE_RESPONSE: &str = r#"{ diff --git a/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai_responses.rs b/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai_responses.rs index 782e0cc40..d98dfdc61 100644 --- a/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai_responses.rs +++ b/src-tauri/crates/aster-rust/crates/aster/src/providers/formats/openai_responses.rs @@ -450,7 +450,7 @@ pub fn create_responses_request( } if !tools.is_empty() { - let tools_spec: Vec = tools + let mut tools_spec: Vec = tools .iter() .map(|tool| { json!({ @@ -461,6 +461,11 @@ pub fn create_responses_request( }) }) .collect(); + for tool in tools_spec.iter_mut() { + if let Some(parameters) = tool.get_mut("parameters") { + super::openai::ensure_valid_json_schema(parameters); + } + } payload .as_object_mut() diff --git a/src-tauri/crates/aster-rust/crates/aster/src/providers/openai.rs b/src-tauri/crates/aster-rust/crates/aster/src/providers/openai.rs index e18e9dae9..394a70fa8 100644 --- a/src-tauri/crates/aster-rust/crates/aster/src/providers/openai.rs +++ b/src-tauri/crates/aster-rust/crates/aster/src/providers/openai.rs @@ -654,6 +654,7 @@ impl EmbeddingCapable for OpenAiProvider { mod tests { use super::OpenAiProvider; use crate::conversation::message::Message; + use crate::model::ModelConfig; use crate::session::TurnContextOverride; use std::collections::HashMap; @@ -675,6 +676,32 @@ mod tests { std::env::remove_var("OPENAI_FORCE_RESPONSES_API"); } + #[tokio::test] + #[serial_test::serial] + async fn test_from_env_applies_openai_custom_headers() { + std::env::set_var("OPENAI_API_KEY", "sk-test"); + std::env::set_var("OPENAI_HOST", "https://llm.limeai.run"); + std::env::remove_var("OPENAI_BASE_PATH"); + std::env::set_var("OPENAI_CUSTOM_HEADERS", "X-Lime-Tenant-ID=tenant-0001"); + + let provider = OpenAiProvider::from_env(ModelConfig::new("gpt-4o").unwrap()) + .await + .expect("provider should build"); + + assert_eq!( + provider + .api_client + .default_headers_for_test() + .get("X-Lime-Tenant-ID") + .and_then(|value| value.to_str().ok()), + Some("tenant-0001") + ); + + std::env::remove_var("OPENAI_API_KEY"); + std::env::remove_var("OPENAI_HOST"); + std::env::remove_var("OPENAI_CUSTOM_HEADERS"); + } + #[test] fn test_responses_path_from_base_path_reuses_existing_prefix() { assert_eq!( diff --git a/src-tauri/crates/aster-rust/crates/aster/src/tools/agent_control.rs b/src-tauri/crates/aster-rust/crates/aster/src/tools/agent_control.rs index 84a9132d0..77aa6ec49 100644 --- a/src-tauri/crates/aster-rust/crates/aster/src/tools/agent_control.rs +++ b/src-tauri/crates/aster-rust/crates/aster/src/tools/agent_control.rs @@ -472,7 +472,7 @@ impl Tool for SendInputTool { "oneOf": [ { "type": "string" }, { "type": "object" }, - { "type": "array" }, + { "type": "array", "items": {} }, { "type": "number" }, { "type": "boolean" }, { "type": "null" } @@ -1181,6 +1181,32 @@ mod tests { )); } + #[test] + fn test_send_message_array_schema_declares_items() { + let tool = SendInputTool::new(Arc::new(|_request| { + Box::pin(async move { + Ok(SendInputResponse { + submission_id: "unused".to_string(), + extra: BTreeMap::new(), + }) + }) + })); + + let schema = tool.input_schema(); + let message_variants = schema["properties"]["message"]["oneOf"] + .as_array() + .expect("message oneOf should be an array"); + let array_variant = message_variants + .iter() + .find(|variant| variant["type"] == "array") + .expect("message should accept arrays"); + + assert!( + array_variant.get("items").is_some(), + "OpenAI-compatible tool schemas require array schemas to declare items" + ); + } + #[tokio::test] async fn test_send_message_tool_accepts_current_surface() { let tool = SendInputTool::new(Arc::new(|request| { diff --git a/src-tauri/crates/core/src/config/mod.rs b/src-tauri/crates/core/src/config/mod.rs index c7947706e..8bcbf5d0b 100644 --- a/src-tauri/crates/core/src/config/mod.rs +++ b/src-tauri/crates/core/src/config/mod.rs @@ -37,14 +37,14 @@ pub use types::{ MultiSearchEngineEntryConfig, NativeAgentConfig, NavigationConfig, OpenAIAsrConfig, PairingSettings, ProviderConfig, ProviderModelsConfig, ProvidersConfig, QuotaExceededConfig, RateLimitSettings, RemoteManagementConfig, ResponseCacheSettings, RetrySettings, RoutingConfig, - ScreenshotChatConfig, SearchEngine, ServerConfig, ShellEnvironmentImportConfig, TaskSchedule, - TelegramAccountConfig, TelegramBotConfig, TelegramGroupConfig, TelegramTopicConfig, TlsConfig, - ToolCallingConfig, ToolExecutionOverrideConfig, ToolExecutionPolicyConfig, - ToolExecutionRestrictionProfileConfig, ToolExecutionSandboxProfileConfig, - ToolExecutionWarningPolicyConfig, UpdateCheckConfig, UserProfile, VertexApiKeyEntry, - VertexModelAlias, VoiceInputConfig, VoiceInstruction, VoiceOutputConfig, VoiceOutputMode, - VoiceProcessorConfig, WebSearchConfig, WebSearchProvider, WechatAccountConfig, WechatBotConfig, - WechatGroupConfig, WhisperLocalConfig, WhisperModelSize, WorkspacePreferencesConfig, - WorkspaceSandboxConfig, XunfeiConfig, DEFAULT_API_KEY, + ScreenshotChatConfig, SearchEngine, SenseVoiceLocalConfig, ServerConfig, + ShellEnvironmentImportConfig, TaskSchedule, TelegramAccountConfig, TelegramBotConfig, + TelegramGroupConfig, TelegramTopicConfig, TlsConfig, ToolCallingConfig, + ToolExecutionOverrideConfig, ToolExecutionPolicyConfig, ToolExecutionRestrictionProfileConfig, + ToolExecutionSandboxProfileConfig, ToolExecutionWarningPolicyConfig, UpdateCheckConfig, + UserProfile, VertexApiKeyEntry, VertexModelAlias, VoiceInputConfig, VoiceInstruction, + VoiceOutputConfig, VoiceOutputMode, VoiceProcessorConfig, WebSearchConfig, WebSearchProvider, + WechatAccountConfig, WechatBotConfig, WechatGroupConfig, WhisperLocalConfig, WhisperModelSize, + WorkspacePreferencesConfig, WorkspaceSandboxConfig, XunfeiConfig, DEFAULT_API_KEY, }; pub use yaml::{load_config, save_config, ConfigError, ConfigManager, YamlService}; diff --git a/src-tauri/crates/core/src/config/types.rs b/src-tauri/crates/core/src/config/types.rs index 2643b9a03..11370c561 100644 --- a/src-tauri/crates/core/src/config/types.rs +++ b/src-tauri/crates/core/src/config/types.rs @@ -49,6 +49,8 @@ pub enum AsrProviderType { /// 本地 Whisper（离线） #[default] WhisperLocal, + /// 本地 SenseVoice Small（离线） + SenseVoiceLocal, /// 讯飞语音识别 Xunfei, /// 百度语音识别 @@ -96,6 +98,9 @@ pub struct AsrCredentialEntry { /// Whisper 本地配置（仅 WhisperLocal） #[serde(default, skip_serializing_if = "Option::is_none")] pub whisper_config: Option, + /// SenseVoice 本地配置（仅 SenseVoiceLocal） + #[serde(default, skip_serializing_if = "Option::is_none")] + pub sensevoice_config: Option, /// 讯飞配置（仅 Xunfei） #[serde(default, skip_serializing_if = "Option::is_none")] pub xunfei_config: Option, @@ -122,6 +127,50 @@ pub struct WhisperLocalConfig { pub model_path: Option, } +/// SenseVoice 本地配置 +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct SenseVoiceLocalConfig { + /// 模型 ID + #[serde(default = "default_sensevoice_model_id")] + pub model_id: String, + /// 模型目录（可选，默认使用应用数据目录） + #[serde(default, skip_serializing_if = "Option::is_none")] + pub model_dir: Option, + /// 是否启用逆文本标准化 + #[serde(default = "default_sensevoice_use_itn")] + pub use_itn: bool, + /// 推理线程数 + #[serde(default = "default_sensevoice_num_threads")] + pub num_threads: u16, + /// VAD 模型 ID（可选） + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vad_model_id: Option, +} + +fn default_sensevoice_model_id() -> String { + "sensevoice-small-int8-2024-07-17".to_string() +} + +fn default_sensevoice_use_itn() -> bool { + true +} + +fn default_sensevoice_num_threads() -> u16 { + 4 +} + +impl Default for SenseVoiceLocalConfig { + fn default() -> Self { + Self { + model_id: default_sensevoice_model_id(), + model_dir: None, + use_itn: default_sensevoice_use_itn(), + num_threads: default_sensevoice_num_threads(), + vad_model_id: Some("silero-vad-onnx".to_string()), + } + } +} + /// 讯飞语音配置 #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] pub struct XunfeiConfig { @@ -3384,6 +3433,7 @@ mod unit_tests { model: WhisperModelSize::Base, model_path: None, }), + sensevoice_config: None, xunfei_config: None, baidu_config: None, openai_config: None, @@ -3437,6 +3487,7 @@ mod unit_tests { disabled: false, language: "zh".to_string(), whisper_config: None, + sensevoice_config: None, xunfei_config: Some(XunfeiConfig { app_id: "test_app_id".to_string(), api_key: "test_api_key".to_string(), diff --git a/src-tauri/crates/core/src/database/dao/agent_timeline.rs b/src-tauri/crates/core/src/database/dao/agent_timeline.rs index c923f9b67..9083cabe5 100644 --- a/src-tauri/crates/core/src/database/dao/agent_timeline.rs +++ b/src-tauri/crates/core/src/database/dao/agent_timeline.rs @@ -6,17 +6,49 @@ use rusqlite::{params, Connection}; use serde::{Deserialize, Serialize}; const HISTORY_FILE_ARTIFACT_INLINE_CONTENT_BYTES_LIMIT: usize = 16 * 1024; +const HISTORY_ITEM_INLINE_OUTPUT_BYTES_LIMIT: usize = 16 * 1024; fn history_item_payload_json_projection_sql() -> String { format!( "CASE + WHEN NOT json_valid(payload_json) THEN payload_json WHEN item_type = 'file_artifact' - AND length(payload_json) > {limit} - AND json_valid(payload_json) + AND length(payload_json) > {file_limit} THEN json_remove(payload_json, '$.content') + WHEN item_type = 'tool_call' + AND json_type(payload_json, '$.output') = 'text' + AND length(json_extract(payload_json, '$.output')) > {output_limit} + THEN json_set( + payload_json, + '$.output', + substr(json_extract(payload_json, '$.output'), 1, {output_limit}) + || char(10) || char(10) + || '[历史输出已截断，完整输出未随首屏加载。]' + ) + WHEN item_type = 'command_execution' + AND json_type(payload_json, '$.aggregated_output') = 'text' + AND length(json_extract(payload_json, '$.aggregated_output')) > {output_limit} + THEN json_set( + payload_json, + '$.aggregated_output', + substr(json_extract(payload_json, '$.aggregated_output'), 1, {output_limit}) + || char(10) || char(10) + || '[历史输出已截断，完整输出未随首屏加载。]' + ) + WHEN item_type = 'web_search' + AND json_type(payload_json, '$.output') = 'text' + AND length(json_extract(payload_json, '$.output')) > {output_limit} + THEN json_set( + payload_json, + '$.output', + substr(json_extract(payload_json, '$.output'), 1, {output_limit}) + || char(10) || char(10) + || '[历史输出已截断，完整输出未随首屏加载。]' + ) ELSE payload_json END", - limit = HISTORY_FILE_ARTIFACT_INLINE_CONTENT_BYTES_LIMIT, + file_limit = HISTORY_FILE_ARTIFACT_INLINE_CONTENT_BYTES_LIMIT, + output_limit = HISTORY_ITEM_INLINE_OUTPUT_BYTES_LIMIT, ) } @@ -876,6 +908,63 @@ mod tests { )); } + #[test] + fn item_tail_query_should_truncate_large_tool_output() { + let conn = setup_conn(); + let turn = AgentThreadTurn { + id: "turn-tool-output".to_string(), + thread_id: "thread-1".to_string(), + prompt_text: "生成封面".to_string(), + status: AgentThreadTurnStatus::Completed, + started_at: "2026-03-13T06:00:00Z".to_string(), + completed_at: Some("2026-03-13T06:00:10Z".to_string()), + error_message: None, + created_at: "2026-03-13T06:00:00Z".to_string(), + updated_at: "2026-03-13T06:00:10Z".to_string(), + }; + AgentTimelineDao::create_turn(&conn, &turn).unwrap(); + + let large_output = format!("封面工具输出开始\n{}", "大结果".repeat(20_000)); + AgentTimelineDao::upsert_item( + &conn, + &AgentThreadItem { + id: "item-large-tool-output".to_string(), + thread_id: "thread-1".to_string(), + turn_id: turn.id, + sequence: 1, + status: AgentThreadItemStatus::Completed, + started_at: "2026-03-13T06:00:01Z".to_string(), + completed_at: Some("2026-03-13T06:00:02Z".to_string()), + updated_at: "2026-03-13T06:00:02Z".to_string(), + payload: AgentThreadItemPayload::ToolCall { + tool_name: "social_generate_cover_image".to_string(), + arguments: Some(serde_json::json!({ "prompt": "咖啡封面" })), + output: Some(large_output.clone()), + success: Some(true), + error: None, + metadata: None, + }, + }, + ) + .unwrap(); + + let full_items = AgentTimelineDao::list_items_by_thread(&conn, "thread-1").unwrap(); + let tail_items = AgentTimelineDao::list_items_by_thread_tail(&conn, "thread-1", 1).unwrap(); + + assert!(matches!( + &full_items[0].payload, + AgentThreadItemPayload::ToolCall { output: Some(output), .. } + if output == &large_output + )); + assert!(matches!( + &tail_items[0].payload, + AgentThreadItemPayload::ToolCall { output: Some(output), .. } + if output.starts_with("封面工具输出开始") + && output.contains("历史输出已截断") + && output.len() < large_output.len() + )); + } + #[test] fn update_turn_status_should_persist_terminal_state() { let conn = setup_conn(); diff --git a/src-tauri/crates/core/src/database/system_providers.rs b/src-tauri/crates/core/src/database/system_providers.rs index 91bde9d5a..566ce6184 100644 --- a/src-tauri/crates/core/src/database/system_providers.rs +++ b/src-tauri/crates/core/src/database/system_providers.rs @@ -293,7 +293,7 @@ pub fn get_system_providers() -> Vec { id: "sensenova", name: "SenseNova", provider_type: ApiProviderType::Openai, - api_host: "https://api.sensenova.cn/compatible-mode/v1", + api_host: "https://api.sensenova.cn/compatible-mode/v2", group: ProviderGroup::Chinese, sort_order: 29, api_version: None, diff --git a/src-tauri/crates/lime-cli/src/main.rs b/src-tauri/crates/lime-cli/src/main.rs index 80c45a5a1..43f5f0fd0 100644 --- a/src-tauri/crates/lime-cli/src/main.rs +++ b/src-tauri/crates/lime-cli/src/main.rs @@ -14,6 +14,12 @@ use lime_media_runtime::{ }; use serde_json::{json, Value}; +const AUDIO_TRANSCRIPTION_CONTRACT_KEY: &str = "audio_transcription"; +const AUDIO_TRANSCRIPTION_MODALITY: &str = "audio"; +const AUDIO_TRANSCRIPTION_ROUTING_SLOT: &str = "audio_transcription_model"; +const AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES: &[&str] = + &["text_generation", "audio_transcription"]; + use crate::catalog::{ find_skill_entry, find_task_entry, ROOT_AFTER_HELP, SKILL_AFTER_HELP, SKILL_ENTRIES, TASK_AFTER_HELP, TASK_ENTRIES, @@ -967,6 +973,21 @@ fn create_transcription_task(args: TranscriptionGenerateArgs) -> Result Result String { - let base = self.get_base_url(); + let base = Self::strip_config_url_metadata(&self.get_base_url()); let base = base.trim_end_matches('/'); // 如果用户输入了带 /v1 的 URL，直接拼接 endpoint @@ -152,6 +156,65 @@ impl ClaudeCustomProvider { } } + fn parse_config_url(base_url: &str) -> Option { + let trimmed = base_url.trim(); + if trimmed.is_empty() { + return None; + } + + Url::parse(trimmed) + .or_else(|_| Url::parse(&format!("https://{trimmed}"))) + .ok() + } + + fn strip_config_url_metadata(base_url: &str) -> String { + if let Some(mut url) = Self::parse_config_url(base_url) { + url.set_query(None); + url.set_fragment(None); + return url.to_string().trim_end_matches('/').to_string(); + } + + let trimmed = base_url.trim(); + let end = [trimmed.find('?'), trimmed.find('#')] + .into_iter() + .flatten() + .min() + .unwrap_or(trimmed.len()); + trimmed[..end].trim_end_matches('/').to_string() + } + + fn normalize_lime_tenant_id(value: &str) -> Option { + let tenant_id = value.trim(); + if tenant_id.is_empty() { + return None; + } + + tenant_id + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_')) + .then(|| tenant_id.to_string()) + } + + fn parse_lime_tenant_id_from_pairs(value: &str) -> Option { + form_urlencoded::parse(value.as_bytes()).find_map(|(key, value)| { + (key == LIME_TENANT_PARAM) + .then(|| Self::normalize_lime_tenant_id(&value)) + .flatten() + }) + } + + fn lime_tenant_id_from_base_url(&self) -> Option { + let base_url = self.config.base_url.as_deref()?; + let url = Self::parse_config_url(base_url)?; + + url.query() + .and_then(Self::parse_lime_tenant_id_from_pairs) + .or_else(|| { + url.fragment() + .and_then(Self::parse_lime_tenant_id_from_pairs) + }) + } + /// 将 OpenAI 图片 URL 格式转换为 Claude 图片格式 /// /// 支持两种格式： @@ -378,14 +441,13 @@ impl ClaudeCustomProvider { } fn effective_runtime_spec(&self) -> ProviderRuntimeSpec { - infer_managed_runtime_spec(self.config.provider_type, &self.get_base_url()) + infer_managed_runtime_spec( + self.config.provider_type, + &Self::strip_config_url_metadata(&self.get_base_url()), + ) } - fn apply_runtime_headers( - &self, - request: reqwest::RequestBuilder, - api_key: &str, - ) -> reqwest::RequestBuilder { + fn apply_runtime_headers(&self, request: RequestBuilder, api_key: &str) -> RequestBuilder { let api_key = api_key.trim(); let runtime_spec = self.effective_runtime_spec(); let auth_value = runtime_spec @@ -412,6 +474,10 @@ impl ClaudeCustomProvider { request = request.header(*name, *value); } + if let Some(tenant_id) = self.lime_tenant_id_from_base_url() { + request = request.header(LIME_TENANT_HEADER, tenant_id); + } + request } @@ -1272,6 +1338,23 @@ mod tests { assert_eq!(spec.auth_prefix, None); } + #[test] + fn test_lime_tenant_id_from_base_url_fragment() { + let provider = ClaudeCustomProvider::with_config( + "test-key".to_string(), + Some("https://llm.limeai.run#lime_tenant_id=tenant-0001".to_string()), + ); + + assert_eq!( + provider.lime_tenant_id_from_base_url().as_deref(), + Some("tenant-0001") + ); + assert_eq!( + provider.build_url("messages"), + "https://llm.limeai.run/v1/messages" + ); + } + #[test] fn test_apply_runtime_headers_adds_dual_auth_headers_for_minimax_host() { let provider = ClaudeCustomProvider::with_config( @@ -1310,6 +1393,30 @@ mod tests { ); } + #[test] + fn test_apply_runtime_headers_adds_lime_tenant_header_from_fragment() { + let provider = ClaudeCustomProvider::with_config( + "test-key".to_string(), + Some("https://llm.limeai.run#lime_tenant_id=tenant-0001".to_string()), + ); + + let request = provider + .apply_runtime_headers( + reqwest::Client::new().post("https://example.com"), + "test-key", + ) + .build() + .expect("构建请求失败"); + + assert_eq!( + request + .headers() + .get(LIME_TENANT_HEADER) + .and_then(|value| value.to_str().ok()), + Some("tenant-0001") + ); + } + #[test] fn test_apply_runtime_headers_keeps_official_anthropic_single_auth_header() { let provider = ClaudeCustomProvider::with_config( diff --git a/src-tauri/crates/providers/src/providers/openai_custom.rs b/src-tauri/crates/providers/src/providers/openai_custom.rs index 8d1eb210d..b3ad86167 100644 --- a/src-tauri/crates/providers/src/providers/openai_custom.rs +++ b/src-tauri/crates/providers/src/providers/openai_custom.rs @@ -58,6 +58,8 @@ impl OpenAICustomProvider { } fn normalize_openai_request_payload(&self, payload: &mut serde_json::Value) { + self.normalize_provider_specific_request_payload(payload); + let model_name = payload .get("model") .and_then(|value| value.as_str()) @@ -151,6 +153,41 @@ impl OpenAICustomProvider { } } + fn normalize_provider_specific_request_payload(&self, payload: &mut serde_json::Value) { + if !self.uses_sensenova_compatible_api() { + return; + } + + let Some(object) = payload.as_object_mut() else { + return; + }; + + if object.contains_key("max_completion_tokens") { + object.remove("max_tokens"); + return; + } + + if let Some(max_tokens) = object.remove("max_tokens") { + object.insert("max_completion_tokens".to_string(), max_tokens); + } + } + + fn uses_sensenova_compatible_api(&self) -> bool { + self.config + .base_url + .as_deref() + .and_then(Self::parse_config_url) + .is_some_and(|url| { + url.host_str() + .map(|host| host.eq_ignore_ascii_case("api.sensenova.cn")) + .unwrap_or(false) + && url + .path() + .trim_end_matches('/') + .eq_ignore_ascii_case("/compatible-mode/v2") + }) + } + fn maybe_log_protocol_mismatch_hint(url: &str, status: StatusCode) { if (status == StatusCode::UNAUTHORIZED || status == StatusCode::FORBIDDEN) && url.contains("/api/anthropic") @@ -943,6 +980,42 @@ mod tests { ); } + #[test] + fn test_normalize_openai_request_payload_uses_sensenova_token_field() { + let provider = OpenAICustomProvider::with_config( + "sk-test".to_string(), + Some("https://api.sensenova.cn/compatible-mode/v2".to_string()), + ); + let mut payload = serde_json::json!({ + "model": "SenseChat-5", + "messages": [{"role":"user","content":"hi"}], + "max_tokens": 64 + }); + + provider.normalize_openai_request_payload(&mut payload); + + assert!(payload.get("max_tokens").is_none()); + assert_eq!(payload["max_completion_tokens"], serde_json::json!(64)); + } + + #[test] + fn test_normalize_openai_request_payload_keeps_standard_token_field() { + let provider = OpenAICustomProvider::with_config( + "sk-test".to_string(), + Some("https://api.deepseek.com".to_string()), + ); + let mut payload = serde_json::json!({ + "model": "deepseek-chat", + "messages": [{"role":"user","content":"hi"}], + "max_tokens": 64 + }); + + provider.normalize_openai_request_payload(&mut payload); + + assert_eq!(payload["max_tokens"], serde_json::json!(64)); + assert!(payload.get("max_completion_tokens").is_none()); + } + #[test] fn test_normalize_openai_request_payload_clears_reasoning_before_latest_user() { let provider = OpenAICustomProvider::default(); diff --git a/src-tauri/crates/services/Cargo.toml b/src-tauri/crates/services/Cargo.toml index b21e04154..7742ec9a9 100644 --- a/src-tauri/crates/services/Cargo.toml +++ b/src-tauri/crates/services/Cargo.toml @@ -8,6 +8,7 @@ repository.workspace = true [features] default = [] local-whisper = ["voice-core/local-whisper"] +local-sensevoice = ["voice-core/local-sensevoice"] [dependencies] # 项目内 crate diff --git a/src-tauri/crates/services/src/api_key_provider_service.rs b/src-tauri/crates/services/src/api_key_provider_service.rs index 983160df4..8e051f9ee 100644 --- a/src-tauri/crates/services/src/api_key_provider_service.rs +++ b/src-tauri/crates/services/src/api_key_provider_service.rs @@ -31,6 +31,10 @@ use std::collections::{HashMap, HashSet}; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::RwLock; +const SENSENOVA_OLD_OPENAI_COMPATIBLE_API_HOST: &str = + "https://api.sensenova.cn/compatible-mode/v1"; +const SENSENOVA_OPENAI_COMPATIBLE_API_HOST: &str = "https://api.sensenova.cn/compatible-mode/v2"; + // ============================================================================ // 连接测试结果 // ============================================================================ @@ -50,7 +54,10 @@ pub struct ConnectionTestResult { #[cfg(test)] mod tests { - use super::ApiKeyProviderService; + use super::{ + ApiKeyProviderService, SENSENOVA_OLD_OPENAI_COMPATIBLE_API_HOST, + SENSENOVA_OPENAI_COMPATIBLE_API_HOST, + }; use base64::{engine::general_purpose::STANDARD as BASE64, Engine}; use chrono::Utc; use lime_core::database::dao::api_key_provider::ApiProviderType; @@ -419,6 +426,74 @@ data: [DONE]\n"; assert_eq!(persisted.provider.provider_type, ApiProviderType::Openai); } + #[test] + fn test_initialize_system_providers_updates_legacy_sensenova_default_host() { + let db = init_test_database(); + let service = ApiKeyProviderService::new(); + + service + .initialize_system_providers(&db) + .expect("初始化系统 Provider 失败"); + + { + let conn = db.lock().expect("获取数据库锁失败"); + let mut provider = ApiKeyProviderDao::get_provider_by_id(&conn, "sensenova") + .expect("读取 SenseNova Provider 失败") + .expect("SenseNova Provider 应存在"); + provider.api_host = SENSENOVA_OLD_OPENAI_COMPATIBLE_API_HOST.to_string(); + ApiKeyProviderDao::update_provider(&conn, &provider) + .expect("写入旧 SenseNova 地址失败"); + } + + service + .initialize_system_providers(&db) + .expect("重新初始化系统 Provider 失败"); + + let persisted = service + .get_provider(&db, "sensenova") + .expect("读取 SenseNova Provider 失败") + .expect("SenseNova Provider 应存在"); + + assert_eq!( + persisted.provider.api_host, + SENSENOVA_OPENAI_COMPATIBLE_API_HOST + ); + } + + #[test] + fn test_initialize_system_providers_keeps_user_custom_sensenova_host() { + let db = init_test_database(); + let service = ApiKeyProviderService::new(); + + service + .initialize_system_providers(&db) + .expect("初始化系统 Provider 失败"); + + { + let conn = db.lock().expect("获取数据库锁失败"); + let mut provider = ApiKeyProviderDao::get_provider_by_id(&conn, "sensenova") + .expect("读取 SenseNova Provider 失败") + .expect("SenseNova Provider 应存在"); + provider.api_host = "https://proxy.example.com/sensenova".to_string(); + ApiKeyProviderDao::update_provider(&conn, &provider) + .expect("写入自定义 SenseNova 地址失败"); + } + + service + .initialize_system_providers(&db) + .expect("重新初始化系统 Provider 失败"); + + let persisted = service + .get_provider(&db, "sensenova") + .expect("读取 SenseNova Provider 失败") + .expect("SenseNova Provider 应存在"); + + assert_eq!( + persisted.provider.api_host, + "https://proxy.example.com/sensenova" + ); + } + #[test] fn test_add_custom_provider_should_force_known_anthropic_compatible_host_to_automatic() { let db = init_test_database(); @@ -1900,27 +1975,54 @@ impl ApiKeyProviderService { let conn = lime_core::database::lock_db(db)?; let system_providers = get_system_providers(); let mut inserted_count = 0; + let mut updated_count = 0; for def in &system_providers { // 检查是否已存在 let existing = ApiKeyProviderDao::get_provider_by_id(&conn, def.id).map_err(|e| e.to_string())?; - if existing.is_none() { - // 插入新的系统 Provider - let provider = to_api_key_provider(def); - ApiKeyProviderDao::insert_provider(&conn, &provider).map_err(|e| e.to_string())?; - inserted_count += 1; + match existing { + Some(mut provider) => { + if Self::should_update_system_provider_default_api_host(&provider, def.api_host) + { + provider.api_host = def.api_host.to_string(); + provider.updated_at = Utc::now(); + ApiKeyProviderDao::update_provider(&conn, &provider) + .map_err(|e| e.to_string())?; + updated_count += 1; + } + } + None => { + // 插入新的系统 Provider + let provider = to_api_key_provider(def); + ApiKeyProviderDao::insert_provider(&conn, &provider) + .map_err(|e| e.to_string())?; + inserted_count += 1; + } } } if inserted_count > 0 { tracing::info!("初始化了 {} 个系统 Provider", inserted_count); } + if updated_count > 0 { + tracing::info!("同步了 {} 个系统 Provider 默认地址", updated_count); + } Ok(inserted_count) } + fn should_update_system_provider_default_api_host( + provider: &ApiKeyProvider, + next_api_host: &str, + ) -> bool { + provider.id == "sensenova" + && provider.is_system + && provider.api_host.trim_end_matches('/') == SENSENOVA_OLD_OPENAI_COMPATIBLE_API_HOST + && next_api_host == SENSENOVA_OPENAI_COMPATIBLE_API_HOST + } + /// 获取所有 Provider（包含 API Keys） /// 首次调用时会自动初始化系统 Provider pub fn get_all_providers(&self, db: &DbConnection) -> Result, String> { diff --git a/src-tauri/crates/services/src/voice_asr_service.rs b/src-tauri/crates/services/src/voice_asr_service.rs index c47dcaa6d..adb069127 100644 --- a/src-tauri/crates/services/src/voice_asr_service.rs +++ b/src-tauri/crates/services/src/voice_asr_service.rs @@ -23,9 +23,9 @@ //! let text = AsrService::transcribe(&credential, &audio_data, 16000).await?; //! ``` -#[cfg(feature = "local-whisper")] use std::path::PathBuf; +use lime_core::app_paths; #[cfg(feature = "local-whisper")] use lime_core::config::WhisperModelSize; use lime_core::config::{AsrCredentialEntry, AsrProviderType}; @@ -34,6 +34,10 @@ use super::voice_config_service; use voice_core::asr_client::{AsrClient, BaiduClient, OpenAIWhisperClient, XunfeiClient}; use voice_core::types::AudioData; +const SENSEVOICE_MODEL_FILE: &str = "model.int8.onnx"; +const SENSEVOICE_TOKENS_FILE: &str = "tokens.txt"; +const SENSEVOICE_VAD_FILE: &str = "silero_vad.onnx"; + /// ASR 服务 pub struct AsrService; @@ -56,9 +60,16 @@ impl AsrService { audio_data: &[u8], sample_rate: u32, ) -> Result { - // 如果是本地 Whisper，直接调用 - if matches!(credential.provider, AsrProviderType::WhisperLocal) { - return Self::transcribe_whisper_local(credential, audio_data, sample_rate).await; + // 本地服务直接调用，不走云端回退链。 + match credential.provider { + AsrProviderType::WhisperLocal => { + return Self::transcribe_whisper_local(credential, audio_data, sample_rate).await; + } + AsrProviderType::SenseVoiceLocal => { + return Self::transcribe_sensevoice_local(credential, audio_data, sample_rate) + .await; + } + _ => {} } // 云端服务：先尝试云端，失败则回退到本地 Whisper @@ -73,6 +84,7 @@ impl AsrService { Self::transcribe_xunfei(credential, audio_data, sample_rate).await } AsrProviderType::WhisperLocal => unreachable!(), // 已在上面处理 + AsrProviderType::SenseVoiceLocal => unreachable!(), // 已在上面处理 }; // 云端成功，直接返回 @@ -178,6 +190,114 @@ impl AsrService { Err("本地 Whisper 功能未启用。请使用云端 ASR 服务（OpenAI、百度、讯飞）".to_string()) } + /// 本地 SenseVoice 识别。 + #[cfg(feature = "local-sensevoice")] + async fn transcribe_sensevoice_local( + credential: &AsrCredentialEntry, + audio_data: &[u8], + sample_rate: u32, + ) -> Result { + let config = credential + .sensevoice_config + .as_ref() + .ok_or("SenseVoice 本地配置缺失")?; + + let model_dir = Self::get_sensevoice_model_dir(config)?; + let model_path = model_dir.join(SENSEVOICE_MODEL_FILE); + let tokens_path = model_dir.join(SENSEVOICE_TOKENS_FILE); + let vad_path = model_dir.join(SENSEVOICE_VAD_FILE); + Self::ensure_sensevoice_required_files(&[ + (&model_path, SENSEVOICE_MODEL_FILE), + (&tokens_path, SENSEVOICE_TOKENS_FILE), + (&vad_path, SENSEVOICE_VAD_FILE), + ])?; + + let audio = Self::build_audio_data(audio_data, sample_rate)?; + if !audio.is_valid() { + return Err("录音时间过短（需要至少 0.5 秒）".to_string()); + } + + let language = if credential.language.trim().is_empty() { + "auto".to_string() + } else { + credential.language.clone() + }; + let use_itn = config.use_itn; + let num_threads = config.num_threads; + + tokio::task::spawn_blocking(move || { + let transcriber = voice_core::SenseVoiceTranscriber::new( + model_path, + tokens_path, + &language, + use_itn, + num_threads, + ) + .map_err(|error| format!("SenseVoice 模型加载失败: {error}"))?; + + let result = transcriber + .transcribe(&audio) + .map_err(|error| format!("SenseVoice 识别失败: {error}"))?; + + Ok::<_, String>(result.text) + }) + .await + .map_err(|error| format!("SenseVoice 识别任务执行失败: {error}"))? + } + + /// 本地 SenseVoice 识别（未启用 local-sensevoice feature 时的 stub） + #[cfg(not(feature = "local-sensevoice"))] + async fn transcribe_sensevoice_local( + _credential: &AsrCredentialEntry, + _audio_data: &[u8], + _sample_rate: u32, + ) -> Result { + Err( + "本地 SenseVoice 功能未启用。请启用 local-sensevoice 构建特性，或使用云端 ASR 服务" + .to_string(), + ) + } + + fn get_sensevoice_model_dir( + config: &lime_core::config::SenseVoiceLocalConfig, + ) -> Result { + if let Some(model_dir) = config.model_dir.as_ref() { + let trimmed = model_dir.trim(); + if !trimmed.is_empty() { + return Ok(PathBuf::from(trimmed)); + } + } + + Ok(app_paths::preferred_data_dir()? + .join("models") + .join("voice") + .join(&config.model_id)) + } + + fn ensure_sensevoice_required_files(files: &[(&PathBuf, &str)]) -> Result<(), String> { + let missing_files = files + .iter() + .filter_map( + |(path, name)| { + if path.is_file() { + None + } else { + Some(*name) + } + }, + ) + .collect::>(); + + if missing_files.is_empty() { + Ok(()) + } else { + Err(format!( + "本地 SenseVoice Small 尚未安装或文件不完整，请先在设置 -> 语音模型中下载；缺失文件: {}", + missing_files.join(", ") + )) + } + } + /// 获取 Whisper 模型文件路径 #[cfg(feature = "local-whisper")] fn get_whisper_model_path(model_size: &WhisperModelSize) -> Result { @@ -308,3 +428,40 @@ impl AsrService { Ok(audio) } } + +#[cfg(test)] +mod tests { + use super::*; + use lime_core::config::SenseVoiceLocalConfig; + + #[test] + fn sensevoice_model_dir_prefers_explicit_config_path() { + let config = SenseVoiceLocalConfig { + model_dir: Some("/tmp/lime-sensevoice".to_string()), + ..SenseVoiceLocalConfig::default() + }; + + let path = AsrService::get_sensevoice_model_dir(&config).expect("model dir"); + + assert_eq!(path, PathBuf::from("/tmp/lime-sensevoice")); + } + + #[test] + fn sensevoice_required_files_reports_missing_names() { + let temp = tempfile::tempdir().expect("tempdir"); + let model_path = temp.path().join(SENSEVOICE_MODEL_FILE); + std::fs::write(&model_path, b"mock").expect("write model"); + let tokens_path = temp.path().join(SENSEVOICE_TOKENS_FILE); + let vad_path = temp.path().join(SENSEVOICE_VAD_FILE); + + let error = AsrService::ensure_sensevoice_required_files(&[ + (&model_path, SENSEVOICE_MODEL_FILE), + (&tokens_path, SENSEVOICE_TOKENS_FILE), + (&vad_path, SENSEVOICE_VAD_FILE), + ]) + .expect_err("missing files should fail"); + + assert!(error.contains(SENSEVOICE_TOKENS_FILE)); + assert!(error.contains(SENSEVOICE_VAD_FILE)); + } +} diff --git a/src-tauri/crates/services/src/voice_config_service.rs b/src-tauri/crates/services/src/voice_config_service.rs index c9240e141..788fecd29 100644 --- a/src-tauri/crates/services/src/voice_config_service.rs +++ b/src-tauri/crates/services/src/voice_config_service.rs @@ -156,6 +156,7 @@ pub fn resolve_output_mode(mode: Option<&str>) -> Result &'static str { match provider { AsrProviderType::WhisperLocal => "本地 Whisper", + AsrProviderType::SenseVoiceLocal => "SenseVoice Small 本地", AsrProviderType::OpenAI => "OpenAI Whisper", AsrProviderType::Baidu => "百度语音", AsrProviderType::Xunfei => "讯飞语音", diff --git a/src-tauri/crates/voice-core/Cargo.toml b/src-tauri/crates/voice-core/Cargo.toml index c53649cd7..72a12e656 100644 --- a/src-tauri/crates/voice-core/Cargo.toml +++ b/src-tauri/crates/voice-core/Cargo.toml @@ -10,6 +10,8 @@ license = "MIT" default = [] # 本地 Whisper 识别（编译很慢，CI 默认不启用） local-whisper = ["dep:whisper-rs"] +# 本地 SenseVoice Small 识别（模型仍按需下载，不进入安装包） +local-sensevoice = ["dep:sherpa-onnx"] [dependencies] # 音频录制 @@ -18,6 +20,9 @@ cpal = "0.15" # Whisper 本地识别（可选，编译耗时） whisper-rs = { version = "0.12", optional = true } +# SenseVoice 本地识别（可选，链接 sherpa-onnx 运行时，不内置模型文件） +sherpa-onnx = { version = "1.13", default-features = false, features = ["static"], optional = true } + # WAV 处理 hound = "3.5" diff --git a/src-tauri/crates/voice-core/src/lib.rs b/src-tauri/crates/voice-core/src/lib.rs index 15eb1f8dc..04c4181ba 100644 --- a/src-tauri/crates/voice-core/src/lib.rs +++ b/src-tauri/crates/voice-core/src/lib.rs @@ -8,6 +8,8 @@ pub mod device; pub mod error; pub mod output; pub mod recorder; +#[cfg(feature = "local-sensevoice")] +pub mod sensevoice_transcriber; pub mod text_polish; pub mod threaded_recorder; #[cfg(feature = "local-whisper")] @@ -18,6 +20,8 @@ pub use device::{list_audio_devices, AudioDeviceInfo}; pub use error::{Result, VoiceError}; pub use output::OutputHandler; pub use recorder::AudioRecorder; +#[cfg(feature = "local-sensevoice")] +pub use sensevoice_transcriber::SenseVoiceTranscriber; pub use threaded_recorder::{RecordingCommand, RecordingResponse, RecordingService}; #[cfg(feature = "local-whisper")] pub use transcriber::WhisperTranscriber; diff --git a/src-tauri/crates/voice-core/src/sensevoice_transcriber.rs b/src-tauri/crates/voice-core/src/sensevoice_transcriber.rs new file mode 100644 index 000000000..41e705321 --- /dev/null +++ b/src-tauri/crates/voice-core/src/sensevoice_transcriber.rs @@ -0,0 +1,158 @@ +//! SenseVoice Small 本地语音识别模块。 +//! +//! 模型文件由上层按需下载；这里只负责把已安装模型接入 sherpa-onnx +//! offline recognizer。 + +use std::path::{Path, PathBuf}; + +use sherpa_onnx::{ + LinearResampler, OfflineRecognizer, OfflineRecognizerConfig, OfflineSenseVoiceModelConfig, +}; + +use crate::error::{Result, VoiceError}; +use crate::types::{AudioData, Segment, TranscribeResult}; + +const SENSEVOICE_SAMPLE_RATE: i32 = 16_000; + +/// SenseVoice 识别器。 +pub struct SenseVoiceTranscriber { + recognizer: OfflineRecognizer, + language: String, +} + +impl SenseVoiceTranscriber { + /// 创建新的 SenseVoice 识别器。 + pub fn new( + model_path: PathBuf, + tokens_path: PathBuf, + language: &str, + use_itn: bool, + num_threads: u16, + ) -> Result { + ensure_file_exists(&model_path, "SenseVoice 模型文件")?; + ensure_file_exists(&tokens_path, "SenseVoice tokens 文件")?; + + let language = normalize_language(language); + let mut config = OfflineRecognizerConfig::default(); + config.model_config.sense_voice = OfflineSenseVoiceModelConfig { + model: Some(model_path.to_string_lossy().to_string()), + language: Some(language.clone()), + use_itn, + }; + config.model_config.tokens = Some(tokens_path.to_string_lossy().to_string()); + config.model_config.num_threads = i32::from(num_threads.max(1)); + + let recognizer = OfflineRecognizer::create(&config).ok_or_else(|| { + VoiceError::TranscriberError("创建 sherpa-onnx SenseVoice 识别器失败".to_string()) + })?; + + Ok(Self { + recognizer, + language, + }) + } + + /// 识别音频。 + pub fn transcribe(&self, audio: &AudioData) -> Result { + if !audio.is_valid() { + return Err(VoiceError::RecordingTooShort); + } + + let samples = prepare_samples(audio)?; + if samples.is_empty() { + return Err(VoiceError::AudioFormatError( + "SenseVoice 输入音频为空".to_string(), + )); + } + + let stream = self.recognizer.create_stream(); + stream.accept_waveform(SENSEVOICE_SAMPLE_RATE, &samples); + self.recognizer.decode(&stream); + + let result = stream.get_result().ok_or_else(|| { + VoiceError::TranscriberError("读取 sherpa-onnx SenseVoice 识别结果失败".to_string()) + })?; + let text = result.text.trim().to_string(); + + let segments = if text.is_empty() { + Vec::new() + } else { + vec![Segment { + start: 0.0, + end: audio.duration_secs, + text: text.clone(), + }] + }; + + Ok(TranscribeResult { + text, + language: (self.language != "auto").then(|| self.language.clone()), + confidence: None, + segments, + }) + } +} + +fn ensure_file_exists(path: &Path, label: &str) -> Result<()> { + if path.is_file() { + Ok(()) + } else { + Err(VoiceError::TranscriberError(format!( + "{label}不存在: {}", + path.display() + ))) + } +} + +fn normalize_language(language: &str) -> String { + let normalized = language.trim(); + if normalized.is_empty() { + "auto".to_string() + } else { + normalized.to_string() + } +} + +fn prepare_samples(audio: &AudioData) -> Result> { + let samples = audio + .samples + .iter() + .map(|sample| *sample as f32 / i16::MAX as f32) + .collect::>(); + + if audio.sample_rate == SENSEVOICE_SAMPLE_RATE as u32 { + return Ok(samples); + } + + let resampler = LinearResampler::create(audio.sample_rate as i32, SENSEVOICE_SAMPLE_RATE) + .ok_or_else(|| { + VoiceError::AudioFormatError(format!( + "创建采样率转换器失败: {} -> {}", + audio.sample_rate, SENSEVOICE_SAMPLE_RATE + )) + })?; + Ok(resampler.resample(&samples, true)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normalize_empty_language_to_auto() { + assert_eq!(normalize_language(""), "auto"); + assert_eq!(normalize_language(" "), "auto"); + assert_eq!(normalize_language("zh"), "zh"); + } + + #[test] + fn prepare_samples_keeps_16khz_length() { + let audio = AudioData::new(vec![0, i16::MAX, i16::MIN], 16_000, 1); + let samples = prepare_samples(&audio).expect("prepare samples"); + + assert_eq!(samples.len(), 3); + assert_eq!(samples[0], 0.0); + assert!(samples[1] > 0.99); + assert!(samples[2] < -0.99); + } +} diff --git a/src-tauri/src/app/runner.rs b/src-tauri/src/app/runner.rs index 9610a6ab2..2ad51b5cc 100644 --- a/src-tauri/src/app/runner.rs +++ b/src-tauri/src/app/runner.rs @@ -1507,6 +1507,7 @@ pub fn run() { commands::video_generation_cmd::cancel_video_generation_task, commands::media_task_cmd::create_image_generation_task_artifact, commands::media_task_cmd::create_audio_generation_task_artifact, + commands::media_task_cmd::complete_audio_generation_task_artifact, commands::media_task_cmd::get_media_task_artifact, commands::media_task_cmd::list_media_task_artifacts, commands::media_task_cmd::cancel_media_task_artifact, @@ -1596,6 +1597,12 @@ pub fn run() { commands::asr_cmd::delete_asr_credential, commands::asr_cmd::set_default_asr_credential, commands::asr_cmd::test_asr_credential, + commands::voice_model_cmd::voice_models_list_catalog, + commands::voice_model_cmd::voice_models_get_install_state, + commands::voice_model_cmd::voice_models_download, + commands::voice_model_cmd::voice_models_delete, + commands::voice_model_cmd::voice_models_set_default, + commands::voice_model_cmd::voice_models_test_transcribe_file, // External Tools commands (Codex CLI 等外部工具) commands::external_tools_cmd::check_codex_cli_status, commands::external_tools_cmd::open_codex_cli_login, diff --git a/src-tauri/src/commands/agent_cmd.rs b/src-tauri/src/commands/agent_cmd.rs index 06c4c60f3..7dab614c3 100644 --- a/src-tauri/src/commands/agent_cmd.rs +++ b/src-tauri/src/commands/agent_cmd.rs @@ -21,6 +21,7 @@ use aster::conversation::message::Message; use futures::StreamExt; use lime_agent::merge_system_prompt_with_runtime_agents; use serde::{Deserialize, Serialize}; +use serde_json::Value; use tauri::{AppHandle, State}; use uuid::Uuid; @@ -172,6 +173,20 @@ fn build_title_generation_system_prompt(title_kind: &str) -> String { } } +fn force_direct_answer_tool_surface(metadata: &mut Option) { + let Some(Value::Object(root)) = metadata else { + return; + }; + let Some(Value::Object(runtime_metadata)) = root.get_mut("lime_runtime") else { + return; + }; + + runtime_metadata.insert( + "tool_surface".to_string(), + Value::String("direct_answer".to_string()), + ); +} + async fn generate_title_with_agent( agent_state: &AsterAgentState, db: &DbConnection, @@ -204,7 +219,7 @@ async fn generate_title_with_agent( )), None => Some(build_title_generation_system_prompt(title_kind)), }; - let auxiliary_runtime_metadata = build_auxiliary_runtime_metadata( + let mut auxiliary_runtime_metadata = build_auxiliary_runtime_metadata( provider_scope.resolution(), if title_kind == "image_task" { "auxiliary_generation_topic" @@ -228,6 +243,7 @@ async fn generate_title_with_agent( }, &["当前为内部标题生成辅助任务，只会使用一条已解析的 provider/model 路线。"], ); + force_direct_answer_tool_surface(&mut auxiliary_runtime_metadata); let session_config = build_auxiliary_session_config_with_turn_context( session_id, system_prompt, @@ -466,3 +482,31 @@ pub async fn agent_generate_title( Ok(result) } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn force_direct_answer_tool_surface_marks_auxiliary_runtime() { + let mut metadata = Some(json!({ + "lime_runtime": { + "task_profile": { + "kind": "title_generation" + } + } + })); + + force_direct_answer_tool_surface(&mut metadata); + + assert_eq!( + metadata + .as_ref() + .and_then(|value| value.get("lime_runtime")) + .and_then(|value| value.get("tool_surface")) + .and_then(Value::as_str), + Some("direct_answer") + ); + } +} diff --git a/src-tauri/src/commands/asr_cmd.rs b/src-tauri/src/commands/asr_cmd.rs index 1d723d406..d534bfabb 100644 --- a/src-tauri/src/commands/asr_cmd.rs +++ b/src-tauri/src/commands/asr_cmd.rs @@ -4,12 +4,18 @@ use crate::config::{ load_config, save_config, AsrCredentialEntry, AsrProviderType, BaiduConfig, OpenAIAsrConfig, - WhisperLocalConfig, XunfeiConfig, + SenseVoiceLocalConfig, WhisperLocalConfig, XunfeiConfig, }; +use lime_core::app_paths; use serde::{Deserialize, Serialize}; +use std::path::{Path, PathBuf}; use tauri::command; use uuid::Uuid; +const SENSEVOICE_MODEL_FILE: &str = "model.int8.onnx"; +const SENSEVOICE_TOKENS_FILE: &str = "tokens.txt"; +const SENSEVOICE_VAD_FILE: &str = "silero_vad.onnx"; + /// 获取所有 ASR 凭证 #[command] pub async fn get_asr_credentials() -> Result, String> { @@ -32,6 +38,8 @@ pub struct AddAsrCredentialRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub whisper_config: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pub sensevoice_config: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub xunfei_config: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub baidu_config: Option, @@ -68,6 +76,7 @@ pub async fn add_asr_credential( disabled: entry.disabled, language: entry.language, whisper_config: entry.whisper_config, + sensevoice_config: entry.sensevoice_config, xunfei_config: entry.xunfei_config, baidu_config: entry.baidu_config, openai_config: entry.openai_config, @@ -189,6 +198,27 @@ pub async fn test_asr_credential(id: String) -> Result { message: "本地 Whisper 已就绪".to_string(), }) } + AsrProviderType::SenseVoiceLocal => { + let Some(sensevoice_config) = credential.sensevoice_config.as_ref() else { + return Ok(TestResult { + success: false, + message: "SenseVoice 本地配置缺失".to_string(), + }); + }; + + match resolve_sensevoice_model_dir(sensevoice_config) + .and_then(|model_dir| ensure_sensevoice_model_files(&model_dir)) + { + Ok(()) => Ok(TestResult { + success: true, + message: "SenseVoice Small 本地模型已就绪".to_string(), + }), + Err(error) => Ok(TestResult { + success: false, + message: error, + }), + } + } AsrProviderType::Xunfei => { // TODO: 实现讯飞 API 测试 if credential.xunfei_config.is_some() { @@ -234,6 +264,42 @@ pub async fn test_asr_credential(id: String) -> Result { } } +fn resolve_sensevoice_model_dir(config: &SenseVoiceLocalConfig) -> Result { + if let Some(model_dir) = config.model_dir.as_ref() { + let trimmed = model_dir.trim(); + if !trimmed.is_empty() { + return Ok(PathBuf::from(trimmed)); + } + } + + Ok(app_paths::preferred_data_dir()? + .join("models") + .join("voice") + .join(&config.model_id)) +} + +fn ensure_sensevoice_model_files(model_dir: &Path) -> Result<(), String> { + let required_files = [ + SENSEVOICE_MODEL_FILE, + SENSEVOICE_TOKENS_FILE, + SENSEVOICE_VAD_FILE, + ]; + let missing_files = required_files + .iter() + .filter(|file_name| !model_dir.join(file_name).is_file()) + .copied() + .collect::>(); + + if missing_files.is_empty() { + Ok(()) + } else { + Err(format!( + "SenseVoice Small 本地模型文件不完整，请先在设置 -> 语音模型中下载；缺失文件: {}", + missing_files.join(", ") + )) + } +} + /// 测试结果 #[derive(serde::Serialize)] pub struct TestResult { diff --git a/src-tauri/src/commands/aster_agent_cmd/mod.rs b/src-tauri/src/commands/aster_agent_cmd/mod.rs index 945b29e3d..cd2d9ed52 100644 --- a/src-tauri/src/commands/aster_agent_cmd/mod.rs +++ b/src-tauri/src/commands/aster_agent_cmd/mod.rs @@ -510,6 +510,7 @@ pub(crate) use tool_runtime::{ pub(crate) use transcription_skill_launch::{ append_transcription_skill_launch_session_permissions, merge_system_prompt_with_transcription_skill_launch, + prepare_transcription_skill_launch_request_metadata, prune_transcription_skill_launch_detour_tools_from_registry, }; pub(crate) use translation_skill_launch::{ diff --git a/src-tauri/src/commands/aster_agent_cmd/runtime_turn.rs b/src-tauri/src/commands/aster_agent_cmd/runtime_turn.rs index 8fb82c108..c73aee98e 100644 --- a/src-tauri/src/commands/aster_agent_cmd/runtime_turn.rs +++ b/src-tauri/src/commands/aster_agent_cmd/runtime_turn.rs @@ -232,7 +232,7 @@ fn quoted_absolute_path_regex() -> &'static Regex { fn unix_absolute_path_regex() -> &'static Regex { static REGEX: OnceLock = OnceLock::new(); REGEX.get_or_init(|| { - Regex::new(r#"(?P/[^\s"'“”‘’,;:(){}\[\]<>]+)"#) + Regex::new(r#"(?P/[^\s"'“”‘’,;:，。；：、()（）{}\[\]【】<>《》]+)"#) .expect("unix absolute path regex should compile") }) } @@ -240,7 +240,7 @@ fn unix_absolute_path_regex() -> &'static Regex { fn windows_absolute_path_regex() -> &'static Regex { static REGEX: OnceLock = OnceLock::new(); REGEX.get_or_init(|| { - Regex::new(r#"(?P[A-Za-z]:\\[^\s"'“”‘’,;:(){}\[\]<>]+)"#) + Regex::new(r#"(?P[A-Za-z]:\\[^\s"'“”‘’,;:，。；：、()（）{}\[\]【】<>《》]+)"#) .expect("windows absolute path regex should compile") }) } @@ -1554,6 +1554,8 @@ async fn prepare_runtime_turn_request( prepare_translation_skill_launch_request_metadata(request.metadata.as_ref()); request.metadata = prepare_analysis_skill_launch_request_metadata(request.metadata.as_ref()); + request.metadata = + prepare_transcription_skill_launch_request_metadata(request.metadata.as_ref()); request.metadata = prepare_typesetting_skill_launch_request_metadata(request.metadata.as_ref()); request.metadata = prepare_webpage_skill_launch_request_metadata(request.metadata.as_ref()); @@ -5822,8 +5824,11 @@ mod tests { #[test] fn resolve_fast_chat_tool_surface_mode_should_use_local_workspace_for_explicit_local_path() { + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); + let repo_dir = temp_dir.path().join("claudecode"); + std::fs::create_dir_all(&repo_dir).expect("create repo dir"); let request = build_runtime_turn_test_request( - "请读取并分析项目 /Users/coso/Documents/dev/js/claudecode", + &format!("请读取并分析项目 {}", repo_dir.display()), None, ); let policy = lime_agent::resolve_request_tool_policy(Some(false), false); @@ -5875,6 +5880,27 @@ mod tests { assert!(paths.contains(&plain_expected)); } + #[test] + fn extract_explicit_local_focus_paths_from_message_should_stop_at_chinese_punctuation() { + let temp_dir = tempfile::TempDir::new().expect("create temp dir"); + let package_json = temp_dir.path().join("package.json"); + std::fs::write(&package_json, r#"{"name":"lime"}"#).expect("write package json"); + + let message = format!( + "必须先调用 Read 工具读取 {}，并确认 JSON 里的 name 字段。", + package_json.display() + ); + + let paths = extract_explicit_local_focus_paths_from_message(&message); + let expected = package_json + .canonicalize() + .expect("canonicalize package json") + .to_string_lossy() + .to_string(); + + assert_eq!(paths, vec![expected]); + } + #[test] fn merge_system_prompt_with_explicit_local_path_focus_should_append_focus_guidance() { let temp_dir = tempfile::TempDir::new().expect("create temp dir"); diff --git a/src-tauri/src/commands/aster_agent_cmd/tests.rs b/src-tauri/src/commands/aster_agent_cmd/tests.rs index 9729f9854..fe20a27e1 100644 --- a/src-tauri/src/commands/aster_agent_cmd/tests.rs +++ b/src-tauri/src/commands/aster_agent_cmd/tests.rs @@ -4824,6 +4824,13 @@ mod tests { assert!(merged.contains("不要先走 ToolSearch / WebSearch / Read / Glob / Grep")); assert!(merged.contains("应立即改为直调 Skill(transcription_generate)")); assert!(merged.contains("不要伪造“转写已完成”")); + assert!(merged.contains( + "modality_contract_key=audio_transcription, modality=audio, routing_slot=audio_transcription_model" + )); + assert!(merged.contains("当前合同所需能力：text_generation, audio_transcription")); + assert!(merged.contains("frontend_direct_asr")); + assert!(merged.contains("runtime_contract(JSON)")); + assert!(merged.contains("contract_key\":\"audio_transcription")); assert!(merged.contains("当前任务已经显式进入转写技能主链")); } @@ -5496,6 +5503,63 @@ mod tests { ); } + #[test] + fn test_prepare_transcription_skill_launch_request_metadata_sets_contract_fields() { + let metadata = serde_json::json!({ + "harness": { + "transcription_skill_launch": { + "skill_name": "transcription_generate", + "kind": "transcription_task", + "transcription_task": { + "source_url": "https://example.com/interview.mp4" + } + } + } + }); + + let prepared = prepare_transcription_skill_launch_request_metadata(Some(&metadata)) + .expect("prepared metadata"); + + let harness = prepared + .get("harness") + .and_then(serde_json::Value::as_object) + .expect("harness"); + let launch = harness + .get("transcription_skill_launch") + .and_then(serde_json::Value::as_object) + .expect("transcription_skill_launch"); + assert_eq!( + launch + .get("modality_contract_key") + .and_then(serde_json::Value::as_str), + Some("audio_transcription") + ); + let transcription_task = launch + .get("transcription_task") + .and_then(serde_json::Value::as_object) + .expect("transcription_task"); + assert_eq!( + transcription_task + .get("modality_contract_key") + .and_then(serde_json::Value::as_str), + Some("audio_transcription") + ); + assert_eq!( + transcription_task + .get("routing_slot") + .and_then(serde_json::Value::as_str), + Some("audio_transcription_model") + ); + assert_eq!( + transcription_task + .get("runtime_contract") + .and_then(|value| value.get("executor_binding")) + .and_then(|value| value.get("binding_key")) + .and_then(serde_json::Value::as_str), + Some("transcription_generate") + ); + } + #[test] fn test_prepare_research_skill_launch_request_metadata_sets_workbench_chat_mode() { let metadata = serde_json::json!({ diff --git a/src-tauri/src/commands/aster_agent_cmd/tool_runtime/creation_tools.rs b/src-tauri/src/commands/aster_agent_cmd/tool_runtime/creation_tools.rs index d46cc97cf..d40bd48a0 100644 --- a/src-tauri/src/commands/aster_agent_cmd/tool_runtime/creation_tools.rs +++ b/src-tauri/src/commands/aster_agent_cmd/tool_runtime/creation_tools.rs @@ -2,14 +2,17 @@ use super::*; use crate::agent_tools::catalog::LIME_CREATE_AUDIO_TASK_TOOL_NAME; use crate::commands::media_task_cmd::{ create_audio_generation_task_artifact_inner, create_image_generation_task_artifact_inner, - finalize_audio_generation_task_creation, finalize_image_generation_task_creation, + create_transcription_task_artifact_inner, finalize_audio_generation_task_creation, + finalize_image_generation_task_creation, finalize_transcription_task_creation, CreateAudioGenerationTaskArtifactRequest, CreateImageGenerationTaskArtifactRequest, - ImageStoryboardSlotInput, + CreateTranscriptionTaskArtifactRequest, ImageStoryboardSlotInput, }; use crate::commands::modality_runtime_contracts::{ - image_generation_required_capabilities, IMAGE_GENERATION_CONTRACT_KEY, - IMAGE_GENERATION_MODALITY, IMAGE_GENERATION_ROUTING_SLOT, VOICE_GENERATION_CONTRACT_KEY, - VOICE_GENERATION_MODALITY, VOICE_GENERATION_ROUTING_SLOT, + audio_transcription_required_capabilities, image_generation_required_capabilities, + AUDIO_TRANSCRIPTION_CONTRACT_KEY, AUDIO_TRANSCRIPTION_MODALITY, + AUDIO_TRANSCRIPTION_ROUTING_SLOT, IMAGE_GENERATION_CONTRACT_KEY, IMAGE_GENERATION_MODALITY, + IMAGE_GENERATION_ROUTING_SLOT, VOICE_GENERATION_CONTRACT_KEY, VOICE_GENERATION_MODALITY, + VOICE_GENERATION_ROUTING_SLOT, }; use lime_media_runtime::{ write_task_artifact, MediaTaskType, TaskRelationships, TaskType, TaskWriteOptions, @@ -684,6 +687,18 @@ struct TranscriptionTaskInput { #[serde(default)] entry_source: Option, #[serde(default)] + modality_contract_key: Option, + #[serde(default)] + modality: Option, + #[serde(default)] + required_capabilities: Vec, + #[serde(default)] + routing_slot: Option, + #[serde(default)] + runtime_contract: Option, + #[serde(default)] + requested_target: Option, + #[serde(default)] output_path: Option, } @@ -1332,10 +1347,11 @@ fn submit_audio_generation_task_record( input: AudioTaskInput, ) -> Result { let request = build_audio_generation_task_request(context, input); + let project_root_path = request.project_root_path.trim().to_string(); let output = create_audio_generation_task_artifact_inner(request) .map_err(|error| ToolError::execution_failed(format!("创建音频任务失败: {error}")))?; - finalize_audio_generation_task_creation(Some(app_handle), &output); + finalize_audio_generation_task_creation(Some(app_handle), project_root_path.as_str(), &output); let serialized = serde_json::to_string_pretty(&output) .unwrap_or_else(|_| serde_json::json!(&output).to_string()); @@ -1449,31 +1465,56 @@ impl Tool for LimeCreateTranscriptionTaskTool { )); } - let payload = serde_json::json!({ - "prompt": input.prompt, - "raw_text": input.raw_text, - "source_url": source_url, - "source_path": source_path, - "language": input.language, - "output_format": input.output_format, - "speaker_labels": input.speaker_labels, - "timestamps": input.timestamps, - "provider_id": input.provider_id, - "model": input.model, - "session_id": input.session_id, - "project_id": input.project_id, - "content_id": input.content_id, - "entry_source": input.entry_source - }); - submit_creation_task_record( - &self.app_handle, - context, - TaskType::TranscriptionGenerate, - input.title, - payload, - None, - input.output_path.as_deref(), - ) + let output = + create_transcription_task_artifact_inner(CreateTranscriptionTaskArtifactRequest { + project_root_path: context.working_directory.to_string_lossy().to_string(), + prompt: input.prompt, + title: input.title, + raw_text: input.raw_text, + source_url: source_url.map(ToString::to_string), + source_path: source_path.map(ToString::to_string), + language: input.language, + output_format: input.output_format, + speaker_labels: input.speaker_labels, + timestamps: input.timestamps, + provider_id: input.provider_id, + model: input.model, + session_id: input.session_id, + project_id: input.project_id, + content_id: input.content_id, + entry_source: input.entry_source, + modality_contract_key: input + .modality_contract_key + .or_else(|| Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY.to_string())), + modality: input + .modality + .or_else(|| Some(AUDIO_TRANSCRIPTION_MODALITY.to_string())), + required_capabilities: if input.required_capabilities.is_empty() { + audio_transcription_required_capabilities() + } else { + input.required_capabilities + }, + routing_slot: input + .routing_slot + .or_else(|| Some(AUDIO_TRANSCRIPTION_ROUTING_SLOT.to_string())), + runtime_contract: input.runtime_contract, + requested_target: input + .requested_target + .or_else(|| Some("transcript".to_string())), + output_path: input.output_path, + }) + .map_err(ToolError::execution_failed)?; + finalize_transcription_task_creation( + Some(&self.app_handle), + context.working_directory.to_string_lossy().as_ref(), + &output, + ); + let serialized = serde_json::to_string_pretty(&output) + .unwrap_or_else(|_| serde_json::json!(&output).to_string()); + Ok(media_cli_bridge::attach_media_task_metadata( + ToolResult::success(serialized), + &output, + )) } } diff --git a/src-tauri/src/commands/aster_agent_cmd/tool_runtime/media_cli_bridge.rs b/src-tauri/src/commands/aster_agent_cmd/tool_runtime/media_cli_bridge.rs index bb7f712f2..0abe2256b 100644 --- a/src-tauri/src/commands/aster_agent_cmd/tool_runtime/media_cli_bridge.rs +++ b/src-tauri/src/commands/aster_agent_cmd/tool_runtime/media_cli_bridge.rs @@ -100,6 +100,13 @@ pub(crate) fn emit_media_creation_task_event(app_handle: &AppHandle, output: &Me "anchor_text": read_payload_string(payload_record, &["anchor_text", "anchorText"]), "requested_count": read_output_result_u64(output, &["requested_count", "requestedCount"]), "received_count": read_output_result_u64(output, &["received_count", "receivedCount"]), + "source_text": read_payload_string(payload_record, &["source_text", "sourceText"]), + "voice": read_payload_string(payload_record, &["voice"]), + "voice_style": read_payload_string(payload_record, &["voice_style", "voiceStyle"]), + "target_language": read_payload_string(payload_record, &["target_language", "targetLanguage"]), + "audio_path": read_payload_string(payload_record, &["audio_path", "audioPath"]), + "mime_type": read_payload_string(payload_record, &["mime_type", "mimeType"]), + "duration_ms": read_payload_u64(payload_record, &["duration_ms", "durationMs"]), }); if let Err(error) = app_handle.emit(CREATION_TASK_EVENT_NAME, &payload) { @@ -167,14 +174,16 @@ pub(crate) fn attach_media_task_metadata( ) .with_metadata( "provider_id", - serde_json::json!(read_output_result_string( - output, - &["provider_id", "providerId"] - )), + serde_json::json!( + read_output_result_string(output, &["provider_id", "providerId"]).or_else(|| { + read_payload_string(&output.record.payload, &["provider_id", "providerId"]) + }) + ), ) .with_metadata( "model", - serde_json::json!(read_output_result_string(output, &["model"])), + serde_json::json!(read_output_result_string(output, &["model"]) + .or_else(|| read_payload_string(&output.record.payload, &["model"]))), ) .with_metadata( "requested_count", @@ -190,6 +199,52 @@ pub(crate) fn attach_media_task_metadata( &["received_count", "receivedCount"] )), ) + .with_metadata( + "source_text", + serde_json::json!(read_payload_string( + &output.record.payload, + &["source_text", "sourceText"] + )), + ) + .with_metadata( + "voice", + serde_json::json!(read_payload_string(&output.record.payload, &["voice"])), + ) + .with_metadata( + "voice_style", + serde_json::json!(read_payload_string( + &output.record.payload, + &["voice_style", "voiceStyle"] + )), + ) + .with_metadata( + "target_language", + serde_json::json!(read_payload_string( + &output.record.payload, + &["target_language", "targetLanguage"] + )), + ) + .with_metadata( + "audio_path", + serde_json::json!(read_payload_string( + &output.record.payload, + &["audio_path", "audioPath"] + )), + ) + .with_metadata( + "mime_type", + serde_json::json!(read_payload_string( + &output.record.payload, + &["mime_type", "mimeType"] + )), + ) + .with_metadata( + "duration_ms", + serde_json::json!(read_payload_u64( + &output.record.payload, + &["duration_ms", "durationMs"] + )), + ) .with_metadata("artifact_paths", serde_json::json!(output.artifact_paths())) } diff --git a/src-tauri/src/commands/aster_agent_cmd/transcription_skill_launch.rs b/src-tauri/src/commands/aster_agent_cmd/transcription_skill_launch.rs index f5e9fb29c..cb9735684 100644 --- a/src-tauri/src/commands/aster_agent_cmd/transcription_skill_launch.rs +++ b/src-tauri/src/commands/aster_agent_cmd/transcription_skill_launch.rs @@ -1,4 +1,9 @@ use super::*; +use crate::commands::modality_runtime_contracts::{ + audio_transcription_required_capabilities, audio_transcription_runtime_contract, + insert_audio_transcription_contract_fields, AUDIO_TRANSCRIPTION_CONTRACT_KEY, + AUDIO_TRANSCRIPTION_MODALITY, AUDIO_TRANSCRIPTION_ROUTING_SLOT, +}; const TRANSCRIPTION_SKILL_LAUNCH_PROMPT_MARKER: &str = "<>"; const TRANSCRIPTION_SKILL_LAUNCH_DETOUR_DENY_PATTERNS: &[&str] = &[ @@ -25,6 +30,65 @@ fn extract_object_string( .map(str::to_string) } +fn extract_harness_nested_object_mut<'a>( + value: &'a mut serde_json::Value, + keys: &[&str], +) -> Option<&'a mut serde_json::Map> { + let root = value.as_object_mut()?; + let harness = if root.contains_key("harness") { + root.get_mut("harness") + .and_then(serde_json::Value::as_object_mut)? + } else { + root + }; + + for key in keys.iter().copied() { + let exists = harness + .get(key) + .and_then(serde_json::Value::as_object) + .is_some(); + if exists { + return harness + .get_mut(key) + .and_then(serde_json::Value::as_object_mut); + } + } + + None +} + +fn ensure_harness_workbench_chat_mode(value: &mut serde_json::Value, launch_keys: &[&str]) { + let Some(root) = value.as_object_mut() else { + return; + }; + let harness = if root.contains_key("harness") { + match root + .get_mut("harness") + .and_then(serde_json::Value::as_object_mut) + { + Some(harness) => harness, + None => return, + } + } else { + root + }; + + let has_launch = launch_keys.iter().any(|key| { + harness + .get(*key) + .and_then(serde_json::Value::as_object) + .is_some() + }); + if !has_launch { + return; + } + + harness.insert( + "chat_mode".to_string(), + serde_json::Value::String("workbench".to_string()), + ); +} + fn truncate_prompt_text(value: String, max_chars: usize) -> String { let total_chars = value.chars().count(); if total_chars <= max_chars { @@ -35,6 +99,68 @@ fn truncate_prompt_text(value: String, max_chars: usize) -> String { format!("{truncated}...(已截断，原始长度 {total_chars} 字)") } +fn extract_object_string_array( + object: &serde_json::Map, + keys: &[&str], +) -> Vec { + keys.iter() + .filter_map(|key| object.get(*key)) + .find_map(serde_json::Value::as_array) + .map(|values| { + values + .iter() + .filter_map(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(str::to_string) + .collect::>() + }) + .unwrap_or_default() +} + +fn extract_object_value( + object: &serde_json::Map, + keys: &[&str], +) -> Option { + keys.iter() + .filter_map(|key| object.get(*key)) + .find(|value| !value.is_null()) + .cloned() +} + +fn ensure_audio_transcription_contract_metadata( + launch: &mut serde_json::Map, +) { + insert_audio_transcription_contract_fields(launch); + let transcription_task = if launch.contains_key("transcription_task") { + launch.get_mut("transcription_task") + } else { + launch.get_mut("transcriptionTask") + }; + if let Some(transcription_task) = transcription_task.and_then(serde_json::Value::as_object_mut) + { + insert_audio_transcription_contract_fields(transcription_task); + } +} + +pub(crate) fn prepare_transcription_skill_launch_request_metadata( + request_metadata: Option<&serde_json::Value>, +) -> Option { + let mut metadata = request_metadata.cloned()?; + ensure_harness_workbench_chat_mode( + &mut metadata, + &["transcription_skill_launch", "transcriptionSkillLaunch"], + ); + if let Some(launch) = extract_harness_nested_object_mut( + &mut metadata, + &["transcription_skill_launch", "transcriptionSkillLaunch"], + ) { + ensure_audio_transcription_contract_metadata(launch); + } + + Some(metadata) +} + pub(crate) fn merge_system_prompt_with_transcription_skill_launch( base_prompt: Option, request_metadata: Option<&serde_json::Value>, @@ -162,6 +288,29 @@ fn build_transcription_skill_launch_system_prompt( let model = extract_object_string(transcription_task, &["model"]); let entry_source = extract_object_string(transcription_task, &["entry_source", "entrySource"]) .unwrap_or_else(|| "at_transcription_command".to_string()); + let modality_contract_key = extract_object_string( + transcription_task, + &["modality_contract_key", "modalityContractKey"], + ) + .unwrap_or_else(|| AUDIO_TRANSCRIPTION_CONTRACT_KEY.to_string()); + let modality = extract_object_string(transcription_task, &["modality"]) + .unwrap_or_else(|| AUDIO_TRANSCRIPTION_MODALITY.to_string()); + let routing_slot = extract_object_string(transcription_task, &["routing_slot", "routingSlot"]) + .unwrap_or_else(|| AUDIO_TRANSCRIPTION_ROUTING_SLOT.to_string()); + let required_capabilities = { + let values = extract_object_string_array( + transcription_task, + &["required_capabilities", "requiredCapabilities"], + ); + if values.is_empty() { + audio_transcription_required_capabilities() + } else { + values + } + }; + let runtime_contract = + extract_object_value(transcription_task, &["runtime_contract", "runtimeContract"]) + .unwrap_or_else(audio_transcription_runtime_contract); let args_payload = serde_json::json!({ "user_input": raw_text .clone() @@ -196,6 +345,21 @@ fn build_transcription_skill_launch_system_prompt( "- 如果某个通用搜索/读文件工具因为 session policy 被拒绝，不要重复同类调用；应立即改为直调 Skill(transcription_generate)。".to_string(), "- Skill 执行后，优先沿 transcription_generate skill 的 Bash / task file 主链提交异步任务；只有 Skill 明确不可用时，才允许直接回退到 lime_create_transcription_task。".to_string(), "- 不要伪造“转写已完成”；在 task file 真正返回结果前，只能汇报任务已提交、排队或执行中。".to_string(), + format!( + "- 当前底层运行合同：modality_contract_key={modality_contract_key}, modality={modality}, routing_slot={routing_slot}；`@转写` 只是 audio_transcription 的上层入口，首刀仍走 Skill(transcription_generate)。" + ), + format!( + "- 当前合同所需能力：{}；不得退回 frontend_direct_asr、generic_file_transcript、tool_search_before_transcription_skill 或 web_search_before_transcription_skill。", + required_capabilities.join(", ") + ), + format!( + "- 当前 runtime_contract(JSON)：{}", + truncate_prompt_text( + serde_json::to_string(&runtime_contract) + .unwrap_or_else(|_| "{}".to_string()), + 4_000, + ) + ), format!("- 当前转写任务上下文(JSON)：{transcription_task_json}"), format!("- 当前入口来源：{entry_source}。"), ]; diff --git a/src-tauri/src/commands/media_task_cmd.rs b/src-tauri/src/commands/media_task_cmd.rs index 44a927d15..2d2127c78 100644 --- a/src-tauri/src/commands/media_task_cmd.rs +++ b/src-tauri/src/commands/media_task_cmd.rs @@ -12,29 +12,52 @@ use serde::{Deserialize, Serialize}; use serde_json::json; use sha2::{Digest, Sha256}; use std::collections::HashSet; +use std::fs; use std::path::{Path, PathBuf}; use std::sync::Mutex; +use std::time::Duration; use tauri::{AppHandle, Manager}; +use crate::commands::api_key_provider_cmd::ApiKeyProviderServiceState; use crate::commands::aster_agent_cmd::tool_runtime::media_cli_bridge; use crate::commands::modality_runtime_contracts::{ - assess_image_generation_model_capability_from_registry, image_generation_runtime_contract, - looks_like_text_model_for_image_generation, normalize_image_generation_contract_key, + assess_image_generation_model_capability_from_registry, audio_transcription_runtime_contract, + image_generation_runtime_contract, looks_like_text_model_for_image_generation, + normalize_audio_transcription_contract_key, normalize_audio_transcription_modality, + normalize_audio_transcription_required_capabilities, + normalize_audio_transcription_routing_slot, normalize_image_generation_contract_key, normalize_image_generation_modality, normalize_image_generation_required_capabilities, normalize_image_generation_routing_slot, normalize_voice_generation_contract_key, normalize_voice_generation_modality, normalize_voice_generation_required_capabilities, normalize_voice_generation_routing_slot, voice_generation_runtime_contract, - ImageGenerationModelCapabilityAssessment, IMAGE_GENERATION_CONTRACT_KEY, - IMAGE_GENERATION_ROUTING_SLOT, VOICE_GENERATION_CONTRACT_KEY, + ImageGenerationModelCapabilityAssessment, AUDIO_TRANSCRIPTION_CONTRACT_KEY, + AUDIO_TRANSCRIPTION_ROUTING_SLOT, IMAGE_GENERATION_CONTRACT_KEY, IMAGE_GENERATION_ROUTING_SLOT, + VOICE_GENERATION_CONTRACT_KEY, VOICE_GENERATION_ROUTING_SLOT, }; use crate::commands::model_registry_cmd::ModelRegistryState; use crate::config::GlobalConfigManagerState; +use crate::database::DbConnection; +use lime_core::models::runtime_provider_model::{RuntimeCredentialData, RuntimeProviderCredential}; const IMAGE_TASK_RUNNER_WORKER_ID: &str = "lime-image-api-worker"; +const AUDIO_TASK_RUNNER_WORKER_ID: &str = "lime-audio-worker"; +const AUDIO_TASK_COMPLETION_WORKER_ID: &str = "lime-audio-output-writer"; +const TRANSCRIPTION_TASK_RUNNER_WORKER_ID: &str = "lime-transcription-worker"; const AUDIO_TASK_DEFAULT_MIME_TYPE: &str = "audio/mpeg"; +const AUDIO_TASK_DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; +const AUDIO_TASK_DEFAULT_VOICE: &str = "alloy"; +const AUDIO_TASK_RUNNER_TIMEOUT_SECS: u64 = 240; +const AUDIO_TASK_OUTPUT_RELATIVE_DIR: &str = ".lime/runtime/audio"; +const TRANSCRIPTION_TASK_DEFAULT_OPENAI_BASE_URL: &str = "https://api.openai.com/v1"; +const TRANSCRIPTION_TASK_RUNNER_TIMEOUT_SECS: u64 = 300; +const TRANSCRIPTION_TASK_OUTPUT_RELATIVE_DIR: &str = ".lime/runtime/transcripts"; static ACTIVE_IMAGE_TASK_EXECUTIONS: Lazy>> = Lazy::new(|| Mutex::new(HashSet::new())); +static ACTIVE_AUDIO_TASK_EXECUTIONS: Lazy>> = + Lazy::new(|| Mutex::new(HashSet::new())); +static ACTIVE_TRANSCRIPTION_TASK_EXECUTIONS: Lazy>> = + Lazy::new(|| Mutex::new(HashSet::new())); const MAX_IMAGE_TASK_COUNT: u32 = 16; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -166,6 +189,56 @@ pub struct CreateAudioGenerationTaskArtifactRequest { pub output_path: Option, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CreateTranscriptionTaskArtifactRequest { + pub project_root_path: String, + #[serde(default)] + pub prompt: Option, + #[serde(default)] + pub title: Option, + #[serde(default)] + pub raw_text: Option, + #[serde(default, alias = "source_url")] + pub source_url: Option, + #[serde(default, alias = "source_path")] + pub source_path: Option, + #[serde(default)] + pub language: Option, + #[serde(default, alias = "output_format")] + pub output_format: Option, + #[serde(default, alias = "speaker_labels")] + pub speaker_labels: Option, + #[serde(default)] + pub timestamps: Option, + #[serde(default, alias = "provider_id")] + pub provider_id: Option, + #[serde(default)] + pub model: Option, + #[serde(default, alias = "session_id")] + pub session_id: Option, + #[serde(default, alias = "project_id")] + pub project_id: Option, + #[serde(default, alias = "content_id")] + pub content_id: Option, + #[serde(default, alias = "entry_source")] + pub entry_source: Option, + #[serde(default, alias = "modality_contract_key")] + pub modality_contract_key: Option, + #[serde(default)] + pub modality: Option, + #[serde(default, alias = "required_capabilities")] + pub required_capabilities: Vec, + #[serde(default, alias = "routing_slot")] + pub routing_slot: Option, + #[serde(default, alias = "runtime_contract")] + pub runtime_contract: Option, + #[serde(default, alias = "requested_target")] + pub requested_target: Option, + #[serde(default, alias = "output_path")] + pub output_path: Option, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct MediaTaskLookupRequest { @@ -173,6 +246,23 @@ pub struct MediaTaskLookupRequest { pub task_ref: String, } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CompleteAudioGenerationTaskArtifactRequest { + pub project_root_path: String, + pub task_ref: String, + #[serde(alias = "audio_path", alias = "audio_url")] + pub audio_path: String, + #[serde(default, alias = "mime_type")] + pub mime_type: Option, + #[serde(default, alias = "duration_ms")] + pub duration_ms: Option, + #[serde(default, alias = "provider_id")] + pub provider_id: Option, + #[serde(default)] + pub model: Option, +} + #[derive(Debug, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ListMediaTaskArtifactsRequest { @@ -217,6 +307,20 @@ pub struct MediaTaskModalityRuntimeContractIndexEntry { pub failure_code: Option, pub model_capability_assessment_source: Option, pub model_supports_image_generation: Option, + pub audio_output_status: Option, + pub audio_output_path: Option, + pub audio_output_mime_type: Option, + pub audio_output_duration_ms: Option, + pub audio_output_error_code: Option, + pub audio_output_retryable: Option, + pub transcript_status: Option, + pub transcript_path: Option, + pub transcript_source_url: Option, + pub transcript_source_path: Option, + pub transcript_language: Option, + pub transcript_output_format: Option, + pub transcript_error_code: Option, + pub transcript_retryable: Option, } #[derive(Debug, Serialize)] @@ -225,6 +329,18 @@ pub struct MediaTaskRoutingOutcomeCount { pub count: usize, } +#[derive(Debug, Serialize)] +pub struct MediaTaskAudioOutputStatusCount { + pub status: String, + pub count: usize, +} + +#[derive(Debug, Serialize)] +pub struct MediaTaskTranscriptStatusCount { + pub status: String, + pub count: usize, +} + #[derive(Debug, Serialize)] pub struct MediaTaskModalityRuntimeContractIndex { pub snapshot_count: usize, @@ -232,6 +348,12 @@ pub struct MediaTaskModalityRuntimeContractIndex { pub blocked_count: usize, pub routing_outcomes: Vec, pub model_registry_assessment_count: usize, + pub audio_output_count: usize, + pub audio_output_statuses: Vec, + pub audio_output_error_codes: Vec, + pub transcript_count: usize, + pub transcript_statuses: Vec, + pub transcript_error_codes: Vec, pub snapshots: Vec, } @@ -263,6 +385,12 @@ struct ImageGenerationPreferenceDefaults { model: Option, } +#[derive(Debug, Clone, Default, PartialEq, Eq)] +struct AudioGenerationPreferenceDefaults { + provider_id: Option, + model: Option, +} + fn load_image_generation_preference_defaults() -> ImageGenerationPreferenceDefaults { #[cfg(test)] { @@ -289,6 +417,53 @@ fn load_image_generation_preference_defaults() -> ImageGenerationPreferenceDefau } } +fn load_audio_generation_preference_defaults() -> AudioGenerationPreferenceDefaults { + #[cfg(test)] + { + return AudioGenerationPreferenceDefaults::default(); + } + + #[cfg(not(test))] + { + let config_path = lime_core::config::ConfigManager::default_config_path(); + let Ok(manager) = lime_core::config::ConfigManager::load(&config_path) else { + return AudioGenerationPreferenceDefaults::default(); + }; + let voice_preference = manager + .config() + .workspace_preferences + .media_defaults + .voice + .clone(); + + AudioGenerationPreferenceDefaults { + provider_id: normalize_optional_string(voice_preference.preferred_provider_id), + model: normalize_optional_string(voice_preference.preferred_model_id), + } + } +} + +fn load_audio_generation_preference_defaults_from_app( + app: Option<&AppHandle>, +) -> AudioGenerationPreferenceDefaults { + let Some(config_manager) = + app.and_then(|app_handle| app_handle.try_state::()) + else { + return AudioGenerationPreferenceDefaults::default(); + }; + let voice_preference = config_manager + .config() + .workspace_preferences + .media_defaults + .voice + .clone(); + + AudioGenerationPreferenceDefaults { + provider_id: normalize_optional_string(voice_preference.preferred_provider_id), + model: normalize_optional_string(voice_preference.preferred_model_id), + } +} + fn apply_image_generation_preference_defaults( provider_id: Option, model: Option, @@ -312,6 +487,29 @@ fn apply_image_generation_preference_defaults( (effective_provider_id, effective_model) } +fn apply_audio_generation_preference_defaults( + provider_id: Option, + model: Option, + defaults: &AudioGenerationPreferenceDefaults, +) -> (Option, Option) { + let provider_missing = provider_id.is_none(); + let effective_provider_id = provider_id.or_else(|| defaults.provider_id.clone()); + let effective_model = match model { + Some(value) => Some(value), + None => { + if effective_provider_id.as_deref() == defaults.provider_id.as_deref() + || provider_missing + { + defaults.model.clone() + } else { + None + } + } + }; + + (effective_provider_id, effective_model) +} + fn normalize_required_string(value: &str, field_name: &str) -> Result { let trimmed = value.trim(); if trimmed.is_empty() { @@ -482,12 +680,84 @@ fn build_audio_task_idempotency_key( Ok(format!("audio-task-{}", hex::encode(&digest[..16]))) } +fn build_transcription_task_idempotency_key( + request: &CreateTranscriptionTaskArtifactRequest, + source_url: Option<&str>, + source_path: Option<&str>, + language: Option<&str>, + output_format: Option<&str>, + provider_id: Option<&str>, + model: Option<&str>, +) -> Result { + let fingerprint = json!({ + "session_id": normalize_optional_string(request.session_id.clone()), + "project_id": normalize_optional_string(request.project_id.clone()), + "content_id": normalize_optional_string(request.content_id.clone()), + "entry_source": normalize_optional_string(request.entry_source.clone()), + "source_url": source_url, + "source_path": source_path, + "language": language, + "output_format": output_format, + "speaker_labels": request.speaker_labels, + "timestamps": request.timestamps, + "provider_id": provider_id, + "model": model, + }); + let serialized = serde_json::to_vec(&fingerprint) + .map_err(|error| format!("序列化转写任务幂等指纹失败: {error}"))?; + let mut hasher = Sha256::new(); + hasher.update(&serialized); + let digest = hasher.finalize(); + Ok(format!("transcription-task-{}", hex::encode(&digest[..16]))) +} + #[derive(Debug, Clone)] struct ImageGenerationRunnerConfig { endpoint: String, api_key: String, } +#[derive(Debug, Clone, PartialEq, Eq)] +struct AudioGenerationRunnerConfig { + provider_id: String, + model: String, + endpoint: String, + api_key: String, +} + +#[derive(Debug, Clone)] +struct GeneratedAudioOutput { + audio_bytes: Vec, + mime_type: String, + provider_id: String, + model: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct TranscriptionRunnerConfig { + provider_id: String, + model: String, + endpoint: String, + api_key: String, +} + +#[derive(Debug, Clone)] +struct TranscriptionSourceBytes { + bytes: Vec, + file_name: String, + mime_type: String, +} + +#[derive(Debug, Clone)] +struct GeneratedTranscriptOutput { + content: String, + text_preview: Option, + language: Option, + output_format: String, + provider_id: String, + model: String, +} + fn normalize_server_host(host: &str) -> String { let trimmed = host.trim(); if trimmed.is_empty() || trimmed == "0.0.0.0" || trimmed == "::" { @@ -540,6 +810,46 @@ fn finish_image_task_execution(task_id: &str) { active.remove(task_id); } +fn should_start_audio_generation_worker(output: &MediaTaskOutput) -> bool { + output.task_type == MediaTaskType::AudioGenerate.as_str() + && media_task_contract_key(output).as_deref() == Some(VOICE_GENERATION_CONTRACT_KEY) + && matches!(output.normalized_status.as_str(), "pending" | "queued") +} + +fn should_start_transcription_worker(output: &MediaTaskOutput) -> bool { + output.task_type == MediaTaskType::TranscriptionGenerate.as_str() + && media_task_contract_key(output).as_deref() == Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY) + && matches!(output.normalized_status.as_str(), "pending" | "queued") +} + +fn mark_audio_task_execution_started(task_id: &str) -> bool { + let mut active = ACTIVE_AUDIO_TASK_EXECUTIONS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + active.insert(task_id.to_string()) +} + +fn finish_audio_task_execution(task_id: &str) { + let mut active = ACTIVE_AUDIO_TASK_EXECUTIONS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + active.remove(task_id); +} + +fn mark_transcription_task_execution_started(task_id: &str) -> bool { + let mut active = ACTIVE_TRANSCRIPTION_TASK_EXECUTIONS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + active.insert(task_id.to_string()) +} + +fn finish_transcription_task_execution(task_id: &str) { + let mut active = ACTIVE_TRANSCRIPTION_TASK_EXECUTIONS + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + active.remove(task_id); +} + fn build_task_progress(phase: &str, message: String, percent: Option) -> TaskProgress { TaskProgress { phase: Some(phase.to_string()), @@ -565,6 +875,339 @@ fn build_task_error( } } +fn build_task_error_with_provider_code( + code: &str, + message: impl Into, + retryable: bool, + stage: &str, + provider_code: Option, +) -> TaskErrorRecord { + let mut error = build_task_error(code, message, retryable, stage); + error.provider_code = provider_code; + error +} + +fn summarize_audio_response_body(raw: &str) -> String { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return "响应体为空".to_string(); + } + let mut summary: String = trimmed.chars().take(320).collect(); + if trimmed.chars().count() > 320 { + summary.push_str("..."); + } + summary +} + +fn extract_audio_provider_error_code(body: &str) -> Option { + let parsed: serde_json::Value = serde_json::from_str(body).ok()?; + parsed + .pointer("/error/code") + .or_else(|| parsed.pointer("/error/type")) + .or_else(|| parsed.get("code")) + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) +} + +fn normalize_openai_audio_base_url(base_url: Option<&str>) -> String { + base_url + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(AUDIO_TASK_DEFAULT_OPENAI_BASE_URL) + .trim_end_matches('/') + .to_string() +} + +fn build_openai_audio_speech_url(base_url: Option<&str>) -> String { + let normalized = normalize_openai_audio_base_url(base_url); + if normalized.ends_with("/audio/speech") { + normalized + } else { + format!("{normalized}/audio/speech") + } +} + +fn audio_response_format_for_mime_type(mime_type: Option<&str>) -> &'static str { + match mime_type + .map(str::trim) + .unwrap_or(AUDIO_TASK_DEFAULT_MIME_TYPE) + .to_ascii_lowercase() + .as_str() + { + "audio/wav" | "audio/x-wav" => "wav", + "audio/aac" => "aac", + "audio/ogg" | "audio/opus" => "opus", + "audio/flac" => "flac", + "audio/pcm" | "audio/l16" => "pcm", + _ => "mp3", + } +} + +fn audio_file_extension_for_mime_type(mime_type: &str) -> &'static str { + match mime_type.trim().to_ascii_lowercase().as_str() { + "audio/wav" | "audio/x-wav" => "wav", + "audio/aac" => "aac", + "audio/ogg" | "audio/opus" => "opus", + "audio/flac" => "flac", + "audio/pcm" | "audio/l16" => "pcm", + _ => "mp3", + } +} + +fn transcription_file_extension_for_output_format(output_format: &str) -> &'static str { + match output_format.trim().to_ascii_lowercase().as_str() { + "srt" => "srt", + "vtt" => "vtt", + "json" | "verbose_json" => "json", + _ => "txt", + } +} + +fn media_file_mime_type_from_name(file_name: &str) -> &'static str { + match Path::new(file_name) + .extension() + .and_then(|value| value.to_str()) + .map(str::trim) + .unwrap_or_default() + .to_ascii_lowercase() + .as_str() + { + "wav" => "audio/wav", + "mp3" => "audio/mpeg", + "m4a" => "audio/mp4", + "aac" => "audio/aac", + "ogg" | "opus" => "audio/ogg", + "flac" => "audio/flac", + "webm" => "video/webm", + "mp4" | "m4v" => "video/mp4", + "mov" => "video/quicktime", + _ => "application/octet-stream", + } +} + +fn sanitize_audio_output_file_stem(raw: &str) -> String { + let mut sanitized = String::new(); + for character in raw.chars() { + if character.is_ascii_alphanumeric() || matches!(character, '-' | '_') { + sanitized.push(character); + } else { + sanitized.push('-'); + } + } + let sanitized = sanitized.trim_matches('-').to_string(); + if sanitized.is_empty() { + let mut hasher = Sha256::new(); + hasher.update(raw.as_bytes()); + let digest = hasher.finalize(); + format!("audio-{}", hex::encode(&digest[..8])) + } else { + sanitized + } +} + +fn build_audio_output_relative_path(task_id: &str, mime_type: &str) -> PathBuf { + let file_stem = sanitize_audio_output_file_stem(task_id); + PathBuf::from(AUDIO_TASK_OUTPUT_RELATIVE_DIR).join(format!( + "{file_stem}.{}", + audio_file_extension_for_mime_type(mime_type) + )) +} + +fn build_transcript_output_relative_path(task_id: &str, output_format: &str) -> PathBuf { + let file_stem = sanitize_audio_output_file_stem(task_id); + PathBuf::from(TRANSCRIPTION_TASK_OUTPUT_RELATIVE_DIR).join(format!( + "{file_stem}.{}", + transcription_file_extension_for_output_format(output_format) + )) +} + +fn path_to_runtime_relative_string(path: &Path) -> String { + path.components() + .filter_map(|component| match component { + std::path::Component::Normal(value) => Some(value.to_string_lossy().to_string()), + _ => None, + }) + .collect::>() + .join("/") +} + +fn write_audio_output_bytes( + workspace_root: &Path, + task_id: &str, + mime_type: &str, + audio_bytes: &[u8], +) -> Result { + if audio_bytes.is_empty() { + return Err(build_task_error( + "audio_provider_empty_response", + "音频 provider 返回了空音频内容，已阻止写入 audio_output。", + true, + "response", + )); + } + + let relative_path = build_audio_output_relative_path(task_id, mime_type); + let absolute_path = workspace_root.join(&relative_path); + if let Some(parent) = absolute_path.parent() { + fs::create_dir_all(parent).map_err(|error| { + build_task_error( + "audio_output_write_failed", + format!("创建音频输出目录失败: {error}"), + true, + "output", + ) + })?; + } + fs::write(&absolute_path, audio_bytes).map_err(|error| { + build_task_error( + "audio_output_write_failed", + format!("写入音频输出文件失败: {error}"), + true, + "output", + ) + })?; + + Ok(path_to_runtime_relative_string(&relative_path)) +} + +fn write_transcript_output_text( + workspace_root: &Path, + task_id: &str, + output_format: &str, + content: &str, +) -> Result { + if content.trim().is_empty() { + return Err(build_task_error( + "transcription_provider_empty_response", + "转写 provider 返回了空 transcript 内容，已阻止写入 transcript。", + true, + "response", + )); + } + + let relative_path = build_transcript_output_relative_path(task_id, output_format); + let absolute_path = workspace_root.join(&relative_path); + if let Some(parent) = absolute_path.parent() { + fs::create_dir_all(parent).map_err(|error| { + build_task_error( + "transcript_output_write_failed", + format!("创建 transcript 输出目录失败: {error}"), + true, + "output", + ) + })?; + } + fs::write(&absolute_path, content).map_err(|error| { + build_task_error( + "transcript_output_write_failed", + format!("写入 transcript 输出文件失败: {error}"), + true, + "output", + ) + })?; + + Ok(path_to_runtime_relative_string(&relative_path)) +} + +fn normalize_openai_transcription_base_url(base_url: Option<&str>) -> String { + base_url + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(TRANSCRIPTION_TASK_DEFAULT_OPENAI_BASE_URL) + .trim_end_matches('/') + .to_string() +} + +fn build_openai_audio_transcriptions_url(base_url: Option<&str>) -> String { + let normalized = normalize_openai_transcription_base_url(base_url); + if normalized.ends_with("/audio/transcriptions") { + normalized + } else { + format!("{normalized}/audio/transcriptions") + } +} + +fn normalize_transcription_response_format(output_format: Option<&str>) -> String { + match output_format + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or("json") + .to_ascii_lowercase() + .as_str() + { + "srt" => "srt".to_string(), + "vtt" => "vtt".to_string(), + "text" | "txt" | "markdown" | "md" => "text".to_string(), + "verbose_json" | "verbose-json" => "verbose_json".to_string(), + _ => "json".to_string(), + } +} + +fn transcript_text_preview(raw: &str) -> Option { + let trimmed = raw.trim(); + if trimmed.is_empty() { + return None; + } + let mut preview: String = trimmed.chars().take(500).collect(); + if trimmed.chars().count() > 500 { + preview.push_str("..."); + } + Some(preview) +} + +fn extract_transcript_text_from_response(raw: &str, response_format: &str) -> Option { + if matches!(response_format, "text" | "srt" | "vtt") { + return transcript_text_preview(raw); + } + + let parsed: serde_json::Value = serde_json::from_str(raw).ok()?; + parsed + .get("text") + .and_then(serde_json::Value::as_str) + .and_then(transcript_text_preview) +} + +fn extract_transcript_language_from_response(raw: &str) -> Option { + serde_json::from_str::(raw) + .ok() + .and_then(|parsed| { + parsed + .get("language") + .and_then(serde_json::Value::as_str) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) + }) +} + +fn source_url_file_name(source_url: &str) -> String { + let candidate = source_url + .split('?') + .next() + .unwrap_or(source_url) + .trim_end_matches('/') + .rsplit('/') + .next() + .unwrap_or("audio") + .trim(); + if candidate.is_empty() { + "audio".to_string() + } else { + candidate.to_string() + } +} + +fn resolve_transcription_source_path(workspace_root: &Path, source_path: &str) -> PathBuf { + let path = PathBuf::from(source_path); + if path.is_absolute() { + path + } else { + workspace_root.join(path) + } +} + fn read_image_task_payload_string<'a>( payload: &'a serde_json::Value, keys: &[&str], @@ -634,7 +1277,10 @@ fn media_task_routing_outcome(output: &MediaTaskOutput) -> (&'static str, &'stat if is_contract_routing_failure { ("routing_not_possible", "blocked") - } else if media_task_contract_key(output).as_deref() == Some(VOICE_GENERATION_CONTRACT_KEY) { + } else if matches!( + media_task_contract_key(output).as_deref(), + Some(VOICE_GENERATION_CONTRACT_KEY) | Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY) + ) { if output.normalized_status == "failed" { ("executor_invoked", "failed") } else { @@ -687,6 +1333,106 @@ fn increment_routing_outcome_count(counts: &mut Vec, + status: &str, +) { + if let Some(item) = counts.iter_mut().find(|item| item.status == status) { + item.count += 1; + return; + } + counts.push(MediaTaskAudioOutputStatusCount { + status: status.to_string(), + count: 1, + }); +} + +fn increment_transcript_status_count( + counts: &mut Vec, + status: &str, +) { + if let Some(item) = counts.iter_mut().find(|item| item.status == status) { + item.count += 1; + return; + } + counts.push(MediaTaskTranscriptStatusCount { + status: status.to_string(), + count: 1, + }); +} + +fn media_task_audio_output(output: &MediaTaskOutput) -> Option<&serde_json::Value> { + output + .record + .payload + .get("audio_output") + .filter(|value| value.is_object()) + .or_else(|| { + output + .record + .result + .as_ref() + .and_then(|value| value.get("audio_output")) + .filter(|value| value.is_object()) + }) +} + +fn media_task_transcript(output: &MediaTaskOutput) -> Option<&serde_json::Value> { + output + .record + .payload + .get("transcript") + .filter(|value| value.is_object()) + .or_else(|| { + output + .record + .result + .as_ref() + .and_then(|value| value.get("transcript")) + .filter(|value| value.is_object()) + }) +} + +fn media_task_audio_output_string( + audio_output: Option<&serde_json::Value>, + keys: &[&str], +) -> Option { + audio_output + .and_then(|value| read_image_task_payload_string(value, keys).map(ToString::to_string)) +} + +fn media_task_audio_output_u64( + audio_output: Option<&serde_json::Value>, + keys: &[&str], +) -> Option { + audio_output.and_then(|value| { + keys.iter() + .find_map(|key| value.get(*key).and_then(serde_json::Value::as_u64)) + }) +} + +fn media_task_audio_output_bool( + audio_output: Option<&serde_json::Value>, + keys: &[&str], +) -> Option { + audio_output.and_then(|value| read_image_task_payload_bool(value, keys)) +} + +fn media_task_transcript_string( + transcript: Option<&serde_json::Value>, + keys: &[&str], +) -> Option { + transcript + .and_then(|value| read_image_task_payload_string(value, keys).map(ToString::to_string)) +} + +fn media_task_transcript_bool( + transcript: Option<&serde_json::Value>, + keys: &[&str], +) -> Option { + transcript.and_then(|value| read_image_task_payload_bool(value, keys)) +} + fn build_modality_runtime_contract_index( tasks: &[MediaTaskOutput], ) -> MediaTaskModalityRuntimeContractIndex { @@ -694,6 +1440,12 @@ fn build_modality_runtime_contract_index( let mut blocked_count = 0; let mut routing_outcomes = Vec::new(); let mut model_registry_assessment_count = 0; + let mut audio_output_count = 0; + let mut audio_output_statuses = Vec::new(); + let mut audio_output_error_codes = Vec::new(); + let mut transcript_count = 0; + let mut transcript_statuses = Vec::new(); + let mut transcript_error_codes = Vec::new(); let mut snapshots = Vec::new(); for output in tasks { @@ -720,6 +1472,31 @@ fn build_modality_runtime_contract_index( if model_capability_assessment_source.as_deref() == Some("model_registry") { model_registry_assessment_count += 1; } + let audio_output = media_task_audio_output(output); + let audio_output_status = media_task_audio_output_string(audio_output, &["status"]); + let audio_output_error_code = + media_task_audio_output_string(audio_output, &["error_code", "errorCode"]); + if audio_output.is_some() { + audio_output_count += 1; + } + if let Some(status) = audio_output_status.as_deref() { + increment_audio_output_status_count(&mut audio_output_statuses, status); + } + push_unique_string( + &mut audio_output_error_codes, + audio_output_error_code.clone(), + ); + let transcript = media_task_transcript(output); + let transcript_status = media_task_transcript_string(transcript, &["status"]); + let transcript_error_code = + media_task_transcript_string(transcript, &["error_code", "errorCode"]); + if transcript.is_some() { + transcript_count += 1; + } + if let Some(status) = transcript_status.as_deref() { + increment_transcript_status_count(&mut transcript_statuses, status); + } + push_unique_string(&mut transcript_error_codes, transcript_error_code.clone()); snapshots.push(MediaTaskModalityRuntimeContractIndexEntry { task_id: output.task_id.clone(), @@ -738,6 +1515,41 @@ fn build_modality_runtime_contract_index( model_supports_image_generation: model_capability_assessment.and_then(|value| { read_image_task_payload_bool(value, &["supports_image_generation"]) }), + audio_output_status, + audio_output_path: media_task_audio_output_string( + audio_output, + &["audio_path", "audioPath"], + ), + audio_output_mime_type: media_task_audio_output_string( + audio_output, + &["mime_type", "mimeType"], + ), + audio_output_duration_ms: media_task_audio_output_u64( + audio_output, + &["duration_ms", "durationMs"], + ), + audio_output_error_code, + audio_output_retryable: media_task_audio_output_bool(audio_output, &["retryable"]), + transcript_status, + transcript_path: media_task_transcript_string( + transcript, + &["transcript_path", "transcriptPath", "path"], + ), + transcript_source_url: media_task_transcript_string( + transcript, + &["source_url", "sourceUrl"], + ), + transcript_source_path: media_task_transcript_string( + transcript, + &["source_path", "sourcePath"], + ), + transcript_language: media_task_transcript_string(transcript, &["language"]), + transcript_output_format: media_task_transcript_string( + transcript, + &["output_format", "outputFormat"], + ), + transcript_error_code, + transcript_retryable: media_task_transcript_bool(transcript, &["retryable"]), }); } @@ -747,6 +1559,12 @@ fn build_modality_runtime_contract_index( blocked_count, routing_outcomes, model_registry_assessment_count, + audio_output_count, + audio_output_statuses, + audio_output_error_codes, + transcript_count, + transcript_statuses, + transcript_error_codes, snapshots, } } @@ -875,6 +1693,44 @@ fn patch_image_task( .map_err(|error| format!("写回图片任务状态失败: {error}")) } +fn load_current_audio_task( + workspace_root: &Path, + task_id: &str, +) -> Result { + get_media_task_artifact_inner(MediaTaskLookupRequest { + project_root_path: workspace_root.to_string_lossy().to_string(), + task_ref: task_id.to_string(), + }) +} + +fn patch_audio_task( + workspace_root: &Path, + task_id: &str, + patch: TaskArtifactPatch, +) -> Result { + patch_task_artifact(workspace_root, task_id, None, patch) + .map_err(|error| format!("写回音频任务状态失败: {error}")) +} + +fn load_current_transcription_task( + workspace_root: &Path, + task_id: &str, +) -> Result { + get_media_task_artifact_inner(MediaTaskLookupRequest { + project_root_path: workspace_root.to_string_lossy().to_string(), + task_ref: task_id.to_string(), + }) +} + +fn patch_transcription_task( + workspace_root: &Path, + task_id: &str, + patch: TaskArtifactPatch, +) -> Result { + patch_task_artifact(workspace_root, task_id, None, patch) + .map_err(|error| format!("写回转写任务状态失败: {error}")) +} + fn image_generation_model_capability_assessment_payload( assessment: &ImageGenerationModelCapabilityAssessment, ) -> serde_json::Value { @@ -909,6 +1765,861 @@ fn emit_image_task_event(app: Option<&AppHandle>, output: &MediaTaskOutput) { emit_creation_task_event_if_needed(app, output); } +fn emit_audio_task_event(app: Option<&AppHandle>, output: &MediaTaskOutput) { + emit_creation_task_event_if_needed(app, output); +} + +fn emit_transcription_task_event(app: Option<&AppHandle>, output: &MediaTaskOutput) { + emit_creation_task_event_if_needed(app, output); +} + +fn build_audio_output_status_summary( + payload: &serde_json::Value, + status: &str, + error: Option<&TaskErrorRecord>, +) -> serde_json::Value { + let mut audio_output = payload + .get("audio_output") + .cloned() + .filter(serde_json::Value::is_object) + .unwrap_or_else(|| { + json!({ + "kind": "audio_output", + "audio_path": payload.get("audio_path").cloned().unwrap_or(serde_json::Value::Null), + "mime_type": read_image_task_payload_string(payload, &["mime_type", "mimeType"]), + "duration_ms": payload.get("duration_ms").cloned().unwrap_or(serde_json::Value::Null), + "source_text": read_image_task_payload_string(payload, &["source_text", "sourceText", "prompt"]), + "voice": read_image_task_payload_string(payload, &["voice"]), + "voice_style": read_image_task_payload_string(payload, &["voice_style", "voiceStyle"]), + "target_language": read_image_task_payload_string(payload, &["target_language", "targetLanguage"]), + }) + }); + + if let Some(audio_output_object) = audio_output.as_object_mut() { + audio_output_object + .entry("kind".to_string()) + .or_insert_with(|| json!("audio_output")); + audio_output_object.insert("status".to_string(), json!(status)); + audio_output_object + .entry("mime_type".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["mime_type", "mimeType"]) + .map(|value| json!(value)) + .unwrap_or_else(|| json!(AUDIO_TASK_DEFAULT_MIME_TYPE)) + }); + audio_output_object + .entry("source_text".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["source_text", "sourceText", "prompt"]) + .map(|value| json!(value)) + .unwrap_or(serde_json::Value::Null) + }); + audio_output_object + .entry("voice".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["voice"]) + .map(|value| json!(value)) + .unwrap_or(serde_json::Value::Null) + }); + audio_output_object.insert( + "modality_contract_key".to_string(), + json!(VOICE_GENERATION_CONTRACT_KEY), + ); + audio_output_object.insert("modality".to_string(), json!("audio")); + audio_output_object.insert( + "routing_slot".to_string(), + json!(VOICE_GENERATION_ROUTING_SLOT), + ); + + if let Some(error) = error { + audio_output_object.insert("error_code".to_string(), json!(error.code.as_str())); + audio_output_object.insert("error_message".to_string(), json!(error.message.as_str())); + audio_output_object.insert("retryable".to_string(), json!(error.retryable)); + audio_output_object.insert("stage".to_string(), json!(error.stage.as_deref())); + } else { + audio_output_object.remove("error_code"); + audio_output_object.remove("error_message"); + audio_output_object.remove("retryable"); + audio_output_object.remove("stage"); + } + } + + audio_output +} + +fn resolve_audio_generation_payload_provider_model( + app: Option<&AppHandle>, + output: &MediaTaskOutput, +) -> Result<(String, String), TaskErrorRecord> { + let payload = &output.record.payload; + let requested_provider_id = + read_image_task_payload_string(payload, &["provider_id", "providerId"]) + .map(ToString::to_string); + let requested_model = + read_image_task_payload_string(payload, &["model"]).map(ToString::to_string); + let defaults = load_audio_generation_preference_defaults_from_app(app); + let (provider_id, model) = apply_audio_generation_preference_defaults( + requested_provider_id, + requested_model, + &defaults, + ); + + match (provider_id, model) { + (Some(provider_id), Some(model)) => Ok((provider_id, model)), + (None, _) => Err(build_task_error( + "audio_provider_unconfigured", + "voice_generation 音频任务缺少 provider_id；请先配置媒体生成 voice provider，不能回退到 legacy TTS。", + true, + "provider_config", + )), + (_, None) => Err(build_task_error( + "audio_provider_model_unconfigured", + "voice_generation 音频任务缺少 model；请先配置媒体生成 voice model，不能回退到 legacy TTS。", + true, + "provider_config", + )), + } +} + +fn audio_generation_runner_config_from_credential( + provider_id: String, + model: String, + credential: RuntimeProviderCredential, +) -> Result { + match credential.credential { + RuntimeCredentialData::OpenAIKey { api_key, base_url } => Ok(AudioGenerationRunnerConfig { + provider_id, + model, + endpoint: build_openai_audio_speech_url(base_url.as_deref()), + api_key, + }), + _ => Err(build_task_error( + "audio_provider_client_missing", + format!( + "voice_generation 当前只接入 OpenAI-compatible speech adapter，provider runtime type {} 尚未提供音频生成 client。", + credential.provider_type + ), + true, + "provider_client", + )), + } +} + +async fn resolve_audio_generation_runner_config( + app: Option<&AppHandle>, + output: &MediaTaskOutput, +) -> Result { + let Some(app_handle) = app else { + return Err(build_task_error( + "audio_provider_resolver_unavailable", + "缺少 AppHandle，无法从 current API Key Provider 主链解析 voice_generation provider 凭证。", + true, + "bootstrap", + )); + }; + let (provider_id, model) = resolve_audio_generation_payload_provider_model(app, output)?; + let Some(db) = app_handle + .try_state::() + .map(|state| state.inner().clone()) + else { + return Err(build_task_error( + "audio_provider_resolver_unavailable", + "DbConnection 未初始化，无法从 current API Key Provider 主链解析音频 provider。", + true, + "bootstrap", + )); + }; + let Some(api_key_provider_service) = app_handle + .try_state::() + .map(|state| state.0.clone()) + else { + return Err(build_task_error( + "audio_provider_resolver_unavailable", + "ApiKeyProviderServiceState 未初始化，无法从 current API Key Provider 主链解析音频 provider。", + true, + "bootstrap", + )); + }; + + let credential = api_key_provider_service + .select_credential_for_provider(&db, provider_id.as_str(), Some(provider_id.as_str()), None) + .await + .map_err(|error| { + build_task_error( + "audio_provider_resolution_failed", + format!("解析 voice_generation provider 凭证失败: {error}"), + true, + "provider_config", + ) + })? + .ok_or_else(|| { + build_task_error( + "audio_provider_unconfigured", + format!( + "未找到可用的 voice_generation provider/API Key: {provider_id}；任务保留在 audio_generate artifact，不能回退 legacy TTS。" + ), + true, + "provider_config", + ) + })?; + + audio_generation_runner_config_from_credential(provider_id, model, credential) +} + +fn build_audio_generation_instructions(payload: &serde_json::Value) -> Option { + let mut instructions = Vec::new(); + if let Some(voice_style) = + read_image_task_payload_string(payload, &["voice_style", "voiceStyle"]) + { + instructions.push(format!("Voice style: {voice_style}.")); + } + if let Some(target_language) = + read_image_task_payload_string(payload, &["target_language", "targetLanguage"]) + { + instructions.push(format!("Speak in {target_language}.")); + } + + if instructions.is_empty() { + None + } else { + Some(instructions.join(" ")) + } +} + +fn audio_model_supports_speech_instructions(model: &str) -> bool { + let normalized = model.trim().to_ascii_lowercase(); + normalized.starts_with("gpt-4o") || normalized.starts_with("gpt-5") +} + +fn content_type_audio_mime_or_fallback( + headers: &reqwest::header::HeaderMap, + fallback_mime_type: &str, +) -> String { + headers + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .map(str::trim) + .filter(|value| value.starts_with("audio/")) + .map(ToString::to_string) + .unwrap_or_else(|| fallback_mime_type.to_string()) +} + +async fn request_openai_compatible_audio_generation( + config: &AudioGenerationRunnerConfig, + output: &MediaTaskOutput, +) -> Result { + let payload = &output.record.payload; + let source_text = + read_image_task_payload_string(payload, &["source_text", "sourceText", "prompt"]) + .ok_or_else(|| { + build_task_error( + "invalid_audio_task_payload", + "audio_generate 任务缺少 source_text，无法调用音频 provider。", + false, + "payload", + ) + })?; + let requested_mime_type = read_image_task_payload_string(payload, &["mime_type", "mimeType"]) + .unwrap_or(AUDIO_TASK_DEFAULT_MIME_TYPE); + let voice = + read_image_task_payload_string(payload, &["voice"]).unwrap_or(AUDIO_TASK_DEFAULT_VOICE); + let response_format = audio_response_format_for_mime_type(Some(requested_mime_type)); + let mut request_body = json!({ + "model": config.model.as_str(), + "input": source_text, + "voice": voice, + "response_format": response_format, + }); + if audio_model_supports_speech_instructions(&config.model) { + if let Some(instructions) = build_audio_generation_instructions(payload) { + if let Some(body_object) = request_body.as_object_mut() { + body_object.insert("instructions".to_string(), json!(instructions)); + } + } + } + + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(AUDIO_TASK_RUNNER_TIMEOUT_SECS)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + let response = client + .post(&config.endpoint) + .header( + reqwest::header::AUTHORIZATION, + format!("Bearer {}", config.api_key), + ) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .json(&request_body) + .send() + .await + .map_err(|error| { + build_task_error( + "audio_provider_request_failed", + format!("调用音频 provider 失败: {error}"), + true, + "request", + ) + })?; + + let status = response.status(); + let response_mime_type = + content_type_audio_mime_or_fallback(response.headers(), requested_mime_type); + let body_bytes = response.bytes().await.map_err(|error| { + build_task_error( + "audio_provider_response_read_failed", + format!("读取音频 provider 响应失败: {error}"), + true, + "response", + ) + })?; + + if !status.is_success() { + let body = String::from_utf8_lossy(&body_bytes).to_string(); + return Err(build_task_error_with_provider_code( + "audio_provider_request_failed", + format!( + "音频 provider 返回错误 {status}: {}", + summarize_audio_response_body(&body) + ), + status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS, + "request", + extract_audio_provider_error_code(&body), + )); + } + + if body_bytes.is_empty() { + return Err(build_task_error( + "audio_provider_empty_response", + "音频 provider 返回了空音频内容。", + true, + "response", + )); + } + + Ok(GeneratedAudioOutput { + audio_bytes: body_bytes.to_vec(), + mime_type: response_mime_type, + provider_id: config.provider_id.clone(), + model: config.model.clone(), + }) +} + +fn validate_audio_generation_task_execution_contract( + output: &MediaTaskOutput, +) -> Result<(), TaskErrorRecord> { + if output.task_type != MediaTaskType::AudioGenerate.as_str() { + return Err(build_task_error( + "voice_generation_task_type_mismatch", + format!( + "voice_generation 执行器只能处理 audio_generate 任务，收到 {}", + output.task_type + ), + false, + "contract", + )); + } + + if media_task_contract_key(output).as_deref() != Some(VOICE_GENERATION_CONTRACT_KEY) { + return Err(build_task_error( + "voice_generation_contract_mismatch", + format!( + "音频任务 contract_key 必须是 {VOICE_GENERATION_CONTRACT_KEY}，已阻止进入执行器。" + ), + false, + "contract", + )); + } + + let required_capabilities = + read_image_task_payload_string_array(&output.record.payload, "required_capabilities") + .unwrap_or_default(); + if !required_capabilities.is_empty() + && !required_capabilities + .iter() + .any(|capability| capability == VOICE_GENERATION_CONTRACT_KEY) + { + return Err(build_task_error( + "voice_generation_capability_gap", + "音频任务缺少 voice_generation required_capability，已阻止进入执行器。", + false, + "routing", + )); + } + + if let Some(routing_slot) = + read_image_task_payload_string(&output.record.payload, &["routing_slot"]) + { + if routing_slot != VOICE_GENERATION_ROUTING_SLOT { + return Err(build_task_error( + "voice_generation_routing_slot_mismatch", + format!( + "音频任务 routing_slot 必须是 {VOICE_GENERATION_ROUTING_SLOT}，收到 {routing_slot}" + ), + false, + "routing", + )); + } + } + + Ok(()) +} + +fn build_transcript_status_summary( + payload: &serde_json::Value, + status: &str, + error: Option<&TaskErrorRecord>, +) -> serde_json::Value { + let mut transcript = payload + .get("transcript") + .cloned() + .filter(serde_json::Value::is_object) + .unwrap_or_else(|| { + json!({ + "kind": "transcript", + "source_url": read_image_task_payload_string(payload, &["source_url", "sourceUrl"]), + "source_path": read_image_task_payload_string(payload, &["source_path", "sourcePath"]), + "language": read_image_task_payload_string(payload, &["language"]), + "output_format": read_image_task_payload_string(payload, &["output_format", "outputFormat"]), + "timestamps": payload.get("timestamps").cloned().unwrap_or(serde_json::Value::Null), + "speaker_labels": payload.get("speaker_labels").cloned().unwrap_or(serde_json::Value::Null), + "provider_id": read_image_task_payload_string(payload, &["provider_id", "providerId"]), + "model": read_image_task_payload_string(payload, &["model"]), + }) + }); + + if let Some(transcript_object) = transcript.as_object_mut() { + transcript_object + .entry("kind".to_string()) + .or_insert_with(|| json!("transcript")); + transcript_object.insert("status".to_string(), json!(status)); + transcript_object + .entry("source_url".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["source_url", "sourceUrl"]) + .map(|value| json!(value)) + .unwrap_or(serde_json::Value::Null) + }); + transcript_object + .entry("source_path".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["source_path", "sourcePath"]) + .map(|value| json!(value)) + .unwrap_or(serde_json::Value::Null) + }); + transcript_object + .entry("language".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["language"]) + .map(|value| json!(value)) + .unwrap_or(serde_json::Value::Null) + }); + transcript_object + .entry("output_format".to_string()) + .or_insert_with(|| { + read_image_task_payload_string(payload, &["output_format", "outputFormat"]) + .map(|value| json!(value)) + .unwrap_or_else(|| json!("json")) + }); + transcript_object.insert( + "modality_contract_key".to_string(), + json!(AUDIO_TRANSCRIPTION_CONTRACT_KEY), + ); + transcript_object.insert("modality".to_string(), json!("audio")); + transcript_object.insert( + "routing_slot".to_string(), + json!(AUDIO_TRANSCRIPTION_ROUTING_SLOT), + ); + + if let Some(error) = error { + transcript_object.insert("error_code".to_string(), json!(error.code.as_str())); + transcript_object.insert("error_message".to_string(), json!(error.message.as_str())); + transcript_object.insert("retryable".to_string(), json!(error.retryable)); + transcript_object.insert("stage".to_string(), json!(error.stage.as_deref())); + } else { + transcript_object.remove("error_code"); + transcript_object.remove("error_message"); + transcript_object.remove("retryable"); + transcript_object.remove("stage"); + } + } + + transcript +} + +fn resolve_transcription_payload_provider_model( + app: Option<&AppHandle>, + output: &MediaTaskOutput, +) -> Result<(String, String), TaskErrorRecord> { + let payload = &output.record.payload; + let provider_id = read_image_task_payload_string(payload, &["provider_id", "providerId"]) + .map(ToString::to_string); + let model = read_image_task_payload_string(payload, &["model"]).map(ToString::to_string); + + match (provider_id, model) { + (Some(provider_id), Some(model)) => Ok((provider_id, model)), + (None, _) if app.is_none() => Err(build_task_error( + "transcription_provider_resolver_unavailable", + "缺少 AppHandle，无法从 current API Key Provider 主链解析 audio_transcription provider 凭证。", + true, + "bootstrap", + )), + (None, _) => Err(build_task_error( + "transcription_provider_unconfigured", + "audio_transcription 转写任务缺少 provider_id；请先配置转写 provider，不能回退 frontend ASR 或 generic_file transcript。", + true, + "provider_config", + )), + (_, None) => Err(build_task_error( + "transcription_provider_model_unconfigured", + "audio_transcription 转写任务缺少 model；请先配置转写 model，不能回退 frontend ASR 或 generic_file transcript。", + true, + "provider_config", + )), + } +} + +fn transcription_runner_config_from_credential( + provider_id: String, + model: String, + credential: RuntimeProviderCredential, +) -> Result { + match credential.credential { + RuntimeCredentialData::OpenAIKey { api_key, base_url } => Ok(TranscriptionRunnerConfig { + provider_id, + model, + endpoint: build_openai_audio_transcriptions_url(base_url.as_deref()), + api_key, + }), + _ => Err(build_task_error( + "transcription_provider_client_missing", + format!( + "audio_transcription 当前只接入 OpenAI-compatible transcription adapter，provider runtime type {} 尚未提供转写 client。", + credential.provider_type + ), + true, + "provider_client", + )), + } +} + +async fn resolve_transcription_runner_config( + app: Option<&AppHandle>, + output: &MediaTaskOutput, +) -> Result { + let Some(app_handle) = app else { + return Err(build_task_error( + "transcription_provider_resolver_unavailable", + "缺少 AppHandle，无法从 current API Key Provider 主链解析 audio_transcription provider 凭证。", + true, + "bootstrap", + )); + }; + let (provider_id, model) = resolve_transcription_payload_provider_model(app, output)?; + let Some(db) = app_handle + .try_state::() + .map(|state| state.inner().clone()) + else { + return Err(build_task_error( + "transcription_provider_resolver_unavailable", + "DbConnection 未初始化，无法从 current API Key Provider 主链解析转写 provider。", + true, + "bootstrap", + )); + }; + let Some(api_key_provider_service) = app_handle + .try_state::() + .map(|state| state.0.clone()) + else { + return Err(build_task_error( + "transcription_provider_resolver_unavailable", + "ApiKeyProviderServiceState 未初始化，无法从 current API Key Provider 主链解析转写 provider。", + true, + "bootstrap", + )); + }; + + let credential = api_key_provider_service + .select_credential_for_provider(&db, provider_id.as_str(), Some(provider_id.as_str()), None) + .await + .map_err(|error| { + build_task_error( + "transcription_provider_resolution_failed", + format!("解析 audio_transcription provider 凭证失败: {error}"), + true, + "provider_config", + ) + })? + .ok_or_else(|| { + build_task_error( + "transcription_provider_unconfigured", + format!( + "未找到可用的 audio_transcription provider/API Key: {provider_id}；任务保留在 transcription_generate artifact，不能回退 frontend ASR。" + ), + true, + "provider_config", + ) + })?; + + transcription_runner_config_from_credential(provider_id, model, credential) +} + +async fn read_transcription_source_bytes( + workspace_root: &Path, + output: &MediaTaskOutput, +) -> Result { + let payload = &output.record.payload; + if let Some(source_path) = + read_image_task_payload_string(payload, &["source_path", "sourcePath"]) + { + let absolute_path = resolve_transcription_source_path(workspace_root, source_path); + let bytes = fs::read(&absolute_path).map_err(|error| { + build_task_error( + "transcription_source_unavailable", + format!("读取转写源文件失败 {}: {error}", absolute_path.display()), + true, + "source", + ) + })?; + let file_name = absolute_path + .file_name() + .and_then(|value| value.to_str()) + .unwrap_or("audio") + .to_string(); + let mime_type = media_file_mime_type_from_name(&file_name).to_string(); + return Ok(TranscriptionSourceBytes { + bytes, + file_name, + mime_type, + }); + } + + let source_url = read_image_task_payload_string(payload, &["source_url", "sourceUrl"]) + .ok_or_else(|| { + build_task_error( + "invalid_transcription_task_payload", + "transcription_generate 任务缺少 source_path/source_url，无法调用转写 provider。", + false, + "payload", + ) + })?; + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(TRANSCRIPTION_TASK_RUNNER_TIMEOUT_SECS)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + let response = client.get(source_url).send().await.map_err(|error| { + build_task_error( + "transcription_source_download_failed", + format!("下载转写源失败: {error}"), + true, + "source", + ) + })?; + let status = response.status(); + let mime_type = response + .headers() + .get(reqwest::header::CONTENT_TYPE) + .and_then(|value| value.to_str().ok()) + .and_then(|value| value.split(';').next()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) + .unwrap_or_else(|| media_file_mime_type_from_name(source_url).to_string()); + let bytes = response.bytes().await.map_err(|error| { + build_task_error( + "transcription_source_download_failed", + format!("读取转写源下载响应失败: {error}"), + true, + "source", + ) + })?; + if !status.is_success() { + return Err(build_task_error( + "transcription_source_download_failed", + format!("下载转写源返回错误 {status}。"), + true, + "source", + )); + } + + Ok(TranscriptionSourceBytes { + bytes: bytes.to_vec(), + file_name: source_url_file_name(source_url), + mime_type, + }) +} + +async fn request_openai_compatible_transcription( + config: &TranscriptionRunnerConfig, + output: &MediaTaskOutput, + workspace_root: &Path, +) -> Result { + let payload = &output.record.payload; + let source = read_transcription_source_bytes(workspace_root, output).await?; + if source.bytes.is_empty() { + return Err(build_task_error( + "transcription_source_empty", + "转写源文件为空，已阻止调用 audio_transcription provider。", + false, + "source", + )); + } + + let response_format = normalize_transcription_response_format(read_image_task_payload_string( + payload, + &["output_format", "outputFormat"], + )); + let output_format = response_format.clone(); + let file_part = reqwest::multipart::Part::bytes(source.bytes) + .file_name(source.file_name) + .mime_str(source.mime_type.as_str()) + .map_err(|error| { + build_task_error( + "transcription_source_mime_invalid", + format!("转写源 MIME 类型无效: {error}"), + false, + "source", + ) + })?; + let mut form = reqwest::multipart::Form::new() + .part("file", file_part) + .text("model", config.model.clone()) + .text("response_format", response_format.clone()); + if let Some(language) = read_image_task_payload_string(payload, &["language"]) { + if !language.eq_ignore_ascii_case("auto") { + form = form.text("language", language.to_string()); + } + } + if let Some(prompt) = read_image_task_payload_string(payload, &["prompt", "raw_text"]) { + form = form.text("prompt", prompt.to_string()); + } + + let client = reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(TRANSCRIPTION_TASK_RUNNER_TIMEOUT_SECS)) + .build() + .unwrap_or_else(|_| reqwest::Client::new()); + let response = client + .post(&config.endpoint) + .header( + reqwest::header::AUTHORIZATION, + format!("Bearer {}", config.api_key), + ) + .multipart(form) + .send() + .await + .map_err(|error| { + build_task_error( + "transcription_provider_request_failed", + format!("调用转写 provider 失败: {error}"), + true, + "request", + ) + })?; + + let status = response.status(); + let body = response.text().await.map_err(|error| { + build_task_error( + "transcription_provider_response_read_failed", + format!("读取转写 provider 响应失败: {error}"), + true, + "response", + ) + })?; + + if !status.is_success() { + return Err(build_task_error_with_provider_code( + "transcription_provider_request_failed", + format!( + "转写 provider 返回错误 {status}: {}", + summarize_audio_response_body(&body) + ), + status.is_server_error() || status == reqwest::StatusCode::TOO_MANY_REQUESTS, + "request", + extract_audio_provider_error_code(&body), + )); + } + + let text_preview = extract_transcript_text_from_response(&body, &response_format) + .or_else(|| transcript_text_preview(&body)); + let language = extract_transcript_language_from_response(&body).or_else(|| { + read_image_task_payload_string(payload, &["language"]) + .map(ToString::to_string) + .filter(|value| !value.eq_ignore_ascii_case("auto")) + }); + + Ok(GeneratedTranscriptOutput { + content: body, + text_preview, + language, + output_format, + provider_id: config.provider_id.clone(), + model: config.model.clone(), + }) +} + +fn validate_transcription_task_execution_contract( + output: &MediaTaskOutput, +) -> Result<(), TaskErrorRecord> { + if output.task_type != MediaTaskType::TranscriptionGenerate.as_str() { + return Err(build_task_error( + "audio_transcription_task_type_mismatch", + format!( + "audio_transcription 执行器只能处理 transcription_generate 任务，收到 {}", + output.task_type + ), + false, + "contract", + )); + } + + if media_task_contract_key(output).as_deref() != Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY) { + return Err(build_task_error( + "audio_transcription_contract_mismatch", + format!( + "转写任务 contract_key 必须是 {AUDIO_TRANSCRIPTION_CONTRACT_KEY}，已阻止进入执行器。" + ), + false, + "contract", + )); + } + + let required_capabilities = + read_image_task_payload_string_array(&output.record.payload, "required_capabilities") + .unwrap_or_default(); + if !required_capabilities.is_empty() + && !required_capabilities + .iter() + .any(|capability| capability == AUDIO_TRANSCRIPTION_CONTRACT_KEY) + { + return Err(build_task_error( + "audio_transcription_capability_gap", + "转写任务缺少 audio_transcription required_capability，已阻止进入执行器。", + false, + "routing", + )); + } + + if let Some(routing_slot) = + read_image_task_payload_string(&output.record.payload, &["routing_slot"]) + { + if routing_slot != AUDIO_TRANSCRIPTION_ROUTING_SLOT { + return Err(build_task_error( + "audio_transcription_routing_slot_mismatch", + format!( + "转写任务 routing_slot 必须是 {AUDIO_TRANSCRIPTION_ROUTING_SLOT}，收到 {routing_slot}" + ), + false, + "routing", + )); + } + } + + Ok(()) +} + fn mark_image_task_failed( app: Option<&AppHandle>, workspace_root: &Path, @@ -935,6 +2646,223 @@ fn mark_image_task_failed( Ok(output) } +fn mark_audio_task_failed( + app: Option<&AppHandle>, + workspace_root: &Path, + task_id: &str, + error: TaskErrorRecord, +) -> Result { + let current = load_current_audio_task(workspace_root, task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + + let audio_output = + build_audio_output_status_summary(¤t.record.payload, "failed", Some(&error)); + let output = patch_audio_task( + workspace_root, + task_id, + TaskArtifactPatch { + status: Some("failed".to_string()), + payload_patch: Some(json!({ + "audio_output": audio_output, + })), + last_error: Some(Some(error.clone())), + progress: Some(build_task_progress("failed", error.message.clone(), None)), + current_attempt_worker_id: Some(Some(AUDIO_TASK_RUNNER_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + )?; + emit_audio_task_event(app, &output); + Ok(output) +} + +fn mark_audio_task_succeeded( + app: Option<&AppHandle>, + workspace_root: &Path, + task_id: &str, + generated: GeneratedAudioOutput, +) -> Result { + let current = load_current_audio_task(workspace_root, task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + + let audio_path = match write_audio_output_bytes( + workspace_root, + task_id, + generated.mime_type.as_str(), + &generated.audio_bytes, + ) { + Ok(audio_path) => audio_path, + Err(error) => return mark_audio_task_failed(app, workspace_root, task_id, error), + }; + let payload = ¤t.record.payload; + let duration_ms = payload + .get("duration_ms") + .and_then(serde_json::Value::as_u64) + .or_else(|| { + payload + .pointer("/audio_output/duration_ms") + .and_then(serde_json::Value::as_u64) + }); + let audio_output = build_audio_output_summary( + payload, + audio_path.as_str(), + generated.mime_type.as_str(), + duration_ms, + Some(generated.provider_id.as_str()), + Some(generated.model.as_str()), + ); + let result = build_audio_generation_result_value(&audio_output); + let output = patch_audio_task( + workspace_root, + task_id, + TaskArtifactPatch { + status: Some("succeeded".to_string()), + payload_patch: Some(json!({ + "audio_path": audio_path, + "mime_type": generated.mime_type, + "duration_ms": duration_ms, + "provider_id": generated.provider_id, + "model": generated.model, + "audio_output": audio_output, + })), + result: Some(Some(result)), + last_error: Some(None), + progress: Some(build_task_progress( + "succeeded", + "音频任务已由 voice_generation provider 完成，audio_output 已回写。".to_string(), + Some(100), + )), + current_attempt_worker_id: Some(Some(AUDIO_TASK_RUNNER_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + )?; + emit_audio_task_event(app, &output); + Ok(output) +} + +fn mark_transcription_task_failed( + app: Option<&AppHandle>, + workspace_root: &Path, + task_id: &str, + error: TaskErrorRecord, +) -> Result { + let current = load_current_transcription_task(workspace_root, task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + + let transcript = + build_transcript_status_summary(¤t.record.payload, "failed", Some(&error)); + let output = patch_transcription_task( + workspace_root, + task_id, + TaskArtifactPatch { + status: Some("failed".to_string()), + payload_patch: Some(json!({ + "transcript": transcript, + })), + last_error: Some(Some(error.clone())), + progress: Some(build_task_progress("failed", error.message.clone(), None)), + current_attempt_worker_id: Some(Some(TRANSCRIPTION_TASK_RUNNER_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + )?; + emit_transcription_task_event(app, &output); + Ok(output) +} + +fn build_transcript_completed_summary( + payload: &serde_json::Value, + transcript_path: &str, + generated: &GeneratedTranscriptOutput, +) -> serde_json::Value { + json!({ + "kind": "transcript", + "status": "completed", + "transcript_path": transcript_path, + "path": transcript_path, + "source_url": read_image_task_payload_string(payload, &["source_url", "sourceUrl"]), + "source_path": read_image_task_payload_string(payload, &["source_path", "sourcePath"]), + "language": generated.language.as_deref().or_else(|| read_image_task_payload_string(payload, &["language"])), + "output_format": generated.output_format.as_str(), + "text_preview": generated.text_preview.as_deref(), + "timestamps": payload.get("timestamps").cloned().unwrap_or(serde_json::Value::Null), + "speaker_labels": payload.get("speaker_labels").cloned().unwrap_or(serde_json::Value::Null), + "provider_id": generated.provider_id.as_str(), + "model": generated.model.as_str(), + "modality_contract_key": AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "modality": "audio", + "routing_slot": AUDIO_TRANSCRIPTION_ROUTING_SLOT, + }) +} + +fn build_transcription_result_value(transcript: &serde_json::Value) -> serde_json::Value { + json!({ + "kind": "transcription_result", + "status": "completed", + "transcript": transcript, + "outputs": [transcript], + "transcript_path": transcript.get("transcript_path").cloned().unwrap_or(serde_json::Value::Null), + "output_format": transcript.get("output_format").cloned().unwrap_or(serde_json::Value::Null), + }) +} + +fn mark_transcription_task_succeeded( + app: Option<&AppHandle>, + workspace_root: &Path, + task_id: &str, + generated: GeneratedTranscriptOutput, +) -> Result { + let current = load_current_transcription_task(workspace_root, task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + + let transcript_path = match write_transcript_output_text( + workspace_root, + task_id, + generated.output_format.as_str(), + generated.content.as_str(), + ) { + Ok(transcript_path) => transcript_path, + Err(error) => return mark_transcription_task_failed(app, workspace_root, task_id, error), + }; + let transcript = build_transcript_completed_summary( + ¤t.record.payload, + transcript_path.as_str(), + &generated, + ); + let result = build_transcription_result_value(&transcript); + let output = patch_transcription_task( + workspace_root, + task_id, + TaskArtifactPatch { + status: Some("succeeded".to_string()), + payload_patch: Some(json!({ + "transcript_path": transcript_path, + "output_format": generated.output_format, + "provider_id": generated.provider_id, + "model": generated.model, + "transcript": transcript, + })), + result: Some(Some(result)), + last_error: Some(None), + progress: Some(build_task_progress( + "succeeded", + "转写任务已由 audio_transcription provider 完成，transcript 已回写。".to_string(), + Some(100), + )), + current_attempt_worker_id: Some(Some(TRANSCRIPTION_TASK_RUNNER_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + )?; + emit_transcription_task_event(app, &output); + Ok(output) +} + async fn execute_image_generation_task( app: Option, workspace_root: PathBuf, @@ -970,6 +2898,155 @@ async fn execute_image_generation_task( .map_err(|error| format!("执行图片任务失败: {error}")) } +async fn execute_audio_generation_task_with_runner_config( + app: Option, + workspace_root: PathBuf, + task_id: String, + runner_config: AudioGenerationRunnerConfig, +) -> Result { + let current = load_current_audio_task(&workspace_root, &task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + if let Err(task_error) = validate_audio_generation_task_execution_contract(¤t) { + return mark_audio_task_failed(app.as_ref(), &workspace_root, &task_id, task_error); + } + + let running_audio_output = + build_audio_output_status_summary(¤t.record.payload, "running", None); + let running = patch_audio_task( + &workspace_root, + &task_id, + TaskArtifactPatch { + status: Some("running".to_string()), + payload_patch: Some(json!({ + "audio_output": running_audio_output, + })), + progress: Some(build_task_progress( + "running", + "音频任务已进入 voice_generation 执行链，正在调用音频 provider。".to_string(), + Some(5), + )), + current_attempt_worker_id: Some(Some(AUDIO_TASK_RUNNER_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + )?; + emit_audio_task_event(app.as_ref(), &running); + + let generated = match request_openai_compatible_audio_generation(&runner_config, &running).await + { + Ok(generated) => generated, + Err(task_error) => { + return mark_audio_task_failed(app.as_ref(), &workspace_root, &task_id, task_error) + } + }; + mark_audio_task_succeeded(app.as_ref(), &workspace_root, &task_id, generated) +} + +async fn execute_audio_generation_task( + app: Option, + workspace_root: PathBuf, + task_id: String, +) -> Result { + let current = load_current_audio_task(&workspace_root, &task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + if let Err(task_error) = validate_audio_generation_task_execution_contract(¤t) { + return mark_audio_task_failed(app.as_ref(), &workspace_root, &task_id, task_error); + } + + let runner_config = match resolve_audio_generation_runner_config(app.as_ref(), ¤t).await { + Ok(runner_config) => runner_config, + Err(task_error) => { + return mark_audio_task_failed(app.as_ref(), &workspace_root, &task_id, task_error) + } + }; + + execute_audio_generation_task_with_runner_config(app, workspace_root, task_id, runner_config) + .await +} + +async fn execute_transcription_task_with_runner_config( + app: Option, + workspace_root: PathBuf, + task_id: String, + runner_config: TranscriptionRunnerConfig, +) -> Result { + let current = load_current_transcription_task(&workspace_root, &task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + if let Err(task_error) = validate_transcription_task_execution_contract(¤t) { + return mark_transcription_task_failed(app.as_ref(), &workspace_root, &task_id, task_error); + } + + let running_transcript = + build_transcript_status_summary(¤t.record.payload, "running", None); + let running = patch_transcription_task( + &workspace_root, + &task_id, + TaskArtifactPatch { + status: Some("running".to_string()), + payload_patch: Some(json!({ + "transcript": running_transcript, + })), + progress: Some(build_task_progress( + "running", + "转写任务已进入 audio_transcription 执行链，正在调用转写 provider。".to_string(), + Some(5), + )), + current_attempt_worker_id: Some(Some(TRANSCRIPTION_TASK_RUNNER_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + )?; + emit_transcription_task_event(app.as_ref(), &running); + + let generated = + match request_openai_compatible_transcription(&runner_config, &running, &workspace_root) + .await + { + Ok(generated) => generated, + Err(task_error) => { + return mark_transcription_task_failed( + app.as_ref(), + &workspace_root, + &task_id, + task_error, + ) + } + }; + mark_transcription_task_succeeded(app.as_ref(), &workspace_root, &task_id, generated) +} + +async fn execute_transcription_task( + app: Option, + workspace_root: PathBuf, + task_id: String, +) -> Result { + let current = load_current_transcription_task(&workspace_root, &task_id)?; + if current.normalized_status == "cancelled" { + return Ok(current); + } + if let Err(task_error) = validate_transcription_task_execution_contract(¤t) { + return mark_transcription_task_failed(app.as_ref(), &workspace_root, &task_id, task_error); + } + + let runner_config = match resolve_transcription_runner_config(app.as_ref(), ¤t).await { + Ok(runner_config) => runner_config, + Err(task_error) => { + return mark_transcription_task_failed( + app.as_ref(), + &workspace_root, + &task_id, + task_error, + ) + } + }; + + execute_transcription_task_with_runner_config(app, workspace_root, task_id, runner_config).await +} + pub(crate) fn start_image_generation_task_worker_if_needed( app_handle: &AppHandle, workspace_root: &str, @@ -1022,6 +3099,60 @@ pub(crate) fn start_image_generation_task_worker_if_needed( } } +pub(crate) fn start_audio_generation_task_worker_if_needed( + app_handle: &AppHandle, + workspace_root: &str, + output: &MediaTaskOutput, +) { + if !should_start_audio_generation_worker(output) { + return; + } + + let normalized_workspace_root = workspace_root.trim(); + if normalized_workspace_root.is_empty() { + return; + } + + let task_id = output.task_id.trim().to_string(); + if task_id.is_empty() || !mark_audio_task_execution_started(&task_id) { + return; + } + + let app = app_handle.clone(); + let workspace_root = PathBuf::from(normalized_workspace_root); + tauri::async_runtime::spawn(async move { + let _ = execute_audio_generation_task(Some(app), workspace_root, task_id.clone()).await; + finish_audio_task_execution(&task_id); + }); +} + +pub(crate) fn start_transcription_task_worker_if_needed( + app_handle: &AppHandle, + workspace_root: &str, + output: &MediaTaskOutput, +) { + if !should_start_transcription_worker(output) { + return; + } + + let normalized_workspace_root = workspace_root.trim(); + if normalized_workspace_root.is_empty() { + return; + } + + let task_id = output.task_id.trim().to_string(); + if task_id.is_empty() || !mark_transcription_task_execution_started(&task_id) { + return; + } + + let app = app_handle.clone(); + let workspace_root = PathBuf::from(normalized_workspace_root); + tauri::async_runtime::spawn(async move { + let _ = execute_transcription_task(Some(app), workspace_root, task_id.clone()).await; + finish_transcription_task_execution(&task_id); + }); +} + fn emit_creation_task_event_if_needed(app: Option<&AppHandle>, output: &MediaTaskOutput) { if let Some(app_handle) = app { media_cli_bridge::emit_media_creation_task_event(app_handle, output); @@ -1041,9 +3172,24 @@ pub(crate) fn finalize_image_generation_task_creation( pub(crate) fn finalize_audio_generation_task_creation( app: Option<&AppHandle>, + workspace_root: &str, output: &MediaTaskOutput, ) { emit_creation_task_event_if_needed(app, output); + if let Some(app_handle) = app { + start_audio_generation_task_worker_if_needed(app_handle, workspace_root, output); + } +} + +pub(crate) fn finalize_transcription_task_creation( + app: Option<&AppHandle>, + workspace_root: &str, + output: &MediaTaskOutput, +) { + emit_creation_task_event_if_needed(app, output); + if let Some(app_handle) = app { + start_transcription_task_worker_if_needed(app_handle, workspace_root, output); + } } pub(crate) fn create_image_generation_task_artifact_inner( @@ -1181,8 +3327,14 @@ pub(crate) fn create_audio_generation_task_artifact_inner( let mime_type = normalize_optional_string(request.mime_type.clone()) .unwrap_or_else(|| AUDIO_TASK_DEFAULT_MIME_TYPE.to_string()); let audio_path = normalize_optional_string(request.audio_path.clone()); - let provider_id = normalize_optional_string(request.provider_id.clone()); - let model = normalize_optional_string(request.model.clone()); + let requested_provider_id = normalize_optional_string(request.provider_id.clone()); + let requested_model = normalize_optional_string(request.model.clone()); + let audio_preference_defaults = load_audio_generation_preference_defaults(); + let (provider_id, model) = apply_audio_generation_preference_defaults( + requested_provider_id, + requested_model, + &audio_preference_defaults, + ); let session_id = normalize_optional_string(request.session_id.clone()); let project_id = normalize_optional_string(request.project_id.clone()); let content_id = normalize_optional_string(request.content_id.clone()); @@ -1263,6 +3415,257 @@ pub(crate) fn create_audio_generation_task_artifact_inner( .map_err(|error| format!("创建音频任务 artifact 失败: {error}")) } +fn build_audio_output_summary( + payload: &serde_json::Value, + audio_path: &str, + mime_type: &str, + duration_ms: Option, + provider_id: Option<&str>, + model: Option<&str>, +) -> serde_json::Value { + json!({ + "kind": "audio_output", + "status": "completed", + "audio_path": audio_path, + "mime_type": mime_type, + "duration_ms": duration_ms, + "source_text": read_image_task_payload_string(payload, &["source_text", "sourceText", "prompt"]), + "voice": read_image_task_payload_string(payload, &["voice"]), + "voice_style": read_image_task_payload_string(payload, &["voice_style", "voiceStyle"]), + "target_language": read_image_task_payload_string(payload, &["target_language", "targetLanguage"]), + "provider_id": provider_id, + "model": model, + "modality_contract_key": VOICE_GENERATION_CONTRACT_KEY, + "modality": "audio", + "routing_slot": VOICE_GENERATION_ROUTING_SLOT, + }) +} + +fn build_audio_generation_result_value(audio_output: &serde_json::Value) -> serde_json::Value { + json!({ + "kind": "audio_generation_result", + "status": "completed", + "audio_output": audio_output, + "outputs": [audio_output], + "audio_path": audio_output.get("audio_path").cloned().unwrap_or(serde_json::Value::Null), + "mime_type": audio_output.get("mime_type").cloned().unwrap_or(serde_json::Value::Null), + "duration_ms": audio_output.get("duration_ms").cloned().unwrap_or(serde_json::Value::Null), + }) +} + +fn build_transcription_task_transcript_summary( + source_url: Option<&str>, + source_path: Option<&str>, + language: Option<&str>, + output_format: Option<&str>, + timestamps: Option, + speaker_labels: Option, + provider_id: Option<&str>, + model: Option<&str>, +) -> serde_json::Value { + json!({ + "kind": "transcript", + "status": "pending", + "source_url": source_url, + "source_path": source_path, + "language": language, + "output_format": output_format, + "timestamps": timestamps, + "speaker_labels": speaker_labels, + "provider_id": provider_id, + "model": model, + "modality_contract_key": AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "modality": "audio", + "routing_slot": AUDIO_TRANSCRIPTION_ROUTING_SLOT, + }) +} + +pub(crate) fn create_transcription_task_artifact_inner( + request: CreateTranscriptionTaskArtifactRequest, +) -> Result { + let project_root_path = + normalize_required_string(&request.project_root_path, "projectRootPath")?; + let source_url = normalize_optional_string(request.source_url.clone()); + let source_path = normalize_optional_string(request.source_path.clone()); + if source_url.is_none() && source_path.is_none() { + return Err("sourceUrl 或 sourcePath 至少需要提供一个".to_string()); + } + let prompt = normalize_optional_string(request.prompt.clone()); + let raw_text = normalize_optional_string(request.raw_text.clone()); + let language = normalize_optional_string(request.language.clone()); + let output_format = normalize_optional_string(request.output_format.clone()); + let provider_id = normalize_optional_string(request.provider_id.clone()); + let model = normalize_optional_string(request.model.clone()); + let session_id = normalize_optional_string(request.session_id.clone()); + let project_id = normalize_optional_string(request.project_id.clone()); + let content_id = normalize_optional_string(request.content_id.clone()); + let entry_source = normalize_optional_string(request.entry_source.clone()) + .or_else(|| Some("at_transcription_command".to_string())); + let runtime_contract_key = request + .runtime_contract + .as_ref() + .and_then(|value| value.get("contract_key")) + .and_then(serde_json::Value::as_str) + .map(ToString::to_string); + if let Some(runtime_contract_key) = runtime_contract_key { + normalize_audio_transcription_contract_key(Some(runtime_contract_key))?; + } + let modality_contract_key = + normalize_audio_transcription_contract_key(request.modality_contract_key.clone())?; + let modality = normalize_audio_transcription_modality(request.modality.clone())?; + let required_capabilities = + normalize_audio_transcription_required_capabilities(request.required_capabilities.clone())?; + let routing_slot = normalize_audio_transcription_routing_slot(request.routing_slot.clone())?; + let requested_target = normalize_optional_string(request.requested_target.clone()) + .or_else(|| Some("transcript".to_string())); + let output_path = normalize_optional_string(request.output_path.clone()); + let transcript = build_transcription_task_transcript_summary( + source_url.as_deref(), + source_path.as_deref(), + language.as_deref(), + output_format.as_deref(), + request.timestamps, + request.speaker_labels, + provider_id.as_deref(), + model.as_deref(), + ); + let idempotency_key = build_transcription_task_idempotency_key( + &request, + source_url.as_deref(), + source_path.as_deref(), + language.as_deref(), + output_format.as_deref(), + provider_id.as_deref(), + model.as_deref(), + )?; + + write_task_artifact( + std::path::Path::new(project_root_path.as_str()), + MediaTaskType::TranscriptionGenerate, + normalize_optional_string(request.title), + json!({ + "prompt": prompt, + "raw_text": raw_text, + "source_url": source_url, + "source_path": source_path, + "language": language, + "output_format": output_format, + "speaker_labels": request.speaker_labels, + "timestamps": request.timestamps, + "provider_id": provider_id, + "model": model, + "session_id": session_id, + "project_id": project_id, + "content_id": content_id, + "entry_source": entry_source, + "modality_contract_key": modality_contract_key, + "modality": modality, + "required_capabilities": required_capabilities, + "routing_slot": routing_slot, + "runtime_contract": audio_transcription_runtime_contract(), + "requested_target": requested_target, + "transcript": transcript, + }), + TaskWriteOptions { + status: Some("pending_submit".to_string()), + output_path: output_path.as_deref(), + artifact_dir: None, + idempotency_key: Some(idempotency_key.as_str()), + relationships: TaskRelationships::default(), + }, + ) + .map_err(|error| format!("创建转写任务 artifact 失败: {error}")) +} + +pub(crate) fn complete_audio_generation_task_artifact_inner( + request: CompleteAudioGenerationTaskArtifactRequest, +) -> Result { + let project_root_path = + normalize_required_string(&request.project_root_path, "projectRootPath")?; + let task_ref = normalize_required_string(&request.task_ref, "taskRef")?; + let audio_path = normalize_required_string(&request.audio_path, "audioPath")?; + let workspace_root = std::path::Path::new(project_root_path.as_str()); + let current = load_task_output(workspace_root, task_ref.as_str(), None) + .map_err(|error| format!("读取音频任务 artifact 失败: {error}"))?; + + if current.task_type != MediaTaskType::AudioGenerate.as_str() { + return Err(format!( + "只能完成 audio_generate 任务，当前任务类型为 {}", + current.task_type + )); + } + + if media_task_contract_key(¤t).as_deref() != Some(VOICE_GENERATION_CONTRACT_KEY) { + return Err("只能完成 voice_generation 合同下的 audio_generate 任务".to_string()); + } + if matches!(current.normalized_status.as_str(), "cancelled" | "failed") { + return Err(format!( + "当前音频任务状态为 {}，不能直接写回完成态", + current.normalized_status + )); + } + + let payload = ¤t.record.payload; + let mime_type = normalize_optional_string(request.mime_type) + .or_else(|| { + read_image_task_payload_string(payload, &["mime_type", "mimeType"]) + .map(ToString::to_string) + }) + .unwrap_or_else(|| AUDIO_TASK_DEFAULT_MIME_TYPE.to_string()); + let provider_id = normalize_optional_string(request.provider_id).or_else(|| { + read_image_task_payload_string(payload, &["provider_id", "providerId"]) + .map(ToString::to_string) + }); + let model = normalize_optional_string(request.model) + .or_else(|| read_image_task_payload_string(payload, &["model"]).map(ToString::to_string)); + let duration_ms = request.duration_ms.or_else(|| { + payload + .get("duration_ms") + .and_then(serde_json::Value::as_u64) + .or_else(|| { + payload + .pointer("/audio_output/duration_ms") + .and_then(serde_json::Value::as_u64) + }) + }); + let audio_output = build_audio_output_summary( + payload, + audio_path.as_str(), + mime_type.as_str(), + duration_ms, + provider_id.as_deref(), + model.as_deref(), + ); + let result = build_audio_generation_result_value(&audio_output); + + patch_task_artifact( + workspace_root, + task_ref.as_str(), + None, + TaskArtifactPatch { + status: Some("succeeded".to_string()), + payload_patch: Some(json!({ + "audio_path": audio_path, + "mime_type": mime_type, + "duration_ms": duration_ms, + "provider_id": provider_id, + "model": model, + "audio_output": audio_output, + })), + result: Some(Some(result)), + last_error: Some(None), + progress: Some(build_task_progress( + "succeeded", + "音频任务已完成，audio_output 已回写。".to_string(), + Some(100), + )), + current_attempt_worker_id: Some(Some(AUDIO_TASK_COMPLETION_WORKER_ID.to_string())), + ..TaskArtifactPatch::default() + }, + ) + .map_err(|error| format!("完成音频任务 artifact 失败: {error}")) +} + pub(crate) fn get_media_task_artifact_inner( request: MediaTaskLookupRequest, ) -> Result { @@ -1368,8 +3771,19 @@ pub fn create_audio_generation_task_artifact( app: AppHandle, request: CreateAudioGenerationTaskArtifactRequest, ) -> Result { + let project_root_path = request.project_root_path.trim().to_string(); let output = create_audio_generation_task_artifact_inner(request)?; - finalize_audio_generation_task_creation(Some(&app), &output); + finalize_audio_generation_task_creation(Some(&app), &project_root_path, &output); + Ok(output) +} + +#[tauri::command] +pub fn complete_audio_generation_task_artifact( + app: AppHandle, + request: CompleteAudioGenerationTaskArtifactRequest, +) -> Result { + let output = complete_audio_generation_task_artifact_inner(request)?; + emit_creation_task_event_if_needed(Some(&app), &output); Ok(output) } @@ -1807,12 +4221,915 @@ mod tests { assert_eq!(listed.total, 1); assert_eq!(listed.modality_runtime_contracts.snapshot_count, 1); + assert_eq!(listed.modality_runtime_contracts.audio_output_count, 1); + assert_eq!( + listed.modality_runtime_contracts.audio_output_statuses[0].status, + "pending" + ); assert_eq!( listed.modality_runtime_contracts.snapshots[0] .routing_event .as_str(), "executor_invoked" ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .audio_output_status + .as_deref(), + Some("pending") + ); + } + + #[test] + fn create_transcription_task_artifact_inner_should_write_audio_transcription_contract_payload() + { + let temp_dir = tempfile::tempdir().expect("create temp dir"); + let first = + create_transcription_task_artifact_inner(CreateTranscriptionTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + prompt: Some("生成逐字稿".to_string()), + title: Some("会议转写".to_string()), + raw_text: Some("@转写 /tmp/interview.wav 生成逐字稿".to_string()), + source_url: None, + source_path: Some("/tmp/interview.wav".to_string()), + language: Some("zh-CN".to_string()), + output_format: Some("srt".to_string()), + speaker_labels: Some(true), + timestamps: Some(true), + provider_id: Some("limecore".to_string()), + model: Some("asr-pro".to_string()), + session_id: Some("session-transcription-1".to_string()), + project_id: Some("project-transcription-1".to_string()), + content_id: Some("content-transcription-1".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("create transcription task"); + let second = + create_transcription_task_artifact_inner(CreateTranscriptionTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + prompt: Some("生成逐字稿".to_string()), + title: Some("会议转写".to_string()), + raw_text: Some("@转写 /tmp/interview.wav 生成逐字稿".to_string()), + source_url: None, + source_path: Some("/tmp/interview.wav".to_string()), + language: Some("zh-CN".to_string()), + output_format: Some("srt".to_string()), + speaker_labels: Some(true), + timestamps: Some(true), + provider_id: Some("limecore".to_string()), + model: Some("asr-pro".to_string()), + session_id: Some("session-transcription-1".to_string()), + project_id: Some("project-transcription-1".to_string()), + content_id: Some("content-transcription-1".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("reuse transcription task"); + + assert_eq!(first.task_id, second.task_id); + assert!(second.reused_existing); + assert!(first + .path + .starts_with(".lime/tasks/transcription_generate/")); + assert_eq!(first.task_type, "transcription_generate"); + assert_eq!(first.task_family, "document"); + assert_eq!( + first + .record + .payload + .get("entry_source") + .and_then(Value::as_str), + Some("at_transcription_command") + ); + assert_eq!( + first + .record + .payload + .get("modality_contract_key") + .and_then(Value::as_str), + Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY) + ); + assert_eq!( + first.record.payload.get("modality").and_then(Value::as_str), + Some("audio") + ); + assert_eq!( + first.record.payload.get("required_capabilities"), + Some(&json!(["text_generation", "audio_transcription"])) + ); + assert_eq!( + first + .record + .payload + .get("routing_slot") + .and_then(Value::as_str), + Some(AUDIO_TRANSCRIPTION_ROUTING_SLOT) + ); + assert_eq!( + first + .record + .payload + .pointer("/runtime_contract/executor_binding/binding_key") + .and_then(Value::as_str), + Some("transcription_generate") + ); + assert_eq!( + first + .record + .payload + .pointer("/transcript/kind") + .and_then(Value::as_str), + Some("transcript") + ); + assert_eq!( + first + .record + .payload + .pointer("/transcript/status") + .and_then(Value::as_str), + Some("pending") + ); + assert_eq!( + first + .record + .payload + .pointer("/transcript/source_path") + .and_then(Value::as_str), + Some("/tmp/interview.wav") + ); + assert_eq!( + first + .record + .payload + .pointer("/transcript/output_format") + .and_then(Value::as_str), + Some("srt") + ); + + let listed = list_media_task_artifacts_inner(ListMediaTaskArtifactsRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + status: Some("pending".to_string()), + task_family: Some("document".to_string()), + task_type: Some("transcription_generate".to_string()), + modality_contract_key: Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY.to_string()), + routing_outcome: Some("accepted".to_string()), + limit: Some(10), + }) + .expect("list transcription tasks"); + + assert_eq!(listed.total, 1); + assert_eq!(listed.modality_runtime_contracts.snapshot_count, 1); + assert_eq!(listed.modality_runtime_contracts.transcript_count, 1); + assert_eq!( + listed.modality_runtime_contracts.transcript_statuses[0].status, + "pending" + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .routing_event + .as_str(), + "executor_invoked" + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .transcript_status + .as_deref(), + Some("pending") + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .transcript_source_path + .as_deref(), + Some("/tmp/interview.wav") + ); + } + + #[tokio::test] + async fn execute_audio_transcription_task_should_mark_provider_resolver_unavailable_without_fabricated_transcript( + ) { + let temp_dir = tempfile::tempdir().expect("create temp dir"); + let source_path = temp_dir.path().join("meeting.wav"); + std::fs::write(&source_path, b"fake wav bytes").expect("write fake source audio"); + let created = + create_transcription_task_artifact_inner(CreateTranscriptionTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + prompt: Some("生成逐字稿".to_string()), + title: Some("转写执行器失败态".to_string()), + raw_text: Some("@转写 meeting.wav".to_string()), + source_url: None, + source_path: Some(source_path.to_string_lossy().to_string()), + language: Some("zh".to_string()), + output_format: Some("json".to_string()), + speaker_labels: Some(false), + timestamps: Some(false), + provider_id: Some("openai-asr".to_string()), + model: Some("whisper-1".to_string()), + session_id: Some("session-transcription-worker-1".to_string()), + project_id: Some("project-transcription-worker-1".to_string()), + content_id: Some("content-transcription-worker-1".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("create transcription task"); + + let result = execute_transcription_task( + None, + temp_dir.path().to_path_buf(), + created.task_id.clone(), + ) + .await + .expect("transcription worker should settle to failed output"); + + assert_eq!(result.normalized_status, "failed"); + assert_eq!( + result.last_error.as_ref().map(|value| value.code.as_str()), + Some("transcription_provider_resolver_unavailable") + ); + assert_eq!( + result + .record + .payload + .pointer("/transcript/status") + .and_then(Value::as_str), + Some("failed") + ); + assert_eq!( + result + .record + .payload + .pointer("/transcript/error_code") + .and_then(Value::as_str), + Some("transcription_provider_resolver_unavailable") + ); + assert_eq!( + result + .record + .payload + .pointer("/transcript/retryable") + .and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + result + .record + .payload + .pointer("/transcript/transcript_path") + .and_then(Value::as_str), + None + ); + assert!(result.record.result.is_none()); + assert_eq!( + result + .record + .attempts + .first() + .and_then(|attempt| attempt.worker_id.as_deref()), + Some(TRANSCRIPTION_TASK_RUNNER_WORKER_ID) + ); + + let listed = list_media_task_artifacts_inner(ListMediaTaskArtifactsRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + status: Some("failed".to_string()), + task_family: Some("document".to_string()), + task_type: Some("transcription_generate".to_string()), + modality_contract_key: Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY.to_string()), + routing_outcome: Some("failed".to_string()), + limit: Some(10), + }) + .expect("list failed transcription tasks"); + assert_eq!(listed.total, 1); + assert_eq!(listed.modality_runtime_contracts.transcript_count, 1); + assert_eq!( + listed.modality_runtime_contracts.transcript_statuses[0].status, + "failed" + ); + assert_eq!( + listed.modality_runtime_contracts.transcript_error_codes, + vec!["transcription_provider_resolver_unavailable".to_string()] + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .transcript_error_code + .as_deref(), + Some("transcription_provider_resolver_unavailable") + ); + } + + #[tokio::test] + async fn execute_audio_transcription_task_with_openai_compatible_provider_should_write_transcript_output( + ) { + let captured_request = Arc::new(Mutex::new(None::<(String, String)>)); + let captured_for_server = Arc::clone(&captured_request); + let app = Router::new().route( + "/v1/audio/transcriptions", + post(move |headers: HeaderMap, body: axum::body::Bytes| { + let captured = Arc::clone(&captured_for_server); + async move { + let authorization = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + let body_text = String::from_utf8_lossy(&body).to_string(); + *captured.lock().expect("lock captured request") = + Some((authorization, body_text)); + Json(json!({ + "text": "这里是会议转写结果。", + "language": "zh" + })) + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock transcription server"); + let address = listener.local_addr().expect("mock server addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("serve mock transcription server"); + }); + + let temp_dir = tempfile::tempdir().expect("create temp dir"); + let source_path = temp_dir.path().join("meeting.wav"); + std::fs::write(&source_path, b"fake wav bytes").expect("write fake source audio"); + let created = + create_transcription_task_artifact_inner(CreateTranscriptionTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + prompt: Some("请转写会议音频".to_string()), + title: Some("真实转写执行器".to_string()), + raw_text: Some("@转写 meeting.wav".to_string()), + source_url: None, + source_path: Some(source_path.to_string_lossy().to_string()), + language: Some("zh".to_string()), + output_format: Some("json".to_string()), + speaker_labels: Some(false), + timestamps: Some(false), + provider_id: Some("openai-asr".to_string()), + model: Some("whisper-1".to_string()), + session_id: Some("session-transcription-worker-success".to_string()), + project_id: Some("project-transcription-worker-success".to_string()), + content_id: Some("content-transcription-worker-success".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("create transcription task"); + + let result = execute_transcription_task_with_runner_config( + None, + temp_dir.path().to_path_buf(), + created.task_id.clone(), + TranscriptionRunnerConfig { + provider_id: "openai-asr".to_string(), + model: "whisper-1".to_string(), + endpoint: format!("http://{address}/v1/audio/transcriptions"), + api_key: "sk-test-transcription".to_string(), + }, + ) + .await + .expect("transcription worker should complete with provider text"); + server.abort(); + + assert_eq!(result.normalized_status, "succeeded"); + assert_eq!( + result + .record + .payload + .pointer("/transcript/status") + .and_then(Value::as_str), + Some("completed") + ); + let transcript_path = result + .record + .payload + .pointer("/transcript/transcript_path") + .and_then(Value::as_str) + .expect("transcript output path"); + assert!(transcript_path.starts_with(".lime/runtime/transcripts/")); + assert!(transcript_path.ends_with(".json")); + let transcript_content = std::fs::read_to_string(temp_dir.path().join(transcript_path)) + .expect("read generated transcript output"); + assert!(transcript_content.contains("这里是会议转写结果。")); + assert_eq!( + result + .record + .payload + .pointer("/transcript/text_preview") + .and_then(Value::as_str), + Some("这里是会议转写结果。") + ); + assert_eq!( + result + .record + .payload + .pointer("/transcript/provider_id") + .and_then(Value::as_str), + Some("openai-asr") + ); + assert_eq!( + result + .record + .payload + .pointer("/transcript/model") + .and_then(Value::as_str), + Some("whisper-1") + ); + assert_eq!( + result + .record + .result + .as_ref() + .and_then(|value| value.pointer("/transcript/transcript_path")) + .and_then(Value::as_str), + Some(transcript_path) + ); + assert_eq!( + result + .record + .attempts + .first() + .and_then(|attempt| attempt.worker_id.as_deref()), + Some(TRANSCRIPTION_TASK_RUNNER_WORKER_ID) + ); + + let (authorization, body_text) = captured_request + .lock() + .expect("lock captured request") + .clone() + .expect("captured transcription request"); + assert_eq!(authorization, "Bearer sk-test-transcription"); + assert!(body_text.contains("name=\"model\"")); + assert!(body_text.contains("whisper-1")); + assert!(body_text.contains("name=\"response_format\"")); + assert!(body_text.contains("json")); + } + + #[test] + fn complete_audio_generation_task_artifact_inner_should_write_audio_output_result() { + let temp_dir = tempfile::tempdir().expect("create temp dir"); + let created = + create_audio_generation_task_artifact_inner(CreateAudioGenerationTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + source_text: "请把这段内容生成温暖旁白。".to_string(), + title: Some("音频完成态".to_string()), + raw_text: Some("@配音 请把这段内容生成温暖旁白。".to_string()), + voice: Some("warm_narrator".to_string()), + voice_style: Some("温暖".to_string()), + target_language: Some("zh-CN".to_string()), + mime_type: None, + audio_path: None, + duration_ms: None, + provider_id: Some("limecore".to_string()), + model: Some("voice-pro".to_string()), + session_id: Some("session-voice-complete".to_string()), + project_id: Some("project-voice-complete".to_string()), + content_id: Some("content-voice-complete".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("create audio task"); + + let completed = complete_audio_generation_task_artifact_inner( + CompleteAudioGenerationTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + task_ref: created.task_id.clone(), + audio_path: ".lime/runtime/audio/voice-complete.mp3".to_string(), + mime_type: Some("audio/mpeg".to_string()), + duration_ms: Some(3200), + provider_id: Some("limecore".to_string()), + model: Some("voice-pro".to_string()), + }, + ) + .expect("complete audio task"); + + assert_eq!(completed.task_id, created.task_id); + assert_eq!(completed.status, "succeeded"); + assert_eq!(completed.normalized_status, "succeeded"); + assert!(completed.record.completed_at.is_some()); + assert_eq!( + completed + .record + .payload + .pointer("/audio_output/status") + .and_then(Value::as_str), + Some("completed") + ); + assert_eq!( + completed + .record + .payload + .pointer("/audio_output/audio_path") + .and_then(Value::as_str), + Some(".lime/runtime/audio/voice-complete.mp3") + ); + assert_eq!( + completed + .record + .payload + .pointer("/audio_output/duration_ms") + .and_then(Value::as_u64), + Some(3200) + ); + assert_eq!( + completed + .record + .payload + .pointer("/audio_output/modality_contract_key") + .and_then(Value::as_str), + Some(VOICE_GENERATION_CONTRACT_KEY) + ); + assert_eq!( + completed + .record + .result + .as_ref() + .and_then(|value| value.pointer("/audio_output/audio_path")) + .and_then(Value::as_str), + Some(".lime/runtime/audio/voice-complete.mp3") + ); + assert_eq!( + completed + .record + .attempts + .first() + .and_then(|attempt| attempt.worker_id.as_deref()), + Some(AUDIO_TASK_COMPLETION_WORKER_ID) + ); + assert_eq!( + completed + .record + .attempts + .first() + .and_then(|attempt| attempt.result_snapshot.as_ref()) + .and_then(|value| value.pointer("/audio_output/status")) + .and_then(Value::as_str), + Some("completed") + ); + + let listed = list_media_task_artifacts_inner(ListMediaTaskArtifactsRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + status: Some("succeeded".to_string()), + task_family: Some("audio".to_string()), + task_type: Some("audio_generate".to_string()), + modality_contract_key: Some(VOICE_GENERATION_CONTRACT_KEY.to_string()), + routing_outcome: Some("accepted".to_string()), + limit: Some(10), + }) + .expect("list completed audio generation tasks"); + assert_eq!(listed.total, 1); + assert_eq!(listed.modality_runtime_contracts.audio_output_count, 1); + assert_eq!( + listed.modality_runtime_contracts.audio_output_statuses[0].status, + "completed" + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .audio_output_path + .as_deref(), + Some(".lime/runtime/audio/voice-complete.mp3") + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0].audio_output_duration_ms, + Some(3200) + ); + } + + #[tokio::test] + async fn execute_audio_generation_task_should_mark_provider_resolver_unavailable_without_fabricated_audio( + ) { + let temp_dir = tempfile::tempdir().expect("create temp dir"); + let created = + create_audio_generation_task_artifact_inner(CreateAudioGenerationTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + source_text: "请生成一段清晰的产品播报。".to_string(), + title: Some("音频执行器骨架".to_string()), + raw_text: Some("@配音 请生成一段清晰的产品播报。".to_string()), + voice: Some("clear_narrator".to_string()), + voice_style: Some("清晰".to_string()), + target_language: Some("zh-CN".to_string()), + mime_type: None, + audio_path: None, + duration_ms: None, + provider_id: Some("limecore".to_string()), + model: Some("voice-pro".to_string()), + session_id: Some("session-audio-worker-1".to_string()), + project_id: Some("project-audio-worker-1".to_string()), + content_id: Some("content-audio-worker-1".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("create audio task"); + + let result = execute_audio_generation_task( + None, + temp_dir.path().to_path_buf(), + created.task_id.clone(), + ) + .await + .expect("audio worker should settle to failed output"); + + assert_eq!(result.normalized_status, "failed"); + assert_eq!( + result.last_error.as_ref().map(|value| value.code.as_str()), + Some("audio_provider_resolver_unavailable") + ); + assert_eq!( + result.last_error.as_ref().map(|value| value.retryable), + Some(true) + ); + assert_eq!( + result + .last_error + .as_ref() + .and_then(|value| value.stage.as_deref()), + Some("bootstrap") + ); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/status") + .and_then(Value::as_str), + Some("failed") + ); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/error_code") + .and_then(Value::as_str), + Some("audio_provider_resolver_unavailable") + ); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/retryable") + .and_then(Value::as_bool), + Some(true) + ); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/audio_path") + .and_then(Value::as_str), + None + ); + assert!(result.record.result.is_none()); + assert_eq!( + result + .record + .attempts + .first() + .and_then(|attempt| attempt.worker_id.as_deref()), + Some(AUDIO_TASK_RUNNER_WORKER_ID) + ); + + let loaded = get_media_task_artifact_inner(MediaTaskLookupRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + task_ref: created.task_id.clone(), + }) + .expect("load failed audio task"); + assert_eq!(loaded.normalized_status, "failed"); + assert_eq!( + loaded + .record + .payload + .pointer("/audio_output/status") + .and_then(Value::as_str), + Some("failed") + ); + + let listed = list_media_task_artifacts_inner(ListMediaTaskArtifactsRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + status: Some("failed".to_string()), + task_family: Some("audio".to_string()), + task_type: Some("audio_generate".to_string()), + modality_contract_key: Some(VOICE_GENERATION_CONTRACT_KEY.to_string()), + routing_outcome: Some("failed".to_string()), + limit: Some(10), + }) + .expect("list failed audio generation tasks"); + assert_eq!(listed.total, 1); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .failure_code + .as_deref(), + Some("audio_provider_resolver_unavailable") + ); + assert_eq!(listed.modality_runtime_contracts.audio_output_count, 1); + assert_eq!( + listed.modality_runtime_contracts.audio_output_statuses[0].status, + "failed" + ); + assert_eq!( + listed.modality_runtime_contracts.audio_output_error_codes, + vec!["audio_provider_resolver_unavailable".to_string()] + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0] + .audio_output_error_code + .as_deref(), + Some("audio_provider_resolver_unavailable") + ); + assert_eq!( + listed.modality_runtime_contracts.snapshots[0].audio_output_retryable, + Some(true) + ); + } + + #[tokio::test] + async fn execute_audio_generation_task_with_openai_compatible_provider_should_write_audio_output( + ) { + let captured_request = Arc::new(Mutex::new(None::<(String, Value)>)); + let captured_for_server = Arc::clone(&captured_request); + let app = Router::new().route( + "/v1/audio/speech", + post(move |headers: HeaderMap, Json(body): Json| { + let captured = Arc::clone(&captured_for_server); + async move { + let authorization = headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .unwrap_or("") + .to_string(); + *captured.lock().expect("lock captured request") = Some((authorization, body)); + ( + [(axum::http::header::CONTENT_TYPE, "audio/mpeg")], + Vec::from(&b"ID3 lime audio bytes"[..]), + ) + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0") + .await + .expect("bind mock audio server"); + let address = listener.local_addr().expect("mock server addr"); + let server = tokio::spawn(async move { + axum::serve(listener, app) + .await + .expect("serve mock audio server"); + }); + + let temp_dir = tempfile::tempdir().expect("create temp dir"); + let created = + create_audio_generation_task_artifact_inner(CreateAudioGenerationTaskArtifactRequest { + project_root_path: temp_dir.path().to_string_lossy().to_string(), + source_text: "请生成一段清晰的产品播报。".to_string(), + title: Some("真实音频执行器".to_string()), + raw_text: Some("@配音 请生成一段清晰的产品播报。".to_string()), + voice: Some("alloy".to_string()), + voice_style: Some("clear".to_string()), + target_language: Some("zh-CN".to_string()), + mime_type: None, + audio_path: None, + duration_ms: None, + provider_id: Some("openai-tts".to_string()), + model: Some("gpt-4o-mini-tts".to_string()), + session_id: Some("session-audio-worker-success".to_string()), + project_id: Some("project-audio-worker-success".to_string()), + content_id: Some("content-audio-worker-success".to_string()), + entry_source: None, + modality_contract_key: None, + modality: None, + required_capabilities: Vec::new(), + routing_slot: None, + runtime_contract: None, + requested_target: None, + output_path: None, + }) + .expect("create audio task"); + + let result = execute_audio_generation_task_with_runner_config( + None, + temp_dir.path().to_path_buf(), + created.task_id.clone(), + AudioGenerationRunnerConfig { + provider_id: "openai-tts".to_string(), + model: "gpt-4o-mini-tts".to_string(), + endpoint: format!("http://{address}/v1/audio/speech"), + api_key: "sk-test-audio".to_string(), + }, + ) + .await + .expect("audio worker should complete with provider bytes"); + server.abort(); + + assert_eq!(result.normalized_status, "succeeded"); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/status") + .and_then(Value::as_str), + Some("completed") + ); + let audio_path = result + .record + .payload + .pointer("/audio_output/audio_path") + .and_then(Value::as_str) + .expect("audio output path"); + assert!(audio_path.starts_with(".lime/runtime/audio/")); + assert!(audio_path.ends_with(".mp3")); + let audio_bytes = + std::fs::read(temp_dir.path().join(audio_path)).expect("read generated audio output"); + assert_eq!(audio_bytes, b"ID3 lime audio bytes"); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/provider_id") + .and_then(Value::as_str), + Some("openai-tts") + ); + assert_eq!( + result + .record + .payload + .pointer("/audio_output/model") + .and_then(Value::as_str), + Some("gpt-4o-mini-tts") + ); + assert_eq!( + result + .record + .result + .as_ref() + .and_then(|value| value.pointer("/audio_output/audio_path")) + .and_then(Value::as_str), + Some(audio_path) + ); + assert_eq!( + result + .record + .attempts + .first() + .and_then(|attempt| attempt.worker_id.as_deref()), + Some(AUDIO_TASK_RUNNER_WORKER_ID) + ); + + let (authorization, body) = captured_request + .lock() + .expect("lock captured request") + .clone() + .expect("captured speech request"); + assert_eq!(authorization, "Bearer sk-test-audio"); + assert_eq!( + body.get("model").and_then(Value::as_str), + Some("gpt-4o-mini-tts") + ); + assert_eq!( + body.get("input").and_then(Value::as_str), + Some("请生成一段清晰的产品播报。") + ); + assert_eq!(body.get("voice").and_then(Value::as_str), Some("alloy")); + assert_eq!( + body.get("response_format").and_then(Value::as_str), + Some("mp3") + ); + assert!(body + .get("instructions") + .and_then(Value::as_str) + .is_some_and(|value| value.contains("clear") && value.contains("zh-CN"))); } #[test] diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs index a5ae017e4..b9f3d7126 100644 --- a/src-tauri/src/commands/mod.rs +++ b/src-tauri/src/commands/mod.rs @@ -58,6 +58,7 @@ pub mod unified_memory_cmd; pub mod update_cmd; pub mod usage_stats_cmd; pub mod video_generation_cmd; +pub mod voice_model_cmd; pub mod voice_test_cmd; pub mod websocket_cmd; pub mod webview_cmd; diff --git a/src-tauri/src/commands/modality_runtime_contracts.rs b/src-tauri/src/commands/modality_runtime_contracts.rs index dd0ca13f4..60b706f7c 100644 --- a/src-tauri/src/commands/modality_runtime_contracts.rs +++ b/src-tauri/src/commands/modality_runtime_contracts.rs @@ -28,6 +28,12 @@ pub(crate) const VOICE_GENERATION_ROUTING_SLOT: &str = "voice_generation_model"; pub(crate) const VOICE_GENERATION_EXECUTOR_BINDING_KEY: &str = "voice_runtime"; pub(crate) const VOICE_GENERATION_REQUIRED_CAPABILITIES: &[&str] = &["text_generation", "voice_generation"]; +pub(crate) const AUDIO_TRANSCRIPTION_CONTRACT_KEY: &str = "audio_transcription"; +pub(crate) const AUDIO_TRANSCRIPTION_MODALITY: &str = "audio"; +pub(crate) const AUDIO_TRANSCRIPTION_ROUTING_SLOT: &str = "audio_transcription_model"; +pub(crate) const AUDIO_TRANSCRIPTION_EXECUTOR_BINDING_KEY: &str = "transcription_generate"; +pub(crate) const AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES: &[&str] = + &["text_generation", "audio_transcription"]; pub(crate) const WEB_RESEARCH_CONTRACT_KEY: &str = "web_research"; pub(crate) const WEB_RESEARCH_MODALITY: &str = "mixed"; pub(crate) const WEB_RESEARCH_ROUTING_SLOT: &str = "report_generation_model"; @@ -144,6 +150,13 @@ pub(crate) fn voice_generation_required_capabilities() -> Vec { .collect() } +pub(crate) fn audio_transcription_required_capabilities() -> Vec { + AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES + .iter() + .map(|value| (*value).to_string()) + .collect() +} + pub(crate) fn web_research_required_capabilities() -> Vec { WEB_RESEARCH_REQUIRED_CAPABILITIES .iter() @@ -256,6 +269,68 @@ pub(crate) fn normalize_voice_generation_required_capabilities( Ok(expected) } +fn normalize_expected_audio_transcription_contract_string( + value: Option, + expected: &str, + field_name: &str, +) -> Result { + match normalize_optional_contract_string(value) { + Some(value) if value == expected => Ok(expected.to_string()), + Some(value) => Err(format!( + "转写任务 {field_name} 必须是 {expected}，收到 {value}" + )), + None => Ok(expected.to_string()), + } +} + +pub(crate) fn normalize_audio_transcription_contract_key( + value: Option, +) -> Result { + normalize_expected_audio_transcription_contract_string( + value, + AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "modality_contract_key", + ) +} + +pub(crate) fn normalize_audio_transcription_modality( + value: Option, +) -> Result { + normalize_expected_audio_transcription_contract_string( + value, + AUDIO_TRANSCRIPTION_MODALITY, + "modality", + ) +} + +pub(crate) fn normalize_audio_transcription_routing_slot( + value: Option, +) -> Result { + normalize_expected_audio_transcription_contract_string( + value, + AUDIO_TRANSCRIPTION_ROUTING_SLOT, + "routing_slot", + ) +} + +pub(crate) fn normalize_audio_transcription_required_capabilities( + values: Vec, +) -> Result, String> { + let expected = audio_transcription_required_capabilities(); + for value in values { + let normalized = value.trim(); + if normalized.is_empty() { + continue; + } + if !expected.iter().any(|item| item == normalized) { + return Err(format!( + "转写任务 required_capabilities 包含不属于 audio_transcription contract 的能力: {normalized}" + )); + } + } + Ok(expected) +} + pub(crate) fn image_generation_runtime_contract() -> Value { json!({ "contract_key": IMAGE_GENERATION_CONTRACT_KEY, @@ -324,6 +399,23 @@ pub(crate) fn voice_generation_runtime_contract() -> Value { }) } +pub(crate) fn audio_transcription_runtime_contract() -> Value { + json!({ + "contract_key": AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "modality": AUDIO_TRANSCRIPTION_MODALITY, + "required_capabilities": AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES, + "routing_slot": AUDIO_TRANSCRIPTION_ROUTING_SLOT, + "executor_binding": { + "executor_kind": "skill", + "binding_key": AUDIO_TRANSCRIPTION_EXECUTOR_BINDING_KEY + }, + "truth_source": ["transcript_artifact", "runtime_timeline_event"], + "artifact_kinds": ["transcript"], + "viewer_surface": ["transcript_viewer", "document_viewer"], + "owner_surface": "agent_runtime" + }) +} + pub(crate) fn web_research_runtime_contract() -> Value { json!({ "contract_key": WEB_RESEARCH_CONTRACT_KEY, @@ -506,6 +598,29 @@ pub(crate) fn insert_voice_generation_contract_fields(record: &mut Map) { + record.insert( + "modality_contract_key".to_string(), + Value::String(AUDIO_TRANSCRIPTION_CONTRACT_KEY.to_string()), + ); + record.insert( + "modality".to_string(), + Value::String(AUDIO_TRANSCRIPTION_MODALITY.to_string()), + ); + record.insert( + "required_capabilities".to_string(), + json!(AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES), + ); + record.insert( + "routing_slot".to_string(), + Value::String(AUDIO_TRANSCRIPTION_ROUTING_SLOT.to_string()), + ); + record.insert( + "runtime_contract".to_string(), + audio_transcription_runtime_contract(), + ); +} + pub(crate) fn insert_web_research_contract_fields(record: &mut Map) { record.insert( "modality_contract_key".to_string(), diff --git a/src-tauri/src/commands/voice_model_cmd.rs b/src-tauri/src/commands/voice_model_cmd.rs new file mode 100644 index 000000000..090375add --- /dev/null +++ b/src-tauri/src/commands/voice_model_cmd.rs @@ -0,0 +1,1075 @@ +//! 语音本地模型管理命令 +//! +//! 只管理用户显式下载到应用数据目录的模型文件，不把大模型随安装包内置。 + +use bzip2::read::BzDecoder; +use futures::StreamExt; +use lime_core::app_paths; +use lime_services::voice_asr_service::AsrService; +use serde::{Deserialize, Serialize}; +use sha2::{Digest, Sha256}; +use std::env; +use std::fs; +use std::path::{Path, PathBuf}; +use std::time::{SystemTime, UNIX_EPOCH}; +use tauri::command; +use tokio::io::AsyncWriteExt; + +use crate::config::{ + load_config, save_config, AsrCredentialEntry, AsrProviderType, SenseVoiceLocalConfig, +}; + +const SENSEVOICE_MODEL_ID: &str = "sensevoice-small-int8-2024-07-17"; +const SILERO_VAD_MODEL_ID: &str = "silero-vad-onnx"; +const MODEL_ARCHIVE_FILE_NAME: &str = + "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2"; +const MODEL_ARCHIVE_DOWNLOAD_PATH: &str = concat!( + "voice/sensevoice-small-int8-2024-07-17/", + "sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2" +); +const VAD_FILE_NAME: &str = "silero_vad.onnx"; +const VAD_DOWNLOAD_PATH: &str = "voice/silero-vad-onnx/silero_vad.onnx"; +const MODEL_ONNX_FILE: &str = "model.int8.onnx"; +const TOKENS_FILE: &str = "tokens.txt"; +const MANIFEST_FILE: &str = "lime-model.json"; +const DEFAULT_MODEL_BYTES: u64 = 250 * 1024 * 1024; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct VoiceModelCatalogEntry { + pub id: String, + pub name: String, + pub provider: String, + pub description: String, + pub version: String, + pub languages: Vec, + pub size_bytes: u64, + pub download_url: String, + pub vad_model_id: Option, + pub vad_download_url: Option, + pub runtime: String, + pub bundled: bool, + pub checksum_sha256: Option, +} + +#[derive(Debug, Deserialize)] +struct VoiceModelCatalogEnvelope { + data: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LimecoreVoiceModelCatalogResponse { + #[serde(default, rename = "assetBaseURL")] + asset_base_url: String, + #[serde(default)] + items: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LimecoreVoiceModelCatalogItem { + id: String, + name: String, + provider: String, + description: String, + version: String, + #[serde(default)] + languages: Vec, + #[serde(default)] + runtime: String, + #[serde(default)] + bundled: bool, + #[serde(default)] + size_bytes: u64, + checksum_sha256: Option, + download: LimecoreVoiceModelDownloadBundle, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LimecoreVoiceModelDownloadBundle { + archive: LimecoreVoiceModelDownloadAsset, + vad: Option, +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct LimecoreVoiceModelDownloadAsset { + model_id: Option, + download_path: Option, + download_url: Option, + sha256: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct VoiceModelInstallState { + pub model_id: String, + pub installed: bool, + pub installing: bool, + pub install_dir: String, + pub model_file: Option, + pub tokens_file: Option, + pub vad_file: Option, + pub installed_bytes: u64, + pub last_verified_at: Option, + pub missing_files: Vec, + pub default_credential_id: Option, +} + +#[derive(Debug, Clone, Serialize)] +pub struct VoiceModelDownloadResult { + pub state: VoiceModelInstallState, +} + +#[derive(Debug, Clone, Serialize)] +pub struct VoiceModelTestTranscribeResult { + pub text: String, + pub duration_secs: f32, + pub sample_rate: u32, + pub language: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct VoiceModelManifest { + model_id: String, + installed_at: u64, + source_url: String, + vad_url: String, + archive_sha256: String, + checksum_verified: bool, + checksum_note: String, +} + +#[command] +pub async fn voice_models_list_catalog() -> Result, String> { + if let Some(entries) = fetch_configured_voice_model_catalog().await? { + return Ok(entries); + } + + Ok(vec![sensevoice_catalog_entry()]) +} + +#[command] +pub async fn voice_models_get_install_state( + model_id: String, +) -> Result { + ensure_supported_model(&model_id)?; + build_install_state(&model_id) +} + +#[command] +pub async fn voice_models_download( + model_id: String, + catalog_entry: Option, +) -> Result { + ensure_supported_model(&model_id)?; + let catalog_entry = resolve_voice_model_catalog_entry(&model_id, catalog_entry).await?; + let archive_url = require_download_url(&catalog_entry.download_url, "SenseVoice Small 归档")?; + let vad_url = require_download_url( + catalog_entry + .vad_download_url + .as_deref() + .unwrap_or_default(), + "Silero VAD", + )?; + + let install_dir = model_install_dir(&model_id)?; + let temp_root = models_root()?.join(".downloads").join(format!( + "{}-{}", + model_id, + current_unix_secs().unwrap_or_default() + )); + let extract_dir = temp_root.join("extract"); + fs::create_dir_all(&extract_dir) + .map_err(|error| format!("创建模型临时目录失败 {}: {error}", extract_dir.display()))?; + + let archive_path = temp_root.join(MODEL_ARCHIVE_FILE_NAME); + let archive_sha256 = download_file(&archive_url, &archive_path).await?; + let checksum_verified = + verify_optional_sha256(&archive_sha256, catalog_entry.checksum_sha256.as_deref())?; + extract_tar_bz2(&archive_path, &extract_dir)?; + + let model_source_dir = find_sensevoice_model_dir(&extract_dir)?; + let staging_dir = temp_root.join("staging"); + if staging_dir.exists() { + fs::remove_dir_all(&staging_dir) + .map_err(|error| format!("清理模型暂存目录失败 {}: {error}", staging_dir.display()))?; + } + fs::create_dir_all(&staging_dir) + .map_err(|error| format!("创建模型暂存目录失败 {}: {error}", staging_dir.display()))?; + + copy_required_file(&model_source_dir, &staging_dir, MODEL_ONNX_FILE)?; + copy_required_file(&model_source_dir, &staging_dir, TOKENS_FILE)?; + let vad_path = staging_dir.join(VAD_FILE_NAME); + let _vad_sha256 = download_file(&vad_url, &vad_path).await?; + + let manifest = VoiceModelManifest { + model_id: model_id.clone(), + installed_at: current_unix_secs().unwrap_or_default(), + source_url: archive_url, + vad_url, + archive_sha256, + checksum_verified, + checksum_note: if checksum_verified { + "后端目录提供 sha256，已完成归档内容校验".to_string() + } else { + "后端目录未提供 sha256，当前记录下载内容摘要但不声明已完成可信校验".to_string() + }, + }; + write_manifest(&staging_dir, &manifest)?; + + if install_dir.exists() { + fs::remove_dir_all(&install_dir) + .map_err(|error| format!("替换旧模型目录失败 {}: {error}", install_dir.display()))?; + } + fs::create_dir_all( + install_dir + .parent() + .ok_or_else(|| format!("模型目录没有父路径: {}", install_dir.display()))?, + ) + .map_err(|error| format!("创建模型父目录失败: {error}"))?; + fs::rename(&staging_dir, &install_dir) + .map_err(|error| format!("安装模型目录失败 {}: {error}", install_dir.display()))?; + + let _ = fs::remove_dir_all(&temp_root); + Ok(VoiceModelDownloadResult { + state: build_install_state(&model_id)?, + }) +} + +#[command] +pub async fn voice_models_delete(model_id: String) -> Result { + ensure_supported_model(&model_id)?; + let install_dir = model_install_dir(&model_id)?; + if install_dir.exists() { + fs::remove_dir_all(&install_dir) + .map_err(|error| format!("删除模型目录失败 {}: {error}", install_dir.display()))?; + } + Ok(build_install_state(&model_id)?) +} + +#[command] +pub async fn voice_models_set_default(model_id: String) -> Result { + ensure_supported_model(&model_id)?; + let state = build_install_state(&model_id)?; + if !state.installed { + return Err(format!( + "SenseVoice Small 尚未安装，缺失文件: {}", + state.missing_files.join(", ") + )); + } + + let mut config = load_config().map_err(|error| error.to_string())?; + let credentials = &mut config.experimental.voice_input.asr_credentials; + for credential in credentials.iter_mut() { + credential.is_default = false; + } + + let install_dir = model_install_dir(&model_id)?; + let credential = match credentials.iter_mut().find(|credential| { + credential.provider == AsrProviderType::SenseVoiceLocal + && credential + .sensevoice_config + .as_ref() + .map(|config| config.model_id.as_str() == model_id) + .unwrap_or(false) + }) { + Some(existing) => { + existing.name = Some("SenseVoice Small 本地".to_string()); + existing.disabled = false; + existing.is_default = true; + existing.language = "auto".to_string(); + existing.sensevoice_config = Some(SenseVoiceLocalConfig { + model_id: model_id.clone(), + model_dir: Some(install_dir.to_string_lossy().to_string()), + use_itn: true, + num_threads: 4, + vad_model_id: Some(SILERO_VAD_MODEL_ID.to_string()), + }); + existing.clone() + } + None => { + let entry = AsrCredentialEntry { + id: format!("sensevoice-local-{model_id}"), + provider: AsrProviderType::SenseVoiceLocal, + name: Some("SenseVoice Small 本地".to_string()), + is_default: true, + disabled: false, + language: "auto".to_string(), + whisper_config: None, + sensevoice_config: Some(SenseVoiceLocalConfig { + model_id: model_id.clone(), + model_dir: Some(install_dir.to_string_lossy().to_string()), + use_itn: true, + num_threads: 4, + vad_model_id: Some(SILERO_VAD_MODEL_ID.to_string()), + }), + xunfei_config: None, + baidu_config: None, + openai_config: None, + }; + credentials.push(entry.clone()); + entry + } + }; + + save_config(&config).map_err(|error| error.to_string())?; + Ok(credential) +} + +#[command] +pub async fn voice_models_test_transcribe_file( + model_id: String, + file_path: String, +) -> Result { + ensure_supported_model(&model_id)?; + let state = build_install_state(&model_id)?; + if !state.installed { + return Err(format!( + "请先在设置 -> 语音模型中下载 SenseVoice Small；缺失文件: {}", + state.missing_files.join(", ") + )); + } + + let file_path = file_path.trim(); + if file_path.is_empty() { + return Err("请提供本机 WAV 文件路径".to_string()); + } + + let audio = read_pcm16_wav(Path::new(file_path))?; + let install_dir = model_install_dir(&model_id)?; + let credential = AsrCredentialEntry { + id: format!("sensevoice-local-test-{model_id}"), + provider: AsrProviderType::SenseVoiceLocal, + name: Some("SenseVoice Small 本地测试".to_string()), + is_default: false, + disabled: false, + language: "auto".to_string(), + whisper_config: None, + sensevoice_config: Some(SenseVoiceLocalConfig { + model_id: model_id.clone(), + model_dir: Some(install_dir.to_string_lossy().to_string()), + use_itn: true, + num_threads: 4, + vad_model_id: Some(SILERO_VAD_MODEL_ID.to_string()), + }), + xunfei_config: None, + baidu_config: None, + openai_config: None, + }; + + let text = AsrService::transcribe(&credential, &audio.pcm16le, audio.sample_rate).await?; + + Ok(VoiceModelTestTranscribeResult { + text, + duration_secs: audio.duration_secs, + sample_rate: audio.sample_rate, + language: Some("auto".to_string()), + }) +} + +fn sensevoice_catalog_entry() -> VoiceModelCatalogEntry { + let asset_base_url = first_non_empty_env(&[ + "LIME_VOICE_MODEL_ASSET_BASE_URL", + "VOICE_MODEL_ASSET_BASE_URL", + "SERVER_VOICE_MODEL_ASSET_BASE_URL", + ]); + VoiceModelCatalogEntry { + id: SENSEVOICE_MODEL_ID.to_string(), + name: "SenseVoice Small INT8".to_string(), + provider: "FunAudioLLM / sherpa-onnx".to_string(), + description: "本地离线 ASR，支持中文、英文、日文、韩文和粤语；模型按需下载到用户数据目录。" + .to_string(), + version: "2024-07-17".to_string(), + languages: vec![ + "zh".to_string(), + "en".to_string(), + "ja".to_string(), + "ko".to_string(), + "yue".to_string(), + ], + size_bytes: DEFAULT_MODEL_BYTES, + download_url: asset_base_url + .as_deref() + .and_then(|base_url| join_url(base_url, MODEL_ARCHIVE_DOWNLOAD_PATH)) + .unwrap_or_default(), + vad_model_id: Some(SILERO_VAD_MODEL_ID.to_string()), + vad_download_url: asset_base_url + .as_deref() + .and_then(|base_url| join_url(base_url, VAD_DOWNLOAD_PATH)), + runtime: "sherpa-onnx".to_string(), + bundled: false, + checksum_sha256: None, + } +} + +async fn resolve_voice_model_catalog_entry( + model_id: &str, + provided_entry: Option, +) -> Result { + if let Some(entry) = provided_entry { + if entry.id != model_id { + return Err(format!( + "语音模型目录 ID 不匹配: expected={model_id}, actual={}", + entry.id + )); + } + return Ok(entry); + } + + if let Some(entries) = fetch_configured_voice_model_catalog().await? { + if let Some(entry) = entries.into_iter().find(|entry| entry.id == model_id) { + return Ok(entry); + } + return Err(format!("后端语音模型目录缺少模型: {model_id}")); + } + + Ok(sensevoice_catalog_entry()) +} + +async fn fetch_configured_voice_model_catalog( +) -> Result>, String> { + let Some(catalog_url) = resolve_voice_model_catalog_url() else { + return Ok(None); + }; + + let client = reqwest::Client::builder() + .build() + .map_err(|error| format!("创建 HTTP 客户端失败: {error}"))?; + let payload = client + .get(&catalog_url) + .send() + .await + .map_err(|error| format!("拉取后端语音模型目录失败: {error}"))? + .error_for_status() + .map_err(|error| format!("后端语音模型目录响应异常: {error}"))? + .json::() + .await + .map_err(|error| format!("解析后端语音模型目录失败: {error}"))?; + + parse_limecore_voice_model_catalog(payload).map(Some) +} + +fn parse_limecore_voice_model_catalog( + payload: serde_json::Value, +) -> Result, String> { + if let Ok(envelope) = serde_json::from_value::(payload.clone()) { + if let Some(data) = envelope.data { + return map_limecore_voice_model_catalog(data); + } + } + + let response = serde_json::from_value::(payload) + .map_err(|error| format!("后端语音模型目录格式非法: {error}"))?; + map_limecore_voice_model_catalog(response) +} + +fn map_limecore_voice_model_catalog( + response: LimecoreVoiceModelCatalogResponse, +) -> Result, String> { + let asset_base_url = response.asset_base_url; + response + .items + .into_iter() + .map(|item| map_limecore_voice_model_item(item, &asset_base_url)) + .collect() +} + +fn map_limecore_voice_model_item( + item: LimecoreVoiceModelCatalogItem, + asset_base_url: &str, +) -> Result { + let archive_download_url = + resolve_asset_download_url(&item.download.archive, asset_base_url).unwrap_or_default(); + let vad_download_url = item + .download + .vad + .as_ref() + .and_then(|asset| resolve_asset_download_url(asset, asset_base_url)); + let vad_model_id = item + .download + .vad + .as_ref() + .and_then(|asset| asset.model_id.clone()) + .or_else(|| Some(SILERO_VAD_MODEL_ID.to_string())); + + Ok(VoiceModelCatalogEntry { + id: item.id, + name: item.name, + provider: item.provider, + description: item.description, + version: item.version, + languages: item.languages, + size_bytes: item.size_bytes, + download_url: archive_download_url, + vad_model_id, + vad_download_url, + runtime: if item.runtime.trim().is_empty() { + "sherpa-onnx".to_string() + } else { + item.runtime + }, + bundled: item.bundled, + checksum_sha256: item + .download + .archive + .sha256 + .filter(|value| !value.trim().is_empty()) + .or(item.checksum_sha256), + }) +} + +fn resolve_asset_download_url( + asset: &LimecoreVoiceModelDownloadAsset, + asset_base_url: &str, +) -> Option { + asset + .download_url + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty()) + .map(ToString::to_string) + .or_else(|| { + let download_path = asset + .download_path + .as_deref() + .map(str::trim) + .filter(|value| !value.is_empty())?; + join_url(asset_base_url, download_path) + }) +} + +fn resolve_voice_model_catalog_url() -> Option { + if let Some(catalog_url) = first_non_empty_env(&[ + "LIME_VOICE_MODEL_CATALOG_URL", + "VOICE_MODEL_CATALOG_URL", + "SERVER_VOICE_MODEL_CATALOG_URL", + ]) { + return Some(catalog_url); + } + + let base_url = first_non_empty_env(&[ + "LIME_OEM_CLOUD_BASE_URL", + "VITE_OEM_CLOUD_BASE_URL", + "OEM_CLOUD_BASE_URL", + ])?; + let tenant_id = + first_non_empty_env(&["LIME_OEM_TENANT_ID", "VITE_OEM_TENANT_ID", "OEM_TENANT_ID"])?; + join_url( + &base_url, + &format!("api/v1/public/tenants/{tenant_id}/client/voice-model-catalog"), + ) +} + +fn require_download_url(url: &str, label: &str) -> Result { + let normalized = url.trim(); + if normalized.is_empty() { + return Err(format!( + "{label} 下载地址未配置。请在 limecore 设置 server.voiceModelAssetBaseUrl 指向对象存储或 CDN 公开域名（例如阿里云 OSS、Cloudflare R2），或通过前端传入后端语音模型目录。" + )); + } + Ok(normalized.to_string()) +} + +fn verify_optional_sha256(actual: &str, expected: Option<&str>) -> Result { + let Some(expected) = expected.map(str::trim).filter(|value| !value.is_empty()) else { + return Ok(false); + }; + + if actual.eq_ignore_ascii_case(expected) { + return Ok(true); + } + + Err(format!( + "模型归档 sha256 校验失败: expected={expected}, actual={actual}" + )) +} + +fn first_non_empty_env(names: &[&str]) -> Option { + names.iter().find_map(|name| { + env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) + }) +} + +fn join_url(base_url: &str, path: &str) -> Option { + let base = base_url.trim().trim_end_matches('/'); + let path = path.trim().trim_start_matches('/'); + if base.is_empty() || path.is_empty() { + return None; + } + Some(format!("{base}/{path}")) +} + +#[derive(Debug)] +struct PcmWavAudio { + pcm16le: Vec, + sample_rate: u32, + duration_secs: f32, +} + +#[derive(Debug, Clone, Copy)] +struct WavFormat { + audio_format: u16, + channels: u16, + sample_rate: u32, + bits_per_sample: u16, +} + +fn read_pcm16_wav(path: &Path) -> Result { + if path + .extension() + .and_then(|extension| extension.to_str()) + .map(|extension| !extension.eq_ignore_ascii_case("wav")) + .unwrap_or(true) + { + return Err("当前测试转写仅支持 .wav 文件".to_string()); + } + + let bytes = + fs::read(path).map_err(|error| format!("读取 WAV 文件失败 {}: {error}", path.display()))?; + parse_pcm16_wav_bytes(&bytes) +} + +fn parse_pcm16_wav_bytes(bytes: &[u8]) -> Result { + if bytes.len() < 12 || &bytes[0..4] != b"RIFF" || &bytes[8..12] != b"WAVE" { + return Err("不是有效的 RIFF/WAVE 文件".to_string()); + } + + let mut offset = 12_usize; + let mut wav_format = None; + let mut data_chunk = None; + + while offset + 8 <= bytes.len() { + let chunk_id = &bytes[offset..offset + 4]; + let chunk_size = u32::from_le_bytes([ + bytes[offset + 4], + bytes[offset + 5], + bytes[offset + 6], + bytes[offset + 7], + ]) as usize; + let chunk_start = offset + 8; + let chunk_end = chunk_start + .checked_add(chunk_size) + .ok_or_else(|| "WAV chunk 长度溢出".to_string())?; + if chunk_end > bytes.len() { + return Err("WAV chunk 长度异常".to_string()); + } + + match chunk_id { + b"fmt " => { + if chunk_size < 16 { + return Err("WAV fmt chunk 不完整".to_string()); + } + wav_format = Some(WavFormat { + audio_format: read_le_u16(bytes, chunk_start)?, + channels: read_le_u16(bytes, chunk_start + 2)?, + sample_rate: read_le_u32(bytes, chunk_start + 4)?, + bits_per_sample: read_le_u16(bytes, chunk_start + 14)?, + }); + } + b"data" => { + data_chunk = Some(&bytes[chunk_start..chunk_end]); + } + _ => {} + } + + offset = chunk_end + (chunk_size % 2); + } + + let wav_format = wav_format.ok_or_else(|| "WAV 文件缺少 fmt chunk".to_string())?; + if wav_format.audio_format != 1 { + return Err("当前测试转写仅支持 16-bit PCM WAV（audio_format=1）".to_string()); + } + if wav_format.channels == 0 { + return Err("WAV 声道数无效".to_string()); + } + if wav_format.sample_rate == 0 { + return Err("WAV 采样率无效".to_string()); + } + if wav_format.bits_per_sample != 16 { + return Err("当前测试转写仅支持 16-bit PCM WAV".to_string()); + } + + let data = data_chunk.ok_or_else(|| "WAV 文件缺少 data chunk".to_string())?; + let frame_size = usize::from(wav_format.channels) * 2; + if data.len() < frame_size { + return Err("WAV 音频数据为空".to_string()); + } + if data.len() % frame_size != 0 { + return Err("WAV PCM 数据长度与声道数不匹配".to_string()); + } + + let frame_count = data.len() / frame_size; + let mut pcm16le = Vec::with_capacity(frame_count * 2); + if wav_format.channels == 1 { + pcm16le.extend_from_slice(data); + } else { + for frame in data.chunks_exact(frame_size) { + pcm16le.extend_from_slice(&frame[0..2]); + } + } + + Ok(PcmWavAudio { + pcm16le, + sample_rate: wav_format.sample_rate, + duration_secs: frame_count as f32 / wav_format.sample_rate as f32, + }) +} + +fn read_le_u16(bytes: &[u8], offset: usize) -> Result { + if offset + 2 > bytes.len() { + return Err("WAV 字段越界".to_string()); + } + Ok(u16::from_le_bytes([bytes[offset], bytes[offset + 1]])) +} + +fn read_le_u32(bytes: &[u8], offset: usize) -> Result { + if offset + 4 > bytes.len() { + return Err("WAV 字段越界".to_string()); + } + Ok(u32::from_le_bytes([ + bytes[offset], + bytes[offset + 1], + bytes[offset + 2], + bytes[offset + 3], + ])) +} + +fn ensure_supported_model(model_id: &str) -> Result<(), String> { + if model_id == SENSEVOICE_MODEL_ID { + Ok(()) + } else { + Err(format!("不支持的语音模型: {model_id}")) + } +} + +fn models_root() -> Result { + Ok(app_paths::preferred_data_dir()? + .join("models") + .join("voice")) +} + +fn model_install_dir(model_id: &str) -> Result { + Ok(models_root()?.join(model_id)) +} + +fn build_install_state(model_id: &str) -> Result { + let install_dir = model_install_dir(model_id)?; + let model_file = install_dir.join(MODEL_ONNX_FILE); + let tokens_file = install_dir.join(TOKENS_FILE); + let vad_file = install_dir.join(VAD_FILE_NAME); + + let required_files = [ + (MODEL_ONNX_FILE, &model_file), + (TOKENS_FILE, &tokens_file), + (VAD_FILE_NAME, &vad_file), + ]; + let missing_files = required_files + .iter() + .filter_map(|(name, path)| { + if path.exists() { + None + } else { + Some((*name).to_string()) + } + }) + .collect::>(); + + let default_credential_id = load_config().ok().and_then(|config| { + config + .experimental + .voice_input + .asr_credentials + .into_iter() + .find(|credential| { + credential.is_default + && !credential.disabled + && credential.provider == AsrProviderType::SenseVoiceLocal + && credential + .sensevoice_config + .as_ref() + .map(|config| config.model_id.as_str() == model_id) + .unwrap_or(false) + }) + .map(|credential| credential.id) + }); + + Ok(VoiceModelInstallState { + model_id: model_id.to_string(), + installed: missing_files.is_empty(), + installing: false, + install_dir: install_dir.to_string_lossy().to_string(), + model_file: model_file + .exists() + .then(|| model_file.to_string_lossy().to_string()), + tokens_file: tokens_file + .exists() + .then(|| tokens_file.to_string_lossy().to_string()), + vad_file: vad_file + .exists() + .then(|| vad_file.to_string_lossy().to_string()), + installed_bytes: directory_size(&install_dir).unwrap_or(0), + last_verified_at: current_unix_secs(), + missing_files, + default_credential_id, + }) +} + +async fn download_file(url: &str, destination: &Path) -> Result { + if let Some(parent) = destination.parent() { + tokio::fs::create_dir_all(parent) + .await + .map_err(|error| format!("创建下载目录失败 {}: {error}", parent.display()))?; + } + + let client = reqwest::Client::builder() + .build() + .map_err(|error| format!("创建 HTTP 客户端失败: {error}"))?; + let response = client + .get(url) + .send() + .await + .map_err(|error| format!("下载模型失败: {error}"))? + .error_for_status() + .map_err(|error| format!("下载模型响应异常: {error}"))?; + + let temp_path = destination.with_extension("download"); + let mut file = tokio::fs::File::create(&temp_path) + .await + .map_err(|error| format!("创建下载文件失败 {}: {error}", temp_path.display()))?; + let mut hasher = Sha256::new(); + let mut stream = response.bytes_stream(); + + while let Some(chunk) = stream.next().await { + let chunk = chunk.map_err(|error| format!("读取下载数据失败: {error}"))?; + hasher.update(&chunk); + file.write_all(&chunk) + .await + .map_err(|error| format!("写入下载文件失败: {error}"))?; + } + file.flush() + .await + .map_err(|error| format!("刷新下载文件失败: {error}"))?; + drop(file); + + tokio::fs::rename(&temp_path, destination) + .await + .map_err(|error| format!("保存下载文件失败 {}: {error}", destination.display()))?; + + Ok(format!("{:x}", hasher.finalize())) +} + +fn extract_tar_bz2(archive_path: &Path, destination: &Path) -> Result<(), String> { + let file = fs::File::open(archive_path) + .map_err(|error| format!("打开模型归档失败 {}: {error}", archive_path.display()))?; + let decoder = BzDecoder::new(file); + let mut archive = tar::Archive::new(decoder); + + for entry in archive + .entries() + .map_err(|error| format!("读取模型归档失败: {error}"))? + { + let mut entry = entry.map_err(|error| format!("读取模型归档条目失败: {error}"))?; + entry + .unpack_in(destination) + .map_err(|error| format!("解包模型归档失败: {error}"))?; + } + Ok(()) +} + +fn find_sensevoice_model_dir(root: &Path) -> Result { + let mut stack = vec![root.to_path_buf()]; + while let Some(path) = stack.pop() { + if path.join(MODEL_ONNX_FILE).exists() && path.join(TOKENS_FILE).exists() { + return Ok(path); + } + + let entries = fs::read_dir(&path) + .map_err(|error| format!("读取模型解包目录失败 {}: {error}", path.display()))?; + for entry in entries { + let entry = entry.map_err(|error| format!("读取模型解包目录项失败: {error}"))?; + let entry_path = entry.path(); + if entry_path.is_dir() { + stack.push(entry_path); + } + } + } + + Err("模型归档中未找到 model.int8.onnx 和 tokens.txt".to_string()) +} + +fn copy_required_file(source_dir: &Path, target_dir: &Path, file_name: &str) -> Result<(), String> { + let source = source_dir.join(file_name); + if !source.exists() { + return Err(format!("模型归档缺少文件: {file_name}")); + } + fs::copy(&source, target_dir.join(file_name)) + .map_err(|error| format!("复制模型文件失败 {}: {error}", source.display()))?; + Ok(()) +} + +fn write_manifest(dir: &Path, manifest: &VoiceModelManifest) -> Result<(), String> { + let content = serde_json::to_string_pretty(manifest) + .map_err(|error| format!("序列化模型清单失败: {error}"))?; + fs::write(dir.join(MANIFEST_FILE), content) + .map_err(|error| format!("写入模型清单失败 {}: {error}", dir.display())) +} + +fn directory_size(path: &Path) -> Result { + if !path.exists() { + return Ok(0); + } + + let mut total = 0_u64; + let mut stack = vec![path.to_path_buf()]; + while let Some(current) = stack.pop() { + for entry in fs::read_dir(¤t) + .map_err(|error| format!("读取目录大小失败 {}: {error}", current.display()))? + { + let entry = entry.map_err(|error| format!("读取目录项失败: {error}"))?; + let metadata = entry.metadata().map_err(|error| { + format!("读取文件元数据失败 {}: {error}", entry.path().display()) + })?; + if metadata.is_dir() { + stack.push(entry.path()); + } else { + total = total.saturating_add(metadata.len()); + } + } + } + Ok(total) +} + +fn current_unix_secs() -> Option { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .ok() + .map(|duration| duration.as_secs()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_pcm16_wav_bytes_reads_mono_audio() { + let wav = build_test_wav(1, 16_000, &[100, -100, 200]); + + let audio = parse_pcm16_wav_bytes(&wav).expect("parse wav"); + + assert_eq!(audio.sample_rate, 16_000); + assert!((audio.duration_secs - 0.0001875).abs() < f32::EPSILON); + assert_eq!(audio.pcm16le, samples_to_pcm16le(&[100, -100, 200])); + } + + #[test] + fn parse_pcm16_wav_bytes_uses_first_channel_for_stereo() { + let wav = build_test_wav(2, 16_000, &[100, 900, -100, -900]); + + let audio = parse_pcm16_wav_bytes(&wav).expect("parse wav"); + + assert_eq!(audio.pcm16le, samples_to_pcm16le(&[100, -100])); + } + + #[test] + fn parse_pcm16_wav_bytes_rejects_non_pcm_format() { + let mut wav = build_test_wav(1, 16_000, &[100]); + wav[20..22].copy_from_slice(&3_u16.to_le_bytes()); + + let error = parse_pcm16_wav_bytes(&wav).expect_err("float wav should fail"); + + assert!(error.contains("16-bit PCM WAV")); + } + + #[test] + fn parse_limecore_voice_model_catalog_maps_object_storage_urls() { + let payload = serde_json::json!({ + "code": 200, + "message": "success", + "data": { + "assetBaseURL": "https://models.example.com", + "items": [ + { + "id": SENSEVOICE_MODEL_ID, + "name": "SenseVoice Small INT8", + "provider": "FunAudioLLM / sherpa-onnx", + "description": "后端下发", + "version": "2024-07-17", + "languages": ["zh", "en"], + "runtime": "sherpa-onnx", + "bundled": false, + "sizeBytes": 262144000, + "download": { + "archive": { + "downloadPath": MODEL_ARCHIVE_DOWNLOAD_PATH, + "sha256": "abc123" + }, + "vad": { + "modelId": SILERO_VAD_MODEL_ID, + "downloadPath": VAD_DOWNLOAD_PATH + } + } + } + ] + } + }); + + let entries = parse_limecore_voice_model_catalog(payload).expect("parse catalog"); + + assert_eq!(entries.len(), 1); + assert_eq!(entries[0].id, SENSEVOICE_MODEL_ID); + assert_eq!( + entries[0].download_url, + "https://models.example.com/voice/sensevoice-small-int8-2024-07-17/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2" + ); + assert_eq!( + entries[0].vad_download_url.as_deref(), + Some("https://models.example.com/voice/silero-vad-onnx/silero_vad.onnx") + ); + assert_eq!(entries[0].checksum_sha256.as_deref(), Some("abc123")); + } + + #[test] + fn verify_optional_sha256_rejects_mismatch() { + let error = verify_optional_sha256("actual", Some("expected")) + .expect_err("sha mismatch should fail"); + + assert!(error.contains("sha256 校验失败")); + } + + fn build_test_wav(channels: u16, sample_rate: u32, samples: &[i16]) -> Vec { + let data = samples_to_pcm16le(samples); + let fmt_chunk_size = 16_u32; + let data_chunk_size = data.len() as u32; + let riff_size = 4 + 8 + fmt_chunk_size + 8 + data_chunk_size; + let byte_rate = sample_rate * u32::from(channels) * 2; + let block_align = channels * 2; + + let mut wav = Vec::new(); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&riff_size.to_le_bytes()); + wav.extend_from_slice(b"WAVE"); + wav.extend_from_slice(b"fmt "); + wav.extend_from_slice(&fmt_chunk_size.to_le_bytes()); + wav.extend_from_slice(&1_u16.to_le_bytes()); + wav.extend_from_slice(&channels.to_le_bytes()); + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&byte_rate.to_le_bytes()); + wav.extend_from_slice(&block_align.to_le_bytes()); + wav.extend_from_slice(&16_u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_chunk_size.to_le_bytes()); + wav.extend_from_slice(&data); + wav + } + + fn samples_to_pcm16le(samples: &[i16]) -> Vec { + samples + .iter() + .flat_map(|sample| sample.to_le_bytes()) + .collect() + } +} diff --git a/src-tauri/src/dev_bridge.rs b/src-tauri/src/dev_bridge.rs index 11b62705b..9ff80d78a 100644 --- a/src-tauri/src/dev_bridge.rs +++ b/src-tauri/src/dev_bridge.rs @@ -267,6 +267,9 @@ async fn stream_events( listener_id, }; + // 先刷新一个 SSE 注释，避免浏览器 EventSource 在首个业务事件到达前一直不触发 open。 + yield Ok::(SseEvent::default().comment("ready")); + while let Some(payload) = rx.recv().await { yield Ok::(SseEvent::default().data(payload)); } diff --git a/src-tauri/src/dev_bridge/dispatcher.rs b/src-tauri/src/dev_bridge/dispatcher.rs index dbe61be34..75ad8dae8 100644 --- a/src-tauri/src/dev_bridge/dispatcher.rs +++ b/src-tauri/src/dev_bridge/dispatcher.rs @@ -832,6 +832,87 @@ mod tests { assert!(error.to_string().contains("模型注册服务未初始化")); } + #[tokio::test] + async fn api_key_provider_write_flow_is_bridged() { + let state = make_test_state(); + + let created = handle_command( + &state, + "add_custom_api_key_provider", + Some(serde_json::json!({ + "request": { + "name": "SenseNova E2E", + "type": "openai", + "api_host": "https://api.sensenova.cn/compatible-mode/v2" + } + })), + ) + .await + .expect("custom provider should be created through bridge"); + let provider_id = created["id"].as_str().unwrap().to_string(); + + handle_command( + &state, + "update_api_key_provider", + Some(serde_json::json!({ + "id": provider_id, + "request": { + "enabled": true, + "custom_models": ["SenseChat-5"] + } + })), + ) + .await + .expect("custom provider should be updated through bridge"); + + let key = handle_command( + &state, + "add_api_key", + Some(serde_json::json!({ + "request": { + "provider_id": provider_id, + "api_key": "sk-bridge-test", + "alias": "bridge" + } + })), + ) + .await + .expect("api key should be added through bridge"); + + assert_eq!(key["provider_id"], created["id"]); + assert_ne!(key["api_key_masked"], "sk-bridge-test"); + + let provider = handle_command( + &state, + "get_api_key_provider", + Some(serde_json::json!({ + "id": created["id"].as_str().unwrap() + })), + ) + .await + .expect("custom provider should be readable through bridge"); + + assert_eq!(provider["id"], created["id"]); + assert_eq!( + provider["api_host"], + "https://api.sensenova.cn/compatible-mode/v2" + ); + assert_eq!(provider["custom_models"][0], "SenseChat-5"); + assert_eq!(provider["api_keys"].as_array().unwrap().len(), 1); + + let deleted = handle_command( + &state, + "delete_custom_api_key_provider", + Some(serde_json::json!({ + "id": created["id"].as_str().unwrap() + })), + ) + .await + .expect("custom provider should be deleted through bridge"); + + assert_eq!(deleted, serde_json::json!(true)); + } + #[tokio::test] async fn test_api_key_provider_connection_is_bridged() { let state = make_test_state(); diff --git a/src-tauri/src/dev_bridge/dispatcher/providers.rs b/src-tauri/src/dev_bridge/dispatcher/providers.rs index 09f665891..4858676ce 100644 --- a/src-tauri/src/dev_bridge/dispatcher/providers.rs +++ b/src-tauri/src/dev_bridge/dispatcher/providers.rs @@ -1,5 +1,8 @@ use super::{args_or_default, get_string_arg, parse_nested_arg, require_app_handle}; use crate::connect::RelayRegistry; +use crate::database::dao::api_key_provider::{ + ApiKeyEntry, ApiKeyProvider, ApiProviderPromptCacheMode, ApiProviderType, +}; use crate::dev_bridge::DevBridgeState; use serde_json::Value as JsonValue; use std::sync::Arc; @@ -22,59 +25,67 @@ fn api_key_provider_with_keys_to_display( provider_with_keys: &crate::database::dao::api_key_provider::ProviderWithKeys, service: &lime_services::api_key_provider_service::ApiKeyProviderService, ) -> crate::commands::api_key_provider_cmd::ProviderWithKeysDisplay { - let api_keys = provider_with_keys + let api_keys: Vec = provider_with_keys .api_keys .iter() - .map(|key| { - let masked = match service.decrypt_api_key(&key.api_key_encrypted) { - Ok(decrypted) => mask_api_key_for_display(&decrypted), - Err(_) => "****".to_string(), - }; - - crate::commands::api_key_provider_cmd::ApiKeyDisplay { - id: key.id.clone(), - provider_id: key.provider_id.clone(), - api_key_masked: masked, - alias: key.alias.clone(), - enabled: key.enabled, - usage_count: key.usage_count, - error_count: key.error_count, - last_used_at: key.last_used_at.map(|value| value.to_rfc3339()), - created_at: key.created_at.to_rfc3339(), - } - }) + .map(|key| api_key_to_display(key, service)) .collect(); crate::commands::api_key_provider_cmd::ProviderWithKeysDisplay { - provider: crate::commands::api_key_provider_cmd::ProviderDisplay { - id: provider_with_keys.provider.id.clone(), - name: provider_with_keys.provider.name.clone(), - provider_type: provider_with_keys - .provider - .effective_provider_type() - .to_string(), - api_host: provider_with_keys.provider.api_host.clone(), - is_system: provider_with_keys.provider.is_system, - group: provider_with_keys.provider.group.to_string(), - enabled: provider_with_keys.provider.enabled, - sort_order: provider_with_keys.provider.sort_order, - api_version: provider_with_keys.provider.api_version.clone(), - project: provider_with_keys.provider.project.clone(), - location: provider_with_keys.provider.location.clone(), - region: provider_with_keys.provider.region.clone(), - custom_models: provider_with_keys.provider.custom_models.clone(), - prompt_cache_mode: provider_with_keys - .provider - .effective_prompt_cache_mode() - .map(|mode| mode.to_string()), - api_key_count: provider_with_keys.api_keys.len(), - created_at: provider_with_keys.provider.created_at.to_rfc3339(), - updated_at: provider_with_keys.provider.updated_at.to_rfc3339(), - }, + provider: api_key_provider_to_display(&provider_with_keys.provider, api_keys.len()), api_keys, } } +fn api_key_provider_to_display( + provider: &ApiKeyProvider, + api_key_count: usize, +) -> crate::commands::api_key_provider_cmd::ProviderDisplay { + crate::commands::api_key_provider_cmd::ProviderDisplay { + id: provider.id.clone(), + name: provider.name.clone(), + provider_type: provider.effective_provider_type().to_string(), + api_host: provider.api_host.clone(), + is_system: provider.is_system, + group: provider.group.to_string(), + enabled: provider.enabled, + sort_order: provider.sort_order, + api_version: provider.api_version.clone(), + project: provider.project.clone(), + location: provider.location.clone(), + region: provider.region.clone(), + custom_models: provider.custom_models.clone(), + prompt_cache_mode: provider + .effective_prompt_cache_mode() + .map(|mode| mode.to_string()), + api_key_count, + created_at: provider.created_at.to_rfc3339(), + updated_at: provider.updated_at.to_rfc3339(), + } +} + +fn api_key_to_display( + key: &ApiKeyEntry, + service: &lime_services::api_key_provider_service::ApiKeyProviderService, +) -> crate::commands::api_key_provider_cmd::ApiKeyDisplay { + let masked = match service.decrypt_api_key(&key.api_key_encrypted) { + Ok(decrypted) => mask_api_key_for_display(&decrypted), + Err(_) => "****".to_string(), + }; + + crate::commands::api_key_provider_cmd::ApiKeyDisplay { + id: key.id.clone(), + provider_id: key.provider_id.clone(), + api_key_masked: masked, + alias: key.alias.clone(), + enabled: key.enabled, + usage_count: key.usage_count, + error_count: key.error_count, + last_used_at: key.last_used_at.map(|value| value.to_rfc3339()), + created_at: key.created_at.to_rfc3339(), + } +} + async fn relay_registry(state: &DevBridgeState) -> Option> { let state_guard = state.connect_state.read().await; state_guard @@ -161,6 +172,134 @@ pub(super) async fn try_handle( serde_json::json!([]) } } + "get_api_key_provider" => { + let args = args_or_default(args); + let id = get_string_arg(&args, "id", "providerId")?; + let db = state + .db + .as_ref() + .ok_or_else(|| "Database not initialized".to_string())?; + let provider = state.api_key_provider_service.get_provider(db, &id)?; + + serde_json::to_value(provider.map(|provider| { + api_key_provider_with_keys_to_display( + &provider, + state.api_key_provider_service.as_ref(), + ) + }))? + } + "add_custom_api_key_provider" => { + let args = args_or_default(args); + let request = parse_nested_arg::< + crate::commands::api_key_provider_cmd::AddCustomProviderRequest, + >(&args, "request")?; + let provider_type: ApiProviderType = request + .provider_type + .parse() + .map_err(|e: String| format!("无效的 Provider 类型: {e}"))?; + let prompt_cache_mode = request + .prompt_cache_mode + .map(|mode| mode.parse::()) + .transpose() + .map_err(|e: String| format!("无效的 Prompt Cache 模式: {e}"))?; + let db = state + .db + .as_ref() + .ok_or_else(|| "Database not initialized".to_string())?; + + let provider = state.api_key_provider_service.add_custom_provider( + db, + request.name, + provider_type, + request.api_host, + request.api_version, + request.project, + request.location, + request.region, + prompt_cache_mode, + )?; + + serde_json::to_value(api_key_provider_to_display(&provider, 0))? + } + "update_api_key_provider" => { + let args = args_or_default(args); + let id = get_string_arg(&args, "id", "providerId")?; + let request = parse_nested_arg::< + crate::commands::api_key_provider_cmd::UpdateProviderRequest, + >(&args, "request")?; + let provider_type: Option = request + .provider_type + .map(|value| value.parse()) + .transpose() + .map_err(|e: String| format!("无效的 Provider 类型: {e}"))?; + let prompt_cache_mode = request + .prompt_cache_mode + .map(|mode| mode.parse::()) + .transpose() + .map_err(|e: String| format!("无效的 Prompt Cache 模式: {e}"))?; + let db = state + .db + .as_ref() + .ok_or_else(|| "Database not initialized".to_string())?; + + let provider = state.api_key_provider_service.update_provider( + db, + &id, + request.name, + provider_type, + request.api_host, + request.enabled, + request.sort_order, + request.api_version, + request.project, + request.location, + request.region, + prompt_cache_mode, + request.custom_models, + )?; + let api_key_count = state + .api_key_provider_service + .get_provider(db, &id)? + .map(|provider| provider.api_keys.len()) + .unwrap_or(0); + + serde_json::to_value(api_key_provider_to_display(&provider, api_key_count))? + } + "add_api_key" => { + let args = args_or_default(args); + let request = parse_nested_arg::< + crate::commands::api_key_provider_cmd::AddApiKeyRequest, + >(&args, "request")?; + let db = state + .db + .as_ref() + .ok_or_else(|| "Database not initialized".to_string())?; + let key = state.api_key_provider_service.add_api_key( + db, + &request.provider_id, + &request.api_key, + request.alias, + )?; + + serde_json::to_value(api_key_to_display( + &key, + state.api_key_provider_service.as_ref(), + ))? + } + "delete_custom_api_key_provider" => { + let args = args_or_default(args); + let id = get_string_arg(&args, "id", "providerId")?; + let db = state + .db + .as_ref() + .ok_or_else(|| "Database not initialized".to_string())?; + + serde_json::to_value( + state + .api_key_provider_service + .delete_custom_provider(db, &id)?, + )? + } "get_system_provider_catalog" => { let catalog = crate::commands::api_key_provider_cmd::get_system_provider_catalog() .map_err(|e| format!("获取系统 Provider Catalog 失败: {e}"))?; diff --git a/src-tauri/src/dev_bridge/dispatcher/voice.rs b/src-tauri/src/dev_bridge/dispatcher/voice.rs index 5ddbb6cec..811d000fa 100644 --- a/src-tauri/src/dev_bridge/dispatcher/voice.rs +++ b/src-tauri/src/dev_bridge/dispatcher/voice.rs @@ -1,4 +1,7 @@ -use super::{args_or_default, get_string_arg, parse_nested_arg, require_app_handle}; +use super::{ + args_or_default, get_string_arg, parse_nested_arg, parse_optional_nested_arg, + require_app_handle, +}; use crate::commands::asr_cmd::AddAsrCredentialRequest; use crate::config::AsrCredentialEntry; use crate::dev_bridge::DevBridgeState; @@ -64,6 +67,51 @@ pub(super) async fn try_handle( let id = get_string_arg(&args, "id", "id")?; serde_json::to_value(crate::commands::asr_cmd::test_asr_credential(id).await?)? } + "voice_models_list_catalog" => serde_json::to_value( + crate::commands::voice_model_cmd::voice_models_list_catalog().await?, + )?, + "voice_models_get_install_state" => { + let args = args_or_default(args); + let model_id = get_string_arg(&args, "modelId", "model_id")?; + serde_json::to_value( + crate::commands::voice_model_cmd::voice_models_get_install_state(model_id).await?, + )? + } + "voice_models_download" => { + let args = args_or_default(args); + let model_id = get_string_arg(&args, "modelId", "model_id")?; + let catalog_entry = parse_optional_nested_arg(&args, "catalogEntry")? + .or(parse_optional_nested_arg(&args, "catalog_entry")?); + serde_json::to_value( + crate::commands::voice_model_cmd::voice_models_download(model_id, catalog_entry) + .await?, + )? + } + "voice_models_delete" => { + let args = args_or_default(args); + let model_id = get_string_arg(&args, "modelId", "model_id")?; + serde_json::to_value( + crate::commands::voice_model_cmd::voice_models_delete(model_id).await?, + )? + } + "voice_models_set_default" => { + let args = args_or_default(args); + let model_id = get_string_arg(&args, "modelId", "model_id")?; + serde_json::to_value( + crate::commands::voice_model_cmd::voice_models_set_default(model_id).await?, + )? + } + "voice_models_test_transcribe_file" => { + let args = args_or_default(args); + let model_id = get_string_arg(&args, "modelId", "model_id")?; + let file_path = get_string_arg(&args, "filePath", "file_path")?; + serde_json::to_value( + crate::commands::voice_model_cmd::voice_models_test_transcribe_file( + model_id, file_path, + ) + .await?, + )? + } "get_voice_input_config" => { serde_json::to_value(crate::voice::commands::get_voice_input_config().await?)? } diff --git a/src-tauri/src/services/runtime_evidence_pack_service.rs b/src-tauri/src/services/runtime_evidence_pack_service.rs index d68084174..aba6d8402 100644 --- a/src-tauri/src/services/runtime_evidence_pack_service.rs +++ b/src-tauri/src/services/runtime_evidence_pack_service.rs @@ -6,6 +6,7 @@ use crate::agent::SessionDetail; use crate::commands::aster_agent_cmd::AgentRuntimeThreadReadModel; use crate::commands::modality_runtime_contracts::{ + AUDIO_TRANSCRIPTION_CONTRACT_KEY, AUDIO_TRANSCRIPTION_ROUTING_SLOT, BROWSER_CONTROL_CONTRACT_KEY, BROWSER_CONTROL_ROUTING_SLOT, IMAGE_GENERATION_CONTRACT_KEY, IMAGE_GENERATION_ROUTING_SLOT, PDF_EXTRACT_CONTRACT_KEY, PDF_EXTRACT_ROUTING_SLOT, TEXT_TRANSFORM_CONTRACT_KEY, TEXT_TRANSFORM_ROUTING_SLOT, VOICE_GENERATION_CONTRACT_KEY, @@ -35,6 +36,7 @@ const ARTIFACTS_FILE_NAME: &str = "artifacts.json"; const MAX_RECENT_ARTIFACTS: usize = 12; const MAX_PREVIEW_CHARS: usize = 200; const MAX_BROWSER_EVIDENCE_ITEMS: usize = 6; +const MAX_BROWSER_ACTION_OBSERVABILITY_ITEMS: usize = 5; const MAX_REQUEST_TELEMETRY_ITEMS: usize = 12; #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -755,18 +757,84 @@ fn build_modality_runtime_contracts_json( }) } +fn build_modality_runtime_contracts_observability_summary_json( + summary: &RuntimeModalityContractSnapshotSummary, +) -> Value { + let snapshot_index = build_modality_runtime_contract_snapshot_index(&summary.snapshots); + let browser_action_index = snapshot_index + .get("browserActionIndex") + .cloned() + .map(compact_browser_action_index_for_observability) + .unwrap_or_else(|| { + json!({ + "actionCount": 0, + "sessionCount": 0, + "observationCount": 0, + "screenshotCount": 0, + "lastUrl": null, + "sessionIds": [], + "targetIds": [], + "profileKeys": [], + "statusCounts": [], + "artifactKindCounts": [], + "actionCounts": [], + "backendCounts": [], + "items": [] + }) + }); + + json!({ + "snapshotCount": summary.snapshots.len(), + "snapshotIndex": { + "browserActionIndex": browser_action_index + } + }) +} + +fn compact_browser_action_index_for_observability(mut index: Value) -> Value { + if let Some(items) = index.get_mut("items").and_then(Value::as_array_mut) { + if items.len() > MAX_BROWSER_ACTION_OBSERVABILITY_ITEMS { + let keep_from = items.len() - MAX_BROWSER_ACTION_OBSERVABILITY_ITEMS; + *items = items.split_off(keep_from); + } + } + + index +} + fn build_modality_runtime_contract_snapshot_index(snapshots: &[Value]) -> Value { let mut contract_keys = BTreeSet::new(); let mut sources: BTreeMap = BTreeMap::new(); let mut routing_outcomes: BTreeMap = BTreeMap::new(); let mut expected_routing_slots = BTreeSet::new(); + let mut execution_profile_keys = BTreeSet::new(); + let mut executor_adapter_keys = BTreeSet::new(); let mut trace_items = Vec::new(); + let mut audio_output_statuses: BTreeMap = BTreeMap::new(); + let mut audio_output_error_codes = BTreeSet::new(); + let mut audio_output_items = Vec::new(); + let mut transcript_statuses: BTreeMap = BTreeMap::new(); + let mut transcript_error_codes = BTreeSet::new(); + let mut transcript_items = Vec::new(); + let mut browser_action_statuses: BTreeMap = BTreeMap::new(); + let mut browser_action_kinds: BTreeMap = BTreeMap::new(); + let mut browser_action_names: BTreeMap = BTreeMap::new(); + let mut browser_session_ids = BTreeSet::new(); + let mut browser_target_ids = BTreeSet::new(); + let mut browser_profile_keys = BTreeSet::new(); + let mut browser_backends: BTreeMap = BTreeMap::new(); + let mut browser_last_url = None; + let mut browser_observation_count = 0usize; + let mut browser_screenshot_count = 0usize; + let mut browser_action_items = Vec::new(); for snapshot in snapshots { let contract_key = snapshot_string(snapshot, "contractKey"); let source = snapshot_string(snapshot, "source"); let routing_outcome = snapshot_string(snapshot, "routingOutcome"); let expected_routing_slot = snapshot_string(snapshot, "expectedRoutingSlot"); + let execution_profile_key = snapshot_string(snapshot, "executionProfileKey"); + let executor_adapter_key = snapshot_string(snapshot, "executorAdapterKey"); if let Some(contract_key) = contract_key.as_deref() { contract_keys.insert(contract_key.to_string()); @@ -782,6 +850,12 @@ fn build_modality_runtime_contract_snapshot_index(snapshots: &[Value]) -> Value if let Some(expected_routing_slot) = expected_routing_slot.as_deref() { expected_routing_slots.insert(expected_routing_slot.to_string()); } + if let Some(execution_profile_key) = execution_profile_key.as_deref() { + execution_profile_keys.insert(execution_profile_key.to_string()); + } + if let Some(executor_adapter_key) = executor_adapter_key.as_deref() { + executor_adapter_keys.insert(executor_adapter_key.to_string()); + } if source .as_deref() @@ -790,11 +864,13 @@ fn build_modality_runtime_contract_snapshot_index(snapshots: &[Value]) -> Value { trace_items.push(json!({ "artifactPath": snapshot.get("artifactPath").cloned().unwrap_or(Value::Null), - "source": source, - "contractKey": contract_key, + "source": source.clone(), + "contractKey": contract_key.clone(), "routingEvent": snapshot.get("routingEvent").cloned().unwrap_or(Value::Null), "routingOutcome": snapshot.get("routingOutcome").cloned().unwrap_or(Value::Null), "expectedRoutingSlot": snapshot.get("expectedRoutingSlot").cloned().unwrap_or(Value::Null), + "executionProfileKey": snapshot.get("executionProfileKey").cloned().unwrap_or(Value::Null), + "executorAdapterKey": snapshot.get("executorAdapterKey").cloned().unwrap_or(Value::Null), "entrySource": snapshot.get("entrySource").cloned().unwrap_or(Value::Null), "executorBindingKey": snapshot .pointer("/runtimeContract/executor_binding/binding_key") @@ -802,6 +878,112 @@ fn build_modality_runtime_contract_snapshot_index(snapshots: &[Value]) -> Value .unwrap_or(Value::Null), })); } + + if let Some(browser_action) = snapshot + .get("browserAction") + .filter(|value| value.is_object()) + { + if let Some(status) = snapshot_string(browser_action, "status") { + *browser_action_statuses.entry(status).or_insert(0) += 1; + } + if let Some(artifact_kind) = snapshot_string(browser_action, "artifactKind") { + *browser_action_kinds.entry(artifact_kind).or_insert(0) += 1; + } + if let Some(action) = snapshot_string(browser_action, "action") { + *browser_action_names.entry(action).or_insert(0) += 1; + } + if let Some(session_id) = snapshot_string(browser_action, "sessionId") { + browser_session_ids.insert(session_id); + } + if let Some(target_id) = snapshot_string(browser_action, "targetId") { + browser_target_ids.insert(target_id); + } + if let Some(profile_key) = snapshot_string(browser_action, "profileKey") { + browser_profile_keys.insert(profile_key); + } + if let Some(backend) = snapshot_string(browser_action, "backend") { + *browser_backends.entry(backend).or_insert(0) += 1; + } + if let Some(last_url) = snapshot_string(browser_action, "lastUrl") { + browser_last_url = Some(last_url); + } + if read_json_bool(browser_action, &[&["observationAvailable"][..]]).unwrap_or(false) { + browser_observation_count += 1; + } + if read_json_bool(browser_action, &[&["screenshotAvailable"][..]]).unwrap_or(false) { + browser_screenshot_count += 1; + } + browser_action_items.push(json!({ + "artifactPath": snapshot.get("artifactPath").cloned().unwrap_or(Value::Null), + "contractKey": contract_key.clone(), + "source": source.clone(), + "entrySource": snapshot.get("entrySource").cloned().unwrap_or(Value::Null), + "artifactKind": browser_action.get("artifactKind").cloned().unwrap_or(Value::Null), + "toolName": browser_action.get("toolName").cloned().unwrap_or(Value::Null), + "action": browser_action.get("action").cloned().unwrap_or(Value::Null), + "status": browser_action.get("status").cloned().unwrap_or(Value::Null), + "success": browser_action.get("success").cloned().unwrap_or(Value::Null), + "sessionId": browser_action.get("sessionId").cloned().unwrap_or(Value::Null), + "targetId": browser_action.get("targetId").cloned().unwrap_or(Value::Null), + "profileKey": browser_action.get("profileKey").cloned().unwrap_or(Value::Null), + "backend": browser_action.get("backend").cloned().unwrap_or(Value::Null), + "requestId": browser_action.get("requestId").cloned().unwrap_or(Value::Null), + "lastUrl": browser_action.get("lastUrl").cloned().unwrap_or(Value::Null), + "title": browser_action.get("title").cloned().unwrap_or(Value::Null), + "attemptCount": browser_action.get("attemptCount").cloned().unwrap_or(Value::Null), + "observationAvailable": browser_action.get("observationAvailable").cloned().unwrap_or(Value::Null), + "screenshotAvailable": browser_action.get("screenshotAvailable").cloned().unwrap_or(Value::Null), + })); + } + + if let Some(audio_output) = snapshot + .get("audioOutput") + .filter(|value| value.is_object()) + { + if let Some(status) = snapshot_string(audio_output, "status") { + *audio_output_statuses.entry(status).or_insert(0) += 1; + } + if let Some(error_code) = snapshot_string(audio_output, "errorCode") { + audio_output_error_codes.insert(error_code); + } + audio_output_items.push(json!({ + "artifactPath": snapshot.get("artifactPath").cloned().unwrap_or(Value::Null), + "taskId": snapshot.get("taskId").cloned().unwrap_or(Value::Null), + "status": audio_output.get("status").cloned().unwrap_or(Value::Null), + "audioPath": audio_output.get("audioPath").cloned().unwrap_or(Value::Null), + "mimeType": audio_output.get("mimeType").cloned().unwrap_or(Value::Null), + "durationMs": audio_output.get("durationMs").cloned().unwrap_or(Value::Null), + "providerId": audio_output.get("providerId").cloned().unwrap_or(Value::Null), + "model": audio_output.get("model").cloned().unwrap_or(Value::Null), + "errorCode": audio_output.get("errorCode").cloned().unwrap_or(Value::Null), + "retryable": audio_output.get("retryable").cloned().unwrap_or(Value::Null), + "workerId": audio_output.get("workerId").cloned().unwrap_or(Value::Null), + })); + } + + if let Some(transcript) = snapshot.get("transcript").filter(|value| value.is_object()) { + if let Some(status) = snapshot_string(transcript, "status") { + *transcript_statuses.entry(status).or_insert(0) += 1; + } + if let Some(error_code) = snapshot_string(transcript, "errorCode") { + transcript_error_codes.insert(error_code); + } + transcript_items.push(json!({ + "artifactPath": snapshot.get("artifactPath").cloned().unwrap_or(Value::Null), + "taskId": snapshot.get("taskId").cloned().unwrap_or(Value::Null), + "status": transcript.get("status").cloned().unwrap_or(Value::Null), + "transcriptPath": transcript.get("transcriptPath").cloned().unwrap_or(Value::Null), + "sourceUrl": transcript.get("sourceUrl").cloned().unwrap_or(Value::Null), + "sourcePath": transcript.get("sourcePath").cloned().unwrap_or(Value::Null), + "language": transcript.get("language").cloned().unwrap_or(Value::Null), + "outputFormat": transcript.get("outputFormat").cloned().unwrap_or(Value::Null), + "providerId": transcript.get("providerId").cloned().unwrap_or(Value::Null), + "model": transcript.get("model").cloned().unwrap_or(Value::Null), + "errorCode": transcript.get("errorCode").cloned().unwrap_or(Value::Null), + "retryable": transcript.get("retryable").cloned().unwrap_or(Value::Null), + "workerId": transcript.get("workerId").cloned().unwrap_or(Value::Null), + })); + } } json!({ @@ -815,9 +997,56 @@ fn build_modality_runtime_contract_snapshot_index(snapshots: &[Value]) -> Value .map(|(outcome, count)| json!({ "outcome": outcome, "count": count })) .collect::>(), "expectedRoutingSlots": expected_routing_slots.into_iter().collect::>(), + "executionProfileKeys": execution_profile_keys.into_iter().collect::>(), + "executorAdapterKeys": executor_adapter_keys.into_iter().collect::>(), "toolTraceIndex": { "traceCount": trace_items.len(), "items": trace_items, + }, + "audioOutputIndex": { + "outputCount": audio_output_items.len(), + "statusCounts": audio_output_statuses + .into_iter() + .map(|(status, count)| json!({ "status": status, "count": count })) + .collect::>(), + "errorCodes": audio_output_error_codes.into_iter().collect::>(), + "items": audio_output_items, + }, + "transcriptIndex": { + "transcriptCount": transcript_items.len(), + "statusCounts": transcript_statuses + .into_iter() + .map(|(status, count)| json!({ "status": status, "count": count })) + .collect::>(), + "errorCodes": transcript_error_codes.into_iter().collect::>(), + "items": transcript_items, + }, + "browserActionIndex": { + "actionCount": browser_action_items.len(), + "sessionCount": browser_session_ids.len(), + "observationCount": browser_observation_count, + "screenshotCount": browser_screenshot_count, + "lastUrl": browser_last_url, + "sessionIds": browser_session_ids.into_iter().collect::>(), + "targetIds": browser_target_ids.into_iter().collect::>(), + "profileKeys": browser_profile_keys.into_iter().collect::>(), + "statusCounts": browser_action_statuses + .into_iter() + .map(|(status, count)| json!({ "status": status, "count": count })) + .collect::>(), + "artifactKindCounts": browser_action_kinds + .into_iter() + .map(|(artifact_kind, count)| json!({ "artifactKind": artifact_kind, "count": count })) + .collect::>(), + "actionCounts": browser_action_names + .into_iter() + .map(|(action, count)| json!({ "action": action, "count": count })) + .collect::>(), + "backendCounts": browser_backends + .into_iter() + .map(|(backend, count)| json!({ "backend": backend, "count": count })) + .collect::>(), + "items": browser_action_items, } }) } @@ -836,6 +1065,102 @@ fn is_runtime_contract_tool_trace_source(source: &str) -> bool { || source.contains("browser_action_trace") || source.contains("service_scene_trace") || source.contains("audio_task") + || source.contains("transcription_task") +} + +fn extract_audio_output_snapshot(document: &Value) -> Option { + let audio_output = find_json_value_at_paths( + document, + &[ + &["audio_output"][..], + &["audioOutput"][..], + &["payload", "audio_output"][..], + &["payload", "audioOutput"][..], + &["result", "audio_output"][..], + &["result", "audioOutput"][..], + &["record", "payload", "audio_output"][..], + &["record", "payload", "audioOutput"][..], + &["record", "result", "audio_output"][..], + &["record", "result", "audioOutput"][..], + ], + ) + .filter(|value| value.is_object())?; + + Some(json!({ + "kind": read_json_string(audio_output, &[&["kind"][..]]).unwrap_or_else(|| "audio_output".to_string()), + "status": read_json_string(audio_output, &[&["status"][..]]), + "audioPath": read_json_string(audio_output, &[&["audio_path"][..], &["audioPath"][..]]), + "mimeType": read_json_string(audio_output, &[&["mime_type"][..], &["mimeType"][..]]), + "durationMs": find_json_value_at_paths(audio_output, &[&["duration_ms"][..], &["durationMs"][..]]) + .cloned() + .unwrap_or(Value::Null), + "sourceText": read_json_string(audio_output, &[&["source_text"][..], &["sourceText"][..]]), + "voice": read_json_string(audio_output, &[&["voice"][..]]), + "providerId": read_json_string(audio_output, &[&["provider_id"][..], &["providerId"][..]]), + "model": read_json_string(audio_output, &[&["model"][..]]), + "errorCode": read_json_string(audio_output, &[&["error_code"][..], &["errorCode"][..]]), + "errorMessage": read_json_string(audio_output, &[&["error_message"][..], &["errorMessage"][..]]), + "retryable": find_json_value_at_paths(audio_output, &[&["retryable"][..]]) + .cloned() + .unwrap_or(Value::Null), + "stage": read_json_string(audio_output, &[&["stage"][..]]), + "workerId": read_json_string( + document, + &[ + &["current_attempt_worker_id"][..], + &["currentAttemptWorkerId"][..], + &["record", "current_attempt_worker_id"][..], + &["record", "currentAttemptWorkerId"][..], + ], + ), + })) +} + +fn extract_transcript_snapshot(document: &Value) -> Option { + let transcript = find_json_value_at_paths( + document, + &[ + &["transcript"][..], + &["payload", "transcript"][..], + &["result", "transcript"][..], + &["record", "payload", "transcript"][..], + &["record", "result", "transcript"][..], + ], + ) + .filter(|value| value.is_object())?; + + Some(json!({ + "kind": read_json_string(transcript, &[&["kind"][..]]).unwrap_or_else(|| "transcript".to_string()), + "status": read_json_string(transcript, &[&["status"][..]]), + "transcriptPath": read_json_string(transcript, &[&["transcript_path"][..], &["transcriptPath"][..], &["path"][..]]), + "sourceUrl": read_json_string(transcript, &[&["source_url"][..], &["sourceUrl"][..]]), + "sourcePath": read_json_string(transcript, &[&["source_path"][..], &["sourcePath"][..]]), + "language": read_json_string(transcript, &[&["language"][..]]), + "outputFormat": read_json_string(transcript, &[&["output_format"][..], &["outputFormat"][..]]), + "timestamps": find_json_value_at_paths(transcript, &[&["timestamps"][..]]) + .cloned() + .unwrap_or(Value::Null), + "speakerLabels": find_json_value_at_paths(transcript, &[&["speaker_labels"][..], &["speakerLabels"][..]]) + .cloned() + .unwrap_or(Value::Null), + "providerId": read_json_string(transcript, &[&["provider_id"][..], &["providerId"][..]]), + "model": read_json_string(transcript, &[&["model"][..]]), + "errorCode": read_json_string(transcript, &[&["error_code"][..], &["errorCode"][..]]), + "errorMessage": read_json_string(transcript, &[&["error_message"][..], &["errorMessage"][..]]), + "retryable": find_json_value_at_paths(transcript, &[&["retryable"][..]]) + .cloned() + .unwrap_or(Value::Null), + "stage": read_json_string(transcript, &[&["stage"][..]]), + "workerId": read_json_string( + document, + &[ + &["current_attempt_worker_id"][..], + &["currentAttemptWorkerId"][..], + &["record", "current_attempt_worker_id"][..], + &["record", "currentAttemptWorkerId"][..], + ], + ), + })) } fn build_known_gaps( @@ -1248,6 +1573,7 @@ fn build_runtime_observability_summary_json( "auxiliaryRuntimeSnapshotCount": auxiliary_runtime.snapshots.len(), "modalityRuntimeContractCount": modality_runtime_contracts.snapshots.len() }, + "modalityRuntimeContracts": build_modality_runtime_contracts_observability_summary_json(modality_runtime_contracts), "latest": { "warning": latest_warning_json(diagnostics), "failedTool": latest_failed_tool_json(diagnostics), @@ -1375,9 +1701,14 @@ fn collect_modality_runtime_contract_snapshots( }; let artifact_path = format!("runtime_timeline/{}/{}", item.id, tool_name); let snapshot = if is_browser_tool_name(tool_name.as_str()) { - metadata.as_ref().and_then(|metadata| { - extract_modality_runtime_contract_snapshot(metadata, artifact_path.as_str()) - }) + extract_browser_control_contract_snapshot( + item.id.as_str(), + tool_name.as_str(), + arguments.as_ref(), + *success, + metadata.as_ref(), + artifact_path.as_str(), + ) } else { extract_pdf_read_skill_contract_snapshot( tool_name.as_str(), @@ -1998,6 +2329,7 @@ fn is_modality_runtime_contract_applicable(artifact: &RuntimeRecentArtifact) -> let normalized_path = artifact.path.replace('\\', "/").to_ascii_lowercase(); if normalized_path.contains(".lime/tasks/image_generate/") || normalized_path.contains(".lime/tasks/audio_generate/") + || normalized_path.contains(".lime/tasks/transcription_generate/") { return true; } @@ -2028,6 +2360,7 @@ fn is_modality_runtime_contract_applicable(artifact: &RuntimeRecentArtifact) -> .map(|value| { value.eq_ignore_ascii_case("image_generate") || value.eq_ignore_ascii_case("audio_generate") + || value.eq_ignore_ascii_case("transcription_generate") }) .unwrap_or(false) }) @@ -2581,6 +2914,212 @@ fn apply_tool_call_status_to_contract_document(document: &mut Value, success: Op .or_insert_with(|| Value::String(status.to_string())); } +fn extract_browser_control_contract_snapshot( + item_id: &str, + tool_name: &str, + arguments: Option<&Value>, + success: Option, + metadata: Option<&Value>, + artifact_path: &str, +) -> Option { + let metadata = metadata?; + let mut snapshot = extract_modality_runtime_contract_snapshot(metadata, artifact_path)?; + if snapshot.get("contractKey").and_then(Value::as_str) != Some(BROWSER_CONTROL_CONTRACT_KEY) { + return Some(snapshot); + } + + if let Value::Object(object) = &mut snapshot { + object.insert( + "browserAction".to_string(), + build_browser_action_contract_index_item( + item_id, tool_name, arguments, success, metadata, + ), + ); + } + + Some(snapshot) +} + +fn build_browser_action_contract_index_item( + item_id: &str, + tool_name: &str, + arguments: Option<&Value>, + success: Option, + metadata: &Value, +) -> Value { + let action = read_json_string(metadata, &[&["action"][..], &["result", "action"][..]]) + .unwrap_or_else(|| infer_browser_action_name(tool_name)); + let artifact_kind = infer_browser_action_artifact_kind(action.as_str()); + let action_success = + read_json_bool(metadata, &[&["result", "success"][..], &["success"][..]]).or(success); + let status = match action_success { + Some(true) => "completed", + Some(false) => "failed", + None => "unknown", + }; + let attempt_count = read_json_usize(metadata, &[&["attempt_count"][..], &["attemptCount"][..]]) + .or_else(|| { + metadata + .get("attempts") + .and_then(Value::as_array) + .map(Vec::len) + }) + .or_else(|| { + metadata + .pointer("/result/attempts") + .and_then(Value::as_array) + .map(Vec::len) + }); + let last_url = read_json_string( + metadata, + &[ + &["browser_session", "target_url"][..], + &["browserSession", "targetUrl"][..], + &["result", "data", "browser_session", "target_url"][..], + &["result", "data", "browserSession", "targetUrl"][..], + &["result", "data", "target_url"][..], + &["result", "data", "targetUrl"][..], + &["result", "data", "url"][..], + &["result", "data", "tab", "url"][..], + &["result", "target_url"][..], + &["result", "targetUrl"][..], + &["result", "url"][..], + ], + ) + .or_else(|| { + arguments.and_then(|arguments| { + read_json_string( + arguments, + &[ + &["url"][..], + &["target_url"][..], + &["targetUrl"][..], + &["page_url"][..], + &["pageUrl"][..], + ], + ) + }) + }); + let screenshot_available = has_browser_screenshot(metadata); + let observation_available = + artifact_kind == "browser_snapshot" || screenshot_available || last_url.is_some(); + + json!({ + "itemId": item_id, + "artifactKind": artifact_kind, + "toolName": tool_name, + "action": action, + "status": status, + "success": action_success, + "sessionId": read_json_string( + metadata, + &[ + &["browser_session", "session_id"][..], + &["browserSession", "sessionId"][..], + &["result", "session_id"][..], + &["result", "sessionId"][..], + &["result", "data", "session_id"][..], + &["result", "data", "sessionId"][..], + &["result", "data", "browser_session", "session_id"][..], + &["result", "data", "browserSession", "sessionId"][..], + ], + ), + "targetId": read_json_string( + metadata, + &[ + &["browser_session", "target_id"][..], + &["browserSession", "targetId"][..], + &["result", "target_id"][..], + &["result", "targetId"][..], + &["result", "data", "target_id"][..], + &["result", "data", "targetId"][..], + &["result", "data", "browser_session", "target_id"][..], + &["result", "data", "browserSession", "targetId"][..], + &["result", "data", "tab", "id"][..], + ], + ), + "profileKey": read_json_string( + metadata, + &[ + &["browser_session", "profile_key"][..], + &["browserSession", "profileKey"][..], + &["result", "data", "profile_key"][..], + &["result", "data", "profileKey"][..], + &["result", "data", "browser_session", "profile_key"][..], + &["result", "data", "browserSession", "profileKey"][..], + ], + ), + "backend": read_json_string( + metadata, + &[&["selected_backend"][..], &["selectedBackend"][..], &["result", "backend"][..]], + ), + "requestId": read_json_string( + metadata, + &[&["result", "request_id"][..], &["result", "requestId"][..]], + ), + "lastUrl": last_url, + "title": read_json_string( + metadata, + &[ + &["browser_session", "target_title"][..], + &["browserSession", "targetTitle"][..], + &["result", "data", "title"][..], + &["result", "data", "target_title"][..], + &["result", "data", "targetTitle"][..], + &["result", "data", "browser_session", "target_title"][..], + &["result", "data", "browserSession", "targetTitle"][..], + ], + ), + "attemptCount": attempt_count, + "observationAvailable": observation_available, + "screenshotAvailable": screenshot_available, + }) +} + +fn infer_browser_action_name(tool_name: &str) -> String { + tool_name + .rsplit("__") + .next() + .map(str::trim) + .filter(|value| !value.is_empty()) + .unwrap_or(tool_name) + .to_string() +} + +fn infer_browser_action_artifact_kind(action: &str) -> &'static str { + let normalized = action.trim().to_ascii_lowercase(); + if normalized.contains("snapshot") + || normalized.contains("read_page") + || normalized.contains("get_page") + || normalized.contains("page_info") + || normalized.contains("page_text") + || normalized.contains("console") + || normalized.contains("network") + || normalized.contains("find") + || normalized.contains("tabs_context") + { + "browser_snapshot" + } else { + "browser_session" + } +} + +fn has_browser_screenshot(metadata: &Value) -> bool { + find_json_value_at_paths( + metadata, + &[ + &["screenshot"][..], + &["screenshot_path"][..], + &["screenshotPath"][..], + &["result", "data", "screenshot"][..], + &["result", "data", "screenshot_path"][..], + &["result", "data", "screenshotPath"][..], + ], + ) + .map(json_value_has_content) + .unwrap_or(false) +} + fn extract_modality_runtime_contract_snapshot( document: &Value, artifact_path: &str, @@ -2645,6 +3184,7 @@ fn extract_modality_runtime_contract_snapshot( let is_browser_control_contract = contract_key == BROWSER_CONTROL_CONTRACT_KEY; let is_pdf_extract_contract = contract_key == PDF_EXTRACT_CONTRACT_KEY; let is_voice_generation_contract = contract_key == VOICE_GENERATION_CONTRACT_KEY; + let is_audio_transcription_contract = contract_key == AUDIO_TRANSCRIPTION_CONTRACT_KEY; let is_web_research_contract = contract_key == WEB_RESEARCH_CONTRACT_KEY; let is_text_transform_contract = contract_key == TEXT_TRANSFORM_CONTRACT_KEY; let is_audio_task_artifact = is_voice_generation_contract @@ -2653,12 +3193,19 @@ fn extract_modality_runtime_contract_snapshot( .replace('\\', "/") .to_ascii_lowercase() .contains(".lime/tasks/audio_generate/")); + let is_transcription_task_artifact = is_audio_transcription_contract + && (task_type.as_deref() == Some("transcription_generate") + || artifact_path + .replace('\\', "/") + .to_ascii_lowercase() + .contains(".lime/tasks/transcription_generate/")); let routing_event = if is_contract_routing_failure { "routing_not_possible" } else if is_browser_control_contract { "browser_action_requested" } else if is_pdf_extract_contract || is_voice_generation_contract + || is_audio_transcription_contract || is_web_research_contract || is_text_transform_contract { @@ -2682,6 +3229,8 @@ fn extract_modality_runtime_contract_snapshot( "pdf_read_skill_trace.modality_runtime_contract" } else if is_audio_task_artifact { "audio_task.modality_runtime_contract" + } else if is_transcription_task_artifact { + "transcription_task.modality_runtime_contract" } else if is_voice_generation_contract { "voice_generation_service_scene_trace.modality_runtime_contract" } else if is_web_research_contract { @@ -2710,7 +3259,7 @@ fn extract_modality_runtime_contract_snapshot( ), "normalizedStatus": normalized_status, "contractKey": contract_key, - "contractMatchedExpected": is_image_generation_contract || is_browser_control_contract || is_pdf_extract_contract || is_voice_generation_contract || is_web_research_contract || is_text_transform_contract, + "contractMatchedExpected": is_image_generation_contract || is_browser_control_contract || is_pdf_extract_contract || is_voice_generation_contract || is_audio_transcription_contract || is_web_research_contract || is_text_transform_contract, "expectedRoutingSlot": if is_image_generation_contract { Some(IMAGE_GENERATION_ROUTING_SLOT) } else if is_browser_control_contract { @@ -2719,6 +3268,8 @@ fn extract_modality_runtime_contract_snapshot( Some(PDF_EXTRACT_ROUTING_SLOT) } else if is_voice_generation_contract { Some(VOICE_GENERATION_ROUTING_SLOT) + } else if is_audio_transcription_contract { + Some(AUDIO_TRANSCRIPTION_ROUTING_SLOT) } else if is_web_research_contract { Some(WEB_RESEARCH_ROUTING_SLOT) } else if is_text_transform_contract { @@ -2767,6 +3318,8 @@ fn extract_modality_runtime_contract_snapshot( &["record", "payload", "routingSlot"][..], ], ), + "executionProfileKey": extract_runtime_contract_execution_profile_key(document), + "executorAdapterKey": extract_runtime_contract_executor_adapter_key(document), "providerId": read_json_string( document, &[ @@ -2815,6 +3368,16 @@ fn extract_modality_runtime_contract_snapshot( "failureCode": failure_code, "failureStage": failure_stage, "lastError": last_error, + "audioOutput": if is_audio_task_artifact { + extract_audio_output_snapshot(document) + } else { + None + }, + "transcript": if is_transcription_task_artifact { + extract_transcript_snapshot(document) + } else { + None + }, "runtimeContract": find_json_value_at_paths( document, &[ @@ -2830,6 +3393,124 @@ fn extract_modality_runtime_contract_snapshot( })) } +fn extract_runtime_contract_execution_profile_key(document: &Value) -> Option { + read_json_string( + document, + &[ + &["execution_profile_key"][..], + &["executionProfileKey"][..], + &["execution_profile", "profile_key"][..], + &["executionProfile", "profileKey"][..], + &["runtime_contract", "execution_profile", "profile_key"][..], + &["runtime_contract", "executionProfile", "profileKey"][..], + &["runtimeContract", "execution_profile", "profile_key"][..], + &["runtimeContract", "executionProfile", "profileKey"][..], + &["payload", "execution_profile_key"][..], + &["payload", "executionProfileKey"][..], + &[ + "payload", + "runtime_contract", + "execution_profile", + "profile_key", + ][..], + &[ + "payload", + "runtimeContract", + "execution_profile", + "profile_key", + ][..], + &[ + "payload", + "runtimeContract", + "executionProfile", + "profileKey", + ][..], + &["record", "payload", "execution_profile_key"][..], + &["record", "payload", "executionProfileKey"][..], + &[ + "record", + "payload", + "runtime_contract", + "execution_profile", + "profile_key", + ][..], + &[ + "record", + "payload", + "runtimeContract", + "execution_profile", + "profile_key", + ][..], + &[ + "record", + "payload", + "runtimeContract", + "executionProfile", + "profileKey", + ][..], + ], + ) +} + +fn extract_runtime_contract_executor_adapter_key(document: &Value) -> Option { + read_json_string( + document, + &[ + &["executor_adapter_key"][..], + &["executorAdapterKey"][..], + &["executor_adapter", "adapter_key"][..], + &["executorAdapter", "adapterKey"][..], + &["runtime_contract", "executor_adapter", "adapter_key"][..], + &["runtime_contract", "executorAdapter", "adapterKey"][..], + &["runtimeContract", "executor_adapter", "adapter_key"][..], + &["runtimeContract", "executorAdapter", "adapterKey"][..], + &["payload", "executor_adapter_key"][..], + &["payload", "executorAdapterKey"][..], + &[ + "payload", + "runtime_contract", + "executor_adapter", + "adapter_key", + ][..], + &[ + "payload", + "runtimeContract", + "executor_adapter", + "adapter_key", + ][..], + &[ + "payload", + "runtimeContract", + "executorAdapter", + "adapterKey", + ][..], + &["record", "payload", "executor_adapter_key"][..], + &["record", "payload", "executorAdapterKey"][..], + &[ + "record", + "payload", + "runtime_contract", + "executor_adapter", + "adapter_key", + ][..], + &[ + "record", + "payload", + "runtimeContract", + "executor_adapter", + "adapter_key", + ][..], + &[ + "record", + "payload", + "runtimeContract", + "executorAdapter", + "adapterKey", + ][..], + ], + ) +} + fn is_modality_contract_routing_failure_code(code: &str) -> bool { matches!( code.trim(), @@ -3241,6 +3922,27 @@ fn read_json_bool(value: &Value, paths: &[&[&str]]) -> Option { find_json_value_at_paths(value, paths).and_then(Value::as_bool) } +fn read_json_usize(value: &Value, paths: &[&[&str]]) -> Option { + let resolved = find_json_value_at_paths(value, paths)?; + match resolved { + Value::Number(number) => number + .as_u64() + .and_then(|value| usize::try_from(value).ok()), + Value::String(text) => text.trim().parse::().ok(), + _ => None, + } +} + +fn json_value_has_content(value: &Value) -> bool { + match value { + Value::Null => false, + Value::String(text) => !text.trim().is_empty(), + Value::Array(items) => !items.is_empty(), + Value::Object(fields) => !fields.is_empty(), + _ => true, + } +} + fn is_browser_tool_name(tool_name: &str) -> bool { let normalized = tool_name.trim().to_ascii_lowercase(); normalized.contains("browser") @@ -3761,6 +4463,12 @@ mod tests { "executor_kind": "skill", "binding_key": "image_generate" }, + "execution_profile": { + "profile_key": "image_generation_profile" + }, + "executor_adapter": { + "adapter_key": "skill:image_generate" + }, "truth_source": ["image_task_artifact", "runtime_timeline_event"] } }, @@ -3835,18 +4543,110 @@ mod tests { }, "audio_output": { "kind": "audio_output", - "status": "pending", - "audio_path": null, + "status": "completed", + "audio_path": ".lime/runtime/audio/task-audio-1.mp3", "mime_type": "audio/mpeg", - "duration_ms": null, + "duration_ms": 128000, "source_text": "请为这段文案生成温暖旁白", - "voice": "warm_narrator" + "voice": "warm_narrator", + "provider_id": "limecore", + "model": "voice-pro" + } + }, + "status": "succeeded", + "normalized_status": "succeeded", + "created_at": "2026-04-30T10:00:00Z", + "updated_at": "2026-04-30T10:00:05Z", + "submitted_at": null, + "started_at": "2026-04-30T10:00:01Z", + "completed_at": "2026-04-30T10:00:05Z", + "cancelled_at": null, + "idempotency_key": null, + "retry_count": 0, + "source_task_id": null, + "result": { + "kind": "audio_generation_result", + "status": "completed", + "audio_output": { + "kind": "audio_output", + "status": "completed", + "audio_path": ".lime/runtime/audio/task-audio-1.mp3", + "mime_type": "audio/mpeg", + "duration_ms": 128000, + "provider_id": "limecore", + "model": "voice-pro" + } + }, + "last_error": null, + "current_attempt_id": "attempt-audio-1", + "current_attempt_worker_id": "lime-audio-worker", + "attempts": [], + "relationships": {}, + "progress": {}, + "ui_hints": {} + })) + .expect("serialize audio task"), + ) + .expect("write audio task"); + } + + fn write_transcription_task_fixture(root: &Path, relative_path: &str) { + let absolute_path = root.join(relative_path.replace('/', std::path::MAIN_SEPARATOR_STR)); + fs::create_dir_all( + absolute_path + .parent() + .expect("transcription task path should have parent"), + ) + .expect("create transcription task dir"); + fs::write( + absolute_path, + serde_json::to_string_pretty(&json!({ + "task_id": "task-transcription-1", + "task_type": "transcription_generate", + "task_family": "document", + "title": "会议转写", + "summary": "会议音频转写任务", + "payload": { + "prompt": "生成逐字稿", + "source_path": "/tmp/interview.wav", + "language": "zh-CN", + "output_format": "srt", + "speaker_labels": true, + "timestamps": true, + "provider_id": "limecore", + "model": "asr-pro", + "entry_source": "at_transcription_command", + "modality_contract_key": AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "modality": "audio", + "required_capabilities": ["text_generation", "audio_transcription"], + "routing_slot": AUDIO_TRANSCRIPTION_ROUTING_SLOT, + "runtime_contract": { + "contract_key": AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "modality": "audio", + "required_capabilities": ["text_generation", "audio_transcription"], + "routing_slot": AUDIO_TRANSCRIPTION_ROUTING_SLOT, + "executor_binding": { + "executor_kind": "skill", + "binding_key": "transcription_generate" + }, + "truth_source": ["transcript_artifact", "runtime_timeline_event"] + }, + "transcript": { + "kind": "transcript", + "status": "pending", + "source_path": "/tmp/interview.wav", + "language": "zh-CN", + "output_format": "srt", + "speaker_labels": true, + "timestamps": true, + "provider_id": "limecore", + "model": "asr-pro" } }, "status": "pending_submit", "normalized_status": "pending", "created_at": "2026-04-30T10:00:00Z", - "updated_at": null, + "updated_at": "2026-04-30T10:00:05Z", "submitted_at": null, "started_at": null, "completed_at": null, @@ -3856,15 +4656,16 @@ mod tests { "source_task_id": null, "result": null, "last_error": null, - "current_attempt_id": "attempt-audio-1", + "current_attempt_id": "attempt-transcription-1", + "current_attempt_worker_id": "lime-transcription-worker", "attempts": [], "relationships": {}, "progress": {}, "ui_hints": {} })) - .expect("serialize audio task"), + .expect("serialize transcription task"), ) - .expect("write audio task"); + .expect("write transcription task"); } #[allow(dead_code)] @@ -4426,6 +5227,30 @@ mod tests { .and_then(Value::as_str), Some("image_generation_model_capability_gap") ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/executionProfileKey") + .and_then(Value::as_str), + Some("image_generation_profile") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/executorAdapterKey") + .and_then(Value::as_str), + Some("skill:image_generate") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/executionProfileKeys/0") + .and_then(Value::as_str), + Some("image_generation_profile") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/executorAdapterKeys/0") + .and_then(Value::as_str), + Some("skill:image_generate") + ); assert_eq!( runtime .pointer("/modalityRuntimeContracts/snapshots/0/modelCapabilityAssessment/source") @@ -4508,9 +5333,32 @@ mod tests { "routing_slot": BROWSER_CONTROL_ROUTING_SLOT, "runtime_contract": { "contract_key": BROWSER_CONTROL_CONTRACT_KEY, - "routing_slot": BROWSER_CONTROL_ROUTING_SLOT + "routing_slot": BROWSER_CONTROL_ROUTING_SLOT, + "executor_binding": { + "executor_kind": "browser_action", + "binding_key": "lime_browser_mcp" + } }, - "entry_source": "at_browser_command" + "entry_source": "at_browser_command", + "action": "navigate", + "selected_backend": "cdp_direct", + "attempt_count": 1, + "result": { + "success": true, + "action": "navigate", + "request_id": "browser-request-1", + "session_id": "browser-session-1", + "target_id": "target-1", + "data": { + "browser_session": { + "session_id": "browser-session-1", + "profile_key": "general_browser_assist", + "target_id": "target-1", + "target_title": "Example", + "target_url": "https://example.com/" + } + } + } })), }, }); @@ -4567,6 +5415,188 @@ mod tests { .and_then(Value::as_str), Some(BROWSER_CONTROL_ROUTING_SLOT) ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/browserAction/artifactKind") + .and_then(Value::as_str), + Some("browser_session") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/browserAction/sessionId") + .and_then(Value::as_str), + Some("browser-session-1") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/actionCount") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/sessionCount") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/lastUrl") + .and_then(Value::as_str), + Some("https://example.com/") + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/browserActionIndex/items/0/action" + ) + .and_then(Value::as_str), + Some("navigate") + ); + assert_eq!( + result + .observability_summary + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/actionCount") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + result + .observability_summary + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/lastUrl") + .and_then(Value::as_str), + Some("https://example.com/") + ); + } + + #[test] + fn should_index_browser_snapshot_observation_from_tool_metadata() { + let temp_dir = TempDir::new().expect("temp dir"); + let mut detail = build_detail(); + let thread_read = build_thread_read(); + + detail.items.push(AgentThreadItem { + id: "browser-snapshot-tool-1".to_string(), + thread_id: "thread-1".to_string(), + turn_id: "turn-1".to_string(), + sequence: 4, + status: AgentThreadItemStatus::Completed, + started_at: "2026-03-27T10:00:40Z".to_string(), + completed_at: Some("2026-03-27T10:00:40Z".to_string()), + updated_at: "2026-03-27T10:00:40Z".to_string(), + payload: AgentThreadItemPayload::ToolCall { + tool_name: "mcp__lime-browser__get_page_info".to_string(), + arguments: Some(json!({})), + output: Some("ok".to_string()), + success: Some(true), + error: None, + metadata: Some(json!({ + "tool_family": "browser", + "modality_contract_key": BROWSER_CONTROL_CONTRACT_KEY, + "modality": "browser", + "required_capabilities": [ + "text_generation", + "browser_reasoning", + "browser_control_planning" + ], + "routing_slot": BROWSER_CONTROL_ROUTING_SLOT, + "runtime_contract": { + "contract_key": BROWSER_CONTROL_CONTRACT_KEY, + "routing_slot": BROWSER_CONTROL_ROUTING_SLOT, + "executor_binding": { + "executor_kind": "browser_action", + "binding_key": "lime_browser_mcp" + } + }, + "entry_source": "at_browser_agent_command", + "action": "get_page_info", + "selected_backend": "lime_extension_bridge", + "result": { + "success": true, + "action": "get_page_info", + "request_id": "browser-request-2", + "data": { + "title": "Example", + "url": "https://example.com/", + "screenshot_path": ".lime/runtime/browser/browser-snapshot-1.png", + "browser_session": { + "session_id": "browser-session-1", + "profile_key": "general_browser_assist", + "target_id": "target-1", + "target_title": "Example", + "target_url": "https://example.com/" + } + } + } + })), + }, + }); + + let result = + export_runtime_evidence_pack(&detail, &thread_read, temp_dir.path()).expect("export"); + + let runtime_path = temp_dir + .path() + .join(".lime/harness/sessions/session-1/evidence/runtime.json"); + let runtime = serde_json::from_str::( + fs::read_to_string(runtime_path).expect("runtime").as_str(), + ) + .expect("parse runtime json"); + + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/browserAction/artifactKind") + .and_then(Value::as_str), + Some("browser_snapshot") + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/browserActionIndex/observationCount" + ) + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/browserActionIndex/screenshotCount" + ) + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/items/0/artifactKind") + .and_then(Value::as_str), + Some("browser_snapshot") + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/browserActionIndex/items/0/backend" + ) + .and_then(Value::as_str), + Some("lime_extension_bridge") + ); + assert_eq!( + result + .observability_summary + .pointer( + "/modalityRuntimeContracts/snapshotIndex/browserActionIndex/items/0/artifactKind" + ) + .and_then(Value::as_str), + Some("browser_snapshot") + ); + assert_eq!( + result + .observability_summary + .pointer( + "/modalityRuntimeContracts/snapshotIndex/browserActionIndex/observationCount" + ) + .and_then(Value::as_u64), + Some(1) + ); } #[test] @@ -4857,6 +5887,38 @@ mod tests { .and_then(Value::as_str), Some("executor_invoked") ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/audioOutput/status") + .and_then(Value::as_str), + Some("completed") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/audioOutput/audioPath") + .and_then(Value::as_str), + Some(".lime/runtime/audio/task-audio-1.mp3") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/audioOutput/workerId") + .and_then(Value::as_str), + Some("lime-audio-worker") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/audioOutputIndex/outputCount") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/audioOutputIndex/statusCounts/0/status" + ) + .and_then(Value::as_str), + Some("completed") + ); assert_eq!( runtime .pointer("/modalityRuntimeContracts/snapshotIndex/toolTraceIndex/items/0/executorBindingKey") @@ -4865,6 +5927,105 @@ mod tests { ); } + #[test] + fn should_export_audio_transcription_contract_snapshot_from_transcription_task_artifact() { + let temp_dir = TempDir::new().expect("temp dir"); + let mut detail = build_detail(); + let thread_read = build_thread_read(); + let transcription_task_relative_path = + ".lime/tasks/transcription_generate/task-transcription-1.json"; + + write_transcription_task_fixture(temp_dir.path(), transcription_task_relative_path); + + if let AgentThreadItemPayload::FileArtifact { path, metadata, .. } = + &mut detail.items[1].payload + { + *path = transcription_task_relative_path.to_string(); + *metadata = Some(json!({ + "task_type": "transcription_generate" + })); + } + + export_runtime_evidence_pack(&detail, &thread_read, temp_dir.path()).expect("export"); + + let runtime_path = temp_dir + .path() + .join(".lime/harness/sessions/session-1/evidence/runtime.json"); + let runtime = serde_json::from_str::( + fs::read_to_string(runtime_path).expect("runtime").as_str(), + ) + .expect("parse runtime json"); + + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotCount") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/source") + .and_then(Value::as_str), + Some("transcription_task.modality_runtime_contract") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/contractKey") + .and_then(Value::as_str), + Some(AUDIO_TRANSCRIPTION_CONTRACT_KEY) + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/taskType") + .and_then(Value::as_str), + Some("transcription_generate") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/expectedRoutingSlot") + .and_then(Value::as_str), + Some(AUDIO_TRANSCRIPTION_ROUTING_SLOT) + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/routingEvent") + .and_then(Value::as_str), + Some("executor_invoked") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/transcript/status") + .and_then(Value::as_str), + Some("pending") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/transcript/sourcePath") + .and_then(Value::as_str), + Some("/tmp/interview.wav") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/transcriptIndex/transcriptCount") + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/transcriptIndex/statusCounts/0/status" + ) + .and_then(Value::as_str), + Some("pending") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshotIndex/toolTraceIndex/items/0/executorBindingKey") + .and_then(Value::as_str), + Some("transcription_generate") + ); + } + #[test] fn should_export_web_research_contract_snapshot_from_skill_args() { let temp_dir = TempDir::new().expect("temp dir"); @@ -4899,11 +6060,17 @@ mod tests { "runtime_contract": { "contract_key": WEB_RESEARCH_CONTRACT_KEY, "routing_slot": WEB_RESEARCH_ROUTING_SLOT, - "executor_binding": { - "executor_kind": "skill", - "binding_key": "research" - } + "executor_binding": { + "executor_kind": "skill", + "binding_key": "research" }, + "execution_profile": { + "profile_key": "web_research_profile" + }, + "executor_adapter": { + "adapter_key": "skill:research" + } + }, "entry_source": "at_search_command" } })).expect("serialize args") @@ -4967,6 +6134,18 @@ mod tests { .and_then(Value::as_str), Some(WEB_RESEARCH_ROUTING_SLOT) ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/executionProfileKey") + .and_then(Value::as_str), + Some("web_research_profile") + ); + assert_eq!( + runtime + .pointer("/modalityRuntimeContracts/snapshots/0/executorAdapterKey") + .and_then(Value::as_str), + Some("skill:research") + ); assert_eq!( runtime .pointer("/modalityRuntimeContracts/snapshotIndex/contractKeys/0") @@ -5007,6 +6186,22 @@ mod tests { .and_then(Value::as_str), Some("research") ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/toolTraceIndex/items/0/executionProfileKey", + ) + .and_then(Value::as_str), + Some("web_research_profile") + ); + assert_eq!( + runtime + .pointer( + "/modalityRuntimeContracts/snapshotIndex/toolTraceIndex/items/0/executorAdapterKey", + ) + .and_then(Value::as_str), + Some("skill:research") + ); assert_eq!( runtime .pointer("/modalityRuntimeContracts/snapshots/0/contractMatchedExpected") diff --git a/src-tauri/src/services/runtime_replay_case_service.rs b/src-tauri/src/services/runtime_replay_case_service.rs index a40cc89b5..5a769d044 100644 --- a/src-tauri/src/services/runtime_replay_case_service.rs +++ b/src-tauri/src/services/runtime_replay_case_service.rs @@ -7,8 +7,8 @@ use crate::agent::SessionDetail; use crate::commands::aster_agent_cmd::AgentRuntimeThreadReadModel; use crate::commands::modality_runtime_contracts::{ - BROWSER_CONTROL_CONTRACT_KEY, PDF_EXTRACT_CONTRACT_KEY, TEXT_TRANSFORM_CONTRACT_KEY, - VOICE_GENERATION_CONTRACT_KEY, WEB_RESEARCH_CONTRACT_KEY, + AUDIO_TRANSCRIPTION_CONTRACT_KEY, BROWSER_CONTROL_CONTRACT_KEY, PDF_EXTRACT_CONTRACT_KEY, + TEXT_TRANSFORM_CONTRACT_KEY, VOICE_GENERATION_CONTRACT_KEY, WEB_RESEARCH_CONTRACT_KEY, }; use crate::services::runtime_evidence_pack_service::{ export_runtime_evidence_pack, RuntimeEvidencePackExportResult, @@ -694,6 +694,10 @@ fn build_success_criteria( "浏览器控制任务必须继续走 Browser Assist / `mcp__lime-browser__*` 主链，不能用 WebSearch 或普通聊天替代真实浏览器动作。" .to_string(), ); + criteria.push( + "Browser Assist 证据必须能在 `snapshotIndex.browserActionIndex` 中定位 action、session、URL 或 observation 摘要，不能只靠人工扫描 raw snapshots。" + .to_string(), + ); } if modality_contract_has_pdf_extract(modality_runtime_contracts) { criteria.push( @@ -706,6 +710,28 @@ fn build_success_criteria( "配音任务必须继续走 `service_scene_launch(scene_key=voice_runtime)` / 本地 ServiceSkill runtime 主链，不能用 `legacy_tts_test_command`、伪造云端已提交、普通聊天文本或通用文件卡替代真实语音生成合同。" .to_string(), ); + if modality_contract_has_voice_audio_output_completed(modality_runtime_contracts) { + criteria.push( + "如果 replay 沿用已有音频任务，必须保留 `audio_output.completed` 与真实 `audio_path`，不能退回只展示文案或通用文件卡。" + .to_string(), + ); + } + } + if modality_contract_has_audio_transcription(modality_runtime_contracts) { + criteria.push( + "转写任务必须继续走 `Skill(transcription_generate)` / `transcription_generate` task file 主链，不能用 `frontend_direct_asr`、`generic_file_transcript`、普通文件读取或普通聊天伪造 transcript。" + .to_string(), + ); + criteria.push( + "转写证据必须能在 `snapshotIndex.transcriptIndex` 中定位 transcript 状态、来源、语言或输出格式，不能只靠人工扫描 raw task JSON。" + .to_string(), + ); + if modality_contract_has_transcript_completed(modality_runtime_contracts) { + criteria.push( + "如果 replay 沿用已有转写结果，必须保留 `transcript.completed` 与真实 `transcriptPath`，不能退回只展示 Markdown 文本。" + .to_string(), + ); + } } if modality_contract_has_web_research(modality_runtime_contracts) { criteria.push( @@ -778,6 +804,14 @@ fn build_blocking_checks( .to_string(), ); } + if modality_contract_has_browser_control(modality_runtime_contracts) + && !modality_contract_has_browser_action_index(modality_runtime_contracts) + { + checks.push( + "`browser_control` 合同缺少 `snapshotIndex.browserActionIndex`；除非 replay 重新导出可查询的 browser session/snapshot 索引，否则不能判 PASS。" + .to_string(), + ); + } if modality_contract_has_pdf_extract(modality_runtime_contracts) && !modality_contract_has_pdf_skill_trace(modality_runtime_contracts) { @@ -794,6 +828,40 @@ fn build_blocking_checks( .to_string(), ); } + if modality_contract_has_voice_audio_output_failed(modality_runtime_contracts) { + checks.push(format!( + "`voice_generation` 的 audio_output 已失败：{}；除非 replay 修复 provider / model / API Key 并产出新的 audio_output.completed，否则不能判 PASS。", + format_text_list( + &modality_contract_voice_audio_output_error_codes(modality_runtime_contracts), + "未记录 audio_output errorCode" + ) + )); + } + if modality_contract_has_audio_transcription(modality_runtime_contracts) + && !modality_contract_has_audio_transcription_task_trace(modality_runtime_contracts) + { + checks.push( + "`audio_transcription` 合同缺少 transcription_generate task/Skill trace；除非 replay 重新产生 Skill(transcription_generate) 或 transcription task artifact 证据，否则不能判 PASS。" + .to_string(), + ); + } + if modality_contract_has_audio_transcription(modality_runtime_contracts) + && !modality_contract_has_transcript_index(modality_runtime_contracts) + { + checks.push( + "`audio_transcription` 合同缺少 `snapshotIndex.transcriptIndex`；除非 replay 重新导出可查询的 transcript 索引，否则不能判 PASS。" + .to_string(), + ); + } + if modality_contract_has_transcript_failed(modality_runtime_contracts) { + checks.push(format!( + "`audio_transcription` 的 transcript 已失败：{}；除非 replay 修复 provider / model / 输入源并产出新的 transcript.completed，否则不能判 PASS。", + format_text_list( + &modality_contract_transcript_error_codes(modality_runtime_contracts), + "未记录 transcript errorCode" + ) + )); + } if modality_contract_has_web_research(modality_runtime_contracts) && !modality_contract_has_web_research_skill_trace(modality_runtime_contracts) { @@ -867,6 +935,10 @@ fn build_modality_contract_checks(modality_runtime_contracts: &Value) -> Vec Vec bool { + modality_runtime_contracts + .pointer("/snapshotIndex/browserActionIndex/actionCount") + .and_then(Value::as_u64) + .is_some_and(|count| count > 0) + || modality_runtime_contracts + .pointer("/snapshotIndex/browserActionIndex/items") + .and_then(Value::as_array) + .is_some_and(|items| !items.is_empty()) +} + fn modality_contract_has_pdf_extract(modality_runtime_contracts: &Value) -> bool { modality_contract_snapshots(modality_runtime_contracts) .iter() @@ -1241,6 +1425,191 @@ fn modality_contract_has_voice_generation_service_trace( }) } +fn modality_contract_voice_audio_outputs(modality_runtime_contracts: &Value) -> Vec<&Value> { + modality_contract_snapshots(modality_runtime_contracts) + .into_iter() + .filter(|snapshot| { + snapshot + .get("contractKey") + .and_then(Value::as_str) + .map(|value| value == VOICE_GENERATION_CONTRACT_KEY) + .unwrap_or(false) + }) + .filter_map(|snapshot| snapshot.get("audioOutput")) + .filter(|value| value.is_object()) + .collect() +} + +fn modality_contract_has_voice_audio_output_completed(modality_runtime_contracts: &Value) -> bool { + modality_contract_voice_audio_outputs(modality_runtime_contracts) + .iter() + .any(|audio_output| { + let completed = audio_output + .get("status") + .and_then(Value::as_str) + .map(|value| value == "completed") + .unwrap_or(false); + let has_audio_path = audio_output + .get("audioPath") + .and_then(Value::as_str) + .and_then(|value| normalize_optional_text(Some(value.to_string()))) + .is_some(); + completed && has_audio_path + }) +} + +fn modality_contract_has_voice_audio_output_failed(modality_runtime_contracts: &Value) -> bool { + modality_contract_voice_audio_outputs(modality_runtime_contracts) + .iter() + .any(|audio_output| { + audio_output + .get("status") + .and_then(Value::as_str) + .map(|value| value == "failed") + .unwrap_or(false) + || audio_output + .get("errorCode") + .and_then(Value::as_str) + .and_then(|value| normalize_optional_text(Some(value.to_string()))) + .is_some() + }) +} + +fn modality_contract_voice_audio_output_error_codes( + modality_runtime_contracts: &Value, +) -> Vec { + let mut error_codes = Vec::new(); + for audio_output in modality_contract_voice_audio_outputs(modality_runtime_contracts) { + if let Some(error_code) = audio_output + .get("errorCode") + .and_then(Value::as_str) + .and_then(|value| normalize_optional_text(Some(value.to_string()))) + { + push_unique_owned_tag(&mut error_codes, error_code); + } + } + error_codes +} + +fn modality_contract_has_audio_transcription(modality_runtime_contracts: &Value) -> bool { + modality_contract_snapshots(modality_runtime_contracts) + .iter() + .any(|snapshot| { + snapshot + .get("contractKey") + .and_then(Value::as_str) + .map(|value| value == AUDIO_TRANSCRIPTION_CONTRACT_KEY) + .unwrap_or(false) + }) +} + +fn modality_contract_has_audio_transcription_task_trace( + modality_runtime_contracts: &Value, +) -> bool { + modality_contract_snapshots(modality_runtime_contracts) + .iter() + .any(|snapshot| { + let is_audio_transcription = snapshot + .get("contractKey") + .and_then(Value::as_str) + .map(|value| value == AUDIO_TRANSCRIPTION_CONTRACT_KEY) + .unwrap_or(false); + if !is_audio_transcription { + return false; + } + + snapshot + .get("source") + .and_then(Value::as_str) + .map(|value| { + value.contains("transcription_task") + || value.contains("transcription_skill_trace") + || value.contains("transcript") + }) + .unwrap_or(false) + || snapshot + .get("routingEvent") + .and_then(Value::as_str) + .map(|value| value == "executor_invoked") + .unwrap_or(false) + }) +} + +fn modality_contract_has_transcript_index(modality_runtime_contracts: &Value) -> bool { + modality_runtime_contracts + .pointer("/snapshotIndex/transcriptIndex/transcriptCount") + .and_then(Value::as_u64) + .is_some_and(|count| count > 0) + || modality_runtime_contracts + .pointer("/snapshotIndex/transcriptIndex/items") + .and_then(Value::as_array) + .is_some_and(|items| !items.is_empty()) +} + +fn modality_contract_transcripts(modality_runtime_contracts: &Value) -> Vec<&Value> { + modality_contract_snapshots(modality_runtime_contracts) + .into_iter() + .filter(|snapshot| { + snapshot + .get("contractKey") + .and_then(Value::as_str) + .map(|value| value == AUDIO_TRANSCRIPTION_CONTRACT_KEY) + .unwrap_or(false) + }) + .filter_map(|snapshot| snapshot.get("transcript")) + .filter(|value| value.is_object()) + .collect() +} + +fn modality_contract_has_transcript_completed(modality_runtime_contracts: &Value) -> bool { + modality_contract_transcripts(modality_runtime_contracts) + .iter() + .any(|transcript| { + let completed = transcript + .get("status") + .and_then(Value::as_str) + .map(|value| value == "completed") + .unwrap_or(false); + let has_transcript_path = transcript + .get("transcriptPath") + .and_then(Value::as_str) + .and_then(|value| normalize_optional_text(Some(value.to_string()))) + .is_some(); + completed && has_transcript_path + }) +} + +fn modality_contract_has_transcript_failed(modality_runtime_contracts: &Value) -> bool { + modality_contract_transcripts(modality_runtime_contracts) + .iter() + .any(|transcript| { + transcript + .get("status") + .and_then(Value::as_str) + .map(|value| value == "failed") + .unwrap_or(false) + || transcript + .get("errorCode") + .and_then(Value::as_str) + .and_then(|value| normalize_optional_text(Some(value.to_string()))) + .is_some() + }) +} + +fn modality_contract_transcript_error_codes(modality_runtime_contracts: &Value) -> Vec { + let mut error_codes = Vec::new(); + for transcript in modality_contract_transcripts(modality_runtime_contracts) { + if let Some(error_code) = transcript + .get("errorCode") + .and_then(Value::as_str) + .and_then(|value| normalize_optional_text(Some(value.to_string()))) + { + push_unique_owned_tag(&mut error_codes, error_code); + } + } + error_codes +} + fn modality_contract_has_web_research(modality_runtime_contracts: &Value) -> bool { modality_contract_snapshots(modality_runtime_contracts) .iter() @@ -2011,28 +2380,43 @@ mod tests { }, "audio_output": { "kind": "audio_output", - "status": "pending", - "audio_path": null, + "status": "completed", + "audio_path": ".lime/runtime/audio/task-audio-1.mp3", "mime_type": "audio/mpeg", - "duration_ms": null, + "duration_ms": 128000, "source_text": "请为这段文案生成温暖旁白", - "voice": "warm_narrator" + "voice": "warm_narrator", + "provider_id": "limecore", + "model": "voice-pro" } }, - "status": "pending_submit", - "normalized_status": "pending", + "status": "succeeded", + "normalized_status": "succeeded", "created_at": "2026-04-30T10:00:00Z", - "updated_at": null, + "updated_at": "2026-04-30T10:00:05Z", "submitted_at": null, - "started_at": null, - "completed_at": null, + "started_at": "2026-04-30T10:00:01Z", + "completed_at": "2026-04-30T10:00:05Z", "cancelled_at": null, "idempotency_key": null, "retry_count": 0, "source_task_id": null, - "result": null, + "result": { + "kind": "audio_generation_result", + "status": "completed", + "audio_output": { + "kind": "audio_output", + "status": "completed", + "audio_path": ".lime/runtime/audio/task-audio-1.mp3", + "mime_type": "audio/mpeg", + "duration_ms": 128000, + "provider_id": "limecore", + "model": "voice-pro" + } + }, "last_error": null, "current_attempt_id": "attempt-audio-1", + "current_attempt_worker_id": "lime-audio-worker", "attempts": [], "relationships": {}, "progress": {}, @@ -2246,9 +2630,31 @@ mod tests { "routing_slot": "browser_reasoning_model", "runtime_contract": { "contract_key": BROWSER_CONTROL_CONTRACT_KEY, - "routing_slot": "browser_reasoning_model" + "routing_slot": "browser_reasoning_model", + "executor_binding": { + "executor_kind": "browser_action", + "binding_key": "lime_browser_mcp" + } }, - "entry_source": "at_browser_command" + "entry_source": "at_browser_command", + "action": "navigate", + "selected_backend": "cdp_direct", + "result": { + "success": true, + "action": "navigate", + "request_id": "browser-request-1", + "session_id": "browser-session-1", + "target_id": "target-1", + "data": { + "browser_session": { + "session_id": "browser-session-1", + "profile_key": "general_browser_assist", + "target_id": "target-1", + "target_title": "Example", + "target_url": "https://example.com/" + } + } + } })), }, }); @@ -2283,6 +2689,22 @@ mod tests { .and_then(Value::as_str), Some("browser_action_requested") ); + assert_eq!( + input + .pointer( + "/runtimeContext/modalityRuntimeContracts/snapshotIndex/browserActionIndex/actionCount" + ) + .and_then(Value::as_u64), + Some(1) + ); + assert_eq!( + input + .pointer( + "/runtimeContext/modalityRuntimeContracts/snapshotIndex/browserActionIndex/items/0/sessionId" + ) + .and_then(Value::as_str), + Some("browser-session-1") + ); let suite_tags = input .pointer("/classification/suiteTags") .and_then(Value::as_array) @@ -2292,6 +2714,7 @@ mod tests { "browser-control", "browser-assist", "browser-action-trace", + "browser-action-index", ] { assert!(suite_tags .iter() @@ -2303,11 +2726,13 @@ mod tests { assert!(expected.contains("mcp__lime-browser__*")); assert!(expected.contains("WebSearch")); assert!(expected.contains("browser_action_trace")); + assert!(expected.contains("browserActionIndex")); assert!(expected.contains("\"requiresHumanReview\": false")); let grader = fs::read_to_string(grader_path).expect("grader"); assert!(grader.contains("多模态运行合同检查")); assert!(grader.contains("browser_action_requested")); + assert!(grader.contains("browserActionIndex")); assert!(grader.contains("WebSearch")); let links = @@ -2319,6 +2744,12 @@ mod tests { .and_then(Value::as_str), Some("browser_action_trace.modality_runtime_contract") ); + assert_eq!( + links + .pointer("/modalityRuntimeContracts/snapshotIndex/browserActionIndex/lastUrl") + .and_then(Value::as_str), + Some("https://example.com/") + ); } #[test] @@ -2599,6 +3030,9 @@ mod tests { let expected_path = temp_dir .path() .join(".lime/harness/sessions/session-1/replay/expected.json"); + let grader_path = temp_dir + .path() + .join(".lime/harness/sessions/session-1/replay/grader.md"); let links_path = temp_dir .path() .join(".lime/harness/sessions/session-1/replay/evidence-links.json"); @@ -2618,6 +3052,26 @@ mod tests { .and_then(Value::as_str), Some("audio_generate") ); + assert_eq!( + input + .pointer("/runtimeContext/modalityRuntimeContracts/snapshots/0/audioOutput/status") + .and_then(Value::as_str), + Some("completed") + ); + assert_eq!( + input + .pointer( + "/runtimeContext/modalityRuntimeContracts/snapshots/0/audioOutput/audioPath" + ) + .and_then(Value::as_str), + Some(".lime/runtime/audio/task-audio-1.mp3") + ); + assert_eq!( + input + .pointer("/runtimeContext/modalityRuntimeContracts/snapshotIndex/audioOutputIndex/outputCount") + .and_then(Value::as_u64), + Some(1) + ); let suite_tags = input .pointer("/classification/suiteTags") .and_then(Value::as_array) @@ -2625,6 +3079,9 @@ mod tests { assert!(suite_tags .iter() .any(|item| item.as_str() == Some("voice-generation-trace"))); + assert!(suite_tags + .iter() + .any(|item| item.as_str() == Some("audio-output-completed"))); let failure_modes = input .pointer("/classification/failureModes") .and_then(Value::as_array) @@ -2635,8 +3092,13 @@ mod tests { let expected = fs::read_to_string(expected_path).expect("expected"); assert!(expected.contains("audio_task/audio_output")); + assert!(expected.contains("audio_output.completed")); assert!(expected.contains("\"requiresHumanReview\": false")); + let grader = fs::read_to_string(grader_path).expect("grader"); + assert!(grader.contains("audio_output.completed")); + assert!(grader.contains("audio_path")); + let links = serde_json::from_str::(fs::read_to_string(links_path).expect("links").as_str()) .expect("parse links"); @@ -2646,6 +3108,112 @@ mod tests { .and_then(Value::as_str), Some("audio_task.modality_runtime_contract") ); + assert_eq!( + links + .pointer( + "/modalityRuntimeContracts/snapshotIndex/audioOutputIndex/items/0/audioPath" + ) + .and_then(Value::as_str), + Some(".lime/runtime/audio/task-audio-1.mp3") + ); + } + + #[test] + fn should_classify_voice_audio_output_provider_failure_for_replay() { + let detail = build_detail(); + let thread_read = build_thread_read(); + let modality_runtime_contracts = json!({ + "snapshotCount": 1, + "snapshots": [{ + "contractKey": VOICE_GENERATION_CONTRACT_KEY, + "source": "audio_task.modality_runtime_contract", + "routingEvent": "executor_invoked", + "audioOutput": { + "status": "failed", + "errorCode": "audio_provider_unconfigured", + "errorMessage": "未找到可用的 voice_generation provider/API Key。", + "retryable": true + } + }] + }); + + let failure_modes = + infer_replay_failure_modes(&detail, &thread_read, &modality_runtime_contracts); + assert!(failure_modes + .iter() + .any(|item| item == "voice_generation_audio_output_failed")); + assert!(failure_modes + .iter() + .any(|item| item == "audio_provider_unconfigured")); + + let blocking_checks = build_blocking_checks(&thread_read, &[], &modality_runtime_contracts); + assert!(blocking_checks + .iter() + .any(|item| item.contains("audio_provider_unconfigured"))); + + let contract_checks = build_modality_contract_checks(&modality_runtime_contracts); + assert!(contract_checks + .iter() + .any(|item| item.contains("audio_output.failed"))); + assert!(contract_checks + .iter() + .any(|item| item.contains("audio_provider_unconfigured"))); + } + + #[test] + fn should_classify_audio_transcription_transcript_failure_for_replay() { + let detail = build_detail(); + let thread_read = build_thread_read(); + let modality_runtime_contracts = json!({ + "snapshotCount": 1, + "snapshotIndex": { + "transcriptIndex": { + "transcriptCount": 1, + "statusCounts": [{ "status": "failed", "count": 1 }], + "errorCodes": ["transcription_provider_unconfigured"], + "items": [{ + "status": "failed", + "errorCode": "transcription_provider_unconfigured" + }] + } + }, + "snapshots": [{ + "contractKey": AUDIO_TRANSCRIPTION_CONTRACT_KEY, + "source": "transcription_task.modality_runtime_contract", + "routingEvent": "executor_invoked", + "transcript": { + "status": "failed", + "errorCode": "transcription_provider_unconfigured", + "errorMessage": "未找到可用的 audio_transcription provider/API Key。", + "retryable": true + } + }] + }); + + let failure_modes = + infer_replay_failure_modes(&detail, &thread_read, &modality_runtime_contracts); + assert!(failure_modes + .iter() + .any(|item| item == "audio_transcription_transcript_failed")); + assert!(failure_modes + .iter() + .any(|item| item == "transcription_provider_unconfigured")); + + let blocking_checks = build_blocking_checks(&thread_read, &[], &modality_runtime_contracts); + assert!(blocking_checks + .iter() + .any(|item| item.contains("transcription_provider_unconfigured"))); + assert!(!blocking_checks + .iter() + .any(|item| item.contains("缺少 `snapshotIndex.transcriptIndex`"))); + + let contract_checks = build_modality_contract_checks(&modality_runtime_contracts); + assert!(contract_checks + .iter() + .any(|item| item.contains("transcript.failed"))); + assert!(contract_checks + .iter() + .any(|item| item.contains("frontend_direct_asr"))); } #[test] diff --git a/src-tauri/src/voice/commands.rs b/src-tauri/src/voice/commands.rs index 3de768607..7f4cf0633 100644 --- a/src-tauri/src/voice/commands.rs +++ b/src-tauri/src/voice/commands.rs @@ -27,6 +27,10 @@ pub struct VoiceShortcutRuntimeStatus { pub registered_shortcut: Option, pub translate_shortcut_registered: bool, pub registered_translate_shortcut: Option, + pub fn_supported: bool, + pub fn_registered: bool, + pub fn_fallback_shortcut: Option, + pub fn_note: String, } /// 获取所有可用的麦克风设备 @@ -44,11 +48,17 @@ pub async fn get_voice_input_config() -> Result { /// 获取语音快捷键运行时状态 #[command] pub async fn get_voice_shortcut_runtime_status() -> Result { + let fn_status = super::fn_shortcut::runtime_status(); Ok(VoiceShortcutRuntimeStatus { shortcut_registered: super::shortcut::is_registered(), registered_shortcut: super::shortcut::get_current(), translate_shortcut_registered: super::shortcut::is_translate_registered(), registered_translate_shortcut: super::shortcut::get_current_translate(), + fn_supported: fn_status.supported, + fn_registered: fn_status.registered, + fn_fallback_shortcut: super::shortcut::get_current() + .or_else(|| Some("CommandOrControl+Shift+V".to_string())), + fn_note: fn_status.note, }) } @@ -63,6 +73,8 @@ pub async fn save_voice_input_config( let new_enabled = voice_config.enabled; if old_enabled && new_enabled { + super::register_fn_shortcut_if_supported(&app); + if old_config.shortcut != voice_config.shortcut { super::shortcut::update(&app, &voice_config.shortcut)?; } @@ -100,8 +112,10 @@ pub async fn save_voice_input_config( } else if old_enabled && !new_enabled { super::shortcut::unregister(&app)?; let _ = super::shortcut::unregister_translate(&app); + let _ = super::fn_shortcut::unregister(); } else if !old_enabled && new_enabled { super::shortcut::register(&app, &voice_config.shortcut)?; + super::register_fn_shortcut_if_supported(&app); if let Some(translate_shortcut) = normalize_shortcut(voice_config.translate_shortcut.clone()) diff --git a/src-tauri/src/voice/fn_shortcut.rs b/src-tauri/src/voice/fn_shortcut.rs new file mode 100644 index 000000000..9ac60db7a --- /dev/null +++ b/src-tauri/src/voice/fn_shortcut.rs @@ -0,0 +1,294 @@ +//! macOS Fn 按住录音监听。 +//! +//! Fn 不走 `tauri_plugin_global_shortcut` 的字符串解析路径；macOS 下通过 +//! NSEvent 的 FlagsChanged 事件做一个窄桥接，其他平台明确报告不支持。 + +#[derive(Debug, Clone)] +pub struct FnShortcutRuntimeStatus { + pub supported: bool, + pub registered: bool, + pub note: String, +} + +#[cfg(target_os = "macos")] +#[allow(deprecated, unexpected_cfgs)] +mod platform { + use super::FnShortcutRuntimeStatus; + use block::{ConcreteBlock, RcBlock}; + use cocoa::appkit::{NSEvent, NSEventMask, NSEventModifierFlags}; + use cocoa::base::{id, nil}; + use objc::{class, msg_send, sel, sel_impl}; + use parking_lot::Mutex; + use std::os::raw::c_uchar; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::OnceLock; + use tauri::AppHandle; + use tracing::{debug, error, info, warn}; + + type GlobalEventBlock = RcBlock<(id,), ()>; + type LocalEventBlock = RcBlock<(id,), id>; + + static REGISTRATION: OnceLock>> = OnceLock::new(); + static IS_REGISTERED: AtomicBool = AtomicBool::new(false); + static FN_IS_DOWN: AtomicBool = AtomicBool::new(false); + static LAST_ERROR: OnceLock>> = OnceLock::new(); + + #[link(name = "ApplicationServices", kind = "framework")] + extern "C" { + fn AXIsProcessTrusted() -> c_uchar; + } + + struct ObjcMonitor(id); + + // AppKit monitor token 和 Block 都只在 register/unregister 边界持有； + // 回调由 AppKit 调度，Rust 侧不跨线程解引用这些裸指针。 + unsafe impl Send for ObjcMonitor {} + + struct GlobalBlock { + _inner: GlobalEventBlock, + } + + unsafe impl Send for GlobalBlock {} + + struct LocalBlock { + _inner: LocalEventBlock, + } + + unsafe impl Send for LocalBlock {} + + struct FnShortcutRegistration { + global_monitor: Option, + local_monitor: Option, + _global_block: GlobalBlock, + _local_block: LocalBlock, + } + + fn registration() -> &'static Mutex> { + REGISTRATION.get_or_init(|| Mutex::new(None)) + } + + fn last_error() -> &'static Mutex> { + LAST_ERROR.get_or_init(|| Mutex::new(None)) + } + + fn set_last_error(value: Option) { + *last_error().lock() = value; + } + + fn accessibility_trusted() -> bool { + // SAFETY: AXIsProcessTrusted 无参数、无副作用，返回 CoreServices Boolean。 + unsafe { AXIsProcessTrusted() != 0 } + } + + fn handle_flags_changed_event(app: &AppHandle, event: id) { + if event == nil { + return; + } + + let fn_down = unsafe { + event + .modifierFlags() + .contains(NSEventModifierFlags::NSFunctionKeyMask) + }; + let was_down = FN_IS_DOWN.swap(fn_down, Ordering::SeqCst); + + match (was_down, fn_down) { + (false, true) => { + info!("[语音输入] Fn 按下"); + if let Err(error) = crate::screenshot::window::open_floating_window_with_voice(app) + { + error!("[语音输入] Fn 打开语音窗口失败: {}", error); + } + } + (true, false) => { + info!("[语音输入] Fn 释放，发送停止录音事件"); + if let Err(error) = crate::screenshot::window::send_voice_stop_event(app) { + error!("[语音输入] Fn 发送停止录音事件失败: {}", error); + } + } + _ => { + debug!("[语音输入] 忽略重复 Fn 修饰键状态: {}", fn_down); + } + } + } + + unsafe fn add_global_monitor(block: &GlobalEventBlock) -> id { + let mask = NSEventMask::NSFlagsChangedMask.bits(); + msg_send![ + class!(NSEvent), + addGlobalMonitorForEventsMatchingMask: mask + handler: &**block + ] + } + + unsafe fn add_local_monitor(block: &LocalEventBlock) -> id { + let mask = NSEventMask::NSFlagsChangedMask.bits(); + msg_send![ + class!(NSEvent), + addLocalMonitorForEventsMatchingMask: mask + handler: &**block + ] + } + + unsafe fn remove_monitor(monitor: ObjcMonitor) { + if monitor.0 != nil { + let _: () = msg_send![class!(NSEvent), removeMonitor: monitor.0]; + let _: () = msg_send![monitor.0, release]; + } + } + + unsafe fn retain_monitor(monitor: id) -> Option { + if monitor == nil { + None + } else { + let retained: id = msg_send![monitor, retain]; + Some(ObjcMonitor(retained)) + } + } + + pub fn register(app: &AppHandle) -> Result<(), String> { + if IS_REGISTERED.load(Ordering::SeqCst) { + return Ok(()); + } + + let app_for_global = app.clone(); + let global_block = ConcreteBlock::new(move |event: id| { + handle_flags_changed_event(&app_for_global, event); + }) + .copy(); + + let app_for_local = app.clone(); + let local_block = ConcreteBlock::new(move |event: id| -> id { + handle_flags_changed_event(&app_for_local, event); + event + }) + .copy(); + + let global_monitor = unsafe { add_global_monitor(&global_block) }; + let local_monitor = unsafe { add_local_monitor(&local_block) }; + + if global_monitor == nil && local_monitor == nil { + let error = "macOS Fn 事件监听注册失败，NSEvent 未返回 monitor token".to_string(); + set_last_error(Some(error.clone())); + return Err(error); + } + + *registration().lock() = Some(FnShortcutRegistration { + global_monitor: unsafe { retain_monitor(global_monitor) }, + local_monitor: unsafe { retain_monitor(local_monitor) }, + _global_block: GlobalBlock { + _inner: global_block, + }, + _local_block: LocalBlock { + _inner: local_block, + }, + }); + FN_IS_DOWN.store(false, Ordering::SeqCst); + IS_REGISTERED.store(true, Ordering::SeqCst); + set_last_error(None); + + if accessibility_trusted() { + info!("[语音输入] Fn 按住录音监听已注册"); + } else { + warn!("[语音输入] Fn 监听已注册；跨应用捕获可能需要 macOS 辅助功能或输入监听权限"); + } + + Ok(()) + } + + pub fn unregister() -> Result<(), String> { + let registration = registration().lock().take(); + if let Some(registration) = registration { + if let Some(monitor) = registration.global_monitor { + unsafe { remove_monitor(monitor) }; + } + if let Some(monitor) = registration.local_monitor { + unsafe { remove_monitor(monitor) }; + } + } + + IS_REGISTERED.store(false, Ordering::SeqCst); + FN_IS_DOWN.store(false, Ordering::SeqCst); + info!("[语音输入] Fn 按住录音监听已注销"); + Ok(()) + } + + pub fn is_supported() -> bool { + true + } + + pub fn is_registered() -> bool { + IS_REGISTERED.load(Ordering::SeqCst) + } + + pub fn runtime_status() -> FnShortcutRuntimeStatus { + let registered = is_registered(); + let note = if let Some(error) = last_error().lock().clone() { + format!("Fn 监听注册失败：{error}；已使用普通语音快捷键回退。") + } else if registered && accessibility_trusted() { + "macOS Fn 监听已注册：按住 Fn 开始录音，松开 Fn 停止并转写。".to_string() + } else if registered { + "macOS Fn 监听已注册；若跨应用按 Fn 无响应，请在系统设置中授予 Lime 辅助功能/输入监听权限，或使用普通快捷键回退。" + .to_string() + } else { + "macOS 支持 Fn 监听；当前语音输入未启用或监听尚未注册，已使用普通语音快捷键回退。" + .to_string() + }; + + FnShortcutRuntimeStatus { + supported: true, + registered, + note, + } + } +} + +#[cfg(not(target_os = "macos"))] +mod platform { + use super::FnShortcutRuntimeStatus; + use tauri::AppHandle; + + pub fn register(_app: &AppHandle) -> Result<(), String> { + Err("Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。".to_string()) + } + + pub fn unregister() -> Result<(), String> { + Ok(()) + } + + pub fn is_supported() -> bool { + false + } + + pub fn is_registered() -> bool { + false + } + + pub fn runtime_status() -> FnShortcutRuntimeStatus { + FnShortcutRuntimeStatus { + supported: false, + registered: false, + note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。".to_string(), + } + } +} + +pub fn register(app: &tauri::AppHandle) -> Result<(), String> { + platform::register(app) +} + +pub fn unregister() -> Result<(), String> { + platform::unregister() +} + +pub fn is_supported() -> bool { + platform::is_supported() +} + +pub fn is_registered() -> bool { + platform::is_registered() +} + +pub fn runtime_status() -> FnShortcutRuntimeStatus { + platform::runtime_status() +} diff --git a/src-tauri/src/voice/mod.rs b/src-tauri/src/voice/mod.rs index 84bbcebdb..5af0c1f96 100644 --- a/src-tauri/src/voice/mod.rs +++ b/src-tauri/src/voice/mod.rs @@ -9,6 +9,7 @@ pub mod asr_service; pub mod commands; pub mod config; +pub mod fn_shortcut; pub mod output_service; pub mod processor; pub mod recording_service; @@ -17,6 +18,17 @@ pub mod window; use tauri::AppHandle; +pub(crate) fn register_fn_shortcut_if_supported(app: &AppHandle) { + if !fn_shortcut::is_supported() { + tracing::info!("[语音输入] 当前平台不支持 Fn 按住录音，使用普通快捷键回退"); + return; + } + + if let Err(error) = fn_shortcut::register(app) { + tracing::warn!("[语音输入] Fn 按住录音监听注册失败: {}", error); + } +} + /// 初始化语音输入模块 pub fn init(app: &AppHandle) -> Result<(), String> { // 加载配置 @@ -30,6 +42,7 @@ pub fn init(app: &AppHandle) -> Result<(), String> { // 注册全局快捷键 shortcut::register(app, &config.shortcut)?; + register_fn_shortcut_if_supported(app); // 注册翻译快捷键（如果配置了） if let Some(translate_shortcut) = &config.translate_shortcut { @@ -62,6 +75,9 @@ pub fn cleanup(app: &AppHandle) -> Result<(), String> { // 注销翻译快捷键 let _ = shortcut::unregister_translate(app); + // 注销 Fn 监听 + let _ = fn_shortcut::unregister(); + // 关闭悬浮窗口 window::close_voice_window(app)?; diff --git a/src-tauri/tauri.conf.headless.json b/src-tauri/tauri.conf.headless.json index 5aa589045..009acd511 100644 --- a/src-tauri/tauri.conf.headless.json +++ b/src-tauri/tauri.conf.headless.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Lime", - "version": "1.24.0", + "version": "1.25.0", "identifier": "com.limecloud.lime.headless", "build": { "beforeDevCommand": "npm run dev:web-bridge", diff --git a/src-tauri/tauri.conf.json b/src-tauri/tauri.conf.json index 5a53fec9b..7e6aa0cd0 100644 --- a/src-tauri/tauri.conf.json +++ b/src-tauri/tauri.conf.json @@ -1,7 +1,7 @@ { "$schema": "https://schema.tauri.app/config/2", "productName": "Lime", - "version": "1.24.0", + "version": "1.25.0", "identifier": "com.limecloud.lime", "build": { "beforeDevCommand": "node scripts/start-tauri-dev-server.mjs", diff --git a/src/App.tsx b/src/App.tsx index aaaf951c2..6a5007828 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -10,7 +10,6 @@ import React, { Suspense, lazy, useState, useCallback } from "react"; import styled from "styled-components"; -import { getCurrentWindow } from "@tauri-apps/api/window"; import { withI18nPatch } from "./i18n/withI18nPatch"; import { AppPageContent } from "./components/AppPageContent"; import { SplashScreen } from "./components/SplashScreen"; @@ -47,6 +46,8 @@ import { resolveWebsiteOpenNavigation } from "./lib/deepLink/websiteLaunch"; import { toast } from "sonner"; import { SettingsTabs } from "./types/settings"; import { hasTauriInvokeCapability } from "./lib/tauri-runtime"; +import { shouldReserveMacWindowControls } from "./lib/windowControls"; +import { startWindowDragFromMouseEvent } from "./lib/windowDrag"; const AppContainer = styled.div` display: flex; @@ -66,15 +67,42 @@ const MainContent = styled.main<{ $withSidebarGap?: boolean }>` background: var(--lime-app-bg, hsl(var(--background))); `; -const WindowDragRegion = styled.div` +const WINDOW_DRAG_TOP_HEIGHT = 30; +const WINDOW_DRAG_EDGE_WIDTH = 8; +const WINDOW_DRAG_DEFAULT_SAFE_LEFT = 160; +const WINDOW_DRAG_MAC_SAFE_LEFT = 92; + +const WindowDragLayer = styled.div` position: fixed; - top: 0; - left: 160px; - right: 0; - height: 28px; + inset: 0; z-index: 1000; - background: transparent; + pointer-events: none; +`; + +const WindowTopDragRegion = styled.div<{ $reserveMacWindowControls?: boolean }>` + position: absolute; + top: 0; + left: ${({ $reserveMacWindowControls }) => + $reserveMacWindowControls + ? `${WINDOW_DRAG_MAC_SAFE_LEFT}px` + : `${WINDOW_DRAG_DEFAULT_SAFE_LEFT}px`}; + right: 0; + height: ${WINDOW_DRAG_TOP_HEIGHT}px; + pointer-events: auto; user-select: none; + app-region: drag; + -webkit-app-region: drag; +`; + +const WindowSideDragRegion = styled.div<{ $side: "left" | "right" }>` + position: absolute; + top: ${WINDOW_DRAG_TOP_HEIGHT}px; + bottom: 0; + ${({ $side }) => $side}: 0; + width: ${WINDOW_DRAG_EDGE_WIDTH}px; + pointer-events: auto; + user-select: none; + app-region: drag; -webkit-app-region: drag; `; @@ -116,6 +144,7 @@ const pageLoadingFallback = ( function AppContent() { const hasTauriDesktopRuntime = hasTauriInvokeCapability(); + const reserveMacWindowControls = shouldReserveMacWindowControls(); const [showSplash, setShowSplash] = useState(true); const { currentPage, @@ -297,21 +326,10 @@ function AppContent() { }, [completeOnboarding]); const handleWindowDragStart = useCallback( - async (event: React.MouseEvent) => { - if (!hasTauriDesktopRuntime || event.button !== 0) { - return; - } - - event.preventDefault(); - - try { - const currentWindow = getCurrentWindow(); - await currentWindow.startDragging(); - } catch (error) { - console.warn("[窗口] 主窗口拖拽启动失败:", error); - } + (event: React.MouseEvent) => { + void startWindowDragFromMouseEvent(event, { source: "app_shell" }); }, - [hasTauriDesktopRuntime], + [], ); if (showSplash) { @@ -335,10 +353,26 @@ function AppContent() { {hasTauriDesktopRuntime ? ( - + ) : null} {shouldShowAppSidebar && ( )} - + { + void startWindowDragFromMouseEvent(event, { + allowDescendantTargets: false, + source: "main_content", + }); + }} + > void) => { task(); @@ -82,6 +85,7 @@ vi.mock("@/lib/api/pluginUI", () => ({ })); vi.mock("@/lib/api/agentRuntime", () => ({ + deleteAgentRuntimeSession: mockDeleteAgentRuntimeSession, listAgentRuntimeSessions: mockListAgentRuntimeSessions, updateAgentRuntimeSession: mockUpdateAgentRuntimeSession, })); @@ -164,6 +168,41 @@ async function flushEffects(times = 1) { } } +function setInputValue(input: HTMLInputElement, value: string) { + const prototype = Object.getPrototypeOf(input) as HTMLInputElement; + const valueSetter = Object.getOwnPropertyDescriptor(prototype, "value")?.set; + if (valueSetter) { + valueSetter.call(input, value); + } else { + input.value = value; + } + input.dispatchEvent(new Event("input", { bubbles: true })); +} + +async function openConversationMenu(title: string) { + await act(async () => { + document + .querySelector( + `button[aria-label="打开 ${title} 操作菜单"]`, + ) + ?.click(); + await Promise.resolve(); + }); + + return document.body.querySelector( + '[data-testid="app-sidebar-conversation-menu"]', + ); +} + +async function clickConversationMenuItem(testId: string) { + await act(async () => { + document.body + .querySelector(`[data-testid="${testId}"]`) + ?.click(); + await Promise.resolve(); + }); +} + async function openAccountMenu(container: HTMLElement) { await act(async () => { container @@ -247,6 +286,7 @@ describe("AppSidebar", () => { mockGetPluginsForSurface.mockResolvedValue([]); mockListAgentRuntimeSessions.mockResolvedValue([]); mockUpdateAgentRuntimeSession.mockResolvedValue(undefined); + mockDeleteAgentRuntimeSession.mockResolvedValue(undefined); mockLogoutClient.mockResolvedValue(undefined); mockGetConfiguredOemCloudTarget.mockReturnValue({ baseUrl: "https://user.limeai.run", @@ -1037,13 +1077,14 @@ describe("AppSidebar", () => { } }); - it("任务中心内点击已有会话应先通知本地标签栏，再继续导航", async () => { + it("任务中心内点击已有会话应交给本地标签栏，不重复触发导航", async () => { const onNavigate = vi.fn(); const receivedDetails: unknown[] = []; const listener = (event: Event) => { receivedDetails.push( event instanceof CustomEvent ? event.detail : undefined, ); + event.preventDefault(); }; window.addEventListener(TASK_CENTER_OPEN_TASK_EVENT, listener); mockListAgentRuntimeSessions.mockResolvedValue([ @@ -1084,19 +1125,462 @@ describe("AppSidebar", () => { source: "sidebar", }, ]); - expect(onNavigate).toHaveBeenCalledWith( - "agent", - expect.objectContaining({ - agentEntry: "claw", - projectId: "project-1", - initialSessionId: "session-target", - }), - ); + expect(onNavigate).not.toHaveBeenCalled(); } finally { window.removeEventListener(TASK_CENTER_OPEN_TASK_EVENT, listener); } }); + it("新建任务首页点击已有会话应进入对应历史对话", async () => { + const onNavigate = vi.fn(); + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-target", + name: "目标历史会话", + created_at: 1713000000, + updated_at: 1713000600, + archived_at: null, + workspace_id: "project-1", + messages_count: 3, + }, + ]); + + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "new-task", + projectId: "project-1", + } as AgentPageParams, + onNavigate, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector('button[title="目标历史会话"]') + ?.click(); + await Promise.resolve(); + }); + + expect(onNavigate).toHaveBeenCalledWith( + "agent", + expect.objectContaining({ + agentEntry: "claw", + projectId: "project-1", + initialSessionId: "session-target", + }), + ); + }); + + it("任务中心内悬停已有会话应延迟通知本地预取旧会话", async () => { + vi.useFakeTimers(); + const receivedDetails: unknown[] = []; + const listener = (event: Event) => { + receivedDetails.push( + event instanceof CustomEvent ? event.detail : undefined, + ); + }; + window.addEventListener(TASK_CENTER_PREFETCH_TASK_EVENT, listener); + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-prefetch", + name: "可预取历史会话", + created_at: 1713000000, + updated_at: 1713000600, + archived_at: null, + workspace_id: "project-1", + messages_count: 3, + }, + ]); + + try { + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "claw", + projectId: "project-1", + initialSessionId: "session-current", + } as AgentPageParams, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector('button[title="可预取历史会话"]') + ?.focus(); + await Promise.resolve(); + }); + + expect(receivedDetails).toEqual([]); + + act(() => { + vi.advanceTimersByTime(920); + }); + + expect(receivedDetails).toEqual([ + { + sessionId: "session-prefetch", + workspaceId: "project-1", + source: "conversation_shelf", + }, + ]); + } finally { + vi.useRealTimers(); + window.removeEventListener(TASK_CENTER_PREFETCH_TASK_EVENT, listener); + } + }); + + it("点击已有会话时不应先触发旧会话预取抢占切换链路", async () => { + vi.useFakeTimers(); + const receivedPrefetchDetails: unknown[] = []; + const listener = (event: Event) => { + receivedPrefetchDetails.push( + event instanceof CustomEvent ? event.detail : undefined, + ); + }; + window.addEventListener(TASK_CENTER_PREFETCH_TASK_EVENT, listener); + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-click", + name: "立即打开历史会话", + created_at: 1713000000, + updated_at: 1713000600, + archived_at: null, + workspace_id: "project-1", + messages_count: 3, + }, + ]); + + try { + const onNavigate = vi.fn(); + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "claw", + projectId: "project-1", + initialSessionId: "session-current", + } as AgentPageParams, + onNavigate, + }); + await flushEffects(2); + + const button = container.querySelector( + 'button[title="立即打开历史会话"]', + ); + + await act(async () => { + button?.focus(); + button?.click(); + await Promise.resolve(); + }); + + act(() => { + vi.advanceTimersByTime(200); + }); + + expect(receivedPrefetchDetails).toEqual([]); + expect(onNavigate).not.toHaveBeenCalled(); + } finally { + vi.useRealTimers(); + window.removeEventListener(TASK_CENTER_PREFETCH_TASK_EVENT, listener); + } + }); + + it("搜索按钮应打开标题搜索弹窗，并按会话标题过滤结果", async () => { + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-agent", + name: "写一篇AI Agent的公众号", + created_at: 1713000000, + updated_at: 1713000600, + archived_at: null, + workspace_id: "project-1", + messages_count: 3, + }, + { + id: "session-daily", + name: "啊啊啊啊啊", + created_at: 1712900000, + updated_at: 1712900600, + archived_at: null, + workspace_id: "project-1", + messages_count: 1, + }, + ]); + + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "new-task", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector( + '[data-testid="app-sidebar-search-button"]', + ) + ?.click(); + await Promise.resolve(); + }); + + const dialog = document.body.querySelector( + '[data-testid="app-sidebar-search-dialog"]', + ); + expect(dialog).not.toBeNull(); + expect(dialog?.textContent).toContain("新建对话"); + expect(dialog?.textContent).toContain("写一篇AI Agent的公众号"); + expect(dialog?.textContent).toContain("啊啊啊啊啊"); + + const input = document.body.querySelector( + '[data-testid="app-sidebar-search-input"]', + ); + expect(input).not.toBeNull(); + + await act(async () => { + setInputValue(input as HTMLInputElement, "Agent"); + await Promise.resolve(); + }); + + expect(dialog?.textContent).toContain("匹配结果"); + expect(dialog?.textContent).toContain("写一篇AI Agent的公众号"); + expect(dialog?.textContent).not.toContain("啊啊啊啊啊"); + + await act(async () => { + setInputValue(input as HTMLInputElement, "不存在"); + await Promise.resolve(); + }); + + expect(dialog?.textContent).toContain("没有匹配的对话标题"); + }); + + it("搜索弹窗应支持查看更多对话并展示后续结果", async () => { + const sessions = Array.from({ length: 12 }, (_, index) => { + const number = String(index + 1).padStart(2, "0"); + return { + id: `session-${number}`, + name: `对话 ${number}`, + created_at: 1713000000 + index, + updated_at: 1713000600 + index, + archived_at: null, + workspace_id: "project-1", + messages_count: 1, + }; + }); + mockListAgentRuntimeSessions.mockResolvedValue(sessions); + + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "new-task", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector( + '[data-testid="app-sidebar-search-button"]', + ) + ?.click(); + await Promise.resolve(); + }); + + let dialog = document.body.querySelector( + '[data-testid="app-sidebar-search-dialog"]', + ); + expect(dialog?.textContent).toContain("查看更多对话"); + expect(dialog?.textContent).toContain("对话 12"); + expect(dialog?.textContent).not.toContain("对话 02"); + + await act(async () => { + dialog + ?.querySelector( + '[data-testid="app-sidebar-search-more"]', + ) + ?.click(); + await Promise.resolve(); + }); + await flushEffects(2); + + dialog = document.body.querySelector( + '[data-testid="app-sidebar-search-dialog"]', + ); + expect(dialog?.textContent).toContain("对话 02"); + expect(dialog?.textContent).not.toContain("查看更多对话"); + expect(mockListAgentRuntimeSessions).toHaveBeenLastCalledWith({ + limit: 21, + workspaceId: "project-1", + }); + }); + + it("搜索结果点击应复用会话导航并关闭弹窗", async () => { + const onNavigate = vi.fn(); + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-target", + name: "目标历史会话", + created_at: 1713000000, + updated_at: 1713000600, + archived_at: null, + workspace_id: "project-1", + messages_count: 3, + }, + ]); + + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + projectId: "project-1", + } as AgentPageParams, + onNavigate, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector( + '[data-testid="app-sidebar-search-button"]', + ) + ?.click(); + await Promise.resolve(); + }); + + const dialog = document.body.querySelector( + '[data-testid="app-sidebar-search-dialog"]', + ); + + await act(async () => { + dialog + ?.querySelector('button[title="目标历史会话"]') + ?.click(); + await Promise.resolve(); + }); + + expect(onNavigate).toHaveBeenCalledWith( + "agent", + expect.objectContaining({ + agentEntry: "claw", + projectId: "project-1", + initialSessionId: "session-target", + }), + ); + expect( + document.body.querySelector('[data-testid="app-sidebar-search-dialog"]'), + ).toBeNull(); + }); + + it("搜索弹窗的新建对话入口应复用现有新建导航", async () => { + const onNavigate = vi.fn(); + mockListAgentRuntimeSessions.mockResolvedValue([]); + + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + projectId: "project-1", + } as AgentPageParams, + onNavigate, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector( + '[data-testid="app-sidebar-search-button"]', + ) + ?.click(); + await Promise.resolve(); + }); + + await act(async () => { + document.body + .querySelector( + '[data-testid="app-sidebar-search-new-conversation"]', + ) + ?.click(); + await Promise.resolve(); + }); + + expect(onNavigate).toHaveBeenCalledWith( + "agent", + expect.objectContaining({ + agentEntry: "new-task", + projectId: "project-1", + }), + ); + expect( + document.body.querySelector('[data-testid="app-sidebar-search-dialog"]'), + ).toBeNull(); + }); + + it("Meta/Ctrl + K 应打开搜索弹窗，Escape 应关闭弹窗", async () => { + mockListAgentRuntimeSessions.mockResolvedValue([]); + + mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "new-task", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + expect( + document.body.querySelector('[data-testid="app-sidebar-search-dialog"]'), + ).toBeNull(); + + await act(async () => { + window.dispatchEvent( + new KeyboardEvent("keydown", { + key: "k", + metaKey: true, + bubbles: true, + cancelable: true, + }), + ); + await Promise.resolve(); + }); + await flushEffects(); + + expect( + document.body.querySelector('[data-testid="app-sidebar-search-dialog"]'), + ).not.toBeNull(); + + await act(async () => { + document.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Escape", + bubbles: true, + cancelable: true, + }), + ); + await Promise.resolve(); + }); + + expect( + document.body.querySelector('[data-testid="app-sidebar-search-dialog"]'), + ).toBeNull(); + + await act(async () => { + window.dispatchEvent( + new KeyboardEvent("keydown", { + key: "k", + ctrlKey: true, + bubbles: true, + cancelable: true, + }), + ); + await Promise.resolve(); + }); + + expect( + document.body.querySelector('[data-testid="app-sidebar-search-dialog"]'), + ).not.toBeNull(); + }); + it("claw 页面不应再外露左侧工作台一级入口", async () => { const onNavigate = vi.fn(); mockListAgentRuntimeSessions.mockResolvedValue([ @@ -1181,7 +1665,7 @@ describe("AppSidebar", () => { expect(container.textContent).not.toContain("归档会话"); expect(mockListAgentRuntimeSessions).toHaveBeenCalledTimes(1); expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ - limit: 37, + limit: 11, workspaceId: "project-1", }); @@ -1194,7 +1678,7 @@ describe("AppSidebar", () => { const recentConversationList = container.querySelector( '[data-testid="app-sidebar-recent-conversations"]', ); - const archivedToggle = container.querySelector( + const archivedToggle = conversationShelf?.querySelector( 'button[aria-expanded="false"]', ) as HTMLButtonElement | null; @@ -1231,7 +1715,7 @@ describe("AppSidebar", () => { ).toBe("auto"); expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ archivedOnly: true, - limit: 17, + limit: 9, workspaceId: "project-1", }); }); @@ -1280,7 +1764,7 @@ describe("AppSidebar", () => { }), ); expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ - limit: 37, + limit: 11, workspaceId: "project-1", }); }); @@ -1315,7 +1799,7 @@ describe("AppSidebar", () => { expect(container.querySelector('button[title="会话 24"]')).toBeNull(); expect(container.textContent).toContain("查看更多对话"); expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ - limit: 37, + limit: 11, workspaceId: "project-1", }); @@ -1359,7 +1843,7 @@ describe("AppSidebar", () => { expect(mounted.container.textContent).not.toContain("正在加载对话"); expect(mockListAgentRuntimeSessions).toHaveBeenCalledTimes(1); expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ - limit: 37, + limit: 11, workspaceId: "project-1", }); @@ -1384,7 +1868,7 @@ describe("AppSidebar", () => { expect(mounted.container.textContent).not.toContain("正在加载对话"); expect(mockListAgentRuntimeSessions).toHaveBeenCalledTimes(2); expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ - limit: 37, + limit: 11, workspaceId: "project-2", }); }); @@ -1446,12 +1930,178 @@ describe("AppSidebar", () => { expect(mockListAgentRuntimeSessions).toHaveBeenCalledTimes(2); expect(mockListAgentRuntimeSessions).toHaveBeenLastCalledWith({ - limit: 37, + limit: 11, workspaceId: "project-1", }); }); - it("点击导航栏归档动作时应走统一 session update 命令", async () => { + it("点击会话菜单归档动作时应走统一 session update 命令", async () => { + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-recent", + name: "最近会话", + created_at: 1714000000, + updated_at: 1714000600, + archived_at: null, + workspace_id: "project-1", + }, + ]); + + mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "claw", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + const menu = await openConversationMenu("最近会话"); + expect(menu).not.toBeNull(); + expect(menu?.textContent).toContain("重命名"); + expect(menu?.textContent).toContain("收藏"); + expect(menu?.textContent).toContain("归档"); + expect(menu?.textContent).toContain("多选"); + expect(menu?.textContent).toContain("删除"); + expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ + limit: 11, + workspaceId: "project-1", + }); + + await clickConversationMenuItem("app-sidebar-conversation-menu-archive"); + + expect(mockUpdateAgentRuntimeSession).toHaveBeenCalledWith({ + session_id: "session-recent", + archived: true, + }); + }); + + it("归档会话菜单应展示恢复动作并走统一 session update 命令", async () => { + mockListAgentRuntimeSessions.mockImplementation( + async (options?: { + archivedOnly?: boolean; + includeArchived?: boolean; + limit?: number; + workspaceId?: string; + }) => + options?.archivedOnly + ? [ + { + id: "session-archived", + name: "归档会话", + created_at: 1713000000, + updated_at: 1713000600, + archived_at: 1713003600, + workspace_id: "project-1", + }, + ] + : [], + ); + + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "claw", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector('button[aria-expanded="false"]') + ?.click(); + await Promise.resolve(); + }); + await flushEffects(2); + + const menu = await openConversationMenu("归档会话"); + expect(menu).not.toBeNull(); + expect(menu?.textContent).toContain("重命名"); + expect(menu?.textContent).toContain("收藏"); + expect(menu?.textContent).toContain("恢复"); + expect(menu?.textContent).toContain("多选"); + expect(menu?.textContent).toContain("删除"); + expect(menu?.textContent).not.toContain("归档"); + + await clickConversationMenuItem("app-sidebar-conversation-menu-archive"); + + expect(mockUpdateAgentRuntimeSession).toHaveBeenCalledWith({ + session_id: "session-archived", + archived: false, + }); + }); + + it("会话菜单应支持重命名并同步更新 session 名称", async () => { + vi.spyOn(window, "prompt").mockReturnValue("重命名后的会话"); + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-recent", + name: "最近会话", + created_at: 1714000000, + updated_at: 1714000600, + archived_at: null, + workspace_id: "project-1", + }, + ]); + + mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "claw", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + await openConversationMenu("最近会话"); + await clickConversationMenuItem("app-sidebar-conversation-menu-rename"); + await flushEffects(2); + + expect(window.prompt).toHaveBeenCalledWith("重命名对话", "最近会话"); + expect(mockUpdateAgentRuntimeSession).toHaveBeenCalledWith({ + session_id: "session-recent", + name: "重命名后的会话", + }); + expect(mockToastSuccess).toHaveBeenCalledWith("已重命名对话"); + }); + + it("会话菜单应支持删除并在执行前要求确认", async () => { + vi.spyOn(window, "confirm").mockReturnValue(true); + mockListAgentRuntimeSessions.mockResolvedValue([ + { + id: "session-recent", + name: "最近会话", + created_at: 1714000000, + updated_at: 1714000600, + archived_at: null, + workspace_id: "project-1", + }, + ]); + + mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "claw", + projectId: "project-1", + } as AgentPageParams, + }); + await flushEffects(2); + + await openConversationMenu("最近会话"); + await clickConversationMenuItem("app-sidebar-conversation-menu-delete"); + await flushEffects(2); + + expect(window.confirm).toHaveBeenCalledWith( + "确定要删除“最近会话”吗？删除后无法恢复。", + ); + expect(mockDeleteAgentRuntimeSession).toHaveBeenCalledWith( + "session-recent", + ); + expect(mockToastSuccess).toHaveBeenCalledWith("已删除对话"); + }); + + it("会话菜单的收藏与多选应提供即时反馈", async () => { mockListAgentRuntimeSessions.mockResolvedValue([ { id: "session-recent", @@ -1472,25 +2122,40 @@ describe("AppSidebar", () => { }); await flushEffects(2); - const archiveButton = container.querySelector( - 'button[aria-label="归档 最近会话"]', - ) as HTMLButtonElement | null; + await openConversationMenu("最近会话"); + await clickConversationMenuItem("app-sidebar-conversation-menu-favorite"); - expect(archiveButton).not.toBeNull(); - expect(mockListAgentRuntimeSessions).toHaveBeenCalledWith({ - limit: 37, - workspaceId: "project-1", - }); + expect( + container.querySelector( + '[data-testid="app-sidebar-conversation-favorite-badge"]', + ), + ).not.toBeNull(); + + const favoriteMenu = await openConversationMenu("最近会话"); + expect(favoriteMenu?.textContent).toContain("取消收藏"); + + await clickConversationMenuItem( + "app-sidebar-conversation-menu-multiselect", + ); + + expect( + container.querySelector( + '[data-testid="app-sidebar-conversation-multiselect-toolbar"]', + )?.textContent, + ).toContain("已选择 1 个对话"); await act(async () => { - archiveButton?.click(); + container + .querySelector('button[title="最近会话"]') + ?.click(); await Promise.resolve(); }); - expect(mockUpdateAgentRuntimeSession).toHaveBeenCalledWith({ - session_id: "session-recent", - archived: true, - }); + expect( + container.querySelector( + '[data-testid="app-sidebar-conversation-multiselect-toolbar"]', + )?.textContent, + ).toContain("已选择 0 个对话"); }); it("显式开启后应显示可选系统扩展入口", async () => { @@ -1669,14 +2334,22 @@ describe("AppSidebar", () => { expect(popover?.textContent).toContain("浅色"); expect(popover?.textContent).toContain("深色"); expect(popover?.textContent).toContain("跟随系统"); - expect(popover?.textContent).toContain("Lime 经典"); - expect(popover?.textContent).toContain("森林"); - expect(popover?.textContent).toContain("海雾"); - expect(popover?.textContent).toContain("砂岩"); + expect(popover?.textContent).toContain("随机"); + expect(popover?.textContent).toContain("墨绿"); + expect(popover?.textContent).toContain("自然"); + expect(popover?.textContent).toContain("海洋"); + expect(popover?.textContent).toContain("复古"); + expect(popover?.textContent).toContain("霓虹"); + expect(popover?.textContent).toContain("青柠"); + expect(popover?.textContent).toContain("黄昏"); + expect(popover?.textContent).toContain("极简"); + expect(popover?.textContent).toContain("活力"); + expect(popover?.textContent).toContain("文艺"); + expect(popover?.textContent).toContain("奢华"); await act(async () => { container - .querySelector('button[aria-label="切换配色为海雾"]') + .querySelector('button[aria-label="切换配色为海洋"]') ?.click(); await Promise.resolve(); }); @@ -1698,6 +2371,43 @@ describe("AppSidebar", () => { expect(document.documentElement.classList.contains("dark")).toBe(true); }); + it("外观弹层的随机配色应持久化到一个真实预设", async () => { + const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0); + + try { + const container = mountSidebarContainer({ + currentPage: "agent", + currentPageParams: { + agentEntry: "new-task", + } as AgentPageParams, + }); + await flushEffects(2); + + await act(async () => { + container + .querySelector('button[aria-label="快速切换外观"]') + ?.click(); + await Promise.resolve(); + }); + + await act(async () => { + container + .querySelector('button[aria-label="随机切换配色"]') + ?.click(); + await Promise.resolve(); + }); + + expect(localStorage.getItem(LIME_COLOR_SCHEME_STORAGE_KEY)).toBe( + "lime-forest", + ); + expect(document.documentElement.dataset.limeColorScheme).toBe( + "lime-forest", + ); + } finally { + randomSpy.mockRestore(); + } + }); + it("桌宠入口开启后，进入 companion 视图应高亮桌宠", async () => { mockGetConfig.mockResolvedValue({ navigation: { diff --git a/src/components/AppSidebar.tsx b/src/components/AppSidebar.tsx index 69a5a87c2..1ad47f99e 100644 --- a/src/components/AppSidebar.tsx +++ b/src/components/AppSidebar.tsx @@ -12,6 +12,7 @@ import { useRef, useCallback, type ReactElement, + type MouseEvent as ReactMouseEvent, } from "react"; import styled from "styled-components"; import { @@ -27,10 +28,13 @@ import { Languages, LogIn, LogOut, + MessageSquare, + MessageSquarePlus, Monitor, Moon, Palette, RefreshCw, + Shuffle, Sun, Search, PanelLeftClose, @@ -54,10 +58,12 @@ import { buildHomeAgentParams, } from "@/lib/workspace/navigation"; import { + notifyTaskCenterTaskPrefetch, notifyTaskCenterTaskOpen, requestTaskCenterDraftTask, } from "@/components/agent/chat/taskCenterDraftTaskEvents"; import { + deleteAgentRuntimeSession, listAgentRuntimeSessions, updateAgentRuntimeSession, type AsterSessionInfo, @@ -81,6 +87,10 @@ import { hasTauriInvokeCapability } from "@/lib/tauri-runtime"; import { LIME_BRAND_LOGO_SRC, LIME_BRAND_NAME } from "@/lib/branding"; import { scheduleMinimumDelayIdleTask } from "@/lib/utils/scheduleMinimumDelayIdleTask"; import { AppSidebarConversationShelf } from "@/components/app-sidebar/AppSidebarConversationShelf"; +import { + formatSidebarSessionMeta, + resolveSidebarSessionTitle, +} from "@/components/app-sidebar/sidebarSessionFormatting"; import { shouldReserveMacWindowControls } from "@/lib/windowControls"; import { clearStoredOemCloudSessionState, @@ -149,6 +159,7 @@ interface AppSidebarProps { requestedPage?: Page; requestedPageParams?: PageParams; onNavigate: (page: Page, params?: PageParams) => void; + onStartWindowDrag?: (event: ReactMouseEvent) => void; } type SidebarNavItem = SidebarNavItemDefinition; @@ -158,8 +169,8 @@ const SIDEBAR_PLUGIN_CENTER_NAV_ITEM_ID = "plugins"; const SIDEBAR_PLUGIN_IDLE_TIMEOUT_MS = 1200; const SIDEBAR_PLUGIN_BROWSER_IDLE_TIMEOUT_MS = 6000; const SIDEBAR_RECENT_SESSION_PAGE_SIZE = 10; -const SIDEBAR_RECENT_SESSION_PREFETCH_LIMIT = 37; const SIDEBAR_ARCHIVED_SESSION_PAGE_SIZE = 8; +const SIDEBAR_SEARCH_RESULT_LIMIT = 8; const SIDEBAR_SESSION_ENTRY_REFRESH_DEFER_MS = 12_000; const APP_SIDEBAR_LANGUAGE_OPTIONS: Array<{ @@ -184,7 +195,7 @@ function buildSidebarSessionRequestLimit( pageSize: number, ): number { const normalizedVisibleCount = Math.max(visibleCount, pageSize); - return normalizedVisibleCount + pageSize + 1; + return normalizedVisibleCount + 1; } function splitSidebarSessionResult(params: { @@ -196,7 +207,7 @@ function splitSidebarSessionResult(params: { hasMore: boolean; } { const { sessions, visibleCount, pageSize } = params; - const targetCount = Math.max(visibleCount, pageSize) + pageSize; + const targetCount = Math.max(visibleCount, pageSize); return { sessions: sessions.slice(0, targetCount), hasMore: sessions.length > targetCount, @@ -496,6 +507,292 @@ const SearchButton = styled.button<{ $collapsed?: boolean }>` } `; +const SidebarSearchSurface = styled.div` + display: flex; + min-height: min(620px, calc(100vh - 96px)); + max-height: calc(100vh - 96px); + flex-direction: column; + overflow: hidden; + border-radius: 28px; + border: 1px solid var(--lime-card-subtle-border, rgba(226, 240, 226, 0.9)); + background: var(--lime-surface, #ffffff); + color: var(--lime-text, #1a3b2b); + box-shadow: + 0 32px 80px rgba(15, 23, 42, 0.18), + 0 1px 0 rgba(255, 255, 255, 0.78) inset; +`; + +const SidebarSearchHeader = styled.div` + display: flex; + align-items: center; + gap: 12px; + padding: 22px 26px 18px; + color: var(--lime-text-muted, #6b826b); + + svg { + width: 22px; + height: 22px; + flex-shrink: 0; + } +`; + +const SidebarSearchInput = styled.input` + min-width: 0; + flex: 1; + border: none; + background: transparent; + color: var(--lime-text-strong, #0f172a); + font-size: 19px; + font-weight: 650; + outline: none; + + &::placeholder { + color: var(--lime-text-soft, #9aa89a); + font-weight: 600; + } +`; + +const SidebarSearchShortcut = styled.div` + display: inline-flex; + align-items: center; + gap: 4px; + flex-shrink: 0; +`; + +const SidebarSearchKey = styled.kbd` + min-width: 28px; + height: 28px; + border-radius: 9px; + border: 1px solid var(--lime-card-subtle-border, rgba(226, 240, 226, 0.86)); + background: var(--lime-muted-surface, #f5f8f3); + color: var(--lime-text-muted, #6b826b); + display: inline-flex; + align-items: center; + justify-content: center; + padding: 0 7px; + font-family: inherit; + font-size: 13px; + font-weight: 800; + line-height: 1; + box-shadow: inset 0 1px 0 rgba(255, 255, 255, 0.76); +`; + +const SidebarSearchCloseButton = styled.button` + width: 34px; + height: 34px; + border: none; + border-radius: 12px; + display: inline-flex; + align-items: center; + justify-content: center; + background: transparent; + color: var(--lime-text-muted, #6b826b); + cursor: pointer; + transition: + background-color 0.16s ease, + color 0.16s ease; + + &:hover { + background: var(--lime-surface-hover, #f4fdf4); + color: var(--lime-text-strong, #0f172a); + } + + svg { + width: 22px; + height: 22px; + } +`; + +const SidebarSearchDivider = styled.div` + height: 1px; + margin: 0 26px; + background: var(--lime-divider-subtle, rgba(226, 240, 226, 0.86)); +`; + +const SidebarSearchBody = styled.div` + display: flex; + min-height: 0; + flex: 1; + flex-direction: column; + gap: 16px; + overflow-y: auto; + padding: 20px 26px 28px; + + &::-webkit-scrollbar { + width: 6px; + } + + &::-webkit-scrollbar-track { + background: transparent; + } + + &::-webkit-scrollbar-thumb { + border-radius: 999px; + background: var(--lime-divider-strong, rgba(180, 196, 180, 0.7)); + } +`; + +const SidebarSearchCreateButton = styled.button` + min-height: 58px; + width: 100%; + border: 1px solid transparent; + border-radius: 18px; + background: var(--lime-surface-hover, #f4f7f2); + color: var(--lime-text-strong, #0f172a); + display: flex; + align-items: center; + gap: 12px; + padding: 0 18px; + cursor: pointer; + text-align: left; + transition: + border-color 0.16s ease, + background-color 0.16s ease, + transform 0.16s ease; + + &:hover { + border-color: var(--lime-card-subtle-border, #bbf7d0); + background: var(--lime-surface, #ffffff); + transform: translateY(-1px); + } + + svg { + width: 22px; + height: 22px; + flex-shrink: 0; + color: var(--lime-text-muted, #6b826b); + } +`; + +const SidebarSearchCreateText = styled.span` + flex: 1; + min-width: 0; + font-size: 16px; + font-weight: 760; +`; + +const SidebarSearchEnterHint = styled.span` + flex-shrink: 0; + color: var(--lime-text-soft, #9aa89a); + font-size: 20px; + font-weight: 800; +`; + +const SidebarSearchSectionLabel = styled.div` + padding: 0 16px; + color: var(--lime-text-soft, #9aa89a); + font-size: 13px; + font-weight: 760; +`; + +const SidebarSearchResultList = styled.div` + display: flex; + flex-direction: column; + gap: 4px; +`; + +const SidebarSearchResultButton = styled.button<{ $active?: boolean }>` + width: 100%; + min-height: 54px; + border: 1px solid + ${({ $active }) => + $active + ? "var(--lime-card-subtle-border, #bbf7d0)" + : "transparent"}; + border-radius: 16px; + background: ${({ $active }) => + $active ? "var(--lime-surface-hover, #f4fdf4)" : "transparent"}; + color: var(--lime-text, #1a3b2b); + display: flex; + align-items: center; + gap: 12px; + padding: 0 16px; + cursor: pointer; + text-align: left; + transition: + border-color 0.16s ease, + background-color 0.16s ease, + color 0.16s ease; + + &:hover { + border-color: var(--lime-card-subtle-border, rgba(187, 247, 208, 0.92)); + background: var(--lime-surface-hover, #f4fdf4); + color: var(--lime-text-strong, #0f172a); + } + + svg { + width: 20px; + height: 20px; + flex-shrink: 0; + color: var(--lime-text-muted, #6b826b); + } +`; + +const SidebarSearchResultTitle = styled.span` + flex: 1; + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + font-size: 15px; + font-weight: 700; +`; + +const SidebarSearchResultMeta = styled.span` + flex-shrink: 0; + color: var(--lime-text-soft, #9aa89a); + font-size: 14px; + font-weight: 650; +`; + +const SidebarSearchEmptyState = styled.div` + display: flex; + min-height: 180px; + align-items: center; + justify-content: center; + border-radius: 18px; + border: 1px dashed var(--lime-card-subtle-border, rgba(226, 240, 226, 0.9)); + color: var(--lime-text-muted, #6b826b); + background: var(--lime-muted-surface, #f8faf7); + font-size: 14px; + font-weight: 680; +`; + +const SidebarSearchMoreButton = styled.button` + min-height: 42px; + border: 1px solid var(--lime-card-subtle-border, rgba(226, 240, 226, 0.86)); + border-radius: 14px; + background: var(--lime-surface, #ffffff); + color: var(--lime-text-muted, #6b826b); + display: inline-flex; + align-items: center; + justify-content: center; + gap: 7px; + cursor: pointer; + font-size: 13px; + font-weight: 760; + transition: + border-color 0.16s ease, + background-color 0.16s ease, + color 0.16s ease; + + &:hover:not(:disabled) { + border-color: var(--lime-card-subtle-border, #bbf7d0); + background: var(--lime-surface-hover, #f4fdf4); + color: var(--lime-text-strong, #0f172a); + } + + &:disabled { + cursor: not-allowed; + opacity: 0.62; + } + + svg { + width: 15px; + height: 15px; + } +`; + const MenuScroll = styled.div` flex: 1; min-height: 0; @@ -964,8 +1261,10 @@ const AppearancePopover = styled.div` left: calc(100% + 12px); bottom: -2px; z-index: 70; - width: 228px; - max-width: min(228px, calc(100vw - 24px)); + width: 252px; + max-width: min(252px, calc(100vw - 24px)); + max-height: min(560px, calc(100vh - 24px)); + overflow-y: auto; border-radius: 18px; border: 1px solid var(--lime-card-subtle-border, rgba(226, 240, 226, 0.92)); background: var(--lime-card-subtle, var(--lime-surface, #ffffff)); @@ -1127,6 +1426,43 @@ const ColorSchemeList = styled.div` gap: 6px; `; +const RandomColorSchemeButton = styled.button` + grid-column: 1 / -1; + min-height: 40px; + border-radius: 13px; + border: 1px solid var(--lime-card-subtle-border, rgba(226, 240, 226, 0.82)); + background: var(--lime-surface, #ffffff); + color: var(--lime-text, #1a3b2b); + cursor: pointer; + display: inline-flex; + align-items: center; + justify-content: center; + gap: 7px; + padding: 8px 10px; + font-size: 12px; + font-weight: 750; + transition: + border-color 0.16s ease, + background 0.16s ease, + color 0.16s ease; + + &:hover { + border-color: var(--lime-card-subtle-border, #bbf7d0); + background: var( + --lime-chrome-tab-hover, + var(--lime-surface-hover, #f4fdf4) + ); + color: var(--lime-text-strong, #0f172a); + } + + svg { + width: 14px; + height: 14px; + flex-shrink: 0; + color: var(--lime-brand-strong, #166534); + } +`; + const ColorSchemeButton = styled.button<{ $active?: boolean }>` display: flex; position: relative; @@ -1824,6 +2160,23 @@ function buildVisibleSidebarSessions(params: { return [...visibleSessions.slice(0, Math.max(limit - 1, 0)), currentSession]; } +function normalizeSidebarSearchText(value: string): string { + return value.trim().toLocaleLowerCase(); +} + +function matchesSidebarSessionTitle( + session: AsterSessionInfo, + normalizedQuery: string, +): boolean { + if (!normalizedQuery) { + return true; + } + + return normalizeSidebarSearchText( + resolveSidebarSessionTitle(session), + ).includes(normalizedQuery); +} + function resolveAccountDisplayName( sessionState: OemCloudStoredSessionState | null, ): string { @@ -1954,6 +2307,7 @@ export function AppSidebar({ requestedPage, requestedPageParams, onNavigate, + onStartWindowDrag, }: AppSidebarProps) { const activePage = requestedPage ?? currentPage; const activePageParams = requestedPageParams ?? currentPageParams; @@ -2033,6 +2387,8 @@ export function AppSidebar({ const [inviteLoading, setInviteLoading] = useState(false); const [inviteError, setInviteError] = useState(null); const [inviteReloadKey, setInviteReloadKey] = useState(0); + const [sidebarSearchOpen, setSidebarSearchOpen] = useState(false); + const [sidebarSearchQuery, setSidebarSearchQuery] = useState(""); const { setLanguage: setI18nLanguage } = useI18nPatch(); const [enabledNavItems, setEnabledNavItems] = useState( @@ -2064,6 +2420,7 @@ export function AppSidebar({ useState(SIDEBAR_ARCHIVED_SESSION_PAGE_SIZE); const [archivedSessionsCollapsed, setArchivedSessionsCollapsed] = useState(true); + const sidebarSearchInputRef = useRef(null); const appearanceControlRef = useRef(null); const accountControlRef = useRef(null); const reserveWindowControls = shouldReserveMacWindowControls(); @@ -2077,6 +2434,56 @@ export function AppSidebar({ currentSessionId, ); + const openSidebarSearchDialog = useCallback(() => { + setAccountMenuOpen(false); + setLanguageMenuOpen(false); + setAppearancePopoverOpen(false); + setSidebarSearchOpen(true); + }, []); + + const closeSidebarSearchDialog = useCallback(() => { + setSidebarSearchOpen(false); + setSidebarSearchQuery(""); + }, []); + + useEffect(() => { + if (typeof window === "undefined") { + return; + } + + const handleSearchShortcut = (event: KeyboardEvent) => { + if ( + event.key.toLowerCase() !== "k" || + (!event.metaKey && !event.ctrlKey) + ) { + return; + } + + event.preventDefault(); + openSidebarSearchDialog(); + }; + + window.addEventListener("keydown", handleSearchShortcut); + return () => { + window.removeEventListener("keydown", handleSearchShortcut); + }; + }, [openSidebarSearchDialog]); + + useEffect(() => { + if (!sidebarSearchOpen || typeof window === "undefined") { + return; + } + + const focusTimer = window.setTimeout(() => { + sidebarSearchInputRef.current?.focus(); + sidebarSearchInputRef.current?.select(); + }, 0); + + return () => { + window.clearTimeout(focusTimer); + }; + }, [sidebarSearchOpen]); + useEffect(() => { const loadNavConfig = async () => { try { @@ -2499,7 +2906,8 @@ export function AppSidebar({ !collapsed && !(activePage === "agent" && activeAgentPageParams?.immersiveHome); const shouldLoadWorkspaceScopedConversations = - shouldShowConversationList && Boolean(currentProjectId); + (shouldShowConversationList || sidebarSearchOpen) && + Boolean(currentProjectId); const shouldShowSessionLoadingState = sidebarSessionsLoading && sidebarSessions.length === 0; const shouldShowArchivedSessionLoadingState = @@ -2522,12 +2930,10 @@ export function AppSidebar({ }, [currentProjectId]); const recentSessionRequestLimit = useMemo(() => { - const requestLimit = buildSidebarSessionRequestLimit( + return buildSidebarSessionRequestLimit( recentSessionsVisibleCount, SIDEBAR_RECENT_SESSION_PAGE_SIZE, ); - - return Math.max(requestLimit, SIDEBAR_RECENT_SESSION_PREFETCH_LIMIT); }, [recentSessionsVisibleCount]); const archivedSessionRequestLimit = useMemo( () => @@ -2810,6 +3216,45 @@ export function AppSidebar({ const hasMoreArchivedSidebarSessions = archivedSessionEntriesHasMore || archivedSessionsVisibleCount < archivedSidebarSessions.length; + const normalizedSidebarSearchQuery = useMemo( + () => normalizeSidebarSearchText(sidebarSearchQuery), + [sidebarSearchQuery], + ); + const sidebarSearchResultLimit = Math.max( + recentSessionsVisibleCount, + SIDEBAR_SEARCH_RESULT_LIMIT, + ); + const sidebarSearchMatchedSessions = useMemo(() => { + if (!normalizedSidebarSearchQuery) { + return recentSidebarSessions; + } + + return recentSidebarSessions.filter((session) => + matchesSidebarSessionTitle(session, normalizedSidebarSearchQuery), + ); + }, [normalizedSidebarSearchQuery, recentSidebarSessions]); + const sidebarSearchResultSessions = useMemo(() => { + if (!normalizedSidebarSearchQuery) { + return buildVisibleSidebarSessions({ + sessions: recentSidebarSessions, + currentSessionId, + limit: sidebarSearchResultLimit, + }); + } + + return sidebarSearchMatchedSessions.slice(0, sidebarSearchResultLimit); + }, [ + currentSessionId, + normalizedSidebarSearchQuery, + recentSidebarSessions, + sidebarSearchMatchedSessions, + sidebarSearchResultLimit, + ]); + const sidebarSearchHasQuery = normalizedSidebarSearchQuery.length > 0; + const sidebarSearchHasMoreResults = sidebarSearchHasQuery + ? sidebarSessionsHasMore || + sidebarSearchResultLimit < sidebarSearchMatchedSessions.length + : hasMoreRecentSidebarSessions; const isActive = (item: SidebarNavItem): boolean => { if (!item.page) { @@ -2950,6 +3395,7 @@ export function AppSidebar({ workspaceId: session.workspace_id ?? currentProjectId ?? null, source: "sidebar", }); + return; } const targetParams = buildClawAgentParams({ @@ -2976,7 +3422,19 @@ export function AppSidebar({ onNavigate(target.page, target.rawParams); }; - const handleNavigateToNewTask = () => { + const handlePrefetchConversation = (session: AsterSessionInfo) => { + if (!isAgentWorkspace) { + return; + } + + notifyTaskCenterTaskPrefetch({ + sessionId: session.id, + workspaceId: session.workspace_id ?? currentProjectId ?? null, + source: "conversation_shelf", + }); + }; + + const handleNavigateToNewTask = useCallback(() => { if (tryOpenTaskCenterDraftFromSidebar()) { return; } @@ -3002,8 +3460,66 @@ export function AppSidebar({ requestedNavigationTargetRef.current = target; onNavigate(target.page, target.rawParams); + }, [currentProjectId, onNavigate, tryOpenTaskCenterDraftFromSidebar]); + + const handleSidebarSearchCreateConversation = () => { + closeSidebarSearchDialog(); + handleNavigateToNewTask(); }; + const handleSidebarSearchNavigateToConversation = ( + session: AsterSessionInfo, + ) => { + closeSidebarSearchDialog(); + handleNavigateToConversation(session); + }; + + const handleRenameConversation = useCallback( + async (session: AsterSessionInfo) => { + const currentTitle = resolveSidebarSessionTitle(session); + const nextTitle = window.prompt("重命名对话", currentTitle)?.trim(); + if (!nextTitle || nextTitle === currentTitle) { + return; + } + + const nextUpdatedAt = Math.floor(Date.now() / 1000); + const nextSession = { + ...session, + name: nextTitle, + updated_at: nextUpdatedAt, + } satisfies AsterSessionInfo; + setSidebarSessionActionId(session.id); + setSidebarSessions((current) => + sortSidebarSessions( + current.map((item) => (item.id === session.id ? nextSession : item)), + ), + ); + setArchivedSessionEntries((current) => + sortSidebarSessions( + current.map((item) => (item.id === session.id ? nextSession : item)), + ), + ); + + try { + await updateAgentRuntimeSession({ + session_id: session.id, + name: nextTitle, + }); + toast.success("已重命名对话"); + await refreshSidebarSessions(); + } catch (error) { + console.error("重命名会话失败:", error); + toast.error("重命名失败，请稍后重试"); + await refreshSidebarSessions(); + } finally { + setSidebarSessionActionId((current) => + current === session.id ? null : current, + ); + } + }, + [refreshSidebarSessions], + ); + const handleToggleSessionArchive = useCallback( async (session: AsterSessionInfo, archived: boolean) => { const nextUpdatedAt = Math.floor(Date.now() / 1000); @@ -3048,6 +3564,45 @@ export function AppSidebar({ [refreshSidebarSessions], ); + const handleDeleteConversation = useCallback( + async (session: AsterSessionInfo) => { + const title = resolveSidebarSessionTitle(session); + const confirmed = window.confirm( + `确定要删除“${title}”吗？删除后无法恢复。`, + ); + if (!confirmed) { + return; + } + + setSidebarSessionActionId(session.id); + setSidebarSessions((current) => + current.filter((item) => item.id !== session.id), + ); + setArchivedSessionEntries((current) => + current.filter((item) => item.id !== session.id), + ); + + try { + await deleteAgentRuntimeSession(session.id); + toast.success("已删除对话"); + if (currentSessionId === session.id) { + handleNavigateToNewTask(); + } else { + await refreshSidebarSessions(); + } + } catch (error) { + console.error("删除会话失败:", error); + toast.error("删除失败，请稍后重试"); + await refreshSidebarSessions(); + } finally { + setSidebarSessionActionId((current) => + current === session.id ? null : current, + ); + } + }, + [currentSessionId, handleNavigateToNewTask, refreshSidebarSessions], + ); + const currentColorScheme = getLimeColorScheme(colorSchemeId); const currentThemeLabel = LIME_THEME_MODE_OPTIONS.find((option) => option.id === themeState.themeMode) @@ -3092,6 +3647,16 @@ export function AppSidebar({ [], ); + const handleRandomColorScheme = useCallback(() => { + const candidates = LIME_COLOR_SCHEMES.filter( + (scheme) => scheme.id !== colorSchemeId, + ); + const nextScheme = + candidates[Math.floor(Math.random() * candidates.length)] ?? + LIME_COLOR_SCHEMES[0]; + handleColorSchemeChange(nextScheme.id); + }, [colorSchemeId, handleColorSchemeChange]); + const handleAccountMenuNavigate = useCallback( (params: PageParams) => { setAccountMenuOpen(false); @@ -3240,7 +3805,9 @@ export function AppSidebar({ $themeMode={themeState.effectiveThemeMode} $reserveWindowControls={reserveWindowControls} data-testid="app-sidebar" + data-lime-window-drag-region data-window-controls-reserved={String(reserveWindowControls)} + onMouseDown={onStartWindowDrag} > @@ -3301,16 +3868,12 @@ export function AppSidebar({ {maybeWrapWithTooltip( - onNavigate( - "agent", - buildHomeAgentParams({ - projectId: currentProjectId ?? undefined, - }), - ) - } + onClick={openSidebarSearchDialog} title="搜索任务" aria-label="搜索任务" + aria-haspopup="dialog" + aria-expanded={sidebarSearchOpen ? true : undefined} + data-testid="app-sidebar-search-button" > 搜索任务 @@ -3337,6 +3900,9 @@ export function AppSidebar({ actionSessionId={sidebarSessionActionId} onCreateConversation={handleNavigateToNewTask} onNavigateToConversation={handleNavigateToConversation} + onPrefetchConversation={handlePrefetchConversation} + onRenameConversation={handleRenameConversation} + onDeleteConversation={handleDeleteConversation} onToggleArchive={(session, archived) => { void handleToggleSessionArchive(session, archived); }} @@ -3461,6 +4027,15 @@ export function AppSidebar({ 配色 + + + 随机 + {LIME_COLOR_SCHEMES.map((scheme) => { const active = scheme.id === colorSchemeId; return ( @@ -3810,6 +4385,120 @@ export function AppSidebar({ + + + + + + + + + 新建对话 + + + + + {sidebarSearchHasQuery ? "匹配结果" : "最近"} + + + {shouldShowSessionLoadingState ? ( + + 正在加载对话 + + ) : sidebarSearchResultSessions.length > 0 ? ( + + {sidebarSearchResultSessions.map((session) => { + const title = resolveSidebarSessionTitle(session); + const isCurrentConversation = currentSessionId === session.id; + return ( + + handleSidebarSearchNavigateToConversation(session) + } + > + + + {title} + + + {formatSidebarSessionMeta(session)} + + + ); + })} + + ) : ( + + {!currentProjectId + ? "请先选择项目工作区" + : sidebarSearchHasQuery + ? "没有匹配的对话标题" + : "还没有最近对话"} + + )} + + {sidebarSearchHasMoreResults ? ( + + setRecentSessionsVisibleCount( + (current) => current + SIDEBAR_RECENT_SESSION_PAGE_SIZE, + ) + } + data-testid="app-sidebar-search-more" + > + {sidebarSessionsLoading + ? "正在加载..." + : sidebarSearchHasQuery + ? "查看更多匹配结果" + : "查看更多对话"} + + + ) : null} + + + setInviteDialogOpen(false)} diff --git a/src/components/agent/chat/AgentChatWorkspace.tsx b/src/components/agent/chat/AgentChatWorkspace.tsx index a3b8ccb97..9f351f450 100644 --- a/src/components/agent/chat/AgentChatWorkspace.tsx +++ b/src/components/agent/chat/AgentChatWorkspace.tsx @@ -184,6 +184,8 @@ import { import { useWorkspaceImageWorkbenchEventRuntime } from "./workspace/useWorkspaceImageWorkbenchEventRuntime"; import { buildImageSkillLaunchRequestMetadata } from "./workspace/imageSkillLaunch"; import { useWorkspaceImageTaskPreviewRuntime } from "./workspace/useWorkspaceImageTaskPreviewRuntime"; +import { useWorkspaceAudioTaskPreviewRuntime } from "./workspace/useWorkspaceAudioTaskPreviewRuntime"; +import { useWorkspaceTranscriptionTaskPreviewRuntime } from "./workspace/useWorkspaceTranscriptionTaskPreviewRuntime"; import { useWorkspaceVideoTaskPreviewRuntime } from "./workspace/useWorkspaceVideoTaskPreviewRuntime"; import { useWorkspaceVideoTaskActionRuntime } from "./workspace/useWorkspaceVideoTaskActionRuntime"; import { useWorkspaceSessionRestore } from "./workspace/useWorkspaceSessionRestore"; @@ -209,7 +211,10 @@ import { useWorkspaceGeneralWorkbenchDocumentPersistenceRuntime } from "./worksp import { useWorkspaceServiceSkillEntryActions } from "./workspace/useWorkspaceServiceSkillEntryActions"; import { useWorkspaceSceneAppEntryActions } from "./workspace/useWorkspaceSceneAppEntryActions"; import { useWorkspaceArtifactViewModeControl } from "./workspace/useWorkspaceArtifactViewModeControl"; -import { useWorkspaceInitialSessionNavigation } from "./workspace/useWorkspaceInitialSessionNavigation"; +import { + rememberInitialSessionNavigationStart, + useWorkspaceInitialSessionNavigation, +} from "./workspace/useWorkspaceInitialSessionNavigation"; import { WorkspaceGeneralWorkbenchSidebar } from "./workspace/WorkspaceGeneralWorkbenchSidebar"; import { GeneralWorkbenchHarnessDialogSection } from "./workspace/WorkspaceHarnessDialogs"; import { WorkspaceShellScene } from "./workspace/WorkspaceShellScene"; @@ -219,6 +224,7 @@ import { } from "./components/TaskCenterTabStrip"; import { subscribeTaskCenterDraftTaskRequests, + subscribeTaskCenterTaskPrefetchRequests, subscribeTaskCenterTaskOpenRequests, } from "./taskCenterDraftTaskEvents"; import type { GeneralWorkbenchFollowUpActionPayload } from "./components/generalWorkbenchSidebarContract"; @@ -2049,6 +2055,7 @@ export function AgentChatWorkspace({ createFreshSession, ensureSession = async () => null, switchTopic: originalSwitchTopic, + prefetchTopic = async () => false, loadFullSessionHistory = async () => false, deleteTopic, renameTopic, @@ -2071,6 +2078,10 @@ export function AgentChatWorkspace({ : undefined, getSyncedSessionRecentPreferences, }); + const topicById = useMemo( + () => new Map(topics.map((topic) => [topic.id, topic])), + [topics], + ); activeSessionIdRef.current = sessionId; const clawSidebarEntryResetKey = shouldAutoCollapseClassicClawSidebar ? JSON.stringify({ @@ -3535,23 +3546,23 @@ export function AgentChatWorkspace({ rememberProjectId, getRememberedProjectId, loadTopicBoundProjectId: (topicId) => - topics.find((topic) => topic.id === topicId)?.workspaceId || + topicById.get(topicId)?.workspaceId || loadPersistedSessionWorkspaceId(topicId) || loadPersistedProjectId(`agent_session_workspace_${topicId}`), resetTopicLocalState, }); const resolveInitialSessionSwitch = useCallback( (topicId: string) => { - const topic = topics.find((item) => item.id === topicId); + const topic = topicById.get(topicId); return { allowDetachedSession: true, - forceRefresh: true, + forceRefresh: topic?.statusReason === "workspace_error", ...(shouldResumeTaskSession(topic) ? { resumeSessionStartHooks: true } : {}), }; }, - [topics], + [topicById], ); useWorkspaceInitialSessionNavigation({ initialSessionId, @@ -3663,7 +3674,9 @@ export function AgentChatWorkspace({ ); setTaskCenterDraftTabs((current) => (current.length > 0 ? [] : current)); setActiveTaskCenterDraftTabId(null); - setTaskCenterLocalSessionOverride(null); + if (agentEntry !== "new-task") { + setTaskCenterLocalSessionOverride(null); + } return; } @@ -3671,7 +3684,7 @@ export function AgentChatWorkspace({ return; } - const hasTopicMatch = topics.some((topic) => topic.id === sessionId); + const hasTopicMatch = topicById.has(sessionId); if (hasTopicMatch) { setTaskCenterDetachedTopicId((current) => current === sessionId ? null : current, @@ -3682,7 +3695,7 @@ export function AgentChatWorkspace({ setTaskCenterDetachedTopicId((current) => current === sessionId ? current : sessionId, ); - }, [agentEntry, normalizedInitialSessionId, sessionId, topics]); + }, [agentEntry, normalizedInitialSessionId, sessionId, topicById]); useEffect(() => { if ( @@ -3758,8 +3771,7 @@ export function AgentChatWorkspace({ } const shouldWaitForInitialSessionTopic = - normalizedInitialSessionId && - !topics.some((topic) => topic.id === normalizedInitialSessionId); + normalizedInitialSessionId && !topicById.has(normalizedInitialSessionId); if (shouldWaitForInitialSessionTopic) { return; } @@ -3800,11 +3812,12 @@ export function AgentChatWorkspace({ taskCenterLocalSessionOverride?.sessionId, taskCenterDetachedTopicId, taskCenterWorkspaceId, + topicById, topics, ]); useEffect(() => { - if (agentEntry !== "claw") { + if (agentEntry !== "claw" && agentEntry !== "new-task") { return; } @@ -4136,6 +4149,7 @@ export function AgentChatWorkspace({ (hasDisplayMessages || isThemeWorkbench || (!shouldUseCompactGeneralWorkbench && isBootstrapDispatchPending) || + isSessionHydrating || isSending || queuedTurns.length > 0)); const shouldRestoreImageTasksFromWorkspace = !( @@ -4428,7 +4442,7 @@ export function AgentChatWorkspace({ replaceOpenTabs?: boolean; }, ) => { - const topic = topics.find((item) => item.id === topicId); + const topic = topicById.get(topicId); const topicWorkspaceId = normalizeProjectId( topic?.workspaceId ?? loadPersistedSessionWorkspaceId(topicId) ?? @@ -4445,18 +4459,9 @@ export function AgentChatWorkspace({ : undefined; const wasOpenInTaskCenter = taskCenterOpenTabIdsRef.current.includes(topicId); - - setTaskCenterTransitionTopicId(topicId); - setTaskCenterDetachedTopicId(null); - setActiveTaskCenterDraftTabId(null); - if (options?.replaceOpenTabs === true) { - replaceTaskCenterOpenTabs(topicId, topicWorkspaceId); - } else if (agentEntry === "claw") { - upsertTaskCenterOpenTab(topicId, topicWorkspaceId); - } - markTaskCenterLocalSessionOverride(topicId); - const switchResult = await switchTopic(topicId, switchOptions); - if (switchResult !== "success" && switchResult !== "deferred") { + const shouldMaintainTaskCenterTab = + agentEntry === "claw" || agentEntry === "new-task"; + const rollbackPendingOpen = () => { if (!wasOpenInTaskCenter && options?.replaceOpenTabs !== true) { setTaskCenterOpenTabMap((currentMap) => updateTaskCenterTabIdsForWorkspace( @@ -4473,12 +4478,47 @@ export function AgentChatWorkspace({ setTaskCenterTransitionTopicId((current) => current === topicId ? null : current, ); + }; + + setTaskCenterTransitionTopicId(topicId); + setTaskCenterDetachedTopicId(null); + setActiveTaskCenterDraftTabId(null); + if (options?.replaceOpenTabs === true) { + replaceTaskCenterOpenTabs(topicId, topicWorkspaceId); + } else if (shouldMaintainTaskCenterTab) { + upsertTaskCenterOpenTab(topicId, topicWorkspaceId); + } + markTaskCenterLocalSessionOverride(topicId); + rememberInitialSessionNavigationStart(topicId); + const switchResult = await switchTopic(topicId, switchOptions); + if (switchResult === "busy") { + scheduleMinimumDelayIdleTask( + () => { + void switchTopic(topicId, switchOptions) + .then((retryResult) => { + if (retryResult !== "success" && retryResult !== "deferred") { + rollbackPendingOpen(); + } + }) + .catch(() => { + rollbackPendingOpen(); + }); + }, + { + minimumDelayMs: 120, + idleTimeoutMs: 600, + }, + ); + return; + } + if (switchResult !== "success" && switchResult !== "deferred") { + rollbackPendingOpen(); return; } if (options?.replaceOpenTabs === true) { replaceTaskCenterOpenTabs(topicId, topicWorkspaceId); } else { - if (agentEntry === "claw") { + if (shouldMaintainTaskCenterTab) { upsertTaskCenterOpenTab(topicId, topicWorkspaceId); } } @@ -4490,9 +4530,9 @@ export function AgentChatWorkspace({ setActiveTaskCenterDraftTabId, switchTopic, taskCenterWorkspaceId, - topics, setTaskCenterDetachedTopicId, setTaskCenterTransitionTopicId, + topicById, upsertTaskCenterOpenTab, ], ); @@ -4503,6 +4543,7 @@ export function AgentChatWorkspace({ setTaskCenterDetachedTopicId(topicId); setTaskCenterTransitionTopicId(topicId); markTaskCenterLocalSessionOverride(topicId); + rememberInitialSessionNavigationStart(topicId); const switchResult = await switchTopic(topicId, { allowDetachedSession: true, }); @@ -4684,20 +4725,54 @@ export function AgentChatWorkspace({ }); }, [agentEntry, handleOpenTaskCenterNewTaskPage]); useEffect(() => { - if (agentEntry !== "claw") { + return subscribeTaskCenterTaskPrefetchRequests( + ({ sessionId: requestedSessionId, workspaceId }) => { + const requestedWorkspaceId = normalizeProjectId(workspaceId); + if ( + requestedWorkspaceId && + taskCenterWorkspaceId && + requestedWorkspaceId !== normalizeProjectId(taskCenterWorkspaceId) + ) { + return; + } + + void prefetchTopic(requestedSessionId); + }, + ); + }, [prefetchTopic, taskCenterWorkspaceId]); + + useEffect(() => { + if (agentEntry !== "claw" && agentEntry !== "new-task") { return; } return subscribeTaskCenterTaskOpenRequests( ({ sessionId: requestedSessionId, workspaceId }) => { - setTaskCenterTransitionTopicId(requestedSessionId); - setTaskCenterDetachedTopicId(null); - setActiveTaskCenterDraftTabId(null); - markTaskCenterLocalSessionOverride(requestedSessionId); - upsertTaskCenterOpenTab(requestedSessionId, workspaceId); + const requestedWorkspaceId = normalizeProjectId(workspaceId); + if ( + requestedWorkspaceId && + requestedWorkspaceId !== normalizeProjectId(taskCenterWorkspaceId) + ) { + setTaskCenterTransitionTopicId(requestedSessionId); + setTaskCenterDetachedTopicId(null); + setActiveTaskCenterDraftTabId(null); + markTaskCenterLocalSessionOverride(requestedSessionId); + upsertTaskCenterOpenTab(requestedSessionId, requestedWorkspaceId); + deferTopicSwitch(requestedSessionId, requestedWorkspaceId); + return; + } + + void handleOpenTaskTopic(requestedSessionId); }, ); - }, [agentEntry, markTaskCenterLocalSessionOverride, upsertTaskCenterOpenTab]); + }, [ + agentEntry, + deferTopicSwitch, + handleOpenTaskTopic, + markTaskCenterLocalSessionOverride, + taskCenterWorkspaceId, + upsertTaskCenterOpenTab, + ]); const taskCenterPreviewTopicId = useMemo( () => resolveTaskCenterPreviewTopicId({ @@ -4800,7 +4875,7 @@ export function AgentChatWorkspace({ isPinned: false, })); const topicItems = taskCenterVisibleTabIds - .map((topicId) => topics.find((topic) => topic.id === topicId)) + .map((topicId) => topicById.get(topicId)) .filter((topic): topic is NonNullable => Boolean(topic)) .map((topic) => ({ id: topic.id, @@ -4825,8 +4900,13 @@ export function AgentChatWorkspace({ taskCenterDraftTabs, taskCenterPreviewTopicId, taskCenterVisibleTabIds, - topics, - ]); + topicById, + ]); + const shouldRenderTaskCenterTabStrip = + agentEntry === "claw" || + (agentEntry === "new-task" && + taskCenterLocalSessionOverride !== null && + taskCenterTabItems.length > 0); useEffect(() => { if ( agentEntry !== "claw" || @@ -4857,7 +4937,7 @@ export function AgentChatWorkspace({ } const currentSessionIsKnownTopic = Boolean( - sessionId && topics.some((topic) => topic.id === sessionId), + sessionId && topicById.has(sessionId), ); if ( !normalizedInitialSessionId && @@ -4913,10 +4993,11 @@ export function AgentChatWorkspace({ taskCenterTransitionTopicId, taskCenterVisibleTabIds, taskCenterWorkspaceId, + topicById, topics, ]); const taskCenterTabsNode = useMemo(() => { - if (agentEntry !== "claw") { + if (!shouldRenderTaskCenterTabStrip) { return null; } @@ -4938,13 +5019,13 @@ export function AgentChatWorkspace({ /> ); }, [ - agentEntry, handleCloseTaskCenterTab, handleOpenTaskCenterNewTaskPage, handleToggleCanvas, handleSwitchTaskTopic, isThemeWorkbench, layoutMode, + shouldRenderTaskCenterTabStrip, taskCenterTabItems, ]); const browserWorkspaceHomeTabsNode = useMemo(() => { @@ -6485,6 +6566,16 @@ export function AgentChatWorkspace({ messages, setChatMessages, }); + useWorkspaceAudioTaskPreviewRuntime({ + projectRootPath: project?.rootPath || null, + messages, + setChatMessages, + }); + useWorkspaceTranscriptionTaskPreviewRuntime({ + projectRootPath: project?.rootPath || null, + messages, + setChatMessages, + }); useWorkspaceVideoTaskActionRuntime({ projectId, contentId, @@ -6504,6 +6595,7 @@ export function AgentChatWorkspace({ hasUnconsumedInitialDispatch, isPreparingSend, isSending, + isSessionHydrating, queuedTurnCount: queuedTurns.length, }); @@ -6559,6 +6651,7 @@ export function AgentChatWorkspace({ isBootstrapDispatchPending, isPreparingSend, isSending, + isSessionHydrating, isSpecializedThemeMode, isThemeWorkbench, layoutMode, @@ -7175,7 +7268,9 @@ export function AgentChatWorkspace({ handleResumeRecentSession, handleOpenSceneAppsDirectory, taskCenterTabsNode: - agentEntry === "claw" ? taskCenterTabsNode : browserWorkspaceHomeTabsNode, + shouldRenderTaskCenterTabStrip + ? taskCenterTabsNode + : browserWorkspaceHomeTabsNode, suppressNavbarUtilityActions: suppressHomeNavbarUtilityActions, hideHistoryToggle, showChatPanel: effectiveShowChatPanel, diff --git a/src/components/agent/chat/components/EmptyState.test.tsx b/src/components/agent/chat/components/EmptyState.test.tsx index 6091774c4..b7f1fe995 100644 --- a/src/components/agent/chat/components/EmptyState.test.tsx +++ b/src/components/agent/chat/components/EmptyState.test.tsx @@ -9,7 +9,6 @@ import type { UnifiedMemory } from "@/lib/api/unifiedMemory"; import type { ServiceSkillHomeItem } from "../service-skills/types"; import type { SceneAppEntryCardItem } from "../sceneappEntryTypes"; import type { InputCapabilitySelection } from "../skill-selection/inputCapabilitySelection"; -import { recordServiceSkillUsage } from "../service-skills/storage"; import { recordSlashEntryUsage } from "../skill-selection/slashEntryUsage"; import { buildCuratedTaskLaunchPrompt, @@ -89,7 +88,8 @@ vi.mock("@/lib/api/unifiedMemory", () => ({ })); vi.mock("@/lib/api/skillCatalog", async (importOriginal) => { - const actual = await importOriginal(); + const actual = + await importOriginal(); return { ...actual, getSkillCatalog: mockGetSkillCatalog, @@ -397,6 +397,27 @@ async function openHomeMoreSkillsDrawer( return drawer; } +function queryHomeDrawerButton( + drawer: HTMLElement, + testId: string, +): HTMLButtonElement | null { + return drawer.querySelector( + `[data-testid="${testId}"]`, + ) as HTMLButtonElement | null; +} + +function expectHomeDrawerButton( + drawer: HTMLElement, + testId: string, +): HTMLButtonElement { + const button = queryHomeDrawerButton(drawer, testId); + expect(button).toBeTruthy(); + if (!button) { + throw new Error(`expected home drawer button ${testId}`); + } + return button; +} + function createGithubSearchServiceSkill(): ServiceSkillHomeItem { return { id: "github-repo-radar", @@ -509,7 +530,7 @@ describe("EmptyState", () => { expect(container.textContent).not.toContain("新建任务"); }); - it("通用首页应展示轻量起手入口与下滑第二屏", async () => { + it("通用首页应挂载轻量起手区与下滑第二屏", async () => { const container = renderEmptyState({ activeTheme: "general", onLaunchBrowserAssist: vi.fn(), @@ -526,16 +547,21 @@ describe("EmptyState", () => { expect( container.querySelector('[data-testid="home-starter-chips"]'), ).toBeTruthy(); - expect(container.textContent).toContain("帮我想选题"); - expect(container.textContent).toContain("写第一版"); - expect(container.textContent).toContain("拆解爆款"); - expect(container.textContent).toContain("改成我的风格"); - expect(container.textContent).toContain("转成视频脚本"); - expect(container.textContent).toContain("复盘账号"); + expect(container.textContent).toContain("引导帮助"); + expect(container.textContent).toContain("写作"); + expect(container.textContent).toContain("PPT"); + expect(container.textContent).toContain("调研报告"); expect(container.textContent).toContain("更多做法"); expect( - container.querySelector('[data-testid="home-scroll-cue"]'), - ).toBeTruthy(); + container.querySelector('[data-testid="home-guide-cards"]'), + ).toBeNull(); + const scrollCue = container.querySelector( + '[data-testid="home-scroll-cue"]', + ); + expect(scrollCue).toBeTruthy(); + expect(scrollCue?.textContent).toContain( + "向下滑，看看 Lime 可以帮你做什么", + ); expect( container.querySelector('[data-testid="home-second-screen"]'), ).toBeTruthy(); @@ -544,43 +570,96 @@ describe("EmptyState", () => { ).toBeTruthy(); expect(container.textContent).toContain("你可以从这些任务开始"); expect(container.textContent).toContain("每日趋势摘要"); - expect(container.textContent).toContain("内容主稿生成"); - expect(container.textContent).toContain("拆解一条爆款内容"); - expect(container.textContent).toContain("长文转多平台发布稿"); - expect(container.textContent).toContain("脚本转口播/字幕稿"); - expect(container.textContent).toContain("复盘这个账号/项目"); - expect(container.textContent).toContain("复制轮播帖"); - expect(container.textContent).toContain("复制视频脚本"); expect(container.textContent).not.toContain("先开始这一轮"); expect(container.textContent).not.toContain("其他起手结果"); expect(container.textContent).not.toContain("也可以直接按做法开工"); - expect(container.textContent).not.toContain("首选结果"); - expect(container.textContent).not.toContain("更多结果"); - expect(container.textContent).not.toContain("快捷做法"); - expect(container.textContent).not.toContain("任务拆分冒烟测试"); - expect(container.textContent).not.toContain("推荐方案"); - expect(container.textContent).not.toContain("通用对话"); - - const recommendationItems = Array.from( - container.querySelectorAll( - '[data-testid^="home-gallery-entry-recommended-"], [data-testid^="home-gallery-entry-service-skill-"]', - ), - ).map((element) => element.textContent?.trim() ?? ""); - - expect(recommendationItems.length).toBeGreaterThanOrEqual(8); - expect(recommendationItems.join(" ")).toContain("每日趋势摘要"); - expect(recommendationItems.join(" ")).toContain("内容主稿生成"); - expect(recommendationItems.join(" ")).toContain("拆解一条爆款内容"); - expect(recommendationItems.join(" ")).toContain("长文转多平台发布稿"); - expect(recommendationItems.join(" ")).toContain("脚本转口播/字幕稿"); - expect(recommendationItems.join(" ")).toContain("复盘这个账号/项目"); - expect(recommendationItems.join(" ")).toContain("复制轮播帖"); - expect(recommendationItems.join(" ")).toContain("复制视频脚本"); expect( container.querySelector('[data-testid^="entry-continuation-"]'), ).toBeNull(); }); + it("点击引导帮助后进入可关闭的帮助模式，关闭后恢复默认起手入口", async () => { + const container = renderEmptyState({ + activeTheme: "general", + serviceSkills: [], + }); + + await act(async () => { + await Promise.resolve(); + }); + + const guideTrigger = container.querySelector( + '[data-testid="home-guide-help-trigger"]', + ) as HTMLButtonElement | null; + expect(guideTrigger).toBeTruthy(); + + act(() => { + guideTrigger?.click(); + }); + + expect( + container.querySelector('[data-testid="home-guide-help-active-badge"]'), + ).toBeTruthy(); + expect( + container.querySelector('[data-testid="home-guide-cards"]'), + ).toBeTruthy(); + expect( + container.querySelector('[data-testid="home-starter-chips"]'), + ).toBeNull(); + expect(container.querySelector("textarea")?.placeholder).toContain( + "想了解什么？试试", + ); + + const closeGuide = container.querySelector( + '[data-testid="home-guide-help-active-badge"] button', + ) as HTMLButtonElement | null; + + act(() => { + closeGuide?.click(); + }); + + expect( + container.querySelector('[data-testid="home-guide-help-active-badge"]'), + ).toBeNull(); + expect( + container.querySelector('[data-testid="home-guide-cards"]'), + ).toBeNull(); + expect( + container.querySelector('[data-testid="home-starter-chips"]'), + ).toBeTruthy(); + expect( + container.querySelector('[data-testid="home-input-tab-suggestion"]'), + ).toBeTruthy(); + }); + + it("点击首屏起手任务只挂载输入区能力，不弹出启动表单", async () => { + const container = renderEmptyState({ + activeTheme: "general", + serviceSkills: [], + }); + + await act(async () => { + await Promise.resolve(); + }); + + const starter = container.querySelector( + '[data-testid="entry-recommended-daily-trend-briefing"]', + ) as HTMLButtonElement | null; + expect(starter).toBeTruthy(); + + act(() => { + starter?.click(); + }); + + expect(document.body.textContent).not.toContain( + "开始这一步前，我先确认几件事。", + ); + expect( + container.querySelector('[data-testid="curated-task-badge"]'), + ).toBeTruthy(); + expect(container.textContent).toContain("每日趋势摘要"); + }); + it("保存判断结论后不应打乱首屏固定起手入口", async () => { const projectId = "project-review-recommendation"; const container = renderEmptyState({ @@ -627,7 +706,7 @@ describe("EmptyState", () => { ).map((element) => element.getAttribute("data-testid")); expect(starterItems).toEqual(initialStarterItems); - expect(starterItems[0]).toBe("entry-recommended-daily-trend-briefing"); + expect(starterItems[0]).toBe("entry-recommended-social-post-starter"); expect( container.querySelector('[data-testid="entry-review-feedback-banner"]'), ).toBeNull(); @@ -797,24 +876,10 @@ describe("EmptyState", () => { }); const drawer = await openHomeMoreSkillsDrawer(container); - expect(drawer.textContent).toContain("最近使用"); - - const continuationItems = Array.from( - drawer.querySelectorAll('[data-testid^="home-drawer-"]'), - ).map((element) => element.getAttribute("data-testid")); - - expect(continuationItems).toContain( + const recentTemplateButton = expectHomeDrawerButton( + drawer, "home-drawer-entry-recommended-social-post-starter", ); - expect(continuationItems).toContain( - "home-drawer-entry-service-skill-content-iteration-flow", - ); - - const recentTemplateButton = drawer.querySelector( - '[data-testid="home-drawer-entry-recommended-social-post-starter"]', - ) as HTMLButtonElement | null; - expect(recentTemplateButton).toBeTruthy(); - expect(recentTemplateButton?.textContent).toContain("内容主稿生成"); await act(async () => { recentTemplateButton?.click(); @@ -871,11 +936,10 @@ describe("EmptyState", () => { }), ); - const recentMethodButton = drawer.querySelector( - '[data-testid="home-drawer-entry-service-skill-content-iteration-flow"]', - ) as HTMLButtonElement | null; - expect(recentMethodButton).toBeTruthy(); - expect(recentMethodButton?.textContent).toContain("内容迭代整理"); + const recentMethodButton = expectHomeDrawerButton( + drawer, + "home-drawer-entry-service-skill-content-iteration-flow", + ); act(() => { recentMethodButton?.click(); @@ -908,11 +972,10 @@ describe("EmptyState", () => { }); const drawer = await openHomeMoreSkillsDrawer(container); - expect(drawer.textContent).toContain("最近使用"); - const sceneContinuationButton = drawer.querySelector( - '[data-testid="home-drawer-entry-service-skill-project-insight-flow"]', - ) as HTMLButtonElement | null; - expect(sceneContinuationButton).toBeTruthy(); + const sceneContinuationButton = expectHomeDrawerButton( + drawer, + "home-drawer-entry-service-skill-project-insight-flow", + ); act(() => { sceneContinuationButton?.click(); @@ -959,14 +1022,10 @@ describe("EmptyState", () => { }); const drawer = await openHomeMoreSkillsDrawer(container); - expect(drawer.textContent).toContain("最近使用"); - - const installedSkillButton = drawer.querySelector( - '[data-testid="home-drawer-entry-installed-skill-content-playbook"]', - ) as HTMLButtonElement | null; - expect(installedSkillButton).toBeTruthy(); - expect(installedSkillButton?.textContent).toContain("内容主稿方法"); - expect(installedSkillButton?.textContent).toContain("继续优化这套内容主稿"); + const installedSkillButton = expectHomeDrawerButton( + drawer, + "home-drawer-entry-installed-skill-content-playbook", + ); await act(async () => { installedSkillButton?.click(); @@ -996,11 +1055,12 @@ describe("EmptyState", () => { await Promise.resolve(); }); - expect( - container.querySelector( - '[data-testid="home-drawer-entry-installed-skill-content-playbook"]', - ), - ).toBeNull(); + const drawer = await openHomeMoreSkillsDrawer(container); + const installedSkillButton = expectHomeDrawerButton( + drawer, + "home-drawer-entry-installed-skill-content-playbook", + ); + expect(installedSkillButton.title).toContain("本地补充技能"); await act(async () => { recordSlashEntryUsage({ @@ -1012,54 +1072,11 @@ describe("EmptyState", () => { await Promise.resolve(); }); - const drawer = await openHomeMoreSkillsDrawer(container); - const installedSkillButton = drawer.querySelector( - '[data-testid="home-drawer-entry-installed-skill-content-playbook"]', - ) as HTMLButtonElement | null; - expect(installedSkillButton).toBeTruthy(); - expect(drawer.textContent).toContain("最近使用"); - expect(installedSkillButton?.textContent).toContain("内容主稿方法"); - expect(installedSkillButton?.textContent).toContain("继续完善这套内容方法"); - }); - - it("最近使用的 service skill 也应显影上次补过的关键信息", async () => { - act(() => { - recordServiceSkillUsage({ - skillId: "github-repo-radar", - runnerType: "instant", - slotValues: { - repository_query: "AI Agent 监控", - }, - }); - }); - - const onSelectServiceSkill = vi.fn<(skill: ServiceSkillHomeItem) => void>(); - const recentMethod: ServiceSkillHomeItem = { - ...createGithubSearchServiceSkill(), - defaultExecutorBinding: "agent_turn", - siteCapabilityBinding: undefined, - recentUsedAt: 1_900_000_000_000, - isRecent: true, - }; - - const container = renderEmptyState({ - activeTheme: "general", - onSelectServiceSkill, - serviceSkills: [recentMethod], - }); - - await act(async () => { - await Promise.resolve(); - }); - - const drawer = await openHomeMoreSkillsDrawer(container); - const recentMethodButton = drawer.querySelector( - '[data-testid="home-drawer-entry-service-skill-github-repo-radar"]', - ) as HTMLButtonElement | null; - expect(recentMethodButton).toBeTruthy(); - expect(recentMethodButton?.textContent).toContain( - "复用 GitHub 登录态检索项目。", + const updatedInstalledSkillButton = expectHomeDrawerButton( + drawer, + "home-drawer-entry-installed-skill-content-playbook", ); + expect(updatedInstalledSkillButton.title).toContain("继续完善这套内容方法"); }); it("首页主体不再重复展示项目选择器入口", async () => { @@ -1084,7 +1101,7 @@ describe("EmptyState", () => { ).toBeNull(); }); - it("通用首页应把补充入口收成轻量按钮，不再展开支撑能力说明", async () => { + it("通用首页不应把浏览器连接提前放到首屏补充入口", async () => { const container = renderEmptyState({ activeTheme: "general", onLaunchBrowserAssist: vi.fn(), @@ -1096,19 +1113,14 @@ describe("EmptyState", () => { expect( container.querySelector('[data-testid="home-supplemental-actions"]'), - ).toBeTruthy(); + ).toBeNull(); expect( container.querySelector('[data-testid="entry-connect-browser"]'), - ).toBeTruthy(); - expect(container.textContent).toContain("连接浏览器"); + ).toBeNull(); + expect(container.textContent).not.toContain("连接浏览器"); expect(container.textContent).not.toContain( "需要网页登录时，也可以先把浏览器接上。", ); - expect(container.textContent).not.toContain("查看支撑能力"); - expect(container.textContent).not.toContain("收起支撑能力"); - expect(container.textContent).not.toContain("重复任务可持续复用"); - expect(container.textContent).not.toContain("复杂任务可拆分并行推进"); - expect(container.textContent).not.toContain("网页登录与网页执行"); expect( container.querySelector('[data-testid="entry-capability-toggle"]'), ).toBeNull(); @@ -1182,7 +1194,7 @@ describe("EmptyState", () => { expect(onOpenMemoryWorkbench).not.toHaveBeenCalled(); }); - it("点击补充入口里的连接浏览器应触发浏览器接入", async () => { + it("浏览器接入不再通过首屏补充入口直接触发", async () => { const onLaunchBrowserAssist = vi.fn(); const container = renderEmptyState({ activeTheme: "general", @@ -1196,13 +1208,9 @@ describe("EmptyState", () => { const mediaButton = container.querySelector( '[data-testid="entry-connect-browser"]', ) as HTMLButtonElement | null; - expect(mediaButton).toBeTruthy(); + expect(mediaButton).toBeNull(); - act(() => { - mediaButton?.click(); - }); - - expect(onLaunchBrowserAssist).toHaveBeenCalledTimes(1); + expect(onLaunchBrowserAssist).not.toHaveBeenCalled(); }); it("点击每日趋势摘要应开启联网搜索并记录最近使用", async () => { @@ -1221,7 +1229,7 @@ describe("EmptyState", () => { }); const card = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLDivElement | null; expect(card).toBeTruthy(); @@ -1279,7 +1287,7 @@ describe("EmptyState", () => { }); const card = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLDivElement | null; expect(card).toBeTruthy(); @@ -1360,7 +1368,7 @@ describe("EmptyState", () => { }); const card = container.querySelector( - '[data-testid="entry-recommended-social-post-starter"]', + '[data-testid="home-gallery-entry-recommended-social-post-starter"]', ) as HTMLDivElement | null; expect(card).toBeTruthy(); @@ -1417,7 +1425,7 @@ describe("EmptyState", () => { }); const card = container.querySelector( - '[data-testid="entry-recommended-longform-multiplatform-rewrite"]', + '[data-testid="home-gallery-entry-recommended-longform-multiplatform-rewrite"]', ) as HTMLDivElement | null; expect(card).toBeTruthy(); @@ -1761,7 +1769,7 @@ describe("EmptyState", () => { }); const templateButton = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLButtonElement | null; expect(templateButton).toBeTruthy(); @@ -1884,7 +1892,7 @@ describe("EmptyState", () => { }); const templateButton = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLButtonElement | null; expect(templateButton).toBeTruthy(); @@ -2008,7 +2016,7 @@ describe("EmptyState", () => { }); const templateButton = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLButtonElement | null; expect(templateButton).toBeTruthy(); @@ -2052,7 +2060,7 @@ describe("EmptyState", () => { }); const templateButton = container.querySelector( - '[data-testid="entry-recommended-account-project-review"]', + '[data-testid="home-gallery-entry-recommended-account-project-review"]', ) as HTMLButtonElement | null; expect(templateButton).toBeTruthy(); expect(container.textContent).not.toContain("当前判断：先补复核与修复"); @@ -2115,11 +2123,15 @@ describe("EmptyState", () => { }); const templateButton = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLButtonElement | null; + await act(async () => { + await Promise.resolve(); + }); + expect(templateButton).toBeTruthy(); - expect(templateButton?.textContent).toContain("帮我想选题"); + expect(templateButton?.textContent).toContain("每日趋势摘要"); expect(container.textContent).not.toContain("当前结果基线：AI 内容周报"); expect(container.textContent).not.toContain("当前判断：先补复核与修复"); expect(container.textContent).not.toContain("当前卡点：复核阻塞"); @@ -2254,7 +2266,7 @@ describe("EmptyState", () => { }); const templateButton = container.querySelector( - '[data-testid="entry-recommended-daily-trend-briefing"]', + '[data-testid="home-gallery-entry-recommended-daily-trend-briefing"]', ) as HTMLButtonElement | null; expect(templateButton).toBeTruthy(); @@ -2388,7 +2400,7 @@ describe("EmptyState", () => { ); }); - it("通用对话默认占位文案应强调本轮任务起手", async () => { + it("通用对话默认展示 Ribbi 式 Tab 起手建议", async () => { const container = renderEmptyState({ activeTheme: "general", serviceSkills: [], @@ -2398,8 +2410,11 @@ describe("EmptyState", () => { await Promise.resolve(); }); - const textarea = container.querySelector("textarea"); - expect(textarea?.getAttribute("placeholder")).toContain("先说这轮要做什么"); + expect( + container.querySelector('[data-testid="home-input-tab-suggestion"]') + ?.textContent, + ).toContain("tab"); + expect(container.textContent).toContain("帮我整理一下会议纪要"); }); it("点击高级设置中的地球按钮应切换联网搜索开关", async () => { @@ -2564,7 +2579,7 @@ describe("EmptyState", () => { expect( container.querySelector('[data-testid="home-start-surface"]'), ).toBeTruthy(); - expect(container.textContent).toContain("帮我想选题"); + expect(container.textContent).toContain("调研报告"); }); it("通用首页发送时不应自动注入任何历史默认 skill", async () => { @@ -2759,7 +2774,7 @@ describe("EmptyState", () => { expect(onSend).toHaveBeenCalledWith("整理最近发布计划", "react", undefined); }); - it("通用主题应提供浏览器协助入口并触发启动回调", async () => { + it("通用主题不应在首屏直接展示浏览器协助入口", async () => { const onLaunchBrowserAssist = vi.fn(); const container = renderEmptyState({ activeTheme: "general", @@ -2772,16 +2787,12 @@ describe("EmptyState", () => { const launchButton = container.querySelector( '[data-testid="entry-connect-browser"]', ) as HTMLButtonElement | null; - expect(launchButton).toBeTruthy(); + expect(launchButton).toBeNull(); - act(() => { - launchButton?.click(); - }); - - expect(onLaunchBrowserAssist).toHaveBeenCalledTimes(1); + expect(onLaunchBrowserAssist).not.toHaveBeenCalled(); }); - it("通用主题应渲染 SceneApp 面板并触发启动回调", async () => { + it("通用主题应把 SceneApp 放到第二屏任务库而不是首屏补充入口", async () => { const onLaunchSceneApp = vi.fn(); const container = renderEmptyState({ activeTheme: "general", @@ -2795,13 +2806,13 @@ describe("EmptyState", () => { expect(container.textContent).toContain("短视频编排"); expect( container.querySelector('[data-testid="home-supplemental-actions"]'), - ).toBeTruthy(); + ).toBeNull(); expect( container.querySelector('[data-testid="sceneapps-home-directory"]'), ).toBeNull(); const launchButton = container.querySelector( - '[data-testid="sceneapp-launch-story-video-suite"]', + '[data-testid="home-gallery-entry-sceneapp-story-video-suite"]', ) as HTMLButtonElement | null; expect(launchButton).toBeTruthy(); @@ -2812,7 +2823,7 @@ describe("EmptyState", () => { expect(onLaunchSceneApp).toHaveBeenCalledWith("story-video-suite"); }); - it("通用主题应提供进入 SceneApp 目录页的入口", async () => { + it("通用主题不应把 SceneApp 目录页入口提前放到首屏", async () => { const onOpenSceneAppsDirectory = vi.fn(); const container = renderEmptyState({ activeTheme: "general", @@ -2826,13 +2837,9 @@ describe("EmptyState", () => { const browseButton = Array.from(container.querySelectorAll("button")).find( (button) => button.textContent?.includes("查看全部做法"), ); - expect(browseButton).toBeTruthy(); + expect(browseButton).toBeFalsy(); - act(() => { - browseButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); - }); - - expect(onOpenSceneAppsDirectory).toHaveBeenCalledTimes(1); + expect(onOpenSceneAppsDirectory).not.toHaveBeenCalled(); }); it("仅有目录入口但没有可直接收益时，不应默认渲染更多起手方式", async () => { diff --git a/src/components/agent/chat/components/EmptyState.tsx b/src/components/agent/chat/components/EmptyState.tsx index d1426d36b..892183d5c 100644 --- a/src/components/agent/chat/components/EmptyState.tsx +++ b/src/components/agent/chat/components/EmptyState.tsx @@ -9,8 +9,10 @@ import styled, { keyframes } from "styled-components"; import { getConfig } from "@/lib/api/appConfig"; import { getSkillCatalog, + listSkillCatalogEntries, listSkillCatalogSceneEntries, subscribeSkillCatalogChanged, + type SkillCatalogEntry, type SkillCatalogSceneEntry, } from "@/lib/api/skillCatalog"; import type { CreationMode } from "./types"; @@ -88,12 +90,19 @@ import { import { HomeSkillGallery } from "../home/HomeSkillGallery"; import { buildHomeGalleryItems, + buildHomeGuideCards, + buildHomeInputSuggestions, buildHomeSkillItems, buildHomeSkillSections, buildHomeStarterChips, } from "../home/buildHomeSkillSurface"; -import { HOME_COMPOSER_PLACEHOLDER } from "../home/homeSurfaceCopy"; +import { + HOME_COMPOSER_PLACEHOLDER, + HOME_GUIDE_HELP_CONTEXT_LABEL, + HOME_GUIDE_HELP_PLACEHOLDER, +} from "../home/homeSurfaceCopy"; import type { + HomeGuideCard, HomeSkillSurfaceItem, HomeStarterChip, } from "../home/homeSurfaceTypes"; @@ -198,7 +207,7 @@ const ComposerGlowFrame = styled.div` const ScrollCue = styled.a` position: absolute; left: 50%; - bottom: clamp(0.62rem, 1.8vh, 1.15rem); + bottom: clamp(0.7rem, 1.9vh, 1.25rem); z-index: 8; display: grid; width: min(680px, calc(100% - 2rem)); @@ -209,9 +218,9 @@ const ScrollCue = styled.a` gap: 0.9rem; transform: translateX(-50%); padding: 0.35rem 0; - color: var(--lime-text-muted, rgb(100 116 139)); - font-size: 12px; - font-weight: 680; + color: var(--lime-brand-strong, rgb(47 83 60)); + font-size: 13px; + font-weight: 760; line-height: 1; text-decoration: none; white-space: nowrap; @@ -240,7 +249,20 @@ const ScrollCueLine = styled.span` const ScrollCueText = styled.span` display: inline-flex; align-items: center; - gap: 0.22rem; + gap: 0.32rem; + border-radius: 999px; + border: 1px solid rgba(187, 247, 208, 0.86); + background: + linear-gradient( + 180deg, + rgba(255, 255, 255, 0.94), + rgba(240, 253, 244, 0.88) + ), + var(--lime-surface, #fff); + padding: 0.42rem 0.78rem; + box-shadow: + 0 10px 28px rgba(15, 23, 42, 0.055), + inset 0 1px 0 rgba(255, 255, 255, 0.92); `; const ScrollCueArrow = styled.span` @@ -443,10 +465,7 @@ export const EmptyState: React.FC = ({ onImportSkill, onRefreshSkills, onLaunchBrowserAssist, - browserAssistLoading = false, featuredSceneApps = [], - sceneAppsLoading = false, - sceneAppLaunchingId = null, onLaunchSceneApp, canResumeRecentSceneApp = false, onResumeRecentSceneApp, @@ -994,10 +1013,19 @@ export const EmptyState: React.FC = ({ return presets; }, []); - const homeStarterChips = useMemo(() => buildHomeStarterChips(), []); + const [homeCatalogEntries, setHomeCatalogEntries] = useState< + SkillCatalogEntry[] + >([]); const [homeCatalogSceneEntries, setHomeCatalogSceneEntries] = useState< SkillCatalogSceneEntry[] >([]); + const [guideHelpActive, setGuideHelpActive] = useState(false); + + useEffect(() => { + if (!isGeneralTheme) { + setGuideHelpActive(false); + } + }, [isGeneralTheme]); useEffect(() => { let cancelled = false; @@ -1007,6 +1035,10 @@ export const EmptyState: React.FC = ({ if (cancelled) { return; } + const entries = listSkillCatalogEntries(catalog).filter((entry) => + (entry.surfaceScopes ?? []).includes("home"), + ); + setHomeCatalogEntries(entries); setHomeCatalogSceneEntries( listSkillCatalogSceneEntries(catalog).filter((entry) => (entry.surfaceScopes ?? []).includes("home"), @@ -1014,6 +1046,7 @@ export const EmptyState: React.FC = ({ ); } catch { if (!cancelled) { + setHomeCatalogEntries([]); setHomeCatalogSceneEntries([]); } } @@ -1034,6 +1067,27 @@ export const EmptyState: React.FC = ({ () => listFeaturedHomeServiceSkills(serviceSkills ?? [], { limit: 6 }), [serviceSkills], ); + const homeStarterChips = useMemo( + () => buildHomeStarterChips(homeCatalogEntries), + [homeCatalogEntries], + ); + const homeInputSuggestions = useMemo( + () => buildHomeInputSuggestions(homeCatalogEntries), + [homeCatalogEntries], + ); + const homeGuideCards = useMemo( + () => buildHomeGuideCards(homeCatalogEntries), + [homeCatalogEntries], + ); + const guideHelpStarterLabel = useMemo( + () => + homeStarterChips.find((chip) => chip.launchKind === "toggle_guide") + ?.label, + [homeStarterChips], + ); + const guideHelpLabel = guideHelpStarterLabel + ? `Lime ${guideHelpStarterLabel}` + : HOME_GUIDE_HELP_CONTEXT_LABEL; const homeSkillItems = useMemo(() => { void slashEntryUsageVersion; @@ -1148,15 +1202,65 @@ export const EmptyState: React.FC = ({ onOpenSceneAppsDirectory?.(); return; } + if (chip.launchKind === "prefill_prompt") { + setGuideHelpActive(false); + const prompt = chip.prompt?.trim(); + if (prompt) { + setInput(prompt); + } + return; + } const targetItem = chip.targetItemId ? homeSkillItems.find((item) => item.id === chip.targetItemId) : null; if (targetItem) { + if (targetItem.launchKind === "curated_task_launcher") { + const template = findCuratedTaskTemplateById(targetItem.id); + if (!template) { + return; + } + const prefill = resolveCuratedTaskTemplateLaunchPrefill(template); + setGuideHelpActive(false); + setActiveCapability({ + kind: "curated_task", + task: template, + launchInputValues: prefill?.inputValues, + referenceMemoryIds: + prefill?.referenceMemoryIds ?? + effectiveDefaultCuratedTaskReferenceMemoryIds, + referenceEntries: + prefill?.referenceEntries ?? + effectiveDefaultCuratedTaskReferenceEntries, + }); + if (prefill?.hint) { + toast.info(prefill.hint); + } + return; + } + setGuideHelpActive(false); handleSelectHomeSkillItem(targetItem); } }, - [handleSelectHomeSkillItem, homeSkillItems, onOpenSceneAppsDirectory], + [ + effectiveDefaultCuratedTaskReferenceEntries, + effectiveDefaultCuratedTaskReferenceMemoryIds, + handleSelectHomeSkillItem, + homeSkillItems, + onOpenSceneAppsDirectory, + setInput, + ], + ); + + const handleSelectHomeGuideCard = useCallback( + (card: HomeGuideCard) => { + setGuideHelpActive(true); + const prompt = card.prompt.trim(); + if (prompt) { + setInput(prompt); + } + }, + [setInput], ); const recentSessionLinkLabel = useMemo(() => { @@ -1197,66 +1301,23 @@ export const EmptyState: React.FC = ({ }); } - featuredSceneApps.slice(0, 2).forEach((item) => { - actions.push({ - id: `sceneapp-${item.id}`, - label: - sceneAppLaunchingId === item.id - ? `${item.title} 准备中…` - : item.title, - title: `${item.businessLabel} · ${item.summary}`, - testId: `sceneapp-launch-${item.id}`, - onSelect: () => { - if (item.disabledReason || sceneAppLaunchingId === item.id) { - return; - } - void onLaunchSceneApp?.(item.id); - }, - }); - }); - - if (onOpenSceneAppsDirectory) { - actions.push({ - id: "sceneapps-directory", - label: sceneAppsLoading ? "做法加载中" : "查看全部做法", - testId: "entry-sceneapps-directory", - onSelect: onOpenSceneAppsDirectory, - }); - } - - if (onLaunchBrowserAssist) { - actions.push({ - id: "connect-browser", - label: browserAssistLoading ? "浏览器连接准备中" : "连接浏览器", - testId: "entry-connect-browser", - onSelect: () => { - void onLaunchBrowserAssist(); - }, - }); - } - return actions; }, [ - browserAssistLoading, canResumeRecentSceneApp, - featuredSceneApps, - onLaunchBrowserAssist, - onLaunchSceneApp, - onOpenSceneAppsDirectory, onResumeRecentSceneApp, onResumeRecentSession, recentSessionLinkLabel, recentSessionLinkTitle, recentSessionTitle, - sceneAppLaunchingId, - sceneAppsLoading, ]); const composerPanel = ( = ({ onFileSelect={handleFileSelect} onPaste={handlePaste} onRemoveImage={handleRemoveImage} + inputSuggestions={ + hasAutoLaunchSiteSkill || guideHelpActive ? [] : homeInputSuggestions + } + guideHelpActive={guideHelpActive} + guideHelpLabel={guideHelpLabel} + onClearGuideHelp={() => setGuideHelpActive(false)} /> ); @@ -1352,9 +1419,13 @@ export const EmptyState: React.FC = ({ const homeStartSurfacePanel = ( ); @@ -1389,12 +1460,12 @@ export const EmptyState: React.FC = ({ - 向下滚动或点击以查看示例 - ⌄ + 向下滑，看看 Lime 可以帮你做什么 + ↓ diff --git a/src/components/agent/chat/components/EmptyStateComposerPanel.test.tsx b/src/components/agent/chat/components/EmptyStateComposerPanel.test.tsx index d666eb809..fe709f77f 100644 --- a/src/components/agent/chat/components/EmptyStateComposerPanel.test.tsx +++ b/src/components/agent/chat/components/EmptyStateComposerPanel.test.tsx @@ -339,6 +339,119 @@ describe("EmptyStateComposerPanel", () => { expect(composer?.className).toContain("floating-composer"); }); + it("输入为空时展示 Tab 起手建议，按 Tab 后填入当前建议", async () => { + const container = renderPanel({ + inputSuggestions: [ + { + id: "suggestion-email", + label: "帮我写一封工作邮件", + prompt: "请帮我写一封工作邮件。", + order: 10, + }, + ], + }); + + expect( + container.querySelector('[data-testid="home-input-tab-suggestion"]') + ?.textContent, + ).toContain("帮我写一封工作邮件"); + + const textarea = container.querySelector("textarea"); + await act(async () => { + textarea?.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Tab", + bubbles: true, + cancelable: true, + }), + ); + await Promise.resolve(); + }); + + expect( + (container.querySelector("textarea") as HTMLTextAreaElement).value, + ).toBe("请帮我写一封工作邮件。"); + expect( + container.querySelector('[data-testid="home-input-tab-suggestion"]'), + ).toBeNull(); + }); + + it("Shift+Tab 保持焦点切换，不填入起手建议", () => { + const container = renderPanel({ + inputSuggestions: [ + { + id: "suggestion-email", + label: "帮我写一封工作邮件", + prompt: "请帮我写一封工作邮件。", + order: 10, + }, + ], + }); + + const textarea = container.querySelector("textarea"); + act(() => { + textarea?.dispatchEvent( + new KeyboardEvent("keydown", { + key: "Tab", + shiftKey: true, + bubbles: true, + cancelable: true, + }), + ); + }); + + expect( + (container.querySelector("textarea") as HTMLTextAreaElement).value, + ).toBe(""); + }); + + it("引导帮助模式应展示可关闭的上下文 badge 并隐藏 Tab 起手建议", () => { + const onClearGuideHelp = vi.fn(); + const container = renderPanel({ + guideHelpActive: true, + guideHelpLabel: "Lime 引导帮助", + onClearGuideHelp, + inputSuggestions: [ + { + id: "suggestion-meeting", + label: "帮我整理一下会议纪要", + prompt: "帮我整理一下会议纪要。", + order: 10, + }, + ], + }); + + expect( + container.querySelector('[data-testid="home-guide-help-active-badge"]') + ?.textContent, + ).toContain("Lime 引导帮助"); + expect( + container.querySelector('[data-testid="home-guide-help-toolbar-badge"]') + ?.textContent, + ).toContain("引导帮助"); + expect( + container.querySelector('[data-testid="home-input-tab-suggestion"]'), + ).toBeNull(); + + const closeButton = container.querySelector( + '[data-testid="home-guide-help-active-badge"] button', + ) as HTMLButtonElement | null; + act(() => { + closeButton?.click(); + }); + + expect(onClearGuideHelp).toHaveBeenCalledTimes(1); + + const toolbarCloseButton = container.querySelector( + '[data-testid="home-guide-help-toolbar-badge"]', + ) as HTMLButtonElement | null; + act(() => { + toolbarCloseButton?.click(); + }); + + expect(onClearGuideHelp).toHaveBeenCalledTimes(2); + }); + it("存在当前带入的灵感时，应在输入区顶部展示被带入的参考对象", () => { const container = renderPanel({ creationReplaySurface: { diff --git a/src/components/agent/chat/components/EmptyStateComposerPanel.tsx b/src/components/agent/chat/components/EmptyStateComposerPanel.tsx index c19157f15..4cefbef4f 100644 --- a/src/components/agent/chat/components/EmptyStateComposerPanel.tsx +++ b/src/components/agent/chat/components/EmptyStateComposerPanel.tsx @@ -1,5 +1,12 @@ import React, { useEffect, useMemo, useRef, useState } from "react"; -import { ChevronDown, ChevronUp, Globe, Settings2 } from "lucide-react"; +import { + ChevronDown, + ChevronUp, + Globe, + Lightbulb, + Settings2, + X, +} from "lucide-react"; import { Select, SelectContent, @@ -48,6 +55,7 @@ import type { import type { CuratedTaskTemplateItem } from "../utils/curatedTaskTemplates"; import type { CuratedTaskReferenceEntry } from "../utils/curatedTaskReferenceSelection"; import type { CreationReplaySurfaceModel } from "../utils/creationReplaySurface"; +import type { HomeInputSuggestion } from "../home/homeSurfaceTypes"; import { getProviderLabel } from "@/lib/constants/providerMappings"; interface EmptyStateComposerPanelProps { @@ -99,6 +107,60 @@ interface EmptyStateComposerPanelProps { onFileSelect: (event: React.ChangeEvent) => void; onPaste?: (event: React.ClipboardEvent) => void; onRemoveImage?: (index: number) => void; + inputSuggestions?: HomeInputSuggestion[]; + guideHelpActive?: boolean; + guideHelpLabel?: string; + onClearGuideHelp?: () => void; +} + +function GuideHelpBadge({ + label, + onClear, +}: { + label: string; + onClear: () => void; +}) { + return ( +
+ + {label} + +
+ ); +} + +function GuideHelpToolbarBadge({ + label, + onClear, +}: { + label: string; + onClear: () => void; +}) { + return ( + + ); } export function EmptyStateComposerPanel({ @@ -148,6 +210,10 @@ export function EmptyStateComposerPanel({ onFileSelect, onPaste, onRemoveImage, + inputSuggestions = [], + guideHelpActive = false, + guideHelpLabel = "Lime 引导帮助", + onClearGuideHelp, }: EmptyStateComposerPanelProps) { const [draftInput, setDraftInput] = useState(input); const textareaRef = useRef(null); @@ -200,6 +266,71 @@ export function EmptyStateComposerPanel({ teamSuggestion.shouldSuggest && dismissedSuggestionKey !== suggestionKey; const shouldShowTeamSelector = isGeneralTheme && subagentEnabled; + const sortedInputSuggestions = useMemo( + () => + [...inputSuggestions].sort((left, right) => { + if (left.order !== right.order) { + return left.order - right.order; + } + return left.label.localeCompare(right.label, "zh-CN"); + }), + [inputSuggestions], + ); + const [inputSuggestionIndex, setInputSuggestionIndex] = useState(0); + const shouldShowInputSuggestion = + sortedInputSuggestions.length > 0 && + !isLoading && + !disabled && + draftInput.trim().length === 0 && + pendingImages.length === 0 && + !guideHelpActive && + !activeBuiltinCommand && + !activeRuntimeScene && + !activeCuratedTask && + !activeSkill && + !creationReplaySurface; + const activeInputSuggestion = + shouldShowInputSuggestion && sortedInputSuggestions.length > 0 + ? sortedInputSuggestions[ + inputSuggestionIndex % sortedInputSuggestions.length + ] + : null; + + useEffect(() => { + if (inputSuggestionIndex < sortedInputSuggestions.length) { + return; + } + setInputSuggestionIndex(0); + }, [inputSuggestionIndex, sortedInputSuggestions.length]); + + useEffect(() => { + if (!shouldShowInputSuggestion || sortedInputSuggestions.length <= 1) { + return; + } + + const timer = window.setInterval(() => { + setInputSuggestionIndex( + (current) => (current + 1) % sortedInputSuggestions.length, + ); + }, 3500); + + return () => window.clearInterval(timer); + }, [shouldShowInputSuggestion, sortedInputSuggestions.length]); + + const handleAcceptInputSuggestion = (suggestion: { + label: string; + prompt: string; + testId?: string; + }) => { + setDraftInput(suggestion.prompt); + window.requestAnimationFrame(() => { + textareaRef.current?.focus(); + textareaRef.current?.setSelectionRange( + suggestion.prompt.length, + suggestion.prompt.length, + ); + }); + }; const handleEnableTeamSuggestion = () => { onSubagentEnabledChange?.(true); @@ -238,6 +369,7 @@ export function EmptyStateComposerPanel({ }; const topExtra = + guideHelpActive || activeBuiltinCommand || activeRuntimeScene || activeCuratedTask || @@ -245,6 +377,13 @@ export function EmptyStateComposerPanel({ creationReplaySurface || shouldShowTeamSuggestion ? ( <> + {guideHelpActive ? ( + undefined)} + /> + ) : null} + {activeBuiltinCommand ? ( + {guideHelpActive ? ( + undefined)} + /> + ) : null} + {!showAdvancedControls && currentModelSummary ? ( ); diff --git a/src/components/agent/chat/components/HarnessStatusPanel.test.tsx b/src/components/agent/chat/components/HarnessStatusPanel.test.tsx index 1742adf60..f06ee5b98 100644 --- a/src/components/agent/chat/components/HarnessStatusPanel.test.tsx +++ b/src/components/agent/chat/components/HarnessStatusPanel.test.tsx @@ -2,6 +2,10 @@ import { act, type ComponentProps } from "react"; import { createRoot, type Root } from "react-dom/client"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import type { AgentRuntimeToolInventory } from "@/lib/api/agentRuntime"; +import { + areLightweightRenderersRegistered, + registerLightweightRenderers, +} from "@/components/artifact/renderers"; import { HarnessStatusPanel } from "./HarnessStatusPanel"; import type { HarnessSessionState } from "../utils/harnessState"; @@ -144,6 +148,18 @@ function findButtonByText(text: string): HTMLButtonElement | null { ) as HTMLButtonElement | null; } +async function flushUntilTextAppears(text: string): Promise { + for (let index = 0; index < 20; index += 1) { + if (document.body.textContent?.includes(text)) { + return; + } + await act(async () => { + await Promise.resolve(); + await new Promise((resolve) => window.setTimeout(resolve, 0)); + }); + } +} + function createToolInventory(): AgentRuntimeToolInventory { return { request: { @@ -610,6 +626,10 @@ beforeEach(() => { } ).IS_REACT_ACT_ENVIRONMENT = true; + if (!areLightweightRenderersRegistered()) { + registerLightweightRenderers(); + } + originalClipboard = navigator.clipboard; Object.defineProperty(navigator, "clipboard", { configurable: true, @@ -949,6 +969,56 @@ describe("HarnessStatusPanel", () => { "Artifact 校验已恢复 1 个产物，fallback 0 次。", ], }, + modality_runtime_contracts: { + snapshot_count: 2, + snapshot_index: { + browser_action_index: { + action_count: 2, + session_count: 1, + observation_count: 1, + screenshot_count: 1, + last_url: "https://example.com/", + session_ids: ["browser-session-1"], + target_ids: ["target-1"], + profile_keys: ["general_browser_assist"], + status_counts: [{ status: "completed", count: 2 }], + artifact_kind_counts: [ + { artifact_kind: "browser_session", count: 1 }, + { artifact_kind: "browser_snapshot", count: 1 }, + ], + action_counts: [ + { action: "navigate", count: 1 }, + { action: "get_page_info", count: 1 }, + ], + backend_counts: [{ backend: "lime_extension_bridge", count: 1 }], + items: [ + { + artifact_kind: "browser_session", + action: "navigate", + status: "completed", + success: true, + session_id: "browser-session-1", + target_id: "target-1", + backend: "cdp_direct", + last_url: "https://example.com/", + }, + { + artifact_kind: "browser_snapshot", + action: "get_page_info", + status: "completed", + success: true, + session_id: "browser-session-1", + target_id: "target-1", + entry_source: "at_browser_agent_command", + backend: "lime_extension_bridge", + last_url: "https://example.com/", + observation_available: true, + screenshot_available: true, + }, + ], + }, + }, + }, }, artifacts: [ { @@ -992,6 +1062,24 @@ describe("HarnessStatusPanel", () => { expect(document.body.textContent).toContain("阻塞失败"); expect(document.body.textContent).toContain("验证失败焦点"); expect(document.body.textContent).toContain("已恢复结果"); + expect(document.body.textContent).toContain("Browser Assist 索引"); + expect(document.body.textContent).toContain("https://example.com/"); + expect(document.body.textContent).toContain("browser_snapshot"); + expect(document.body.textContent).toContain("get_page_info"); + expect(document.body.textContent).toContain("observation / screenshot"); + const replayButton = document.body.querySelector( + 'button[aria-label="打开 Browser Assist 复盘"]', + ) as HTMLButtonElement | null; + expect(replayButton).not.toBeNull(); + + await act(async () => { + replayButton?.click(); + await Promise.resolve(); + }); + await flushUntilTextAppears("最近浏览器动作"); + + expect(document.body.textContent).toContain("最近浏览器动作"); + expect(document.body.textContent).toContain("browser_replay_viewer"); expect(document.body.textContent).toContain("当前已知缺口"); expect(document.body.textContent).toContain( ".lime/harness/sessions/session-evidence-1/evidence/summary.md", diff --git a/src/components/agent/chat/components/HarnessStatusPanel.tsx b/src/components/agent/chat/components/HarnessStatusPanel.tsx index 4d63744ee..914b565bf 100644 --- a/src/components/agent/chat/components/HarnessStatusPanel.tsx +++ b/src/components/agent/chat/components/HarnessStatusPanel.tsx @@ -37,6 +37,8 @@ import { toast } from "sonner"; import type { StepStatus } from "@/lib/workspace/workbenchContract"; import type { AgentRuntimeAnalysisHandoff, + AgentRuntimeEvidenceBrowserActionIndex, + AgentRuntimeEvidenceBrowserActionItem, AgentRuntimeEvidencePack, AgentRuntimeHandoffBundle, AgentRuntimeSaveReviewDecisionRequest, @@ -60,6 +62,8 @@ import { saveAgentRuntimeReviewDecision, } from "@/lib/api/agentRuntime"; import { getMcpInnerToolName } from "@/lib/api/mcp"; +import { ArtifactRenderer } from "@/components/artifact/ArtifactRenderer"; +import type { Artifact } from "@/lib/artifact/types"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; import { @@ -192,6 +196,7 @@ interface PreviewDialogState { displayName: string; content?: string; preview?: string; + artifact?: Artifact; error?: string; isBinary: boolean; size?: number; @@ -490,6 +495,44 @@ function formatEvidenceArtifactKindLabel( } } +function formatBrowserActionArtifactKindLabel(kind?: string): string { + switch (kind?.trim()) { + case "browser_session": + return "browser_session"; + case "browser_snapshot": + return "browser_snapshot"; + default: + return kind?.trim() || "未知产物"; + } +} + +function formatBrowserActionStatusLabel( + item: AgentRuntimeEvidenceBrowserActionItem, +): string { + if (item.success === true && !item.status) { + return "成功"; + } + if (item.success === false && !item.status) { + return "失败"; + } + + switch (item.status?.trim()) { + case "completed": + case "success": + case "succeeded": + return "成功"; + case "failed": + case "error": + return "失败"; + case "running": + return "执行中"; + case "pending": + return "待处理"; + default: + return item.status?.trim() || "未知状态"; + } +} + function formatReplayArtifactKindLabel( kind: AgentRuntimeReplayCase["artifacts"][number]["kind"], ): string { @@ -1596,6 +1639,209 @@ function InventoryStatCard({ ); } +function BrowserActionIndexSummarySection({ + index, + onOpenReplay, +}: { + index: AgentRuntimeEvidenceBrowserActionIndex; + onOpenReplay?: () => void; +}) { + if (index.action_count <= 0 && index.items.length === 0) { + return null; + } + + const recentItems = index.items.slice(-3).reverse(); + const latestUrl = + index.last_url || + recentItems.find((item) => item.last_url)?.last_url || + "暂无 URL"; + + return ( +
+
+ + Browser Assist 索引 +
+

+ 来自 modalityRuntimeContracts.snapshotIndex.browserActionIndex，复盘 + browser_session / browser_snapshot 的执行证据。 +

+ + {onOpenReplay ? ( +
+ +
+ ) : null} + +
+ + 0 + ? `profile ${index.profile_keys.slice(0, 2).join(" / ")}` + : "session / target" + } + /> + + +
+ + {recentItems.length > 0 ? ( +
+ {recentItems.map((item, indexInList) => { + const itemKey = [ + item.request_id, + item.session_id, + item.action, + indexInList, + ] + .filter(Boolean) + .join(":"); + return ( +
+
+ + {item.action || item.tool_name || "browser action"} + + + {formatBrowserActionArtifactKindLabel(item.artifact_kind)} + + + {formatBrowserActionStatusLabel(item)} + + {item.backend ? ( + {item.backend} + ) : null} +
+
+ {item.last_url ? ( +
+ URL： + + {item.last_url} + +
+ ) : null} +
+ {item.session_id ? ( + + session： + + {item.session_id} + + + ) : null} + {item.target_id ? ( + + target： + + {item.target_id} + + + ) : null} + {item.entry_source ? ( + + entry： + + {item.entry_source} + + + ) : null} +
+
+
+ ); + })} +
+ ) : null} +
+ ); +} + +function buildBrowserReplayArtifact( + evidencePack: AgentRuntimeEvidencePack, + index: AgentRuntimeEvidenceBrowserActionIndex, +): Artifact { + const timestamp = Date.parse(evidencePack.exported_at); + return { + id: `browser-replay:${evidencePack.session_id}`, + type: "browser_assist", + title: "Browser Assist 复盘", + content: "", + status: "complete", + meta: { + browserActionIndex: { + actionCount: index.action_count, + sessionCount: index.session_count, + observationCount: index.observation_count, + screenshotCount: index.screenshot_count, + lastUrl: index.last_url, + sessionIds: index.session_ids, + targetIds: index.target_ids, + profileKeys: index.profile_keys, + items: index.items.map((item) => ({ + artifactKind: item.artifact_kind, + toolName: item.tool_name, + action: item.action, + status: item.status, + success: item.success, + sessionId: item.session_id, + targetId: item.target_id, + profileKey: item.profile_key, + backend: item.backend, + requestId: item.request_id, + lastUrl: item.last_url, + title: item.title, + entrySource: item.entry_source, + observationAvailable: item.observation_available, + screenshotAvailable: item.screenshot_available, + })), + }, + modalityContractKey: "browser_control", + viewerSurface: "browser_replay_viewer", + evidencePackRoot: evidencePack.pack_relative_root, + sessionId: index.session_ids[0] || evidencePack.session_id, + profileKey: index.profile_keys[0], + targetId: index.target_ids[0], + url: index.last_url, + }, + position: { start: 0, end: 0 }, + createdAt: Number.isFinite(timestamp) ? timestamp : Date.now(), + updatedAt: Number.isFinite(timestamp) ? timestamp : Date.now(), + }; +} + function Section({ sectionKey, title, @@ -2526,6 +2772,7 @@ export function HarnessStatusPanel({ displayName: path ? getFileName(path) : title, content: content?.trim() || preview?.trim(), preview, + artifact: undefined, error: content?.trim() || preview?.trim() ? undefined @@ -2635,6 +2882,25 @@ export function HarnessStatusPanel({ } }, [previewDialog.content]); + const openBrowserReplayPreview = useCallback( + ( + pack: AgentRuntimeEvidencePack, + index: AgentRuntimeEvidenceBrowserActionIndex, + ) => { + setPreviewDialog({ + open: true, + title: "Browser Assist 复盘", + description: + "来自 evidence browserActionIndex 的 browser_session / browser_snapshot 复盘。", + displayName: "browser_replay_viewer", + artifact: buildBrowserReplayArtifact(pack, index), + isBinary: false, + loading: false, + }); + }, + [], + ); + const handleOpenPathValue = useCallback( async (path: string) => { const normalizedPath = path.trim(); @@ -3408,6 +3674,28 @@ export function HarnessStatusPanel({ + {evidencePack.observability_summary + ?.modality_runtime_contracts?.snapshot_index + ?.browser_action_index + ? (() => { + const browserActionIndex = + evidencePack.observability_summary + .modality_runtime_contracts.snapshot_index + .browser_action_index; + return ( + + openBrowserReplayPreview( + evidencePack, + browserActionIndex, + ) + } + /> + ); + })() + : null} + {evidencePack.observability_summary ?.verification_summary ? ( - {previewDialog.isBinary ? ( + {previewDialog.artifact ? ( +
+ +
+ ) : previewDialog.isBinary ? (
该文件为二进制内容，暂不支持文本预览。 diff --git a/src/components/agent/chat/components/Inputbar/components/InputbarCore.tsx b/src/components/agent/chat/components/Inputbar/components/InputbarCore.tsx index c6a1d4f63..fa54562c2 100644 --- a/src/components/agent/chat/components/Inputbar/components/InputbarCore.tsx +++ b/src/components/agent/chat/components/Inputbar/components/InputbarCore.tsx @@ -5,6 +5,9 @@ import { InputBarContainer, InputColumn, InputIconButton, + InputSuggestionKeycap, + InputSuggestionLayer, + InputSuggestionText, MainRow, MetaSlot, StyledTextarea, @@ -78,6 +81,16 @@ interface InputbarCoreProps { onPromoteQueuedTurn?: (queuedTurnId: string) => void | Promise; onRemoveQueuedTurn?: (queuedTurnId: string) => void | Promise; showMetaTools?: boolean; + inputSuggestion?: { + label: string; + prompt: string; + testId?: string; + } | null; + onAcceptInputSuggestion?: (suggestion: { + label: string; + prompt: string; + testId?: string; + }) => void; } export const InputbarCore: React.FC = ({ @@ -105,6 +118,8 @@ export const InputbarCore: React.FC = ({ onPromoteQueuedTurn, onRemoveQueuedTurn, showMetaTools = true, + inputSuggestion = null, + onAcceptInputSuggestion, }) => { const [isTextareaExpanded, setIsTextareaExpanded] = useState(false); const inputBarContainerRef = useRef(null); @@ -184,6 +199,57 @@ export const InputbarCore: React.FC = ({ : dictationEnabled || !voiceConfigLoaded ? "开始语音输入" : "语音输入未启用"; + const shouldShowInputSuggestion = + Boolean(inputSuggestion) && text.trim().length === 0 && !disabled; + + const handleInputSuggestionKeyDown = useCallback( + (event: React.KeyboardEvent) => { + if ( + !inputSuggestion || + event.key !== "Tab" || + event.shiftKey || + event.metaKey || + event.ctrlKey || + event.altKey || + text.trim().length > 0 || + disabled + ) { + return; + } + + const nativeEvent = event.nativeEvent as KeyboardEvent & { + isComposing?: boolean; + }; + if ( + nativeEvent.isComposing || + nativeEvent.key === "Process" || + nativeEvent.keyCode === 229 + ) { + return; + } + + event.preventDefault(); + const acceptedText = inputSuggestion.prompt; + if (onAcceptInputSuggestion) { + onAcceptInputSuggestion(inputSuggestion); + } else { + setText(acceptedText); + } + window.requestAnimationFrame(() => { + const textarea = resolvedTextareaRef.current; + textarea?.focus(); + textarea?.setSelectionRange(acceptedText.length, acceptedText.length); + }); + }, + [ + disabled, + inputSuggestion, + onAcceptInputSuggestion, + resolvedTextareaRef, + setText, + text, + ], + ); const handleRemoveImageMouseDown = useCallback( (event: React.MouseEvent) => { @@ -215,6 +281,7 @@ export const InputbarCore: React.FC = ({ isLoading={isLoading} disabled={disabled} onPaste={onPaste} + onKeyDown={handleInputSuggestionKeyDown} isFullscreen={isFullscreen} fillHeightWhenFullscreen hasAdditionalContent={pendingImages.length > 0} @@ -224,10 +291,12 @@ export const InputbarCore: React.FC = ({ allowSendWhileLoading rows={isTextareaExpanded ? 7 : isFloatingVariant ? 3 : 1} placeholder={ - placeholder || - (isFullscreen - ? "全屏编辑模式，按 ESC 退出，Enter 发送" - : "在这里输入消息, 按 Enter 发送") + shouldShowInputSuggestion + ? "" + : placeholder || + (isFullscreen + ? "全屏编辑模式，按 ESC 退出，Enter 发送" + : "在这里输入消息, 按 Enter 发送") } > {({ textareaProps, textareaRef, isPrimaryDisabled, onPrimaryAction }) => { @@ -295,6 +364,18 @@ export const InputbarCore: React.FC = ({ + {shouldShowInputSuggestion && inputSuggestion ? ( + + + {inputSuggestion.label} + + tab + + ) : null} void; + /** 历史恢复等冷路径可用轻量模式，避开高成本 HTML/Katex/语法高亮。 */ + renderMode?: MarkdownRenderMode; } +export type MarkdownRenderMode = "standard" | "light"; + function normalizeCodeLanguage(language: string): string { const normalized = language.trim().toLowerCase(); if (!normalized) { @@ -567,6 +571,7 @@ export const MarkdownRenderer: React.FC = memo( isStreaming = false, showBlockActions = false, onQuoteContent, + renderMode = "standard", }) => { const [copied, setCopied] = React.useState(null); const [bundleImageOverrides, setBundleImageOverrides] = React.useState< @@ -577,6 +582,8 @@ export const MarkdownRenderer: React.FC = memo( const selectionSnapshotRef = React.useRef(null); const useLightweightStreamingRender = isStreaming && content.length >= STREAMING_LIGHT_RENDER_THRESHOLD; + const useLightweightMarkdownRender = + renderMode === "light" || useLightweightStreamingRender; const debouncedStreamingContent = useDebouncedValue( content, useLightweightStreamingRender @@ -592,13 +599,13 @@ export const MarkdownRenderer: React.FC = memo( const remarkPlugins = React.useMemo( () => - useLightweightStreamingRender ? [remarkGfm] : [remarkGfm, remarkMath], - [useLightweightStreamingRender], + useLightweightMarkdownRender ? [remarkGfm] : [remarkGfm, remarkMath], + [useLightweightMarkdownRender], ); const rehypePlugins = React.useMemo( - () => (useLightweightStreamingRender ? [] : [rehypeRaw, rehypeKatex]), - [useLightweightStreamingRender], + () => (useLightweightMarkdownRender ? [] : [rehypeRaw, rehypeKatex]), + [useLightweightMarkdownRender], ); const hasRemoteImageReferences = React.useMemo( () => /https?:\/\//i.test(content), @@ -995,7 +1002,7 @@ export const MarkdownRenderer: React.FC = memo( = memo( ); } - if (useLightweightStreamingRender) { + if (useLightweightMarkdownRender) { return ( 
                         {codeContent}
diff --git a/src/components/agent/chat/components/MessageList.test.tsx b/src/components/agent/chat/components/MessageList.test.tsx
index 1bd373207..672f48950 100644
--- a/src/components/agent/chat/components/MessageList.test.tsx
+++ b/src/components/agent/chat/components/MessageList.test.tsx
@@ -4,6 +4,10 @@ import { createRoot, type Root } from "react-dom/client";
 import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
 import { MessageList } from "./MessageList";
 import type { AgentThreadItem, AgentThreadTurn, Message } from "../types";
+import {
+  clearAgentUiPerformanceMetrics,
+  getAgentUiPerformanceMetrics,
+} from "@/lib/agentUiPerformanceMetrics";
 
 const IMAGE_WORKBENCH_FOCUS_EVENT = "lime:image-workbench-focus";
 const VIDEO_WORKBENCH_TASK_ACTION_EVENT = "lime:video-workbench-task-action";
@@ -101,6 +105,7 @@ const mockStreamingRenderer = vi.fn(
     renderProposedPlanBlocks,
     showContentBlockActions,
     onQuoteContent,
+    markdownRenderMode,
   }: {
     content: string;
     contentParts?: unknown[];
@@ -113,6 +118,7 @@ const mockStreamingRenderer = vi.fn(
     renderProposedPlanBlocks?: boolean;
     showContentBlockActions?: boolean;
     onQuoteContent?: (content: string) => void;
+    markdownRenderMode?: string;
     onOpenSavedSiteContent?: (target: {
       projectId: string;
       contentId: string;
@@ -130,6 +136,7 @@ const mockStreamingRenderer = vi.fn(
       data-render-proposed-plan-blocks={renderProposedPlanBlocks ? "yes" : "no"}
       data-show-content-block-actions={showContentBlockActions ? "yes" : "no"}
       data-has-on-quote-content={onQuoteContent ? "yes" : "no"}
+      data-markdown-render-mode={markdownRenderMode || "standard"}
     >
       {content || ""}
@@ -164,6 +171,7 @@ vi.mock("./StreamingRenderer", () => ({ content: string; renderA2UIInline?: boolean; suppressedActionRequestId?: string | null; + markdownRenderMode?: string; }) => mockStreamingRenderer(props), })); @@ -200,6 +208,7 @@ beforeEach(() => { if (!HTMLElement.prototype.scrollIntoView) { HTMLElement.prototype.scrollIntoView = () => {}; } + clearAgentUiPerformanceMetrics(); }); afterEach(() => { @@ -213,6 +222,7 @@ afterEach(() => { } vi.useRealTimers(); vi.clearAllMocks(); + clearAgentUiPerformanceMetrics(); mockUseConfiguredProviders.mockImplementation(() => ({ providers: [], loading: false, @@ -452,7 +462,7 @@ describe("MessageList", () => { expect(mockAgentThreadTimeline).not.toHaveBeenCalled(); act(() => { - vi.advanceTimersByTime(100); + vi.advanceTimersByTime(920); }); expect(mockAgentThreadTimeline).toHaveBeenCalled(); @@ -481,14 +491,611 @@ describe("MessageList", () => { expect(container.textContent).toContain("最近 40 / 188 条消息"); expect(container.textContent).toContain("消息 40"); - expect(container.textContent).toContain("消息 21"); - expect(container.textContent).not.toContain("消息 20"); + expect(container.textContent).toContain("消息 31"); + expect(container.textContent).not.toContain("消息 30"); + expect(container.textContent).toContain("更早的 30 条可按需展开"); act(() => { - vi.advanceTimersByTime(130); + vi.advanceTimersByTime(2_000); }); - expect(container.textContent).toContain("消息 20"); + expect(container.textContent).not.toContain("消息 30"); + + const expandButton = container.querySelector( + '[data-testid="message-list-expand-history"]', + ) as HTMLButtonElement | null; + + act(() => { + expandButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + expect(container.textContent).toContain("消息 30"); + }); + + it("旧会话消息较少但执行过程很多时也应延后构建 timeline", () => { + vi.useFakeTimers(); + const turn: AgentThreadTurn = { + id: "turn-history-many-items", + thread_id: "thread-history-many-items", + prompt_text: "检查慢历史", + status: "completed", + started_at: "2026-04-25T10:00:00.000Z", + completed_at: "2026-04-25T10:01:00.000Z", + created_at: "2026-04-25T10:00:00.000Z", + updated_at: "2026-04-25T10:01:00.000Z", + }; + const threadItems: AgentThreadItem[] = Array.from( + { length: 30 }, + (_, index): AgentThreadItem => { + const base = { + id: `history-heavy-item-${index + 1}`, + thread_id: turn.thread_id, + turn_id: turn.id, + sequence: index + 1, + status: "completed" as const, + started_at: "2026-04-25T10:00:00.000Z", + completed_at: "2026-04-25T10:01:00.000Z", + updated_at: "2026-04-25T10:01:00.000Z", + }; + + if (index % 2 === 0) { + return { + ...base, + type: "tool_call", + tool_name: "Bash", + arguments: { command: `echo ${index}` }, + output: `输出 ${index}`, + }; + } + + return { + ...base, + type: "reasoning", + text: `思考 ${index}`, + }; + }, + ); + const container = render( + [ + { + id: "msg-user-history-many-items", + role: "user", + content: "检查慢历史", + timestamp: new Date("2026-04-25T10:00:00.000Z"), + } as Message, + { + id: "msg-assistant-history-many-items", + role: "assistant", + content: "历史结果", + timestamp: new Date("2026-04-25T10:01:00.000Z"), + } as Message, + ], + { + currentTurnId: turn.id, + turns: [turn], + threadItems, + sessionHistoryWindow: { + loadedMessages: 2, + totalMessages: 170, + isLoadingFull: false, + error: null, + }, + }, + ); + + expect(container.textContent).toContain("历史结果"); + expect( + container.querySelector( + '[data-testid="message-list-historical-timeline-preview:leading"]', + ), + ).toBeNull(); + expect( + container.querySelector('[data-testid="inputbar-runtime-status-line"]'), + ).toBeNull(); + expect(mockAgentThreadTimeline).not.toHaveBeenCalled(); + + act(() => { + vi.advanceTimersByTime(880); + }); + + expect( + container.querySelector( + '[data-testid="message-list-historical-timeline-preview:leading"]', + ), + ).toBeNull(); + + act(() => { + vi.advanceTimersByTime(60); + }); + + expect( + container.querySelector( + '[data-testid="message-list-historical-timeline-preview:leading"]', + ), + ).not.toBeNull(); + expect( + container.querySelector('[data-testid="inputbar-runtime-status-line"]'), + ).not.toBeNull(); + expect(mockAgentThreadTimeline).not.toHaveBeenCalled(); + }); + + it("已分页旧会话首帧应只把尾部相关 turns 的 threadItems 纳入计算", async () => { + vi.useFakeTimers(); + const turns: AgentThreadTurn[] = Array.from({ length: 8 }, (_, index) => { + const minute = String(index + 1).padStart(2, "0"); + return { + id: `turn-window-${index + 1}`, + thread_id: "thread-windowed-history", + prompt_text: `历史问题 ${index + 1}`, + status: "completed", + started_at: `2026-04-25T10:${minute}:00.000Z`, + completed_at: `2026-04-25T10:${minute}:30.000Z`, + created_at: `2026-04-25T10:${minute}:00.000Z`, + updated_at: `2026-04-25T10:${minute}:30.000Z`, + }; + }); + const threadItems: AgentThreadItem[] = turns.flatMap((turn, turnIndex) => + Array.from({ length: 5 }, (_, itemIndex): AgentThreadItem => ({ + id: `turn-window-${turnIndex + 1}-item-${itemIndex + 1}`, + thread_id: turn.thread_id, + turn_id: turn.id, + sequence: itemIndex + 1, + status: "completed", + started_at: turn.started_at, + completed_at: turn.completed_at, + updated_at: turn.updated_at, + type: "tool_call", + tool_name: "Read", + arguments: { file_path: `/repo/file-${itemIndex + 1}.ts` }, + })), + ); + + render( + [ + { + id: "msg-user-windowed-history", + role: "user", + content: "打开尾部旧会话", + timestamp: new Date("2026-04-25T10:08:00.000Z"), + } as Message, + { + id: "msg-assistant-windowed-history", + role: "assistant", + content: "这是尾部旧会话结果", + timestamp: new Date("2026-04-25T10:08:30.000Z"), + } as Message, + ], + { + sessionId: "session-windowed-history", + currentTurnId: "turn-window-8", + turns, + threadItems, + sessionHistoryWindow: { + loadedMessages: 2, + totalMessages: 220, + isLoadingFull: false, + error: null, + }, + }, + ); + + await act(async () => { + await Promise.resolve(); + }); + + const commit = getAgentUiPerformanceMetrics().find( + (entry) => entry.phase === "messageList.commit", + ); + + expect(commit?.metrics).toEqual( + expect.objectContaining({ + renderedTurnsCount: 2, + shouldDeferHistoricalTimeline: true, + threadItemsCount: 0, + threadItemsScanDeferred: true, + turnsCount: 8, + }), + ); + + act(() => { + vi.advanceTimersByTime(940); + }); + + await act(async () => { + await Promise.resolve(); + }); + + const hydratedCommit = getAgentUiPerformanceMetrics() + .filter((entry) => entry.phase === "messageList.commit") + .find( + (entry) => + entry.metrics.threadItemsScanDeferred === false && + entry.metrics.threadItemsCount === 10, + ); + + expect(hydratedCommit?.metrics).toEqual( + expect.objectContaining({ + renderedTurnsCount: 2, + threadItemsCount: 10, + turnsCount: 8, + }), + ); + }); + + it("旧会话首帧应延后历史助手 contentParts 与 Markdown 细节扫描", async () => { + vi.useFakeTimers(); + const turn: AgentThreadTurn = { + id: "turn-history-content-parts", + thread_id: "thread-history-content-parts", + prompt_text: "检查 content parts", + status: "completed", + started_at: "2026-04-25T10:00:00.000Z", + completed_at: "2026-04-25T10:01:00.000Z", + created_at: "2026-04-25T10:00:00.000Z", + updated_at: "2026-04-25T10:01:00.000Z", + }; + const threadItems: AgentThreadItem[] = Array.from( + { length: 30 }, + (_, index): AgentThreadItem => ({ + id: `history-content-parts-tool-${index + 1}`, + thread_id: turn.thread_id, + turn_id: turn.id, + sequence: index + 1, + status: "completed", + started_at: turn.started_at, + completed_at: turn.completed_at, + updated_at: turn.updated_at, + type: "tool_call", + tool_name: "Read", + arguments: { file_path: `/repo/history-${index + 1}.ts` }, + }), + ); + const container = render( + [ + { + id: "msg-user-history-content-parts", + role: "user", + content: "检查 content parts", + timestamp: new Date("2026-04-25T10:00:00.000Z"), + } as Message, + { + id: "msg-assistant-history-content-parts", + role: "assistant", + content: "历史 content parts 正文", + contentParts: [ + { + type: "text", + text: "历史 content parts 正文", + }, + { + type: "tool_use", + toolCall: { + id: "tool-history-content-parts", + name: "Read", + arguments: JSON.stringify({ file_path: "/repo/history.ts" }), + status: "completed", + result: { success: true, output: "ok" }, + startTime: new Date("2026-04-25T10:00:10.000Z"), + endTime: new Date("2026-04-25T10:00:11.000Z"), + }, + }, + ], + timestamp: new Date("2026-04-25T10:01:00.000Z"), + } as Message, + ], + { + sessionId: "session-history-content-parts", + currentTurnId: turn.id, + turns: [turn], + threadItems, + sessionHistoryWindow: { + loadedMessages: 2, + totalMessages: 180, + isLoadingFull: false, + error: null, + }, + }, + ); + + expect( + container.querySelector( + '[data-testid="message-list-historical-markdown-preview"]', + ), + ).not.toBeNull(); + expect( + container.querySelector('[data-testid="streaming-renderer"]'), + ).toBeNull(); + expect(mockStreamingRenderer).not.toHaveBeenCalled(); + const markdownPreview = container.querySelector( + '[data-testid="message-list-historical-markdown-preview"]', + ); + expect(markdownPreview?.textContent).toContain( + "历史 content parts 正文", + ); + const commit = getAgentUiPerformanceMetrics().find( + (entry) => entry.phase === "messageList.commit", + ); + expect(commit?.metrics).toEqual( + expect.objectContaining({ + historicalContentPartsDeferredCount: 1, + historicalMarkdownDeferredCount: 1, + threadItemsScanDeferred: true, + }), + ); + + act(() => { + vi.advanceTimersByTime(940); + }); + + await act(async () => { + await Promise.resolve(); + }); + + const hydratedCommit = getAgentUiPerformanceMetrics() + .filter((entry) => entry.phase === "messageList.commit") + .find( + (entry) => entry.metrics.historicalContentPartsDeferredCount === 0, + ); + expect( + container.querySelector( + '[data-testid="message-list-historical-markdown-preview"]', + ), + ).toBeNull(); + expect(mockStreamingRenderer).toHaveBeenCalled(); + expect(hydratedCommit?.metrics).toEqual( + expect.objectContaining({ + historicalContentPartsDeferredCount: 0, + historicalMarkdownDeferredCount: 0, + threadItemsCount: 30, + }), + ); + }); + + it("已分页旧会话的完成执行过程应先折叠为轻量摘要，点击后再挂载真实 timeline", () => { + const turn: AgentThreadTurn = { + id: "turn-history-heavy", + thread_id: "thread-history-heavy", + prompt_text: "打开慢历史", + status: "completed", + started_at: "2026-04-25T10:00:00.000Z", + completed_at: "2026-04-25T10:01:00.000Z", + created_at: "2026-04-25T10:00:00.000Z", + updated_at: "2026-04-25T10:01:00.000Z", + }; + const threadItems: AgentThreadItem[] = Array.from( + { length: 10 }, + (_, index) => ({ + id: `history-tool-${index + 1}`, + thread_id: turn.thread_id, + turn_id: turn.id, + sequence: index + 1, + status: "completed", + started_at: "2026-04-25T10:00:00.000Z", + completed_at: "2026-04-25T10:01:00.000Z", + updated_at: "2026-04-25T10:01:00.000Z", + type: "tool_call", + tool_name: "Bash", + arguments: { command: `echo ${index + 1}` }, + output: `输出 ${index + 1}`, + }), + ); + const container = render( + [ + { + id: "msg-user-heavy-history", + role: "user", + content: "打开慢历史", + timestamp: new Date("2026-04-25T10:00:00.000Z"), + } as Message, + { + id: "msg-assistant-heavy-history", + role: "assistant", + content: "这是旧会话的最终回复", + contentParts: [ + { + type: "text", + text: "这是旧会话的最终回复", + }, + ], + timestamp: new Date("2026-04-25T10:01:00.000Z"), + } as Message, + ], + { + turns: [turn], + threadItems, + currentTurnId: turn.id, + sessionHistoryWindow: { + loadedMessages: 2, + totalMessages: 170, + isLoadingFull: false, + error: null, + }, + }, + ); + + expect( + container.querySelector( + '[data-testid="message-list-historical-timeline-preview:leading"]', + ), + ).not.toBeNull(); + expect(mockAgentThreadTimeline).not.toHaveBeenCalled(); + expect(mockStreamingRenderer).toHaveBeenLastCalledWith( + expect.objectContaining({ + contentParts: undefined, + markdownRenderMode: "light", + }), + ); + + const expandButton = container.querySelector( + '[data-testid="message-list-historical-timeline-preview:leading"]', + ) as HTMLButtonElement | null; + + act(() => { + expandButton?.click(); + }); + + expect(mockAgentThreadTimeline).toHaveBeenCalledWith( + expect.objectContaining({ + placement: "leading", + isCurrentTurn: false, + }), + ); + }); + + it("旧会话里的超长历史助手消息应先渲染轻量预览，点击后再展开完整正文", () => { + const longContent = `开头内容 ${"长历史 ".repeat(8000)} 末尾完整内容`; + const container = render( + [ + { + id: "msg-user-long-history", + role: "user", + content: "打开超长历史", + timestamp: new Date("2026-04-25T10:00:00.000Z"), + } as Message, + { + id: "msg-assistant-long-history", + role: "assistant", + content: longContent, + timestamp: new Date("2026-04-25T10:00:01.000Z"), + } as Message, + ], + { + sessionHistoryWindow: { + loadedMessages: 2, + totalMessages: 120, + isLoadingFull: false, + error: null, + }, + }, + ); + + const preview = container.querySelector( + '[data-testid="message-list-long-history-preview"]', + ); + + expect(preview).not.toBeNull(); + expect(preview?.textContent).toContain("此历史消息较长"); + expect(preview?.textContent).toContain("纯文本预览"); + expect(preview?.textContent).not.toContain("末尾完整内容"); + expect(mockStreamingRenderer).not.toHaveBeenCalled(); + + const expandButton = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent?.includes("展开完整内容"), + ) as HTMLButtonElement | undefined; + + act(() => { + expandButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + expect( + container.querySelector( + '[data-testid="message-list-long-history-preview"]', + ), + ).toBeNull(); + expect(mockStreamingRenderer).toHaveBeenLastCalledWith( + expect.objectContaining({ + content: expect.stringContaining("末尾完整内容"), + markdownRenderMode: "light", + }), + ); + }); + + it("旧会话里的长助手回复应先展示纯文本预览，避免首帧挂载 Markdown", () => { + const oldAssistantContent = `旧回复开头 ${"历史分析 ".repeat(360)} 旧回复末尾完整内容`; + const latestAssistantContent = "最新回复保持完整"; + const container = render( + [ + { + id: "msg-user-old-compact", + role: "user", + content: "旧问题", + timestamp: new Date("2026-04-25T10:00:00.000Z"), + } as Message, + { + id: "msg-assistant-old-compact", + role: "assistant", + content: oldAssistantContent, + timestamp: new Date("2026-04-25T10:00:01.000Z"), + } as Message, + { + id: "msg-user-latest-compact", + role: "user", + content: "最新问题", + timestamp: new Date("2026-04-25T10:01:00.000Z"), + } as Message, + { + id: "msg-assistant-latest-compact", + role: "assistant", + content: latestAssistantContent, + timestamp: new Date("2026-04-25T10:01:01.000Z"), + } as Message, + ], + { + sessionHistoryWindow: { + loadedMessages: 4, + totalMessages: 88, + isLoadingFull: false, + error: null, + }, + }, + ); + + const preview = container.querySelector( + '[data-testid="message-list-historical-assistant-preview"]', + ); + + expect(preview).not.toBeNull(); + expect(preview?.textContent).toContain("历史助手回复较长"); + expect(preview?.textContent).not.toContain("旧回复末尾完整内容"); + expect(container.textContent).toContain(latestAssistantContent); + expect(mockStreamingRenderer).not.toHaveBeenCalledWith( + expect.objectContaining({ + content: expect.stringContaining("旧回复末尾完整内容"), + }), + ); + + const expandButton = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent?.includes("展开完整内容"), + ) as HTMLButtonElement | undefined; + + act(() => { + expandButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + }); + + expect( + container.querySelector( + '[data-testid="message-list-historical-assistant-preview"]', + ), + ).toBeNull(); + expect(mockStreamingRenderer).toHaveBeenCalledWith( + expect.objectContaining({ + content: expect.stringContaining("旧回复末尾完整内容"), + markdownRenderMode: "light", + }), + ); + }); + + it("非旧会话助手正文应保持标准 Markdown 渲染模式", () => { + render([ + { + id: "msg-user-live-standard-markdown", + role: "user", + content: "实时对话", + timestamp: new Date("2026-04-25T10:00:00.000Z"), + } as Message, + { + id: "msg-assistant-live-standard-markdown", + role: "assistant", + content: "```ts\nconsole.log('live')\n```", + timestamp: new Date("2026-04-25T10:00:01.000Z"), + } as Message, + ]); + + expect(mockStreamingRenderer).toHaveBeenLastCalledWith( + expect.objectContaining({ + markdownRenderMode: "standard", + }), + ); }); it("任务中心空列表时应展示最近对话空态而不是普通新对话文案", () => { @@ -747,6 +1354,70 @@ describe("MessageList", () => { ); }); + it("旧会话恢复首帧不应立即自动加载 Provider 缓存提示配置", () => { + const now = new Date(); + const messages: Message[] = [ + { + id: "msg-assistant-restored-usage", + role: "assistant", + content: "旧会话结果。", + timestamp: now, + usage: { + input_tokens: 1_200, + output_tokens: 300, + cached_input_tokens: 0, + }, + }, + ]; + + render(messages, { + providerType: "custom-provider-id", + sessionHistoryWindow: { + loadedMessages: 40, + totalMessages: 320, + isLoadingFull: false, + error: null, + }, + }); + + expect(mockUseConfiguredProviders).toHaveBeenCalledWith({ + autoLoad: false, + }); + }); + + it("旧会话首帧应记录可汇总的渲染采样数值", async () => { + const messages = createConversationMessages(32); + + render(messages, { + sessionId: "session-metrics", + sessionHistoryWindow: { + loadedMessages: 32, + totalMessages: 160, + isLoadingFull: false, + error: null, + }, + }); + + await act(async () => { + await Promise.resolve(); + }); + + const commit = getAgentUiPerformanceMetrics().find( + (entry) => entry.phase === "messageList.commit", + ); + expect(commit).toEqual( + expect.objectContaining({ + sessionId: "session-metrics", + metrics: expect.objectContaining({ + hiddenHistoryCount: expect.any(Number), + messagesCount: 32, + persistedHiddenHistoryCount: 128, + renderedMessagesCount: expect.any(Number), + }), + }), + ); + }); + it("复杂任务完成后应把运行状态、耗时与 token 结算收口到最后一条 assistant 消息尾部", () => { const now = new Date(); const messages: Message[] = [ @@ -938,6 +1609,11 @@ describe("MessageList", () => { expect( container.querySelector('[data-testid="inputbar-runtime-status-line"]'), ).toBeNull(); + expect( + container.querySelector( + '[data-testid="assistant-first-token-placeholder"]', + ), + ).toBeNull(); }); it("首个文本分片到来前，不应把运行态当作 assistant 回复渲染", () => { @@ -1016,7 +1692,7 @@ describe("MessageList", () => { expect(container.textContent).not.toContain("Built-in Tool"); }); - it("assistant 占位消息只有启动态 runtimeStatus 时，也不应保留状态回复", () => { + it("assistant 占位消息只有启动态 runtimeStatus 时，应渲染轻量首字前占位", () => { const now = new Date(); const messages: Message[] = [ { @@ -1058,8 +1734,15 @@ describe("MessageList", () => { expect( container.querySelector('[data-testid="inputbar-runtime-status-line"]'), ).toBeNull(); - expect(container.textContent).not.toContain("处理中"); - expect(container.textContent).not.toContain("正在启动处理流程"); + expect( + container.querySelector( + '[data-testid="assistant-first-token-placeholder"]', + ), + ).not.toBeNull(); + expect(container.textContent).toContain("正在启动处理流程"); + expect(container.textContent).toContain( + "已开始处理，正在准备环境并等待第一条进展。", + ); }); it("assistant 消息结算区应以内联模式承载 token usage", () => { @@ -1541,6 +2224,211 @@ describe("MessageList", () => { ); }); + it("配音任务消息卡应展示 audio_generate 预览并支持打开运行时文档", () => { + const now = new Date(); + const onOpenMessagePreview = vi.fn(); + const messages: Message[] = [ + { + id: "msg-assistant-audio-task", + role: "assistant", + content: "配音任务已提交。", + timestamp: now, + taskPreview: { + kind: "audio_generate", + taskId: "task-audio-1", + taskType: "audio_generate", + prompt: "欢迎来到 Lime 多模态工作台。", + title: "配音生成任务", + status: "running", + artifactPath: ".lime/runtime/audio-generate/task-audio-1.md", + taskFilePath: ".lime/tasks/audio_generate/task-audio-1.json", + metaItems: ["warm_female", "8 秒"], + voice: "warm_female", + durationMs: 8200, + }, + }, + ]; + + const container = render(messages, { onOpenMessagePreview }); + const previewCard = container.querySelector( + '[data-testid="task-message-preview-task-audio-1"]', + ) as HTMLButtonElement | null; + + expect(previewCard?.textContent).toContain("配音生成"); + expect(previewCard?.textContent).toContain("欢迎来到 Lime 多模态工作台"); + expect(previewCard?.textContent).toContain("warm_female"); + expect(previewCard?.textContent).toContain("源任务"); + + act(() => { + previewCard?.click(); + }); + + expect(onOpenMessagePreview).toHaveBeenCalledWith( + { + kind: "task", + preview: expect.objectContaining({ + kind: "audio_generate", + taskId: "task-audio-1", + }), + }, + expect.objectContaining({ + id: "msg-assistant-audio-task", + }), + ); + }); + + it("失败的配音任务卡应展示 provider 错误码与原因", () => { + const now = new Date(); + const messages: Message[] = [ + { + id: "msg-assistant-audio-task-failed", + role: "assistant", + content: "配音任务失败。", + timestamp: now, + taskPreview: { + kind: "audio_generate", + taskId: "task-audio-failed-1", + taskType: "audio_generate", + prompt: "欢迎来到 Lime 多模态工作台。", + title: "配音生成任务", + status: "failed", + artifactPath: ".lime/runtime/audio-generate/task-audio-failed-1.md", + taskFilePath: ".lime/tasks/audio_generate/task-audio-failed-1.json", + errorCode: "audio_provider_unconfigured", + errorMessage: + "未找到可用的 voice_generation provider/API Key: missing-provider。", + statusMessage: + "配音 Provider 未配置，请先在语音生成设置中选择可用 Provider；任务保留在 audio_generate，不会回退 legacy TTS。", + }, + }, + ]; + + const container = render(messages); + const previewCard = container.querySelector( + '[data-testid="task-message-preview-task-audio-failed-1"]', + ) as HTMLButtonElement | null; + + expect(previewCard?.textContent).toContain("执行失败"); + expect(previewCard?.textContent).toContain("audio_provider_unconfigured"); + expect(previewCard?.textContent).toContain( + "未找到可用的 voice_generation provider/API Key", + ); + expect(previewCard?.textContent).toContain("不会回退 legacy TTS"); + }); + + it("转写任务消息卡应展示 transcript 路径与 provider 错误", () => { + const now = new Date(); + const onOpenMessagePreview = vi.fn(); + const messages: Message[] = [ + { + id: "msg-assistant-transcription-task", + role: "assistant", + content: "转写任务已同步。", + timestamp: now, + taskPreview: { + kind: "transcription_generate", + taskId: "task-transcription-1", + taskType: "transcription_generate", + prompt: "请转写访谈音频", + title: "内容转写任务", + status: "complete", + artifactPath: + ".lime/runtime/transcription-generate/task-transcription-1.md", + taskFilePath: + ".lime/tasks/transcription_generate/task-transcription-1.json", + transcriptPath: ".lime/runtime/transcripts/task-transcription-1.txt", + language: "zh-CN", + outputFormat: "txt", + transcriptSegments: [ + { + id: "segment-1", + index: 1, + startMs: 1000, + endMs: 3500, + speaker: "主持人", + text: "欢迎来到 Lime 访谈。", + }, + ], + statusMessage: + "转写结果已同步，工作区已从 transcript 读取可校对文本。", + }, + }, + ]; + + const container = render(messages, { onOpenMessagePreview }); + const previewCard = container.querySelector( + '[data-testid="task-message-preview-task-transcription-1"]', + ) as HTMLButtonElement | null; + + expect(previewCard?.textContent).toContain("内容转写"); + expect(previewCard?.textContent).toContain("请转写访谈音频"); + expect(previewCard?.textContent).toContain("转写结果"); + expect(previewCard?.textContent).toContain("task-transcription-1.txt"); + expect(previewCard?.textContent).toContain("1 段时间轴"); + expect(previewCard?.textContent).toContain("时间轴预览"); + expect(previewCard?.textContent).toContain("主持人：欢迎来到 Lime 访谈。"); + + act(() => { + previewCard?.click(); + }); + + expect(onOpenMessagePreview).toHaveBeenCalledWith( + { + kind: "task", + preview: expect.objectContaining({ + kind: "transcription_generate", + taskId: "task-transcription-1", + }), + }, + expect.objectContaining({ + id: "msg-assistant-transcription-task", + }), + ); + }); + + it("失败的转写任务卡应展示 transcript 错误码与原因", () => { + const now = new Date(); + const messages: Message[] = [ + { + id: "msg-assistant-transcription-task-failed", + role: "assistant", + content: "转写任务失败。", + timestamp: now, + taskPreview: { + kind: "transcription_generate", + taskId: "task-transcription-failed-1", + taskType: "transcription_generate", + prompt: "请转写访谈音频", + title: "内容转写任务", + status: "failed", + artifactPath: + ".lime/runtime/transcription-generate/task-transcription-failed-1.md", + taskFilePath: + ".lime/tasks/transcription_generate/task-transcription-failed-1.json", + errorCode: "transcription_provider_unconfigured", + errorMessage: + "未找到可用的 audio_transcription provider/API Key: missing-provider。", + statusMessage: + "转写 Provider 未配置，请先在转写设置中选择可用 Provider；任务保留在 transcription_generate，不会回退 frontend ASR。", + }, + }, + ]; + + const container = render(messages); + const previewCard = container.querySelector( + '[data-testid="task-message-preview-task-transcription-failed-1"]', + ) as HTMLButtonElement | null; + + expect(previewCard?.textContent).toContain("执行失败"); + expect(previewCard?.textContent).toContain( + "transcription_provider_unconfigured", + ); + expect(previewCard?.textContent).toContain( + "未找到可用的 audio_transcription provider/API Key", + ); + expect(previewCard?.textContent).toContain("不会回退 frontend ASR"); + }); + it("联网搜图结果消息卡应展示缩略图候选", () => { const now = new Date(); const messages: Message[] = [ @@ -2823,6 +3711,12 @@ describe("MessageList", () => { expect( container.querySelector('[data-testid="inputbar-runtime-status-line"]'), ).toBeNull(); + expect( + container.querySelector( + '[data-testid="assistant-first-token-placeholder"]', + ), + ).not.toBeNull(); + expect(container.textContent).toContain("正在打开 GitHub"); }); it("本地工具批次的阶段结论不应再进入主消息流时间线", () => { diff --git a/src/components/agent/chat/components/MessageList.tsx b/src/components/agent/chat/components/MessageList.tsx index ed397e58c..c88baa2a1 100644 --- a/src/components/agent/chat/components/MessageList.tsx +++ b/src/components/agent/chat/components/MessageList.tsx @@ -22,6 +22,7 @@ import { Button } from "@/components/ui/button"; import { toast } from "sonner"; import type { Artifact } from "@/lib/artifact/types"; import { scheduleMinimumDelayIdleTask } from "@/lib/utils/scheduleMinimumDelayIdleTask"; +import { recordAgentUiPerformanceMetric } from "@/lib/agentUiPerformanceMetrics"; import { resolveConfiguredProviderPromptCacheSupportNotice, useConfiguredProviders, @@ -89,6 +90,7 @@ import { isPureRuntimePeerMessageText } from "../utils/runtimePeerMessageDisplay import { LIME_BRAND_LOGO_SRC, LIME_BRAND_NAME } from "@/lib/branding"; interface MessageListProps { + sessionId?: string | null; messages: Message[]; leadingContent?: React.ReactNode; emptyStateVariant?: "default" | "task-center"; @@ -190,11 +192,180 @@ const MESSAGE_LIST_PROGRESSIVE_RENDER_THRESHOLD = 72; const MESSAGE_LIST_INITIAL_RENDER_COUNT = 36; const MESSAGE_LIST_RENDER_BATCH_SIZE = 48; const MESSAGE_LIST_RESTORED_PROGRESSIVE_RENDER_THRESHOLD = 20; -const MESSAGE_LIST_RESTORED_INITIAL_RENDER_COUNT = 20; -const MESSAGE_LIST_RESTORED_RENDER_BATCH_SIZE = 12; -const MESSAGE_LIST_TIMELINE_ITEMS_FACTOR = 4; +const MESSAGE_LIST_RESTORED_INITIAL_RENDER_COUNT = 10; +const MESSAGE_LIST_RESTORED_RENDER_BATCH_SIZE = 6; +const MESSAGE_LIST_PROGRESSIVE_RENDER_MINIMUM_DELAY_MS = 120; +const MESSAGE_LIST_RESTORED_PROGRESSIVE_RENDER_MINIMUM_DELAY_MS = 600; const MESSAGE_LIST_TIMELINE_DEFER_MESSAGE_THRESHOLD = 24; const MESSAGE_LIST_TIMELINE_DEFER_ITEM_THRESHOLD = 24; +const MESSAGE_LIST_HISTORICAL_TIMELINE_COMPACT_ITEM_THRESHOLD = 8; +const MESSAGE_LIST_HISTORICAL_TIMELINE_IDLE_DELAY_MS = 80; +const MESSAGE_LIST_RESTORED_HISTORICAL_TIMELINE_IDLE_DELAY_MS = 900; +const MESSAGE_LIST_COMPACT_HISTORICAL_ASSISTANT_THRESHOLD = 900; +const MESSAGE_LIST_COMPACT_HISTORICAL_ASSISTANT_PREVIEW_CHARS = 900; +const MESSAGE_LIST_LONG_HISTORICAL_MESSAGE_THRESHOLD = 24_000; +const MESSAGE_LIST_LONG_HISTORICAL_MESSAGE_PREVIEW_CHARS = 2_000; +const MESSAGE_LIST_STRUCTURED_HISTORY_CONTENT_RE = + / void; +} + +function HistoricalAssistantMessagePreview({ + content, + contentLength, + variant, + onExpand, +}: HistoricalAssistantMessagePreviewProps) { + const isLong = variant === "long"; + + return ( +
+
+ {content} +
+
+ + {isLong ? "此历史消息较长" : "历史助手回复较长"}（约{" "} + {formatContentLength(contentLength)} 字），已先展示纯文本预览。 + + +
+
+ ); +} + +const HistoricalMarkdownHydrationPreview: React.FC<{ content: string }> = ({ + content, +}) => ( +
+ {content} +
+); + +function summarizeHistoricalTimelineItems(items: AgentThreadItem[]): { + stepsCount: number; + metaText: string; +} { + const visibleItems = items.filter((item) => { + if (item.type === "user_message" || item.type === "agent_message") { + return false; + } + + return !( + item.type === "file_artifact" && + isHiddenConversationArtifactPath(item.path) + ); + }); + const toolStepsCount = visibleItems.filter( + (item) => + item.type === "tool_call" || + item.type === "command_execution" || + item.type === "web_search", + ).length; + const thinkingStepsCount = visibleItems.filter( + (item) => + item.type === "reasoning" || + item.type === "plan" || + item.type === "turn_summary" || + item.type === "context_compaction", + ).length; + const artifactStepsCount = visibleItems.filter( + (item) => item.type === "file_artifact", + ).length; + const metaParts = [ + toolStepsCount > 0 ? `${toolStepsCount} 个工具步骤` : null, + thinkingStepsCount > 0 ? `${thinkingStepsCount} 条思路` : null, + artifactStepsCount > 0 ? `${artifactStepsCount} 份产物` : null, + ].filter((part): part is string => Boolean(part)); + + return { + stepsCount: visibleItems.length, + metaText: metaParts.length > 0 ? metaParts.join("，") : "执行细节已折叠", + }; +} + +const HistoricalTimelinePreview: React.FC<{ + items: AgentThreadItem[]; + placement: "leading" | "trailing" | "default"; + onExpand: () => void; +}> = ({ items, placement, onExpand }) => { + const summary = useMemo( + () => summarizeHistoricalTimelineItems(items), + [items], + ); + + if (summary.stepsCount <= 0) { + return null; + } + + return ( + + ); +}; function normalizeRuntimeStatusMetaText(value?: string | null): string { return (value || "").trim().replace(/\s+/g, " "); @@ -243,6 +414,40 @@ const MessageRuntimeStatusPill: React.FC<{ ); }; +function truncateRuntimeStatusText(value: string, maxLength = 96): string { + const normalized = normalizeRuntimeStatusMetaText(value); + if (normalized.length <= maxLength) { + return normalized; + } + + return `${Array.from(normalized).slice(0, maxLength).join("")}...`; +} + +const AssistantFirstTokenPlaceholder: React.FC<{ + status: AgentRuntimeStatus; +}> = ({ status }) => { + const title = truncateRuntimeStatusText(status.title || "正在准备处理", 48); + const detail = + truncateRuntimeStatusText(status.detail, 120) || + "已提交请求，等待首个响应。"; + + return ( +
+ + + {title} + + {detail} + + +
+ ); +}; + function shouldRenderRuntimeStatusPill( status?: AgentRuntimeStatus | null, ): boolean { @@ -481,6 +686,7 @@ function filterConversationDisplayContentParts( } const MessageListInner: React.FC = ({ + sessionId = null, messages, leadingContent, emptyStateVariant = "default", @@ -525,38 +731,21 @@ const MessageListInner: React.FC = ({ const scrollRef = useRef(null); const containerRef = useRef(null); const previousVisibleMessageCountRef = useRef(null); + const restoredSessionMetricRef = useRef(null); const isTaskCenterEmptyState = emptyStateVariant === "task-center"; const [copiedId, setCopiedId] = useState(null); const [isUserScrolling, setIsUserScrolling] = useState(false); const [shouldAutoScroll, setShouldAutoScroll] = useState(true); - - const shouldInspectPromptCacheNotice = useMemo( - () => - Boolean( - providerType?.trim() && - messages.some( - (msg) => - msg.role === "assistant" && - !msg.isThinking && - msg.usage && - resolvePromptCacheActivity(msg.usage) <= 0, - ), - ), - [messages, providerType], - ); - const { providers } = useConfiguredProviders({ - autoLoad: shouldInspectPromptCacheNotice, - }); - const promptCacheNotice = useMemo( - () => - shouldInspectPromptCacheNotice - ? resolveConfiguredProviderPromptCacheSupportNotice( - providers, - providerType, - ) - : null, - [providerType, providers, shouldInspectPromptCacheNotice], - ); + const [ + expandedLongHistoricalMessageIds, + setExpandedLongHistoricalMessageIds, + ] = useState>(() => new Set()); + const [ + expandedHistoricalAssistantMessageIds, + setExpandedHistoricalAssistantMessageIds, + ] = useState>(() => new Set()); + const [expandedHistoricalTimelineKeys, setExpandedHistoricalTimelineKeys] = + useState>(() => new Set()); const visibleMessages = useMemo( () => @@ -577,6 +766,26 @@ const MessageListInner: React.FC = ({ : 0; const isRestoredHistoryWindow = isRestoringSession || persistedHiddenHistoryCount > 0; + const [restoredPromptCacheNoticeReady, setRestoredPromptCacheNoticeReady] = + useState(() => !isRestoredHistoryWindow); + + useEffect(() => { + if (!isRestoredHistoryWindow) { + setRestoredPromptCacheNoticeReady(true); + return; + } + + setRestoredPromptCacheNoticeReady(false); + return scheduleMinimumDelayIdleTask( + () => { + setRestoredPromptCacheNoticeReady(true); + }, + { + minimumDelayMs: 1_500, + idleTimeoutMs: 3_000, + }, + ); + }, [isRestoredHistoryWindow, visibleMessageFirstId, visibleMessageLastId]); const progressiveRenderThreshold = isRestoredHistoryWindow ? MESSAGE_LIST_RESTORED_PROGRESSIVE_RENDER_THRESHOLD : MESSAGE_LIST_PROGRESSIVE_RENDER_THRESHOLD; @@ -586,6 +795,9 @@ const MessageListInner: React.FC = ({ const progressiveRenderBatchSize = isRestoredHistoryWindow ? MESSAGE_LIST_RESTORED_RENDER_BATCH_SIZE : MESSAGE_LIST_RENDER_BATCH_SIZE; + const progressiveRenderMinimumDelayMs = isRestoredHistoryWindow + ? MESSAGE_LIST_RESTORED_PROGRESSIVE_RENDER_MINIMUM_DELAY_MS + : MESSAGE_LIST_PROGRESSIVE_RENDER_MINIMUM_DELAY_MS; const shouldUseProgressiveRender = !isSending && visibleMessages.length > progressiveRenderThreshold; const visibleMessageWindowRef = useRef<{ @@ -647,10 +859,12 @@ const MessageListInner: React.FC = ({ const hiddenHistoryCount = shouldUseProgressiveRender ? Math.max(0, visibleMessages.length - renderedMessageCount) : 0; + const shouldAutoHydrateHiddenHistory = + shouldUseProgressiveRender && !isRestoredHistoryWindow; useEffect(() => { if ( - !shouldUseProgressiveRender || + !shouldAutoHydrateHiddenHistory || hiddenHistoryCount <= 0 || isUserScrolling ) { @@ -667,7 +881,7 @@ const MessageListInner: React.FC = ({ ); }, { - minimumDelayMs: 120, + minimumDelayMs: progressiveRenderMinimumDelayMs, idleTimeoutMs: 1_200, }, ); @@ -675,7 +889,8 @@ const MessageListInner: React.FC = ({ hiddenHistoryCount, isUserScrolling, progressiveRenderBatchSize, - shouldUseProgressiveRender, + progressiveRenderMinimumDelayMs, + shouldAutoHydrateHiddenHistory, visibleMessages.length, ]); @@ -686,37 +901,57 @@ const MessageListInner: React.FC = ({ : visibleMessages, [hiddenHistoryCount, renderedMessageCount, visibleMessages], ); - const renderedTurns = useMemo( + const renderedAssistantMessageCount = useMemo( () => - hiddenHistoryCount > 0 - ? turns.slice( - -Math.max(renderedMessageCount, progressiveInitialRenderCount), - ) - : turns, + renderedMessages.reduce( + (count, message) => count + (message.role === "assistant" ? 1 : 0), + 0, + ), + [renderedMessages], + ); + const restoredTurnWindowSize = Math.max(1, renderedAssistantMessageCount + 1); + const renderedTurns = useMemo( + () => { + const shouldWindowTurns = + hiddenHistoryCount > 0 || isRestoredHistoryWindow; + if (!shouldWindowTurns) { + return turns; + } + + const turnWindowSize = isRestoredHistoryWindow + ? restoredTurnWindowSize + : Math.max(renderedMessageCount, progressiveInitialRenderCount); + const tailTurns = + turnWindowSize > 0 + ? turns.slice(-Math.min(turns.length, turnWindowSize)) + : []; + if ( + !currentTurnId || + tailTurns.some((turn) => turn.id === currentTurnId) + ) { + return tailTurns; + } + + const selectedTurnIds = new Set(tailTurns.map((turn) => turn.id)); + selectedTurnIds.add(currentTurnId); + return turns.filter((turn) => selectedTurnIds.has(turn.id)); + }, [ + currentTurnId, hiddenHistoryCount, + isRestoredHistoryWindow, progressiveInitialRenderCount, renderedMessageCount, + restoredTurnWindowSize, turns, ], ); - const renderedThreadItems = useMemo( - () => - hiddenHistoryCount > 0 - ? threadItems.slice( - -Math.max( - renderedMessageCount * MESSAGE_LIST_TIMELINE_ITEMS_FACTOR, - progressiveInitialRenderCount, - ), - ) - : threadItems, - [ - hiddenHistoryCount, - progressiveInitialRenderCount, - renderedMessageCount, - threadItems, - ], - ); + const renderedTurnIdSet = useMemo(() => { + if (hiddenHistoryCount <= 0 && !isRestoredHistoryWindow) { + return null; + } + return new Set(renderedTurns.map((turn) => turn.id)); + }, [hiddenHistoryCount, isRestoredHistoryWindow, renderedTurns]); const activeCurrentTurn = useMemo(() => { if (!currentTurnId) { return null; @@ -733,14 +968,17 @@ const MessageListInner: React.FC = ({ const timelineHydrationKey = [ renderedMessages[renderedMessages.length - 1]?.id ?? "no-message", renderedTurns[renderedTurns.length - 1]?.id ?? "no-turn", - renderedThreadItems[renderedThreadItems.length - 1]?.id ?? "no-item", + `${threadItems.length}:${ + threadItems[threadItems.length - 1]?.id ?? "no-item" + }`, ].join("|"); const shouldDeferHistoricalTimeline = !isSending && !activeCurrentTurnId && !focusedTimelineItemId && - renderedMessages.length >= MESSAGE_LIST_TIMELINE_DEFER_MESSAGE_THRESHOLD && - renderedThreadItems.length >= MESSAGE_LIST_TIMELINE_DEFER_ITEM_THRESHOLD; + threadItems.length >= MESSAGE_LIST_TIMELINE_DEFER_ITEM_THRESHOLD && + (isRestoredHistoryWindow || + renderedMessages.length >= MESSAGE_LIST_TIMELINE_DEFER_MESSAGE_THRESHOLD); const shouldDeferHistoricalTimelineDetails = !focusedTimelineItemId && (shouldDeferHistoricalTimeline || @@ -761,13 +999,55 @@ const MessageListInner: React.FC = ({ setIsHistoricalTimelineReady(true); }, { - minimumDelayMs: 80, - idleTimeoutMs: 900, + minimumDelayMs: isRestoredHistoryWindow + ? MESSAGE_LIST_RESTORED_HISTORICAL_TIMELINE_IDLE_DELAY_MS + : MESSAGE_LIST_HISTORICAL_TIMELINE_IDLE_DELAY_MS, + idleTimeoutMs: isRestoredHistoryWindow ? 1_800 : 900, }, ); - }, [shouldDeferHistoricalTimeline, timelineHydrationKey]); + }, [ + isRestoredHistoryWindow, + shouldDeferHistoricalTimeline, + timelineHydrationKey, + ]); const canBuildHistoricalTimeline = !shouldDeferHistoricalTimeline || isHistoricalTimelineReady; + const shouldDeferTailRuntimeStatusLine = + isRestoredHistoryWindow && + shouldDeferHistoricalTimeline && + !isHistoricalTimelineReady && + !isSending && + !activeCurrentTurnId && + pendingActions.length === 0 && + queuedTurns.length === 0 && + (threadRead?.pending_requests?.length ?? 0) === 0; + const shouldDeferThreadItemsScan = + shouldDeferHistoricalTimeline && + !isHistoricalTimelineReady && + !activeCurrentTurnId; + const renderedThreadItems = useMemo( + () => { + if (shouldDeferThreadItemsScan) { + return []; + } + + if (!renderedTurnIdSet) { + return threadItems; + } + if (renderedTurnIdSet.size === 0) { + return []; + } + + const scopedItems: AgentThreadItem[] = []; + for (const item of threadItems) { + if (renderedTurnIdSet.has(item.turn_id)) { + scopedItems.push(item); + } + } + return scopedItems; + }, + [renderedTurnIdSet, shouldDeferThreadItemsScan, threadItems], + ); const timelineByMessageId = useMemo(() => { if (!canBuildHistoricalTimeline) { return new Map(); @@ -784,15 +1064,42 @@ const MessageListInner: React.FC = ({ renderedThreadItems, renderedTurns, ]); - const lastAssistantMessageId = useMemo( + const lastAssistantMessage = useMemo(() => { + for (let index = renderedMessages.length - 1; index >= 0; index -= 1) { + const message = renderedMessages[index]; + if (message?.role === "assistant") { + return message; + } + } + return null; + }, [renderedMessages]); + const lastAssistantMessageId = lastAssistantMessage?.id ?? null; + const shouldInspectPromptCacheNotice = useMemo( () => - [...renderedMessages] - .reverse() - .find((message) => message.role === "assistant")?.id ?? null, - [renderedMessages], + Boolean( + providerType?.trim() && + restoredPromptCacheNoticeReady && + lastAssistantMessage?.usage && + !lastAssistantMessage.isThinking && + resolvePromptCacheActivity(lastAssistantMessage.usage) <= 0, + ), + [lastAssistantMessage, providerType, restoredPromptCacheNoticeReady], + ); + const { providers } = useConfiguredProviders({ + autoLoad: shouldInspectPromptCacheNotice, + }); + const promptCacheNotice = useMemo( + () => + shouldInspectPromptCacheNotice + ? resolveConfiguredProviderPromptCacheSupportNotice( + providers, + providerType, + ) + : null, + [providerType, providers, shouldInspectPromptCacheNotice], ); const tailRuntimeStatusLine = useMemo(() => { - if (!lastAssistantMessageId) { + if (!lastAssistantMessageId || shouldDeferTailRuntimeStatusLine) { return null; } @@ -817,6 +1124,7 @@ const MessageListInner: React.FC = ({ renderedMessages, renderedThreadItems, renderedTurns, + shouldDeferTailRuntimeStatusLine, threadRead, ]); const currentTurnTimeline = useMemo(() => { @@ -824,10 +1132,13 @@ const MessageListInner: React.FC = ({ return null; } - const mappedMessageId = - [...timelineByMessageId.values()].find( - (entry) => entry.turn.id === activeCurrentTurnId, - )?.messageId ?? null; + let mappedMessageId: string | null = null; + for (const entry of timelineByMessageId.values()) { + if (entry.turn.id === activeCurrentTurnId) { + mappedMessageId = entry.messageId; + break; + } + } return { messageId: mappedMessageId || lastAssistantMessageId, @@ -853,10 +1164,13 @@ const MessageListInner: React.FC = ({ const lastAssistantId = group.assistantMessages[group.assistantMessages.length - 1]?.id ?? null; - const mappedTimeline = - group.assistantMessages - .map((message) => timelineByMessageId.get(message.id)) - .find(Boolean) ?? null; + let mappedTimeline: MessageTurnTimeline | null = null; + for (const message of group.assistantMessages) { + mappedTimeline = timelineByMessageId.get(message.id) ?? null; + if (mappedTimeline) { + break; + } + } const isCurrentTurnGroup = Boolean(lastAssistantId) && currentTurnTimeline?.messageId === lastAssistantId; @@ -881,6 +1195,170 @@ const MessageListInner: React.FC = ({ timelineByMessageId, ], ); + const shouldDeferHistoricalAssistantMessageDetails = useCallback( + (message: Message): boolean => + isRestoredHistoryWindow && + !isHistoricalTimelineReady && + !focusedTimelineItemId && + !isSending && + !activeCurrentTurnId && + message.role === "assistant" && + !message.isThinking && + !message.thinkingContent && + (message.toolCalls?.length ?? 0) === 0 && + (message.actionRequests?.length ?? 0) === 0, + [ + activeCurrentTurnId, + focusedTimelineItemId, + isHistoricalTimelineReady, + isRestoredHistoryWindow, + isSending, + ], + ); + const historicalContentPartsDeferredCount = useMemo(() => { + if (!isRestoredHistoryWindow || isHistoricalTimelineReady) { + return 0; + } + + let count = 0; + for (const message of renderedMessages) { + if ( + (message.contentParts?.length ?? 0) > 0 && + shouldDeferHistoricalAssistantMessageDetails(message) + ) { + count += 1; + } + } + return count; + }, [ + isHistoricalTimelineReady, + isRestoredHistoryWindow, + renderedMessages, + shouldDeferHistoricalAssistantMessageDetails, + ]); + const historicalMarkdownDeferredCount = useMemo(() => { + if (!isRestoredHistoryWindow || isHistoricalTimelineReady) { + return 0; + } + + let count = 0; + for (const message of renderedMessages) { + const content = message.content.trim(); + if ( + content && + !hasStructuredHistoricalContentHint(content) && + shouldDeferHistoricalAssistantMessageDetails(message) + ) { + count += 1; + } + } + return count; + }, [ + isHistoricalTimelineReady, + isRestoredHistoryWindow, + renderedMessages, + shouldDeferHistoricalAssistantMessageDetails, + ]); + useEffect(() => { + if (!sessionId) { + restoredSessionMetricRef.current = null; + return; + } + + if ( + restoredSessionMetricRef.current && + restoredSessionMetricRef.current !== sessionId + ) { + restoredSessionMetricRef.current = null; + } + + const shouldTrackRestoredSession = + isRestoringSession || + isRestoredHistoryWindow || + hiddenHistoryCount > 0 || + persistedHiddenHistoryCount > 0; + if (shouldTrackRestoredSession) { + restoredSessionMetricRef.current = sessionId; + } + + const shouldRecordRestoredFollowUp = + restoredSessionMetricRef.current === sessionId && + visibleMessages.length > 0; + const shouldRecord = + shouldTrackRestoredSession || shouldRecordRestoredFollowUp; + if (!shouldRecord) { + return; + } + const shouldFinishRestoredFollowUp = + shouldRecordRestoredFollowUp && + renderedMessages.length >= visibleMessages.length; + + const metricContext = { + canBuildHistoricalTimeline, + hiddenHistoryCount, + isHistoricalTimelineReady, + isRestoredHistoryWindow, + isRestoringSession, + historicalContentPartsDeferredCount, + historicalMarkdownDeferredCount, + messagesCount: messages.length, + persistedHiddenHistoryCount, + renderedMessagesCount: renderedMessages.length, + renderedTurnsCount: renderedTurns.length, + recordReason: shouldTrackRestoredSession + ? "restored-window" + : "restored-follow-up", + sessionId, + shouldDeferHistoricalTimeline, + tailRuntimeStatusDeferred: shouldDeferTailRuntimeStatusLine, + threadItemsScanDeferred: shouldDeferThreadItemsScan, + threadItemsCount: renderedThreadItems.length, + timelineGroupsCount: renderGroups.length, + turnsCount: turns.length, + visibleMessagesCount: visibleMessages.length, + }; + + recordAgentUiPerformanceMetric("messageList.commit", metricContext); + + if (typeof window === "undefined" || !window.requestAnimationFrame) { + recordAgentUiPerformanceMetric("messageList.paint", metricContext); + if (shouldFinishRestoredFollowUp) { + restoredSessionMetricRef.current = null; + } + return; + } + + const frameId = window.requestAnimationFrame(() => { + recordAgentUiPerformanceMetric("messageList.paint", metricContext); + if (shouldFinishRestoredFollowUp) { + restoredSessionMetricRef.current = null; + } + }); + + return () => { + window.cancelAnimationFrame?.(frameId); + }; + }, [ + canBuildHistoricalTimeline, + hiddenHistoryCount, + historicalContentPartsDeferredCount, + historicalMarkdownDeferredCount, + isHistoricalTimelineReady, + isRestoredHistoryWindow, + isRestoringSession, + messages.length, + persistedHiddenHistoryCount, + renderGroups.length, + renderedMessages.length, + renderedThreadItems.length, + renderedTurns.length, + sessionId, + shouldDeferHistoricalTimeline, + shouldDeferTailRuntimeStatusLine, + shouldDeferThreadItemsScan, + turns.length, + visibleMessages.length, + ]); const shouldKeepInlineProcessForActiveAssistant = useCallback( (message: Message, isConversationTailAssistant: boolean): boolean => { if (message.role !== "assistant") { @@ -927,6 +1405,42 @@ const MessageListInner: React.FC = ({ const handleExpandAllHistory = useCallback(() => { setRenderedMessageCount(visibleMessages.length); }, [visibleMessages.length]); + const handleExpandLongHistoricalMessage = useCallback((messageId: string) => { + setExpandedLongHistoricalMessageIds((current) => { + if (current.has(messageId)) { + return current; + } + + const next = new Set(current); + next.add(messageId); + return next; + }); + }, []); + const handleExpandHistoricalAssistantMessage = useCallback( + (messageId: string) => { + setExpandedHistoricalAssistantMessageIds((current) => { + if (current.has(messageId)) { + return current; + } + + const next = new Set(current); + next.add(messageId); + return next; + }); + }, + [], + ); + const handleExpandHistoricalTimeline = useCallback((timelineKey: string) => { + setExpandedHistoricalTimelineKeys((current) => { + if (current.has(timelineKey)) { + return current; + } + + const next = new Set(current); + next.add(timelineKey); + return next; + }); + }, []); // 检测用户是否在手动滚动 useEffect(() => { @@ -1001,17 +1515,18 @@ const MessageListInner: React.FC = ({ role: msg.role, hasImages, }); + const shouldDeferMessageDetails = + shouldDeferHistoricalAssistantMessageDetails(msg); const rawRuntimePeerContent = (msg.content || "").trim(); const shouldRenderRuntimePeerCards = rawRuntimePeerContent.length > 0 && isPureRuntimePeerMessageText(rawRuntimePeerContent); - const displayContentParts = sanitizeContentPartsForDisplay( - msg.contentParts, - { - role: msg.role, - hasImages, - }, - ); + const displayContentParts = shouldDeferMessageDetails + ? undefined + : sanitizeContentPartsForDisplay(msg.contentParts, { + role: msg.role, + hasImages, + }); const isConversationTailAssistant = msg.role === "assistant" && msg.id === group.lastAssistantId; const timeline = @@ -1030,6 +1545,7 @@ const MessageListInner: React.FC = ({ ), ); const includeInlineProcessFlow = + !shouldDeferMessageDetails && msg.role === "assistant" && shouldKeepInlineProcessForActiveAssistant( msg, @@ -1063,16 +1579,16 @@ const MessageListInner: React.FC = ({ }), ) : []; + const timelineConversationItemIds = + timelineConversationItems.length > 0 + ? new Set(timelineConversationItems.map((item) => item.id)) + : null; const isInlineCoveredTimelineItem = createInlineCoverageMatcher( inlineProcessCoverage, ); const primaryTimelineItems = timeline ? timeline.items.filter((item) => { - if ( - !timelineConversationItems.some( - (timelineItem) => timelineItem.id === item.id, - ) - ) { + if (!timelineConversationItemIds?.has(item.id)) { return false; } @@ -1135,6 +1651,75 @@ const MessageListInner: React.FC = ({ : null; const actionContent = displayContent.trim(); const hasVisibleAssistantText = Boolean(actionContent); + const shouldCollapseLongHistoricalMessage = + isRestoredHistoryWindow && + msg.role === "assistant" && + !msg.isThinking && + actionContent.length > MESSAGE_LIST_LONG_HISTORICAL_MESSAGE_THRESHOLD && + !expandedLongHistoricalMessageIds.has(msg.id); + const hasNonTextConversationContentParts = Boolean( + conversationContentParts?.some((part) => part.type !== "text"), + ); + const shouldFlattenHistoricalAssistantContent = + isRestoredHistoryWindow && + msg.role === "assistant" && + !msg.isThinking && + !includeInlineProcessFlow && + !hasNonTextConversationContentParts && + actionContent.length > 0 && + !shouldCollapseLongHistoricalMessage; + const shouldCompactHistoricalAssistantMessage = + isRestoredHistoryWindow && + msg.role === "assistant" && + !msg.isThinking && + !focusedTimelineItemId && + !includeInlineProcessFlow && + !hasNonTextConversationContentParts && + !((msg.actionRequests || []).length > 0) && + !actionContent.includes("```a2ui") && + actionContent.length > + MESSAGE_LIST_COMPACT_HISTORICAL_ASSISTANT_THRESHOLD && + !expandedLongHistoricalMessageIds.has(msg.id) && + !expandedHistoricalAssistantMessageIds.has(msg.id); + const shouldPreviewHistoricalAssistantMessage = + shouldCollapseLongHistoricalMessage || + shouldCompactHistoricalAssistantMessage; + const historicalAssistantPreviewContent = + shouldCollapseLongHistoricalMessage + ? buildLongHistoricalMessagePreview(displayContent) + : shouldCompactHistoricalAssistantMessage + ? buildHistoricalMessagePreview( + displayContent, + MESSAGE_LIST_COMPACT_HISTORICAL_ASSISTANT_PREVIEW_CHARS, + ) + : ""; + const rendererContent = shouldCollapseLongHistoricalMessage + ? buildLongHistoricalMessagePreview(displayContent) + : displayContent; + const rendererRawContent = + shouldCollapseLongHistoricalMessage || + shouldFlattenHistoricalAssistantContent + ? rendererContent + : msg.content || ""; + const rendererContentParts = + shouldCollapseLongHistoricalMessage || + shouldFlattenHistoricalAssistantContent + ? undefined + : conversationContentParts; + const rendererThinkingContent = shouldCollapseLongHistoricalMessage + ? undefined + : conversationThinkingContent; + const rendererToolCalls = shouldCollapseLongHistoricalMessage + ? undefined + : conversationToolCalls; + const rendererActionRequests = shouldCollapseLongHistoricalMessage + ? undefined + : msg.actionRequests; + const rendererMarkdownRenderMode = + shouldCollapseLongHistoricalMessage || + shouldFlattenHistoricalAssistantContent + ? "light" + : "standard"; const canQuoteMessage = Boolean(onQuoteMessage && actionContent); const canCopyMessage = Boolean(actionContent); const canSaveMessageAsSkill = Boolean( @@ -1181,8 +1766,38 @@ const MessageListInner: React.FC = ({ const messageCanvasShortcutPath = messageSavedSiteContentTarget ? resolveSiteSavedContentTargetRelativePath(messageSavedSiteContentTarget) : null; + const shouldDeferHistoricalMarkdownRender = + shouldDeferMessageDetails && + msg.role === "assistant" && + hasVisibleAssistantText && + !shouldPreviewHistoricalAssistantMessage && + !hasImages && + !hasNonTextConversationContentParts && + visibleAssistantArtifacts.length === 0 && + !shouldRenderMessageCanvasShortcut && + !msg.imageWorkbenchPreview && + !msg.taskPreview && + !hasStructuredHistoricalContentHint(actionContent); + const shouldRenderFirstTokenPlaceholder = + msg.role === "assistant" && + msg.isThinking && + Boolean(msg.runtimeStatus) && + !shouldRenderRuntimeStatusPill(msg.runtimeStatus) && + !hasVisibleAssistantText && + !conversationContentParts?.length && + !conversationThinkingContent?.trim() && + !conversationToolCalls?.length && + !((msg.actionRequests || []).length > 0) && + !primaryTimeline && + !trailingTimeline && + !((msg.images || []).length > 0) && + visibleAssistantArtifacts.length === 0 && + !shouldRenderMessageCanvasShortcut && + !msg.imageWorkbenchPreview && + !msg.taskPreview; const shouldCollapseAssistantShell = msg.role === "assistant" && + !shouldRenderFirstTokenPlaceholder && !hasVisibleAssistantText && !conversationContentParts?.length && !conversationThinkingContent?.trim() && @@ -1259,28 +1874,48 @@ const MessageListInner: React.FC = ({ return null; } + const primaryTimelineKey = primaryTimeline + ? `leading:${primaryTimeline.turn.id}` + : null; + const shouldRenderCompactPrimaryTimeline = + Boolean(primaryTimelineKey) && + isRestoredHistoryWindow && + !focusedTimelineItemId && + primaryTimeline?.turn.status === "completed" && + primaryTimeline.turn.id !== activeCurrentTurnId && + primaryTimeline.items.length >= + MESSAGE_LIST_HISTORICAL_TIMELINE_COMPACT_ITEM_THRESHOLD && + !expandedHistoricalTimelineKeys.has(primaryTimelineKey!); const primaryTimelineNode = msg.role === "assistant" && primaryTimeline ? ( - + shouldRenderCompactPrimaryTimeline && primaryTimelineKey ? ( + handleExpandHistoricalTimeline(primaryTimelineKey)} + /> + ) : ( + + ) ) : null; const shouldRenderPrimaryTimelineOutsideBubble = msg.role === "assistant" && @@ -1314,60 +1949,87 @@ const MessageListInner: React.FC = ({ ? null : primaryTimelineNode} - onA2UISubmit(formData, msg.id) - : undefined - } - a2uiFormId={a2uiFormDataMap?.[msg.id]?.formId} - a2uiInitialFormData={a2uiFormDataMap?.[msg.id]?.formData} - onA2UIFormChange={onA2UIFormChange} - renderA2UIInline={ - renderA2UIInline && !shouldSuppressInlineA2UI - } - onWriteFile={ - onWriteFile - ? (content, fileName, context) => - onWriteFile(content, fileName, { - ...context, - sourceMessageId: - context?.sourceMessageId || msg.id, - source: context?.source || "message_content", - }) - : undefined - } - onFileClick={onFileClick} - onOpenSavedSiteContent={onOpenSavedSiteContent} - onPermissionResponse={onPermissionResponse} - collapseCodeBlocks={collapseCodeBlocks} - shouldCollapseCodeBlock={shouldCollapseCodeBlock} - onCodeBlockClick={onCodeBlockClick} - promoteActionRequestsToA2UI={promoteActionRequestsToA2UI} - suppressedActionRequestId={suppressedActionRequestId} - showRuntimeStatusInline={true} - renderProposedPlanBlocks={ - !primaryTimeline || - inlineProcessCoverage.hasInlineProcessEntries - } - suppressProcessFlow={shouldSuppressImageProcessFlow} - showContentBlockActions={Boolean(actionContent)} - onQuoteContent={ - onQuoteMessage - ? (quotedContent) => - onQuoteMessage(quotedContent, msg.id) - : undefined - } - /> + {shouldRenderFirstTokenPlaceholder && msg.runtimeStatus ? ( + + ) : shouldPreviewHistoricalAssistantMessage ? ( + { + if (shouldCollapseLongHistoricalMessage) { + handleExpandLongHistoricalMessage(msg.id); + return; + } + + handleExpandHistoricalAssistantMessage(msg.id); + }} + /> + ) : shouldDeferHistoricalMarkdownRender ? ( + + ) : ( + onA2UISubmit(formData, msg.id) + : undefined + } + a2uiFormId={a2uiFormDataMap?.[msg.id]?.formId} + a2uiInitialFormData={a2uiFormDataMap?.[msg.id]?.formData} + onA2UIFormChange={onA2UIFormChange} + renderA2UIInline={ + renderA2UIInline && !shouldSuppressInlineA2UI + } + onWriteFile={ + onWriteFile + ? (content, fileName, context) => + onWriteFile(content, fileName, { + ...context, + sourceMessageId: + context?.sourceMessageId || msg.id, + source: context?.source || "message_content", + }) + : undefined + } + onFileClick={onFileClick} + onOpenSavedSiteContent={onOpenSavedSiteContent} + onPermissionResponse={onPermissionResponse} + collapseCodeBlocks={collapseCodeBlocks} + shouldCollapseCodeBlock={shouldCollapseCodeBlock} + onCodeBlockClick={onCodeBlockClick} + promoteActionRequestsToA2UI={promoteActionRequestsToA2UI} + suppressedActionRequestId={suppressedActionRequestId} + showRuntimeStatusInline={true} + renderProposedPlanBlocks={ + !primaryTimeline || + inlineProcessCoverage.hasInlineProcessEntries + } + suppressProcessFlow={shouldSuppressImageProcessFlow} + showContentBlockActions={Boolean(actionContent)} + markdownRenderMode={rendererMarkdownRenderMode} + onQuoteContent={ + onQuoteMessage + ? (quotedContent) => + onQuoteMessage(quotedContent, msg.id) + : undefined + } + /> + )} {shouldRenderMessageCanvasShortcut ? ( + + ) : null, +})); + +const mountedRoots: Array<{ root: Root; container: HTMLDivElement }> = []; + +function createItem(): HomeSkillSurfaceItem { + return { + id: "daily-trend-briefing", + title: "每日趋势摘要", + summary: "先收一版内容趋势。", + category: "social", + sourceKind: "curated_task", + launchKind: "curated_task_launcher", + coverToken: "trend", + isRecent: false, + isRecommended: true, + usedAt: null, + testId: "entry-recommended-daily-trend-briefing", + }; +} + +function createStarterChips(): HomeStarterChip[] { + return [ + { + id: "starter-guide", + label: "引导帮助", + launchKind: "toggle_guide", + testId: "home-guide-help-trigger", + }, + { + id: "starter-daily-trend", + label: "帮我想选题", + launchKind: "curated_task_launcher", + targetItemId: "daily-trend-briefing", + testId: "entry-recommended-daily-trend-briefing", + }, + { + id: "starter-more", + label: "更多做法", + launchKind: "open_drawer", + testId: "home-more-skills-trigger", + }, + { + id: "starter-manager", + label: "⚙", + launchKind: "open_manager", + testId: "home-skill-manager-trigger", + }, + ]; +} + +function createGuideCards(): HomeGuideCard[] { + return [ + { + id: "guide-voice", + title: "语音输入怎么设置？", + summary: "把灵感直接说进生成容器。", + prompt: "请告诉我语音输入怎么设置。", + testId: "home-guide-voice", + }, + ]; +} + +function renderSurface(options?: { + starterChips?: HomeStarterChip[]; + sections?: HomeSkillSection[]; + supplementalActions?: React.ComponentProps< + typeof HomeStartSurface + >["supplementalActions"]; + guideCards?: HomeGuideCard[]; + onSelectStarterChip?: (chip: HomeStarterChip) => void; + onSelectGuideCard?: (card: HomeGuideCard) => void; + onSelectSkillItem?: (item: HomeSkillSurfaceItem) => void; +}) { + const item = createItem(); + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + const onSelectStarterChip = options?.onSelectStarterChip ?? vi.fn(); + const onSelectGuideCard = options?.onSelectGuideCard ?? vi.fn(); + const onSelectSkillItem = options?.onSelectSkillItem ?? vi.fn(); + mountedRoots.push({ root, container }); + + act(() => { + root.render( + , + ); + }); + + return { + container, + item, + onSelectStarterChip, + onSelectGuideCard, + onSelectSkillItem, + }; +} + +beforeEach(() => { + ( + globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; + } + ).IS_REACT_ACT_ENVIRONMENT = true; +}); + +afterEach(() => { + while (mountedRoots.length > 0) { + const mounted = mountedRoots.pop(); + if (!mounted) break; + act(() => { + mounted.root.unmount(); + }); + mounted.container.remove(); + } +}); + +describe("HomeStartSurface", () => { + it("普通起手 chip 只透传给上层，不打开抽屉或管理弹窗", () => { + const { container, onSelectStarterChip } = renderSurface(); + const chip = container.querySelector( + '[data-testid="entry-recommended-daily-trend-briefing"]', + ) as HTMLButtonElement | null; + + act(() => { + chip?.click(); + }); + + expect(onSelectStarterChip).toHaveBeenCalledWith( + expect.objectContaining({ id: "starter-daily-trend" }), + ); + expect( + container.querySelector('[data-testid="home-more-skills-drawer"]'), + ).toBeNull(); + expect( + container.querySelector('[data-testid="home-scene-skill-manager-mock"]'), + ).toBeNull(); + }); + + it("引导帮助 chip 展开帮助卡并把卡片选择交给上层", () => { + const { container, onSelectGuideCard } = renderSurface(); + const guide = container.querySelector( + '[data-testid="home-guide-help-trigger"]', + ) as HTMLButtonElement | null; + + act(() => { + guide?.click(); + }); + + expect( + container.querySelector('[data-testid="home-guide-cards"]'), + ).toBeTruthy(); + + const card = container.querySelector( + '[data-testid="home-guide-voice"]', + ) as HTMLButtonElement | null; + act(() => { + card?.click(); + }); + + expect(onSelectGuideCard).toHaveBeenCalledWith( + expect.objectContaining({ id: "guide-voice" }), + ); + }); + + it("更多做法 chip 切换抽屉，抽屉条目继续透传选择", () => { + const { container, item, onSelectSkillItem } = renderSurface(); + const more = container.querySelector( + '[data-testid="home-more-skills-trigger"]', + ) as HTMLButtonElement | null; + + act(() => { + more?.click(); + }); + + expect( + container.querySelector('[data-testid="home-more-skills-drawer"]'), + ).toBeTruthy(); + + const drawerItem = container.querySelector( + '[data-testid="home-drawer-entry-recommended-daily-trend-briefing"]', + ) as HTMLButtonElement | null; + act(() => { + drawerItem?.click(); + }); + + expect(onSelectSkillItem).toHaveBeenCalledWith(item); + }); + + it("抽屉打开后按 Escape 可关闭", () => { + const { container } = renderSurface(); + const more = container.querySelector( + '[data-testid="home-more-skills-trigger"]', + ) as HTMLButtonElement | null; + + act(() => { + more?.click(); + }); + expect( + container.querySelector('[data-testid="home-more-skills-drawer"]'), + ).toBeTruthy(); + + act(() => { + window.dispatchEvent(new KeyboardEvent("keydown", { key: "Escape" })); + }); + + expect( + container.querySelector('[data-testid="home-more-skills-drawer"]'), + ).toBeNull(); + }); + + it("管理 chip 打开场景管理弹窗并允许关闭", () => { + const { container } = renderSurface(); + const manager = container.querySelector( + '[data-testid="home-skill-manager-trigger"]', + ) as HTMLButtonElement | null; + + act(() => { + manager?.click(); + }); + + const dialog = container.querySelector( + '[data-testid="home-scene-skill-manager-mock"]', + ); + expect(dialog).toBeTruthy(); + + const close = container.querySelector( + '[data-testid="home-scene-skill-manager-mock"] button', + ) as HTMLButtonElement | null; + act(() => { + close?.click(); + }); + + expect( + container.querySelector('[data-testid="home-scene-skill-manager-mock"]'), + ).toBeNull(); + }); + + it("补充入口使用轻按钮呈现并触发自身动作", () => { + const onSelect = vi.fn(); + const { container } = renderSurface({ + supplementalActions: [ + { + id: "connect-browser", + label: "连接浏览器", + testId: "entry-connect-browser", + onSelect, + }, + ], + }); + + const action = container.querySelector( + '[data-testid="entry-connect-browser"]', + ) as HTMLButtonElement | null; + expect(action?.textContent).toBe("连接浏览器"); + + act(() => { + action?.click(); + }); + + expect(onSelect).toHaveBeenCalledTimes(1); + }); +}); diff --git a/src/components/agent/chat/home/HomeStartSurface.tsx b/src/components/agent/chat/home/HomeStartSurface.tsx index 3c92d5889..acb8e09ab 100644 --- a/src/components/agent/chat/home/HomeStartSurface.tsx +++ b/src/components/agent/chat/home/HomeStartSurface.tsx @@ -2,8 +2,10 @@ import { useEffect, useState } from "react"; import styled from "styled-components"; import { HomeStarterChips } from "./HomeStarterChips"; import { HomeMoreSkillsDrawer } from "./HomeMoreSkillsDrawer"; +import { HomeGuideCards } from "./HomeGuideCards"; import { HomeSceneSkillManagerDialog } from "./HomeSceneSkillManagerDialog"; import type { + HomeGuideCard, HomeSkillSection, HomeSkillSurfaceItem, HomeStarterChip, @@ -60,21 +62,41 @@ export interface HomeSupplementalAction { interface HomeStartSurfaceProps { starterChips: HomeStarterChip[]; + guideCards?: HomeGuideCard[]; + guideOpen?: boolean; sections: HomeSkillSection[]; supplementalActions?: HomeSupplementalAction[]; + onGuideOpenChange?: (open: boolean) => void; onSelectStarterChip: (chip: HomeStarterChip) => void; + onSelectGuideCard?: (card: HomeGuideCard) => void; onSelectSkillItem: (item: HomeSkillSurfaceItem) => void; } export function HomeStartSurface({ starterChips, + guideCards = [], + guideOpen, sections, supplementalActions = [], + onGuideOpenChange, onSelectStarterChip, + onSelectGuideCard, onSelectSkillItem, }: HomeStartSurfaceProps) { const [drawerOpen, setDrawerOpen] = useState(false); const [managerOpen, setManagerOpen] = useState(false); + const [internalGuideOpen, setInternalGuideOpen] = useState(false); + const resolvedGuideOpen = guideOpen ?? internalGuideOpen; + const updateGuideOpen = ( + nextOpen: boolean | ((current: boolean) => boolean), + ) => { + const resolvedNextOpen = + typeof nextOpen === "function" ? nextOpen(resolvedGuideOpen) : nextOpen; + if (guideOpen === undefined) { + setInternalGuideOpen(resolvedNextOpen); + } + onGuideOpenChange?.(resolvedNextOpen); + }; useEffect(() => { if (!drawerOpen) { @@ -94,23 +116,39 @@ export function HomeStartSurface({ const handleSelectStarterChip = (chip: HomeStarterChip) => { if (chip.launchKind === "open_drawer") { setDrawerOpen((current) => !current); + updateGuideOpen(false); return; } if (chip.launchKind === "open_manager") { setManagerOpen(true); return; } + if (chip.launchKind === "toggle_guide") { + updateGuideOpen((current) => !current); + setDrawerOpen(false); + return; + } + updateGuideOpen(false); onSelectStarterChip(chip); }; return ( - + {!resolvedGuideOpen ? ( + + ) : null} - {supplementalActions.length > 0 ? ( + {resolvedGuideOpen ? ( + onSelectGuideCard?.(card)} + /> + ) : null} + + {!resolvedGuideOpen && supplementalActions.length > 0 ? ( {supplementalActions.map((action) => ( = []; + +beforeEach(() => { + ( + globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; + } + ).IS_REACT_ACT_ENVIRONMENT = true; +}); + +function renderChips(chips: HomeStarterChip[], onSelect = vi.fn()) { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + mountedRoots.push({ root, container }); + + act(() => { + root.render(); + }); + + return { container, onSelect }; +} + +afterEach(() => { + while (mountedRoots.length > 0) { + const mounted = mountedRoots.pop(); + if (!mounted) break; + act(() => { + mounted.root.unmount(); + }); + mounted.container.remove(); + } +}); + +describe("HomeStarterChips", () => { + it("渲染轻量起手入口并把选择交给上层", () => { + const chips: HomeStarterChip[] = [ + { + id: "starter-draft", + label: "写第一版", + launchKind: "curated_task_launcher", + targetItemId: "social-post-starter", + testId: "entry-recommended-social-post-starter", + }, + { + id: "starter-manager", + label: "⚙", + launchKind: "open_manager", + testId: "home-skill-manager-trigger", + }, + ]; + const { container, onSelect } = renderChips(chips); + + const draft = container.querySelector( + '[data-testid="entry-recommended-social-post-starter"]', + ) as HTMLButtonElement | null; + const manager = container.querySelector( + '[data-testid="home-skill-manager-trigger"]', + ) as HTMLButtonElement | null; + + expect(draft?.textContent).toBe("写第一版"); + expect(manager?.getAttribute("aria-label")).toBe("管理做法"); + + act(() => { + draft?.click(); + }); + + expect(onSelect).toHaveBeenCalledWith(chips[0]); + }); +}); diff --git a/src/components/agent/chat/home/HomeStarterChips.tsx b/src/components/agent/chat/home/HomeStarterChips.tsx index 494fd417e..36260a851 100644 --- a/src/components/agent/chat/home/HomeStarterChips.tsx +++ b/src/components/agent/chat/home/HomeStarterChips.tsx @@ -1,5 +1,6 @@ import type { HomeStarterChip } from "./homeSurfaceTypes"; import styled from "styled-components"; +import { Lightbulb, Settings } from "lucide-react"; const StarterRow = styled.div` display: flex; @@ -51,11 +52,44 @@ const StarterButton = styled.button<{ $primary?: boolean }>` } `; +const StarterIcon = styled.span` + display: inline-flex; + width: 14px; + height: 14px; + flex: 0 0 auto; + align-items: center; + justify-content: center; + + svg { + width: 14px; + height: 14px; + } +`; + interface HomeStarterChipsProps { chips: HomeStarterChip[]; onSelect: (chip: HomeStarterChip) => void; } +function renderStarterIcon(chip: HomeStarterChip) { + const token = chip.iconToken?.trim(); + if (token === "lightbulb" || chip.groupKey === "guide_help") { + return ( + + + + ); + } + if (chip.label === "⚙" || token === "settings") { + return ( + + + + ); + } + return null; +} + export function HomeStarterChips({ chips, onSelect }: HomeStarterChipsProps) { return ( @@ -69,7 +103,8 @@ export function HomeStarterChips({ chips, onSelect }: HomeStarterChipsProps) { title={chip.label === "⚙" ? "管理做法" : chip.label} onClick={() => onSelect(chip)} > - {chip.label} + {renderStarterIcon(chip)} + {chip.label === "⚙" ? null : chip.label} ))} diff --git a/src/components/agent/chat/home/buildHomeSkillSurface.test.ts b/src/components/agent/chat/home/buildHomeSkillSurface.test.ts new file mode 100644 index 000000000..0157a07ea --- /dev/null +++ b/src/components/agent/chat/home/buildHomeSkillSurface.test.ts @@ -0,0 +1,265 @@ +import { describe, expect, it } from "vitest"; +import type { Skill } from "@/lib/api/skills"; +import type { + SkillCatalogEntry, + SkillCatalogSceneEntry, +} from "@/lib/api/skillCatalog"; +import type { ServiceSkillHomeItem } from "../service-skills/types"; +import type { CuratedTaskTemplateItem } from "../utils/curatedTaskTemplates"; +import { + buildHomeGalleryItems, + buildHomeGuideCards, + buildHomeInputSuggestions, + buildHomeSkillItems, + buildHomeSkillSections, + buildHomeStarterChips, +} from "./buildHomeSkillSurface"; + +function createCuratedTask( + id: string, + title: string, + recentUsedAt: number | null = null, +): CuratedTaskTemplateItem { + return { + id, + title, + summary: `${title} 摘要`, + outputHint: "输出", + resultDestination: "当前内容", + categoryLabel: "社交媒体", + prompt: "prompt", + requiredInputs: [], + requiredInputFields: [], + optionalReferences: [], + outputContract: [], + followUpActions: [], + badge: recentUsedAt ? "最近使用" : "推荐", + actionLabel: "进入生成", + statusLabel: "可直接开始", + statusTone: "emerald", + recentUsedAt, + isRecent: typeof recentUsedAt === "number", + }; +} + +function createServiceSkill(): ServiceSkillHomeItem { + return { + id: "project-insight-flow", + title: "项目线索整理", + summary: "围绕当前项目整理线索。", + category: "研究与方案", + outputHint: "线索清单", + source: "cloud_catalog", + runnerType: "instant", + defaultExecutorBinding: "agent_turn", + executionLocation: "client_default", + version: "seed-v1", + badge: "云目录", + recentUsedAt: null, + isRecent: false, + runnerLabel: "立即开始", + runnerTone: "emerald", + runnerDescription: "围绕当前项目继续整理线索。", + actionLabel: "继续整理", + automationStatus: null, + slotSchema: [], + sceneBinding: { + sceneKey: "project-insight-flow", + commandPrefix: "/project-insight-flow", + title: "项目线索整理", + summary: "围绕当前项目整理线索。", + }, + }; +} + +describe("buildHomeSkillSurface", () => { + it("服务端未下发首页展示时使用 Ribbi 式本地兜底入口", () => { + const labels = buildHomeStarterChips().map((chip) => chip.label); + + expect(labels).toEqual([ + "引导帮助", + "写作", + "PPT", + "调研报告", + "需求分析", + "视频", + "设计", + "Excel", + "编程", + "更多做法", + "⚙", + ]); + }); + + it("优先使用服务端下发的首页展示入口、Tab 建议与帮助卡", () => { + const entries: SkillCatalogEntry[] = [ + { + id: "home:starter:poster", + kind: "command", + title: "做海报", + summary: "把主题变成海报方向。", + commandKey: "home_poster", + surfaceScopes: ["home"], + triggers: [], + homePresentation: { + slot: "starter_chip", + label: "做海报", + order: 20, + prompt: "请帮我做一张海报。", + }, + }, + { + id: "home:input-suggestion:email", + kind: "command", + title: "帮我写一封工作邮件", + summary: "Tab 起手建议。", + commandKey: "home_input_email", + surfaceScopes: ["home"], + triggers: [], + homePresentation: { + slot: "input_suggestion", + label: "帮我写一封工作邮件", + order: 10, + prompt: "请帮我写一封工作邮件。", + }, + }, + { + id: "home:guide:voice", + kind: "command", + title: "语音输入怎么设置？", + summary: "了解语音输入。", + commandKey: "home_guide_voice", + surfaceScopes: ["home"], + triggers: [], + homePresentation: { + slot: "guide_card", + title: "语音输入怎么设置？", + summary: "把灵感直接说进生成容器。", + order: 10, + groupKey: "guide_help", + prompt: "请告诉我语音输入怎么设置。", + }, + }, + ]; + + expect(buildHomeStarterChips(entries).map((chip) => chip.label)).toEqual([ + "做海报", + "更多做法", + "⚙", + ]); + expect(buildHomeStarterChips(entries)[0]).toMatchObject({ + launchKind: "prefill_prompt", + prompt: "请帮我做一张海报。", + }); + expect(buildHomeInputSuggestions(entries)).toEqual([ + expect.objectContaining({ + label: "帮我写一封工作邮件", + prompt: "请帮我写一封工作邮件。", + }), + ]); + expect(buildHomeGuideCards(entries)).toEqual([ + expect.objectContaining({ + title: "语音输入怎么设置？", + prompt: "请告诉我语音输入怎么设置。", + }), + ]); + }); + + it("把多来源条目归一成首页模型，并按最近使用优先排序", () => { + const installedSkill: Skill = { + key: "content-playbook", + name: "内容主稿方法", + description: "本地补充技能", + directory: "content-playbook", + installed: true, + sourceKind: "other", + }; + const catalogScene: SkillCatalogSceneEntry = { + id: "custom_scene:daily-review", + kind: "scene", + title: "每日复盘", + summary: "把趋势技能变成复盘入口。", + sceneKey: "daily-review", + commandPrefix: "/daily-review", + linkedSkillId: "project-insight-flow", + surfaceScopes: ["home"], + placeholder: "今天想复盘哪个账号？", + templates: [ + { + id: "default", + title: "开始复盘", + prompt: "请帮我复盘今天的小红书账号表现。", + }, + ], + }; + + const items = buildHomeSkillItems({ + curatedTasks: [ + createCuratedTask("daily-trend-briefing", "每日趋势摘要"), + createCuratedTask("social-post-starter", "内容主稿生成", 10), + ], + serviceSkills: [createServiceSkill()], + installedSkills: [installedSkill], + catalogSceneEntries: [catalogScene], + slashEntryUsage: [ + { + kind: "skill", + entryId: "content-playbook", + usedAt: 30, + replayText: "继续优化内容主稿", + }, + { + kind: "scene", + entryId: "project-insight-flow", + usedAt: 20, + replayText: "继续整理项目线索", + }, + { + kind: "scene", + entryId: "custom_scene:daily-review", + usedAt: 40, + replayText: "继续复盘账号", + }, + ], + }); + + expect(items.slice(0, 4).map((item) => item.id)).toEqual([ + "custom_scene:daily-review", + "content-playbook", + "project-insight-flow", + "social-post-starter", + ]); + expect(items[0]).toMatchObject({ + launchKind: "skill_catalog_scene", + linkedSkillId: "project-insight-flow", + launchPrompt: "请帮我复盘今天的小红书账号表现。", + }); + expect(items[1]).toMatchObject({ + launchKind: "installed_skill", + summary: "继续优化内容主稿", + }); + expect(items[2]).toMatchObject({ + launchKind: "service_skill", + summary: "继续整理项目线索", + }); + }); + + it("按分类生成 drawer 分组，并为 gallery 截取最多 12 个任务", () => { + const items = buildHomeSkillItems({ + curatedTasks: [ + createCuratedTask("daily-trend-briefing", "每日趋势摘要", 10), + createCuratedTask("script-to-voiceover", "脚本转口播"), + ], + }); + + const sections = buildHomeSkillSections(items); + const gallery = buildHomeGalleryItems(items); + + expect(sections[0]).toMatchObject({ + id: "recent", + title: "最近使用", + }); + expect(sections.some((section) => section.id === "video")).toBe(true); + expect(gallery).toHaveLength(2); + }); +}); diff --git a/src/components/agent/chat/home/buildHomeSkillSurface.ts b/src/components/agent/chat/home/buildHomeSkillSurface.ts index 66b5cd62d..2a5868500 100644 --- a/src/components/agent/chat/home/buildHomeSkillSurface.ts +++ b/src/components/agent/chat/home/buildHomeSkillSurface.ts @@ -1,5 +1,8 @@ import type { Skill } from "@/lib/api/skills"; -import type { SkillCatalogSceneEntry } from "@/lib/api/skillCatalog"; +import type { + SkillCatalogEntry, + SkillCatalogSceneEntry, +} from "@/lib/api/skillCatalog"; import type { ServiceSkillHomeItem } from "../service-skills/types"; import type { SceneAppEntryCardItem } from "../sceneappEntryTypes"; import type { CuratedTaskTemplateItem } from "../utils/curatedTaskTemplates"; @@ -7,6 +10,8 @@ import type { SlashEntryUsageRecord } from "../skill-selection/slashEntryUsage"; import { HOME_CATEGORY_LABELS, HOME_CATEGORY_ORDER, + HOME_GUIDE_CARDS, + HOME_INPUT_SUGGESTIONS, HOME_STARTER_CHIPS, } from "./homeSurfaceCopy"; import { @@ -18,8 +23,11 @@ import { } from "./homeSkillSurfaceAdapters"; import type { HomeSkillCategory, + HomeGuideCard, + HomeInputSuggestion, HomeSkillSection, HomeSkillSurfaceItem, + HomeStarterChip, } from "./homeSurfaceTypes"; interface BuildHomeSkillSurfaceInput { @@ -141,6 +149,163 @@ export function buildHomeGalleryItems( .slice(0, 12); } -export function buildHomeStarterChips() { - return HOME_STARTER_CHIPS; +function hasHomeScope(entry: SkillCatalogEntry): boolean { + return (entry.surfaceScopes ?? []).includes("home"); +} + +function normalizeHomePresentationOrder(entry: SkillCatalogEntry): number { + return entry.homePresentation?.order ?? Number.MAX_SAFE_INTEGER; +} + +function compareHomePresentationEntries( + left: SkillCatalogEntry, + right: SkillCatalogEntry, +): number { + const orderDelta = + normalizeHomePresentationOrder(left) - + normalizeHomePresentationOrder(right); + if (orderDelta !== 0) { + return orderDelta; + } + return left.title.localeCompare(right.title, "zh-CN"); +} + +function getSceneEntryPrompt(entry: SkillCatalogEntry): string { + if (entry.kind !== "scene") { + return ""; + } + return ( + entry.templates?.[0]?.prompt?.trim() || + entry.placeholder?.trim() || + entry.summary.trim() + ); +} + +function getHomePresentationPrompt(entry: SkillCatalogEntry): string { + return ( + entry.homePresentation?.prompt?.trim() || + getSceneEntryPrompt(entry) || + entry.summary.trim() + ); +} + +function buildHomeTestId(prefix: string, entry: SkillCatalogEntry): string { + return `${prefix}-${entry.id.replace(/[^a-z0-9_-]/gi, "-")}`; +} + +function listHomePresentationEntries( + entries: SkillCatalogEntry[] | undefined, + slot: NonNullable["slot"], +): SkillCatalogEntry[] { + return (entries ?? []) + .filter( + (entry) => hasHomeScope(entry) && entry.homePresentation?.slot === slot, + ) + .sort(compareHomePresentationEntries); +} + +export function buildHomeStarterChips( + entries?: SkillCatalogEntry[], +): HomeStarterChip[] { + const dynamicChips = listHomePresentationEntries(entries, "starter_chip").map( + (entry) => { + const presentation = entry.homePresentation; + const groupKey = presentation?.groupKey?.trim(); + const prompt = getHomePresentationPrompt(entry); + const isGuideHelp = groupKey === "guide_help"; + + return { + id: `home-starter-${entry.id}`, + label: presentation?.label?.trim() || entry.title, + launchKind: isGuideHelp ? "toggle_guide" : "prefill_prompt", + primary: + presentation?.order === 0 || + isGuideHelp || + presentation?.label === "引导帮助", + prompt, + groupKey, + iconToken: presentation?.iconToken, + testId: buildHomeTestId("home-starter", entry), + } satisfies HomeStarterChip; + }, + ); + + if (dynamicChips.length === 0) { + return HOME_STARTER_CHIPS; + } + + return [ + ...dynamicChips, + { + id: "starter-more", + label: "更多做法", + launchKind: "open_drawer", + testId: "home-more-skills-trigger", + }, + { + id: "starter-manager", + label: "⚙", + launchKind: "open_manager", + testId: "home-skill-manager-trigger", + }, + ]; +} + +export function buildHomeInputSuggestions( + entries?: SkillCatalogEntry[], +): HomeInputSuggestion[] { + const dynamicSuggestions = listHomePresentationEntries( + entries, + "input_suggestion", + ) + .map((entry) => { + const prompt = getHomePresentationPrompt(entry); + const label = entry.homePresentation?.label?.trim() || entry.title; + if (!prompt || !label) { + return null; + } + + const suggestion: HomeInputSuggestion = { + id: entry.id, + label, + prompt, + order: normalizeHomePresentationOrder(entry), + testId: buildHomeTestId("home-input-suggestion", entry), + }; + return suggestion; + }) + .filter((item): item is HomeInputSuggestion => Boolean(item)); + + return dynamicSuggestions.length > 0 + ? dynamicSuggestions + : HOME_INPUT_SUGGESTIONS; +} + +export function buildHomeGuideCards( + entries?: SkillCatalogEntry[], +): HomeGuideCard[] { + const dynamicCards = listHomePresentationEntries(entries, "guide_card") + .map((entry) => { + const prompt = getHomePresentationPrompt(entry); + if (!prompt) { + return null; + } + + const card: HomeGuideCard = { + id: entry.id, + title: entry.homePresentation?.title?.trim() || entry.title, + summary: + entry.homePresentation?.summary?.trim() || + entry.summary.trim() || + prompt, + prompt, + groupKey: entry.homePresentation?.groupKey, + iconToken: entry.homePresentation?.iconToken, + testId: buildHomeTestId("home-guide", entry), + }; + return card; + }) + .filter((item): item is HomeGuideCard => Boolean(item)); + + return dynamicCards.length > 0 ? dynamicCards : HOME_GUIDE_CARDS; } diff --git a/src/components/agent/chat/home/homeSurfaceCopy.ts b/src/components/agent/chat/home/homeSurfaceCopy.ts index 894b88ee1..23616cddf 100644 --- a/src/components/agent/chat/home/homeSurfaceCopy.ts +++ b/src/components/agent/chat/home/homeSurfaceCopy.ts @@ -1,7 +1,15 @@ -import type { HomeSkillCategory, HomeStarterChip } from "./homeSurfaceTypes"; +import type { + HomeGuideCard, + HomeInputSuggestion, + HomeSkillCategory, + HomeStarterChip, +} from "./homeSurfaceTypes"; export const HOME_COMPOSER_PLACEHOLDER = "先说这轮要做什么，目标、对象或限制都可以。"; +export const HOME_GUIDE_HELP_CONTEXT_LABEL = "Lime 引导帮助"; +export const HOME_GUIDE_HELP_PLACEHOLDER = + "想了解什么？试试：怎么创建长期计划 / 如何添加模型 / 语音输入怎么用"; export const HOME_CATEGORY_LABELS: Record = { recent: "最近使用", @@ -25,53 +33,82 @@ export const HOME_CATEGORY_ORDER: HomeSkillCategory[] = [ export const HOME_STARTER_CHIPS: HomeStarterChip[] = [ { - id: "starter-daily-trend", - label: "帮我想选题", - launchKind: "curated_task_launcher", - targetItemId: "daily-trend-briefing", - category: "social", + id: "starter-guide-help", + label: "引导帮助", + launchKind: "toggle_guide", + groupKey: "guide_help", + iconToken: "lightbulb", primary: true, - testId: "entry-recommended-daily-trend-briefing", + testId: "home-guide-help-trigger", }, { - id: "starter-first-draft", - label: "写第一版", + id: "starter-writing", + label: "写作", launchKind: "curated_task_launcher", targetItemId: "social-post-starter", category: "social", + primary: true, testId: "entry-recommended-social-post-starter", }, { - id: "starter-breakdown", - label: "拆解爆款", - launchKind: "curated_task_launcher", - targetItemId: "viral-content-breakdown", - category: "social", - testId: "entry-recommended-viral-content-breakdown", + id: "starter-ppt", + label: "PPT", + launchKind: "prefill_prompt", + category: "editor", + prompt: + "请帮我做一份 PPT，先确认主题、听众、页数、结构和每页核心表达，再给出大纲与页面文案。", + testId: "entry-home-ppt", }, { - id: "starter-rewrite-style", - label: "改成我的风格", + id: "starter-research-report", + label: "调研报告", launchKind: "curated_task_launcher", - targetItemId: "longform-multiplatform-rewrite", + targetItemId: "daily-trend-briefing", category: "social", - testId: "entry-recommended-longform-multiplatform-rewrite", + testId: "entry-recommended-daily-trend-briefing", }, { - id: "starter-video-script", - label: "转成视频脚本", + id: "starter-requirement-analysis", + label: "需求分析", + launchKind: "curated_task_launcher", + targetItemId: "account-project-review", + category: "social", + testId: "entry-recommended-account-project-review", + }, + { + id: "starter-video", + label: "视频", launchKind: "curated_task_launcher", targetItemId: "script-to-voiceover", category: "video", testId: "entry-recommended-script-to-voiceover", }, { - id: "starter-account-review", - label: "复盘账号", - launchKind: "curated_task_launcher", - targetItemId: "account-project-review", - category: "social", - testId: "entry-recommended-account-project-review", + id: "starter-design", + label: "设计", + launchKind: "prefill_prompt", + category: "visual_design", + prompt: + "请帮我设计一个视觉方案，先确认使用场景、目标受众、风格关键词、主视觉方向和可执行的版式建议。", + testId: "entry-home-design", + }, + { + id: "starter-excel", + label: "Excel", + launchKind: "prefill_prompt", + category: "editor", + prompt: + "请帮我整理一个 Excel 表格方案，先确认字段、数据来源、统计口径、公式和最终要看的结论。", + testId: "entry-home-excel", + }, + { + id: "starter-code", + label: "编程", + launchKind: "prefill_prompt", + category: "other", + prompt: + "请帮我完成一个编程任务，先确认目标、运行环境、输入输出、约束和验收方式，再给出实现步骤。", + testId: "entry-home-code", }, { id: "starter-more", @@ -86,3 +123,85 @@ export const HOME_STARTER_CHIPS: HomeStarterChip[] = [ testId: "home-skill-manager-trigger", }, ]; + +export const HOME_INPUT_SUGGESTIONS: HomeInputSuggestion[] = [ + { + id: "suggestion-meeting-notes", + label: "帮我整理一下会议纪要", + prompt: + "帮我整理一下会议纪要，提炼议题、关键结论、待办事项、负责人、截止时间和需要继续确认的问题。", + order: 5, + testId: "home-input-suggestion-meeting-notes", + }, + { + id: "suggestion-research-report", + label: "帮我写一份调研报告", + prompt: + "帮我写一份调研报告，先明确研究问题、目标读者、资料来源、分析框架和最终输出结构。", + order: 10, + testId: "home-input-suggestion-research-report", + }, + { + id: "suggestion-ppt-outline", + label: "帮我做一份 PPT 大纲", + prompt: + "帮我做一份 PPT 大纲，先确认主题、听众、页数、叙事顺序和每页要表达的核心信息。", + order: 20, + testId: "home-input-suggestion-ppt-outline", + }, + { + id: "suggestion-requirement-analysis", + label: "帮我梳理一下需求分析", + prompt: + "帮我梳理一下需求分析，包含目标用户、核心场景、边界条件、优先级、风险和验收标准。", + order: 30, + testId: "home-input-suggestion-requirement-analysis", + }, + { + id: "suggestion-video-script", + label: "帮我把内容改成视频脚本", + prompt: + "帮我把这段内容改成短视频口播脚本，包含开头钩子、分镜节奏、字幕重点和结尾行动引导。", + order: 40, + testId: "home-input-suggestion-video-script", + }, +]; + +export const HOME_GUIDE_CARDS: HomeGuideCard[] = [ + { + id: "guide-long-term-plan", + title: "怎么创建长期计划？", + summary: "让 AI 自动规划任务、定期执行。", + prompt: + "请告诉我怎么创建和使用长期计划功能。我想让 Lime 帮我围绕一个内容目标持续执行、提醒和复盘。", + groupKey: "guide_help", + testId: "home-guide-long-term-plan", + }, + { + id: "guide-add-model", + title: "如何添加新的 AI 模型？", + summary: "配置 Ollama、Kimi 等第三方模型。", + prompt: + "请告诉我如何在 Lime 里添加新的 AI 模型，包括供应商配置、默认模型选择和验证方式。", + groupKey: "guide_help", + testId: "home-guide-add-model", + }, + { + id: "guide-install-skill", + title: "技能怎么安装和使用？", + summary: "从商店安装技能、在对话中启用。", + prompt: + "请告诉我 Lime 的技能怎么安装和使用。我想知道如何把一个高频创作流程沉淀为可复用 skill。", + groupKey: "guide_help", + testId: "home-guide-install-skill", + }, + { + id: "guide-voice-input", + title: "语音输入怎么设置？", + summary: "下载语音模型、配置快捷键。", + prompt: + "请告诉我 Lime 的语音输入怎么设置，包括启用方式、识别模型配置和常见问题排查。", + groupKey: "guide_help", + testId: "home-guide-voice-input", + }, +]; diff --git a/src/components/agent/chat/home/homeSurfaceTypes.ts b/src/components/agent/chat/home/homeSurfaceTypes.ts index 0b1f21b2b..15cb42788 100644 --- a/src/components/agent/chat/home/homeSurfaceTypes.ts +++ b/src/components/agent/chat/home/homeSurfaceTypes.ts @@ -20,6 +20,8 @@ export type HomeSkillLaunchKind = | "installed_skill" | "scene_app" | "skill_catalog_scene" + | "prefill_prompt" + | "toggle_guide" | "open_drawer" | "open_manager" | "launch_browser" @@ -52,6 +54,9 @@ export interface HomeStarterChip { category?: HomeSkillCategory; primary?: boolean; testId?: string; + prompt?: string; + groupKey?: string; + iconToken?: string; } export interface HomeSkillSection { @@ -59,3 +64,21 @@ export interface HomeSkillSection { title: string; items: HomeSkillSurfaceItem[]; } + +export interface HomeGuideCard { + id: string; + title: string; + summary: string; + prompt: string; + groupKey?: string; + iconToken?: string; + testId?: string; +} + +export interface HomeInputSuggestion { + id: string; + label: string; + prompt: string; + order: number; + testId?: string; +} diff --git a/src/components/agent/chat/hooks/agentChatHistory.test.ts b/src/components/agent/chat/hooks/agentChatHistory.test.ts index 5a82c1df9..c70b97cfd 100644 --- a/src/components/agent/chat/hooks/agentChatHistory.test.ts +++ b/src/components/agent/chat/hooks/agentChatHistory.test.ts @@ -6,6 +6,7 @@ import { extractThinkingContentFromParts, hydrateSessionDetailMessages, mergeHydratedMessagesWithLocalState, + shouldCompactCompletedSessionHistory, } from "./agentChatHistory"; describe("agentChatHistory", () => { @@ -136,6 +137,137 @@ describe("agentChatHistory", () => { ]); }); + it("已完成旧会话压缩水合时应跳过工具过程，仅保留可见正文", () => { + const detail: AsterSessionDetail = { + id: "session-compact-history", + created_at: 1, + updated_at: 2, + history_limit: 40, + turns: [ + { + id: "turn-compact-history", + thread_id: "session-compact-history", + prompt_text: "恢复旧会话", + status: "completed", + started_at: "2026-04-30T10:00:00.000Z", + completed_at: "2026-04-30T10:00:05.000Z", + created_at: "2026-04-30T10:00:00.000Z", + updated_at: "2026-04-30T10:00:05.000Z", + }, + ], + items: [ + { + id: "item-compact-history", + thread_id: "session-compact-history", + turn_id: "turn-compact-history", + sequence: 1, + type: "tool_call", + tool_name: "Bash", + arguments: { command: "printf slow" }, + output: "x".repeat(12_000), + status: "completed", + started_at: "2026-04-30T10:00:01.000Z", + completed_at: "2026-04-30T10:00:02.000Z", + updated_at: "2026-04-30T10:00:02.000Z", + } as never, + ], + messages: [ + { + role: "assistant", + timestamp: 1710000005, + content: [ + { type: "thinking", thinking: "大量思考过程" } as never, + { + type: "tool_request", + id: "call-heavy", + tool_name: "Bash", + arguments: { command: "printf slow" }, + } as never, + { + type: "tool_response", + id: "call-heavy", + output: "x".repeat(12_000), + success: true, + } as never, + { type: "output_text", text: "最终回复正文" } as never, + ], + }, + ], + }; + + expect(shouldCompactCompletedSessionHistory(detail)).toBe(true); + + const messages = hydrateSessionDetailMessages( + detail, + "session-compact-history", + { compactCompletedHistory: true }, + ); + + expect(messages).toHaveLength(1); + expect(messages[0]).toMatchObject({ + role: "assistant", + content: "最终回复正文", + thinkingContent: undefined, + toolCalls: undefined, + }); + expect(messages[0]?.contentParts).toEqual([ + { + type: "text", + text: "最终回复正文", + }, + ]); + }); + + it("仍在运行的会话即使请求压缩水合，也应保留工具过程", () => { + const detail: AsterSessionDetail = { + id: "session-running-history", + created_at: 1, + updated_at: 2, + history_limit: 40, + turns: [ + { + id: "turn-running-history", + thread_id: "session-running-history", + prompt_text: "继续执行", + status: "running", + started_at: "2026-04-30T10:00:00.000Z", + created_at: "2026-04-30T10:00:00.000Z", + updated_at: "2026-04-30T10:00:01.000Z", + }, + ], + messages: [ + { + role: "assistant", + timestamp: 1710000005, + content: [ + { + type: "tool_request", + id: "call-running", + tool_name: "Bash", + arguments: { command: "sleep 1" }, + } as never, + ], + }, + ], + }; + + expect(shouldCompactCompletedSessionHistory(detail)).toBe(false); + + const messages = hydrateSessionDetailMessages( + detail, + "session-running-history", + { compactCompletedHistory: true }, + ); + + expect(messages[0]?.toolCalls?.[0]).toMatchObject({ + id: "call-running", + status: "running", + }); + expect( + messages[0]?.contentParts?.some((part) => part.type === "tool_use"), + ).toBe(true); + }); + it("分页历史消息应使用历史窗口绝对位置生成稳定 ID", () => { const detail: AsterSessionDetail = { id: "session-page", diff --git a/src/components/agent/chat/hooks/agentChatHistory.ts b/src/components/agent/chat/hooks/agentChatHistory.ts index 5d3e9a0b4..e78844e44 100644 --- a/src/components/agent/chat/hooks/agentChatHistory.ts +++ b/src/components/agent/chat/hooks/agentChatHistory.ts @@ -345,6 +345,71 @@ export const normalizeHistoricalTopicSnapshotMessages = ( messages: Message[], ): Message[] => messages.map(normalizeHistoricalTopicSnapshotMessage); +function compactHistoricalRestoreMessage(message: Message): Message { + const normalized = normalizeHistoricalTopicSnapshotMessage(message); + if (normalized.role !== "assistant") { + return normalized; + } + + return { + ...normalized, + toolCalls: undefined, + actionRequests: undefined, + contextTrace: undefined, + }; +} + +export const compactHistoricalRestoreMessages = ( + messages: Message[], +): Message[] => + normalizeHistoryMessages(messages) + .map(compactHistoricalRestoreMessage) + .filter((message) => { + if (message.role !== "assistant") { + return true; + } + + if (hasRenderableAssistantTextContent(message)) { + return true; + } + + return ( + (message.images?.length || 0) > 0 || + (message.artifacts?.length || 0) > 0 || + Boolean(message.imageWorkbenchPreview) || + Boolean(message.taskPreview) + ); + }); + +export const shouldCompactCompletedSessionHistory = ( + detail: AsterSessionDetail, +): boolean => { + const historyLimit = + typeof detail.history_limit === "number" && + Number.isFinite(detail.history_limit) && + detail.history_limit > 0 + ? Math.trunc(detail.history_limit) + : null; + + if (historyLimit === null) { + return false; + } + + const hasActiveTurn = (detail.turns || []).some( + (turn) => turn.status === "running", + ); + const hasActiveItem = (detail.items || []).some( + (item) => item.status === "in_progress", + ); + const hasQueuedTurn = (detail.queued_turns || []).length > 0; + + return !hasActiveTurn && !hasActiveItem && !hasQueuedTurn; +}; + +interface HydrateSessionDetailMessagesOptions { + compactCompletedHistory?: boolean; +} + function normalizePreviewSignatureValue(value: unknown): string { if (typeof value === "string") { return normalizeSignatureText(value); @@ -453,6 +518,17 @@ function taskPreviewSignature(preview?: MessageTaskPreview): string { ) .join("|") : ""; + const audioFields = + preview.kind === "audio_generate" + ? [ + preview.taskFilePath, + preview.audioUrl, + preview.mimeType, + preview.durationMs, + preview.sourceText, + preview.voice, + ] + : []; return [ preview.kind, @@ -469,6 +545,7 @@ function taskPreviewSignature(preview?: MessageTaskPreview): string { preview.phase, preview.statusMessage, ...videoFields, + ...audioFields, metaItems, imageCandidates, ] @@ -1099,9 +1176,13 @@ export const dedupeAdjacentHistoryMessages = ( export const hydrateSessionDetailMessages = ( detail: AsterSessionDetail, topicId: string, + options: HydrateSessionDetailMessagesOptions = {}, ): Message[] => { const historyToolNameById = new Map(); const historyToolArgumentsById = new Map(); + const compactCompletedHistory = + options.compactCompletedHistory === true && + shouldCompactCompletedSessionHistory(detail); const historyOffset = typeof detail.history_offset === "number" && Number.isFinite(detail.history_offset) && @@ -1178,6 +1259,16 @@ export const hydrateSessionDetailMessages = ( continue; } + if ( + compactCompletedHistory && + (partType === "thinking" || + partType === "reasoning" || + partType === "tool_request" || + partType === "tool_response") + ) { + continue; + } + if (partType === "thinking" || partType === "reasoning") { const rawThinking = typeof part.thinking === "string" @@ -1368,26 +1459,26 @@ export const hydrateSessionDetailMessages = ( return []; } + const hydratedMessage: Message = { + id: `${topicId}-${historyAbsoluteStartIndex + index}`, + role: normalizedRole, + content, + images: images.length > 0 ? images : undefined, + contentParts: + sanitizedContentParts.length > 0 ? sanitizedContentParts : undefined, + toolCalls: toolCalls.length > 0 ? toolCalls : undefined, + timestamp: messageTimestamp, + isThinking: false, + usage: normalizedRole === "assistant" ? usage : undefined, + thinkingContent: extractThinkingContentFromParts(sanitizedContentParts), + imageWorkbenchPreview, + taskPreview, + }; + return [ - { - id: `${topicId}-${historyAbsoluteStartIndex + index}`, - role: normalizedRole, - content, - images: images.length > 0 ? images : undefined, - contentParts: - sanitizedContentParts.length > 0 - ? sanitizedContentParts - : undefined, - toolCalls: toolCalls.length > 0 ? toolCalls : undefined, - timestamp: messageTimestamp, - isThinking: false, - usage: normalizedRole === "assistant" ? usage : undefined, - thinkingContent: extractThinkingContentFromParts( - sanitizedContentParts, - ), - imageWorkbenchPreview, - taskPreview, - }, + compactCompletedHistory + ? compactHistoricalRestoreMessage(hydratedMessage) + : hydratedMessage, ]; }); diff --git a/src/components/agent/chat/hooks/agentSessionScopedStorage.test.ts b/src/components/agent/chat/hooks/agentSessionScopedStorage.test.ts index 7dc07107a..546777774 100644 --- a/src/components/agent/chat/hooks/agentSessionScopedStorage.test.ts +++ b/src/components/agent/chat/hooks/agentSessionScopedStorage.test.ts @@ -44,6 +44,57 @@ function createItem(index: number): AgentThreadItem { } as AgentThreadItem; } +function createHeavyAssistantMessage(): Message { + const timestamp = new Date("2026-04-24T00:00:02.000Z"); + + return { + id: "message-heavy-assistant", + role: "assistant", + content: "最终回复正文", + timestamp, + thinkingContent: "大量思考过程", + contentParts: [ + { + type: "thinking", + text: "大量思考过程", + }, + { + type: "tool_use", + toolCall: { + id: "tool-heavy", + name: "Bash", + arguments: '{"command":"printf slow"}', + status: "completed", + startTime: timestamp, + endTime: timestamp, + result: { + success: true, + output: "x".repeat(12_000), + }, + }, + }, + { + type: "text", + text: "最终回复正文", + }, + ], + toolCalls: [ + { + id: "tool-heavy", + name: "Bash", + arguments: '{"command":"printf slow"}', + status: "completed", + startTime: timestamp, + endTime: timestamp, + result: { + success: true, + output: "x".repeat(12_000), + }, + }, + ], + }; +} + describe("agentSessionScopedStorage", () => { beforeEach(() => { sessionStorage.clear(); @@ -73,6 +124,71 @@ describe("agentSessionScopedStorage", () => { expect(restored?.currentTurnId).toBe("turn-35"); }); + it("保存已完成会话快照时应压缩 assistant 过程字段，避免旧会话首帧恢复过重", () => { + const workspaceId = "ws-session-snapshot-compact"; + const sessionId = "topic-compact"; + + saveAgentSessionCachedSnapshot(workspaceId, sessionId, { + messages: [createMessage(1), createHeavyAssistantMessage()], + threadTurns: [createTurn(1)], + threadItems: [createItem(1)], + currentTurnId: "turn-1", + }); + + const restored = loadAgentSessionCachedSnapshot(workspaceId, sessionId); + const restoredAssistant = restored?.messages.find( + (message) => message.id === "message-heavy-assistant", + ); + + expect(restoredAssistant).toMatchObject({ + role: "assistant", + content: "最终回复正文", + thinkingContent: undefined, + toolCalls: undefined, + }); + expect(restoredAssistant?.contentParts).toEqual([ + { + type: "text", + text: "最终回复正文", + }, + ]); + }); + + it("保存运行中会话快照时应保留过程字段，避免切回执行中会话丢状态", () => { + const workspaceId = "ws-session-snapshot-running"; + const sessionId = "topic-running"; + const runningTurn: AgentThreadTurn = { + ...createTurn(1), + status: "running", + completed_at: undefined, + }; + const runningItem: AgentThreadItem = { + ...createItem(1), + status: "in_progress", + } as AgentThreadItem; + + saveAgentSessionCachedSnapshot(workspaceId, sessionId, { + messages: [createHeavyAssistantMessage()], + threadTurns: [runningTurn], + threadItems: [runningItem], + currentTurnId: "turn-1", + }); + + const restored = loadAgentSessionCachedSnapshot(workspaceId, sessionId); + const restoredAssistant = restored?.messages.find( + (message) => message.id === "message-heavy-assistant", + ); + + expect(restoredAssistant?.thinkingContent).toBe("大量思考过程"); + expect(restoredAssistant?.toolCalls?.[0]).toMatchObject({ + id: "tool-heavy", + status: "completed", + }); + expect( + restoredAssistant?.contentParts?.some((part) => part.type === "tool_use"), + ).toBe(true); + }); + it("同标签页快照丢失后应回退到持久化 tail，避免重开应用时仍然整段慢恢复", () => { const workspaceId = "ws-session-snapshot-persisted"; const sessionId = "topic-persisted"; @@ -98,6 +214,74 @@ describe("agentSessionScopedStorage", () => { expect(restored?.currentTurnId).toBe("turn-15"); }); + it("持久化 tail 超过热缓存窗口后仍应作为 stale 回放，避免隔天恢复只能等待后端详情", () => { + const workspaceId = "ws-session-snapshot-persisted-stale"; + const sessionId = "topic-persisted-stale"; + const nowMs = Date.parse("2026-04-24T00:00:00.000Z"); + + saveAgentSessionCachedSnapshot( + workspaceId, + sessionId, + { + messages: [createMessage(1)], + threadTurns: [], + threadItems: [], + currentTurnId: null, + }, + { nowMs, sessionUpdatedAt: nowMs, messagesCount: 12 }, + ); + + sessionStorage.clear(); + + const restored = loadAgentSessionCachedSnapshot(workspaceId, sessionId, { + nowMs: nowMs + 2 * 24 * 60 * 60 * 1000, + topicUpdatedAt: nowMs, + messagesCount: 12, + }); + + expect(restored).not.toBeNull(); + expect(restored?.cacheMetadata?.storageKind).toBe("persisted"); + expect(restored?.cacheMetadata?.freshness).toBe("stale"); + expect(restored?.messages[0]?.id).toBe("message-1"); + }); + + it("读取旧版秒级 sessionUpdatedAt 时应归一到毫秒，避免新写缓存被误判过期", () => { + const workspaceId = "ws-session-snapshot-legacy-seconds"; + const sessionId = "topic-legacy-seconds"; + const nowMs = Date.parse("2026-04-24T00:00:00.000Z"); + const sessionUpdatedAtMs = Date.parse("2026-04-24T00:01:00.000Z"); + const cacheKey = `aster_session_snapshots_${workspaceId}`; + + sessionStorage.setItem( + cacheKey, + JSON.stringify({ + [sessionId]: { + messages: [createMessage(1)], + threadTurns: [], + threadItems: [], + currentTurnId: null, + updatedAt: nowMs, + lastAccessedAt: nowMs, + expiresAt: nowMs + 10 * 60 * 1000, + staleUntil: nowMs + 12 * 60 * 1000, + sessionUpdatedAt: Math.floor(sessionUpdatedAtMs / 1000), + messagesCount: 1, + historyTruncated: false, + }, + }), + ); + + const restored = loadAgentSessionCachedSnapshot(workspaceId, sessionId, { + nowMs: nowMs + 1_000, + topicUpdatedAt: sessionUpdatedAtMs, + messagesCount: 1, + }); + + expect(restored).not.toBeNull(); + expect(restored?.cacheMetadata?.sessionUpdatedAt).toBe(sessionUpdatedAtMs); + expect(restored?.cacheMetadata?.freshness).toBe("fresh"); + }); + it("快照超过热缓存 TTL 但仍在 grace 内时应作为 stale 返回并要求后台刷新", () => { const workspaceId = "ws-session-snapshot-stale"; const sessionId = "topic-stale"; @@ -176,6 +360,7 @@ describe("agentSessionScopedStorage", () => { }, { nowMs, sessionUpdatedAt: nowMs }, ); + localStorage.clear(); const restored = loadAgentSessionCachedSnapshot(workspaceId, sessionId, { nowMs: nowMs + 32 * 60 * 1000 + 1, diff --git a/src/components/agent/chat/hooks/agentSessionScopedStorage.ts b/src/components/agent/chat/hooks/agentSessionScopedStorage.ts index 967bbb9a7..c7e5cd426 100644 --- a/src/components/agent/chat/hooks/agentSessionScopedStorage.ts +++ b/src/components/agent/chat/hooks/agentSessionScopedStorage.ts @@ -1,7 +1,10 @@ import { normalizeLegacyThreadItems } from "@/lib/api/agentTextNormalization"; import type { AgentThreadItem, AgentThreadTurn, Message } from "../types"; import { filterConversationThreadItems } from "../utils/threadTimelineView"; -import { normalizeHistoryMessages } from "./agentChatHistory"; +import { + compactHistoricalRestoreMessages, + normalizeHistoryMessages, +} from "./agentChatHistory"; import { loadPersisted, loadTransient, @@ -62,8 +65,9 @@ const MAX_PERSISTED_CACHED_SESSION_MESSAGES = 12; const MAX_PERSISTED_CACHED_SESSION_TURNS = 8; const MAX_PERSISTED_CACHED_SESSION_ITEMS = 32; const TRANSIENT_SNAPSHOT_TTL_MS = 10 * 60 * 1000; -const PERSISTED_SNAPSHOT_TTL_MS = 30 * 60 * 1000; +const PERSISTED_SNAPSHOT_TTL_MS = 12 * 60 * 60 * 1000; const SNAPSHOT_STALE_GRACE_MS = 2 * 60 * 1000; +const PERSISTED_SNAPSHOT_STALE_GRACE_MS = 7 * 24 * 60 * 60 * 1000; interface AgentSessionCachedSnapshotTrimLimits { maxMessages: number; @@ -114,16 +118,20 @@ const PERSISTED_SNAPSHOT_POLICY: AgentSessionCachedSnapshotPolicy = { limits: PERSISTED_SNAPSHOT_LIMITS, maxEntries: MAX_PERSISTED_CACHED_SESSION_SNAPSHOTS, ttlMs: PERSISTED_SNAPSHOT_TTL_MS, - staleGraceMs: SNAPSHOT_STALE_GRACE_MS, + staleGraceMs: PERSISTED_SNAPSHOT_STALE_GRACE_MS, }; function trimCachedSnapshot( snapshot: AgentSessionCachedSnapshot, limits: AgentSessionCachedSnapshotTrimLimits = TRANSIENT_SNAPSHOT_LIMITS, ): AgentSessionCachedSnapshot { - const messages = normalizeHistoryMessages( - snapshot.messages.slice(-limits.maxMessages), - ); + const shouldCompactMessages = + !(snapshot.threadTurns || []).some((turn) => turn.status === "running") && + !(snapshot.threadItems || []).some((item) => item.status === "in_progress"); + const rawMessages = snapshot.messages.slice(-limits.maxMessages); + const messages = shouldCompactMessages + ? compactHistoricalRestoreMessages(rawMessages) + : normalizeHistoryMessages(rawMessages); const threadTurns = snapshot.threadTurns.slice(-limits.maxTurns); const retainedTurnIds = new Set( threadTurns @@ -225,10 +233,17 @@ function normalizeOptionalTimeMs( } if (typeof value === "number" && Number.isFinite(value)) { - return value; + return value > 0 && value < 10_000_000_000 ? value * 1000 : value; } if (typeof value === "string" && value.trim()) { + const numericValue = Number(value); + if (Number.isFinite(numericValue)) { + return numericValue > 0 && numericValue < 10_000_000_000 + ? numericValue * 1000 + : numericValue; + } + const timestamp = Date.parse(value); return Number.isFinite(timestamp) ? timestamp : null; } @@ -258,9 +273,9 @@ function normalizeCachedSnapshotRecord( readFiniteNumber(record.expiresAt) ?? updatedAt + policy.ttlMs; const staleUntil = readFiniteNumber(record.staleUntil) ?? expiresAt + policy.staleGraceMs; - const sessionUpdatedAt = - readFiniteNumber(record.sessionUpdatedAt) ?? - normalizeOptionalTimeMs(record.sessionUpdatedAt as string | null); + const sessionUpdatedAt = normalizeOptionalTimeMs( + record.sessionUpdatedAt as number | string | null, + ); const messagesCount = normalizeOptionalCount(record.messagesCount); const snapshot = trimCachedSnapshot( { diff --git a/src/components/agent/chat/hooks/agentSessionState.ts b/src/components/agent/chat/hooks/agentSessionState.ts index 4be567e3d..cdffb2732 100644 --- a/src/components/agent/chat/hooks/agentSessionState.ts +++ b/src/components/agent/chat/hooks/agentSessionState.ts @@ -17,6 +17,7 @@ import { hydrateSessionDetailMessages, mergeHydratedMessagesWithLocalState, normalizeHistoricalTopicSnapshotMessages, + shouldCompactCompletedSessionHistory, } from "./agentChatHistory"; import { filterConversationThreadItems, @@ -189,7 +190,9 @@ export function buildHydratedAgentSessionSnapshot( localSnapshotOverride?.threadTurns ?? currentThreadTurns; const effectiveCurrentThreadItems = localSnapshotOverride?.threadItems ?? currentThreadItems; - const hydratedMessages = hydrateSessionDetailMessages(detail, topicId); + const hydratedMessages = hydrateSessionDetailMessages(detail, topicId, { + compactCompletedHistory: shouldCompactCompletedSessionHistory(detail), + }); const incomingTurns = detail.turns || []; const incomingItems = normalizeLegacyThreadItems(detail.items || []); const hasRecoverableLocalSessionCache = diff --git a/src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts b/src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts index 17d91d516..c8984c925 100644 --- a/src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts +++ b/src/components/agent/chat/hooks/agentStreamRuntimeHandler.test.ts @@ -175,6 +175,183 @@ describe("agentStreamRuntimeHandler", () => { expect(setMessages).not.toHaveBeenCalled(); }); + it("thinking 关闭时不应把 reasoning_delta 渲染进助手正文", () => { + let messages: Message[] = [ + { + id: "assistant-1", + role: "assistant", + content: "", + timestamp: new Date("2026-04-30T10:00:00.000Z"), + isThinking: true, + contentParts: [{ type: "thinking", text: "隐藏推理" }], + }, + ]; + const requestState = { + accumulatedContent: "", + queuedTurnId: null, + requestLogId: null, + requestStartedAt: 0, + requestFinished: false, + }; + const setMessages = vi.fn( + (value: Message[] | ((prev: Message[]) => Message[])) => { + messages = typeof value === "function" ? value(messages) : value; + }, + ); + const activateStream = vi.fn(); + const baseOptions = { + requestState, + callbacks: { + activateStream, + isStreamActivated: () => true, + clearOptimisticItem: () => {}, + clearOptimisticTurn: () => {}, + disposeListener: () => {}, + removeQueuedDraftMessages: () => {}, + clearActiveStreamIfMatch: () => true, + upsertQueuedTurn: () => {}, + removeQueuedTurnState: () => {}, + playToolcallSound: () => {}, + playTypewriterSound: () => {}, + appendThinkingToParts: () => { + throw new Error("thinking 关闭时不应追加 thinking part"); + }, + }, + eventName: "agent-runtime-thinking-disabled-test", + pendingTurnKey: "pending-turn", + pendingItemKey: "pending-item", + assistantMsgId: "assistant-1", + activeSessionId: "session-1", + resolvedWorkspaceId: "workspace-1", + effectiveExecutionStrategy: "react" as const, + surfaceThinkingDeltas: false, + content: "只回复一个字：好", + runtime: {} as never, + warnedKeysRef: { current: new Set() }, + actionLoggedKeys: new Set(), + toolLogIdByToolId: new Map(), + toolStartedAtByToolId: new Map(), + toolNameByToolId: new Map(), + setMessages: setMessages as never, + setPendingActions: vi.fn() as never, + setThreadItems: vi.fn() as never, + setThreadTurns: vi.fn() as never, + setCurrentTurnId: vi.fn() as never, + setExecutionRuntime: vi.fn() as never, + setIsSending: vi.fn() as never, + }; + + handleTurnStreamEvent({ + ...baseOptions, + data: { + type: "thinking_delta", + text: "我们只：好。", + } as AgentEvent, + }); + + expect(activateStream).toHaveBeenCalledTimes(1); + expect(setMessages).not.toHaveBeenCalled(); + + handleTurnStreamEvent({ + ...baseOptions, + data: { + type: "text_delta", + text: "好", + } as AgentEvent, + }); + + expect(messages[0]?.content).toBe("好"); + expect(messages[0]?.thinkingContent).toBeUndefined(); + expect(messages[0]?.contentParts).toEqual([{ type: "text", text: "好" }]); + }); + + it("连续 text_delta 应合并到低频渲染，避免每个字符都刷新消息树", () => { + vi.useFakeTimers(); + let messages: Message[] = [ + { + id: "assistant-1", + role: "assistant", + content: "", + timestamp: new Date("2026-04-30T10:00:00.000Z"), + isThinking: true, + contentParts: [], + }, + ]; + const requestState = { + accumulatedContent: "", + queuedTurnId: null, + requestLogId: null, + requestStartedAt: 0, + requestFinished: false, + }; + const setMessages = vi.fn( + (value: Message[] | ((prev: Message[]) => Message[])) => { + messages = typeof value === "function" ? value(messages) : value; + }, + ); + const baseOptions = { + requestState, + callbacks: { + activateStream: vi.fn(), + isStreamActivated: () => true, + clearOptimisticItem: () => {}, + clearOptimisticTurn: () => {}, + disposeListener: () => {}, + removeQueuedDraftMessages: () => {}, + clearActiveStreamIfMatch: () => true, + upsertQueuedTurn: () => {}, + removeQueuedTurnState: () => {}, + playToolcallSound: () => {}, + playTypewriterSound: () => {}, + appendThinkingToParts: (parts: NonNullable) => + parts, + }, + eventName: "agent-runtime-text-batch-test", + pendingTurnKey: "pending-turn", + pendingItemKey: "pending-item", + assistantMsgId: "assistant-1", + activeSessionId: "session-1", + resolvedWorkspaceId: "workspace-1", + effectiveExecutionStrategy: "react" as const, + content: "数数", + runtime: {} as never, + warnedKeysRef: { current: new Set() }, + actionLoggedKeys: new Set(), + toolLogIdByToolId: new Map(), + toolStartedAtByToolId: new Map(), + toolNameByToolId: new Map(), + setMessages: setMessages as never, + setPendingActions: vi.fn() as never, + setThreadItems: vi.fn() as never, + setThreadTurns: vi.fn() as never, + setCurrentTurnId: vi.fn() as never, + setExecutionRuntime: vi.fn() as never, + setIsSending: vi.fn() as never, + }; + + handleTurnStreamEvent({ + ...baseOptions, + data: { type: "text_delta", text: "1" } as AgentEvent, + }); + handleTurnStreamEvent({ + ...baseOptions, + data: { type: "text_delta", text: "2" } as AgentEvent, + }); + handleTurnStreamEvent({ + ...baseOptions, + data: { type: "text_delta", text: "3" } as AgentEvent, + }); + + expect(setMessages).toHaveBeenCalledTimes(1); + expect(messages[0]?.content).toBe("1"); + + vi.advanceTimersByTime(32); + + expect(setMessages).toHaveBeenCalledTimes(2); + expect(messages[0]?.content).toBe("123"); + expect(messages[0]?.contentParts).toEqual([{ type: "text", text: "123" }]); + }); + it("高频 reasoning item_updated 事件不应持续刷新时间线状态", () => { const setThreadItems = vi.fn(); const activateStream = vi.fn(); @@ -309,6 +486,9 @@ describe("agentStreamRuntimeHandler", () => { }); expect(messages[0]?.content).toBe("已保存到项目目录。"); + expect(messages[0]?.contentParts).toEqual([ + { type: "text", text: "已保存到项目目录。" }, + ]); }); it("收到空 final_done 且没有真实产物信号时应落成失败态", () => { diff --git a/src/components/agent/chat/hooks/agentStreamRuntimeHandler.ts b/src/components/agent/chat/hooks/agentStreamRuntimeHandler.ts index d33569262..e0804f93b 100644 --- a/src/components/agent/chat/hooks/agentStreamRuntimeHandler.ts +++ b/src/components/agent/chat/hooks/agentStreamRuntimeHandler.ts @@ -11,6 +11,7 @@ import type { QueuedTurnSnapshot, } from "@/lib/api/agentRuntime"; import { activityLogger } from "@/lib/workspace/workbenchRuntime"; +import { logAgentDebug } from "@/lib/agentDebug"; import type { ActionRequired, Message } from "../types"; import { appendTextToParts, @@ -71,8 +72,21 @@ interface StreamRequestState { queuedTurnId: string | null; requestLogId: string | null; requestStartedAt: number; + submissionDispatchedAt?: number | null; + listenerBoundAt?: number | null; + firstEventReceivedAt?: number | null; + firstRuntimeStatusAt?: number | null; + firstTextDeltaAt?: number | null; + firstTextPaintAt?: number | null; + firstTextRenderFlushAt?: number | null; + lastTextRenderFlushAt?: number | null; + textDeltaBufferedCount?: number; + textDeltaFlushCount?: number; + maxTextDeltaBacklogChars?: number; requestFinished: boolean; queuedDraftCleanupTimerId?: ReturnType | null; + pendingTextRenderTimerId?: ReturnType | null; + renderedContent?: string; } const EMPTY_FINAL_REPLY_ERROR_HINT = "模型未输出最终答复"; @@ -80,6 +94,7 @@ const EMPTY_FINAL_REPLY_ERROR_MESSAGE = "模型未输出最终答复，请重试 const EMPTY_FINAL_REPLY_FALLBACK_CONTENT = "本轮执行已完成，详细过程与产物已保留在当前对话中。"; const QUEUED_DRAFT_CLEANUP_GRACE_MS = 1800; +const TEXT_DELTA_RENDER_FLUSH_MS = 32; interface StreamLifecycleCallbacks { activateStream: () => void; @@ -111,6 +126,7 @@ interface HandleTurnStreamEventOptions { activeSessionId: string; resolvedWorkspaceId: string; effectiveExecutionStrategy: AsterExecutionStrategy; + surfaceThinkingDeltas?: boolean; content: string; runtime: AgentRuntimeAdapter; webSearch?: boolean; @@ -167,6 +183,47 @@ function shouldDeferHighFrequencyThreadItemUpdate( ); } +function reconcileFinalContentParts(params: { + parts: Message["contentParts"]; + finalContent: string; + rawContent: string; + surfaceThinkingDeltas: boolean; +}): Message["contentParts"] { + if (!params.parts?.length) { + return params.parts; + } + + const visibleParts = params.surfaceThinkingDeltas + ? params.parts + : params.parts.filter((part) => part.type !== "thinking"); + if (visibleParts.length === 0) { + return undefined; + } + + const textContent = visibleParts + .filter((part) => part.type === "text") + .map((part) => part.text) + .join(""); + const finalTextChanged = + params.finalContent !== params.rawContent || + (textContent.length > 0 && textContent !== params.finalContent); + + if (!finalTextChanged) { + return visibleParts; + } + + const processParts = visibleParts.filter((part) => part.type !== "text"); + if (processParts.length === 0) { + return params.finalContent + ? [{ type: "text", text: params.finalContent }] + : undefined; + } + + return params.finalContent + ? [...processParts, { type: "text", text: params.finalContent }] + : processParts; +} + function hasMeaningfulCompletionSignalFromToolResult(params: { toolId: string; toolName: string; @@ -214,6 +271,7 @@ export function handleTurnStreamEvent({ activeSessionId, resolvedWorkspaceId, effectiveExecutionStrategy, + surfaceThinkingDeltas = true, content, runtime, webSearch, @@ -253,6 +311,125 @@ export function handleTurnStreamEvent({ } }; + const clearPendingTextRenderTimer = () => { + if (requestState.pendingTextRenderTimerId) { + clearTimeout(requestState.pendingTextRenderTimerId); + requestState.pendingTextRenderTimerId = null; + } + }; + + const resolvePendingRenderedTextDelta = ( + renderedContent: string, + accumulatedContent: string, + ): string => { + if (!renderedContent) { + return accumulatedContent; + } + if (accumulatedContent.startsWith(renderedContent)) { + return accumulatedContent.slice(renderedContent.length); + } + return accumulatedContent; + }; + + const flushPendingTextRender = () => { + clearPendingTextRenderTimer(); + const renderedContent = requestState.renderedContent || ""; + const nextContent = requestState.accumulatedContent; + if (nextContent === renderedContent) { + return; + } + + const flushStartedAt = Date.now(); + const textDelta = resolvePendingRenderedTextDelta( + renderedContent, + nextContent, + ); + requestState.renderedContent = nextContent; + requestState.textDeltaFlushCount = + (requestState.textDeltaFlushCount ?? 0) + 1; + requestState.lastTextRenderFlushAt = flushStartedAt; + if (!requestState.firstTextRenderFlushAt) { + requestState.firstTextRenderFlushAt = flushStartedAt; + } + if (!requestState.firstTextPaintAt && nextContent.trim().length > 0) { + requestState.firstTextPaintAt = flushStartedAt; + logAgentDebug("AgentStream", "firstTextPaint", { + elapsedMs: flushStartedAt - requestState.requestStartedAt, + eventName, + firstTextDeltaDeltaMs: requestState.firstTextDeltaAt + ? flushStartedAt - requestState.firstTextDeltaAt + : null, + sessionId: activeSessionId, + }); + } + const backlogChars = Math.max( + 0, + nextContent.length - renderedContent.length, + ); + requestState.maxTextDeltaBacklogChars = Math.max( + requestState.maxTextDeltaBacklogChars ?? 0, + backlogChars, + ); + if (backlogChars >= 80 || (requestState.textDeltaFlushCount ?? 0) === 1) { + logAgentDebug( + "AgentStream", + "textRenderFlush", + { + accumulatedChars: nextContent.length, + backlogChars, + elapsedMs: flushStartedAt - requestState.requestStartedAt, + eventName, + flushCount: requestState.textDeltaFlushCount, + maxBacklogChars: requestState.maxTextDeltaBacklogChars ?? 0, + sessionId: activeSessionId, + }, + { + dedupeKey: `AgentStream:textRenderFlush:${eventName}:${requestState.textDeltaFlushCount}`, + throttleMs: 250, + }, + ); + } + setMessages((prev) => + prev.map((msg) => + msg.id === assistantMsgId + ? { + ...msg, + content: nextContent, + thinkingContent: undefined, + contentParts: textDelta + ? appendTextToParts( + surfaceThinkingDeltas + ? msg.contentParts || [] + : (msg.contentParts || []).filter( + (part) => part.type !== "thinking", + ), + textDelta, + ) + : msg.contentParts, + } + : msg, + ), + ); + }; + + const scheduleTextRenderFlush = () => { + const renderedContent = requestState.renderedContent || ""; + const hasVisibleFirstText = + !renderedContent && requestState.accumulatedContent.trim().length > 0; + if (hasVisibleFirstText) { + flushPendingTextRender(); + return; + } + + if (requestState.pendingTextRenderTimerId) { + return; + } + requestState.pendingTextRenderTimerId = setTimeout(() => { + requestState.pendingTextRenderTimerId = null; + flushPendingTextRender(); + }, TEXT_DELTA_RENDER_FLUSH_MS); + }; + const scheduleQueuedDraftCleanup = (shouldWatchCurrentRequest: boolean) => { clearQueuedDraftCleanupTimer(); if (!shouldWatchCurrentRequest || isStreamActivated()) { @@ -330,6 +507,7 @@ export function handleTurnStreamEvent({ errorMessage: string, usage?: Message["usage"], ) => { + clearPendingTextRenderTimer(); markFailedTimelineState(errorMessage); removeQueuedTurnState( requestState.queuedTurnId ? [requestState.queuedTurnId] : [], @@ -497,6 +675,21 @@ export function handleTurnStreamEvent({ case "runtime_status": activateStream(); { + if (!requestState.firstRuntimeStatusAt) { + requestState.firstRuntimeStatusAt = Date.now(); + logAgentDebug("AgentStream", "firstRuntimeStatus", { + elapsedMs: + requestState.firstRuntimeStatusAt - requestState.requestStartedAt, + eventName, + firstEventDeltaMs: requestState.firstEventReceivedAt + ? requestState.firstRuntimeStatusAt - + requestState.firstEventReceivedAt + : null, + phase: data.status.phase, + sessionId: activeSessionId, + title: data.status.title, + }); + } const normalizedStatus = { ...data.status, title: normalizeLegacyRuntimeStatusTitle(data.status.title), @@ -557,6 +750,9 @@ export function handleTurnStreamEvent({ case "thinking_delta": activateStream(); + if (!surfaceThinkingDeltas) { + break; + } setMessages((prev) => prev.map((msg) => msg.id === assistantMsgId @@ -580,27 +776,31 @@ export function handleTurnStreamEvent({ case "text_delta": activateStream(); clearOptimisticItem(); + requestState.textDeltaBufferedCount = + (requestState.textDeltaBufferedCount ?? 0) + 1; + if (!requestState.firstTextDeltaAt) { + requestState.firstTextDeltaAt = Date.now(); + logAgentDebug("AgentStream", "firstTextDelta", { + deltaChars: data.text.length, + elapsedMs: + requestState.firstTextDeltaAt - requestState.requestStartedAt, + eventName, + firstEventDeltaMs: requestState.firstEventReceivedAt + ? requestState.firstTextDeltaAt - requestState.firstEventReceivedAt + : null, + firstRuntimeStatusDeltaMs: requestState.firstRuntimeStatusAt + ? requestState.firstTextDeltaAt - requestState.firstRuntimeStatusAt + : null, + sessionId: activeSessionId, + }); + } requestState.accumulatedContent = appendTextWithOverlapDetection( requestState.accumulatedContent, data.text, ); observer?.onTextDelta?.(data.text, requestState.accumulatedContent); playTypewriterSound(); - setMessages((prev) => - prev.map((msg) => - msg.id === assistantMsgId - ? { - ...msg, - content: requestState.accumulatedContent, - thinkingContent: undefined, - contentParts: appendTextToParts( - msg.contentParts || [], - data.text, - ), - } - : msg, - ), - ); + scheduleTextRenderFlush(); break; case "tool_start": @@ -694,6 +894,7 @@ export function handleTurnStreamEvent({ case "final_done": { clearQueuedDraftCleanupTimer(); + flushPendingTextRender(); clearOptimisticItem(); clearOptimisticTurn(); const rawFinalContent = requestState.accumulatedContent.trim(); @@ -732,6 +933,12 @@ export function handleTurnStreamEvent({ ...updateMessageArtifactsStatus(msg, "complete"), isThinking: false, content: finalContent, + contentParts: reconcileFinalContentParts({ + parts: msg.contentParts, + finalContent, + rawContent: requestState.accumulatedContent, + surfaceThinkingDeltas, + }), runtimeStatus: undefined, usage: data.usage ?? msg.usage, }; @@ -744,6 +951,7 @@ export function handleTurnStreamEvent({ case "error": { clearQueuedDraftCleanupTimer(); + flushPendingTextRender(); if (data.message.includes(EMPTY_FINAL_REPLY_ERROR_HINT)) { clearOptimisticItem(); clearOptimisticTurn(); @@ -768,6 +976,12 @@ export function handleTurnStreamEvent({ ...updateMessageArtifactsStatus(msg, "complete"), isThinking: false, content: gracefulContent, + contentParts: reconcileFinalContentParts({ + parts: msg.contentParts, + finalContent: gracefulContent, + rawContent: requestState.accumulatedContent, + surfaceThinkingDeltas, + }), runtimeStatus: undefined, } : msg, diff --git a/src/components/agent/chat/hooks/agentStreamSubmissionLifecycle.ts b/src/components/agent/chat/hooks/agentStreamSubmissionLifecycle.ts index 2221f7aa6..a11eb4d63 100644 --- a/src/components/agent/chat/hooks/agentStreamSubmissionLifecycle.ts +++ b/src/components/agent/chat/hooks/agentStreamSubmissionLifecycle.ts @@ -27,9 +27,21 @@ export interface StreamRequestState { requestLogId: string | null; requestStartedAt: number; submissionDispatchedAt?: number | null; + listenerBoundAt?: number | null; + firstEventReceivedAt?: number | null; + firstRuntimeStatusAt?: number | null; + firstTextDeltaAt?: number | null; + firstTextPaintAt?: number | null; + firstTextRenderFlushAt?: number | null; + lastTextRenderFlushAt?: number | null; + textDeltaBufferedCount?: number; + textDeltaFlushCount?: number; + maxTextDeltaBacklogChars?: number; requestFinished: boolean; queuedTurnId: string | null; queuedDraftCleanupTimerId?: ReturnType | null; + pendingTextRenderTimerId?: ReturnType | null; + renderedContent?: string; } interface CreateSubmissionLifecycleOptions { @@ -83,9 +95,21 @@ export function createAgentStreamSubmissionLifecycle( requestLogId: null, requestStartedAt: 0, submissionDispatchedAt: null, + listenerBoundAt: null, + firstEventReceivedAt: null, + firstRuntimeStatusAt: null, + firstTextDeltaAt: null, + firstTextPaintAt: null, + firstTextRenderFlushAt: null, + lastTextRenderFlushAt: null, + textDeltaBufferedCount: 0, + textDeltaFlushCount: 0, + maxTextDeltaBacklogChars: 0, requestFinished: false, queuedTurnId: null, queuedDraftCleanupTimerId: null, + pendingTextRenderTimerId: null, + renderedContent: "", }; const optimisticStartedAt = assistantMsg.timestamp.toISOString(); const pendingTurnKey = createPendingTurnKey(); diff --git a/src/components/agent/chat/hooks/agentStreamSubmitExecution.ts b/src/components/agent/chat/hooks/agentStreamSubmitExecution.ts index a090da84d..93ba97825 100644 --- a/src/components/agent/chat/hooks/agentStreamSubmitExecution.ts +++ b/src/components/agent/chat/hooks/agentStreamSubmitExecution.ts @@ -16,6 +16,7 @@ import type { AgentAccessMode } from "./agentChatStorage"; import type { StreamRequestState } from "./agentStreamSubmissionLifecycle"; import type { ActionRequired, Message, MessageImage } from "../types"; import type { ChatToolPreferences } from "../utils/chatToolPreferences"; +import { logAgentDebug } from "@/lib/agentDebug"; import { buildUserInputSubmitOp } from "../utils/buildUserInputSubmitOp"; import { resolveAgentStreamSubmitContext } from "./agentStreamSubmitContext"; import { registerAgentStreamTurnEventBinding } from "./agentStreamTurnEventBinding"; @@ -189,6 +190,7 @@ export async function executeAgentStreamSubmit( effectiveProviderType, effectiveModel, effectiveExecutionStrategy, + thinking, content, webSearch, autoContinue, @@ -231,30 +233,66 @@ export async function executeAgentStreamSubmit( callbacks.registerListener(unlisten); requestState.submissionDispatchedAt = Date.now(); + logAgentDebug("AgentStream", "submitDispatched", { + elapsedMs: + requestState.submissionDispatchedAt - requestState.requestStartedAt, + eventName, + expectingQueue, + listenerBoundDeltaMs: requestState.listenerBoundAt + ? requestState.submissionDispatchedAt - requestState.listenerBoundAt + : null, + sessionId: activeSessionId, + }); - await runtime.submitOp( - buildUserInputSubmitOp({ - content, - images, - sessionId: activeSessionId, + try { + await runtime.submitOp( + buildUserInputSubmitOp({ + content, + images, + sessionId: activeSessionId, + eventName, + workspaceId: submitWorkspaceId, + turnId: requestTurnId, + systemPrompt, + queueIfBusy: true, + requestMetadata, + executionRuntime, + syncedRecentPreferences, + syncedSessionModelPreference, + syncedExecutionStrategy, + effectiveExecutionStrategy, + effectiveAccessMode, + effectiveProviderType, + effectiveModel, + modelOverride, + webSearch, + thinking, + autoContinue, + }), + ); + logAgentDebug("AgentStream", "submitAccepted", { + elapsedMs: Date.now() - requestState.requestStartedAt, eventName, - workspaceId: submitWorkspaceId, - turnId: requestTurnId, - systemPrompt, - queueIfBusy: true, - requestMetadata, - executionRuntime, - syncedRecentPreferences, - syncedSessionModelPreference, - syncedExecutionStrategy, - effectiveExecutionStrategy, - effectiveAccessMode, - effectiveProviderType, - effectiveModel, - modelOverride, - webSearch, - thinking, - autoContinue, - }), - ); + sessionId: activeSessionId, + submitInvokeMs: requestState.submissionDispatchedAt + ? Date.now() - requestState.submissionDispatchedAt + : null, + }); + } catch (error) { + logAgentDebug( + "AgentStream", + "submitFailed", + { + elapsedMs: Date.now() - requestState.requestStartedAt, + error, + eventName, + sessionId: activeSessionId, + submitInvokeMs: requestState.submissionDispatchedAt + ? Date.now() - requestState.submissionDispatchedAt + : null, + }, + { level: "error" }, + ); + throw error; + } } diff --git a/src/components/agent/chat/hooks/agentStreamTurnEventBinding.ts b/src/components/agent/chat/hooks/agentStreamTurnEventBinding.ts index 2dd5daffc..b5d6c4184 100644 --- a/src/components/agent/chat/hooks/agentStreamTurnEventBinding.ts +++ b/src/components/agent/chat/hooks/agentStreamTurnEventBinding.ts @@ -11,6 +11,7 @@ import type { AutoContinueRequestPayload, QueuedTurnSnapshot, } from "@/lib/api/agentRuntime"; +import { logAgentDebug } from "@/lib/agentDebug"; import { activityLogger } from "@/lib/workspace/workbenchRuntime"; import type { ActionRequired, Message } from "../types"; import { mapProviderName } from "./agentChatCoreUtils"; @@ -45,6 +46,7 @@ interface RegisterAgentStreamTurnEventBindingOptions { effectiveProviderType: string; effectiveModel: string; effectiveExecutionStrategy: AsterExecutionStrategy; + thinking?: boolean; content: string; webSearch?: boolean; autoContinue?: AutoContinueRequestPayload; @@ -120,6 +122,7 @@ export async function registerAgentStreamTurnEventBinding( effectiveProviderType, effectiveModel, effectiveExecutionStrategy, + thinking, content, webSearch, autoContinue, @@ -285,6 +288,7 @@ export async function registerAgentStreamTurnEventBinding( activeSessionId, resolvedWorkspaceId, effectiveExecutionStrategy, + surfaceThinkingDeltas: thinking !== false, content, runtime, webSearch, @@ -343,6 +347,7 @@ export async function registerAgentStreamTurnEventBinding( const unlisten = await runtime.listenToTurnEvents( eventName, (event: { payload: unknown }) => { + const eventReceivedAt = Date.now(); const data = parseAgentEvent(event.payload); const eventType = extractRuntimeEventType(event.payload); if (!data) { @@ -351,9 +356,20 @@ export async function registerAgentStreamTurnEventBinding( } if (!firstEventReceived) { firstEventReceived = true; + requestState.firstEventReceivedAt = eventReceivedAt; + logAgentDebug("AgentStream", "firstEvent", { + elapsedMs: eventReceivedAt - requestState.requestStartedAt, + eventName, + eventType, + recognized: false, + sessionId: activeSessionId, + submissionDispatchedDeltaMs: requestState.submissionDispatchedAt + ? eventReceivedAt - requestState.submissionDispatchedAt + : null, + }); clearFirstEventWatchdog(); } - lastEventReceivedAt = Date.now(); + lastEventReceivedAt = eventReceivedAt; callbacks.activateStream( activeSessionId, effectiveWaitingRuntimeStatus, @@ -369,9 +385,20 @@ export async function registerAgentStreamTurnEventBinding( } if (!firstEventReceived) { firstEventReceived = true; + requestState.firstEventReceivedAt = eventReceivedAt; + logAgentDebug("AgentStream", "firstEvent", { + elapsedMs: eventReceivedAt - requestState.requestStartedAt, + eventName, + eventType: data.type, + recognized: true, + sessionId: activeSessionId, + submissionDispatchedDeltaMs: requestState.submissionDispatchedAt + ? eventReceivedAt - requestState.submissionDispatchedAt + : null, + }); clearFirstEventWatchdog(); } - lastEventReceivedAt = Date.now(); + lastEventReceivedAt = eventReceivedAt; handleTurnStreamEvent({ data, @@ -402,6 +429,7 @@ export async function registerAgentStreamTurnEventBinding( activeSessionId, resolvedWorkspaceId, effectiveExecutionStrategy, + surfaceThinkingDeltas: thinking !== false, content, runtime, webSearch, @@ -423,6 +451,14 @@ export async function registerAgentStreamTurnEventBinding( }, ); + requestState.listenerBoundAt = Date.now(); + logAgentDebug("AgentStream", "listenerBound", { + elapsedMs: requestState.listenerBoundAt - requestState.requestStartedAt, + eventName, + expectingQueue, + sessionId: activeSessionId, + }); + return () => { clearFirstEventWatchdog(); clearInactivityWatchdog(); diff --git a/src/components/agent/chat/hooks/useAgentContext.test.tsx b/src/components/agent/chat/hooks/useAgentContext.test.tsx index 4cf540dd0..0d7984e2d 100644 --- a/src/components/agent/chat/hooks/useAgentContext.test.tsx +++ b/src/components/agent/chat/hooks/useAgentContext.test.tsx @@ -9,7 +9,6 @@ const { mockTopicsUpdater, mockToastError, mockUpdateProject, - mockWechatChannelSetRuntimeModel, } = vi.hoisted(() => ({ mockNotifyProjectRuntimeAgentsGuide: vi.fn(), mockSetSessionExecutionStrategy: vi.fn(async () => undefined), @@ -17,7 +16,6 @@ const { mockTopicsUpdater: vi.fn(), mockToastError: vi.fn(), mockUpdateProject: vi.fn(async () => undefined), - mockWechatChannelSetRuntimeModel: vi.fn(async () => undefined), })); vi.mock("sonner", () => ({ @@ -34,10 +32,6 @@ vi.mock("@/components/workspace/services/runtimeAgentsGuideService", () => ({ notifyProjectRuntimeAgentsGuide: mockNotifyProjectRuntimeAgentsGuide, })); -vi.mock("@/lib/api/channelsRuntime", () => ({ - wechatChannelSetRuntimeModel: mockWechatChannelSetRuntimeModel, -})); - import { useAgentContext } from "./useAgentContext"; import { loadPersistedSessionWorkspaceId } from "./agentProjectStorage"; @@ -115,7 +109,6 @@ describe("useAgentContext", () => { mockTopicsUpdater.mockReset(); mockToastError.mockReset(); mockUpdateProject.mockReset(); - mockWechatChannelSetRuntimeModel.mockReset(); localStorage.clear(); sessionStorage.clear(); }); @@ -147,7 +140,7 @@ describe("useAgentContext", () => { harness.unmount(); }); - it("切换 provider 和 model 时应同步微信运行时模型", async () => { + it("切换 provider 和 model 时不应写入微信运行时配置", async () => { const harness = mountHook(); await act(async () => { @@ -156,10 +149,12 @@ describe("useAgentContext", () => { await Promise.resolve(); }); - expect(mockWechatChannelSetRuntimeModel).toHaveBeenCalledWith({ - providerId: "deepseek", - modelId: "deepseek-reasoner", - }); + expect(localStorage.getItem("agent_pref_provider_workspace-1")).toBe( + JSON.stringify("deepseek"), + ); + expect(localStorage.getItem("agent_pref_model_workspace-1")).toBe( + JSON.stringify("deepseek-reasoner"), + ); harness.unmount(); }); diff --git a/src/components/agent/chat/hooks/useAgentContext.ts b/src/components/agent/chat/hooks/useAgentContext.ts index 501b1e1a8..9f021e2e5 100644 --- a/src/components/agent/chat/hooks/useAgentContext.ts +++ b/src/components/agent/chat/hooks/useAgentContext.ts @@ -38,7 +38,6 @@ import { normalizeProjectId, } from "../utils/topicProjectResolution"; import { normalizeExecutionStrategy } from "./agentChatCoreUtils"; -import { useWechatRuntimeModelSync } from "./useWechatRuntimeModelSync"; interface UseAgentContextOptions { workspaceId: string; @@ -128,12 +127,6 @@ export function useAgentContext(options: UseAgentContextOptions) { providerTypeRef.current = providerType; modelRef.current = model; - useWechatRuntimeModelSync({ - providerId: providerType, - modelId: model, - source: "workspace", - }); - const persistSessionModelPreference = useCallback( ( targetSessionId: string, diff --git a/src/components/agent/chat/hooks/useAgentSession.ts b/src/components/agent/chat/hooks/useAgentSession.ts index 9bfa599be..c278732d1 100644 --- a/src/components/agent/chat/hooks/useAgentSession.ts +++ b/src/components/agent/chat/hooks/useAgentSession.ts @@ -20,6 +20,7 @@ import type { QueuedTurnSnapshot, } from "@/lib/api/agentRuntime"; import { logAgentDebug } from "@/lib/agentDebug"; +import { recordAgentUiPerformanceMetric } from "@/lib/agentUiPerformanceMetrics"; import { normalizeLegacyThreadItems } from "@/lib/api/agentTextNormalization"; import { isAsterSessionNotFoundError } from "@/lib/asterSessionRecovery"; import type { AgentThreadItem, AgentThreadTurn, Message } from "../types"; @@ -37,6 +38,7 @@ import { hydrateSessionDetailMessages, mergeHydratedMessagesWithLocalState, normalizeHistoryMessages, + shouldCompactCompletedSessionHistory, } from "./agentChatHistory"; import { getAgentSessionScopedKeys, @@ -88,7 +90,7 @@ import { scheduleMinimumDelayIdleTask } from "@/lib/utils/scheduleMinimumDelayId import { hasTauriInvokeCapability } from "@/lib/tauri-runtime"; const INITIAL_TOPICS_IDLE_TIMEOUT_MS = 1_500; -const INITIAL_TOPICS_SESSION_REQUEST_LIMIT = 60; +const INITIAL_TOPICS_SESSION_REQUEST_LIMIT = 21; const SESSION_HISTORY_LOAD_PAGE_SIZE = 50; const ACTIVE_SESSION_TRANSIENT_MESSAGES_LIMIT = 48; const ACTIVE_SESSION_TRANSIENT_TURNS_LIMIT = 48; @@ -97,6 +99,29 @@ const ACTIVE_SESSION_TRANSIENT_SAVE_DELAY_MS = 180; const ACTIVE_SESSION_TRANSIENT_SAVE_IDLE_TIMEOUT_MS = 1_800; const SESSION_METADATA_SYNC_DELAY_MS = 8_000; const SESSION_METADATA_SYNC_IDLE_TIMEOUT_MS = 15_000; +const SESSION_DETAIL_PREFETCH_HISTORY_LIMIT = 40; +const SESSION_DETAIL_PREFETCH_RECENT_LIMIT = 1; +const SESSION_DETAIL_PREFETCH_DELAY_MS = 5_000; +const SESSION_DETAIL_PREFETCH_IDLE_TIMEOUT_MS = 8_000; +const SESSION_DETAIL_DEFERRED_HYDRATION_DELAY_MS = 1_200; + +type AgentSessionRuntimeDetail = Awaited< + ReturnType +>; + +interface SessionDetailPrefetchEntry { + signature: string; + promise: Promise; +} + +const sessionDetailPrefetchRegistry = new Map< + string, + SessionDetailPrefetchEntry +>(); + +function buildSessionDetailPrefetchKey(workspaceId: string, topicId: string) { + return `${workspaceId.trim() || "global"}:${topicId.trim()}`; +} function mapSessionDetailToTopic( sessionId: string, @@ -148,6 +173,17 @@ function upsertTopicFromSessionDetail( }); } +function buildSessionDetailPrefetchSignature( + topicId: string, + topic?: Topic, +): string { + return [ + topicId, + topic?.updatedAt?.getTime() ?? "unknown-updated", + topic?.messagesCount ?? "unknown-count", + ].join(":"); +} + export interface AgentSessionHistoryWindow { loadedMessages: number; totalMessages: number; @@ -485,6 +521,10 @@ export function useAgentSession(options: UseAgentSessionOptions) { const hydratedSessionRef = useRef(null); const skipAutoRestoreRef = useRef(false); const sessionSwitchRequestVersionRef = useRef(0); + const activeSessionSwitchRef = useRef<{ + topicId: string; + promise: Promise; + } | null>(null); const deferredSessionHydrationCancelRef = useRef<(() => void) | null>(null); const pendingSessionMetadataSyncCancelRef = useRef<(() => void) | null>(null); const createFreshSessionPromiseRef = useRef | null>( @@ -1309,7 +1349,7 @@ export function useAgentSession(options: UseAgentSessionOptions) { } : null, ); - logAgentDebug("useAgentSession", "switchTopic.cachedSnapshotApplied", { + const cachedSnapshotMetricContext = { cacheFreshness: metadata?.freshness ?? null, cacheStorageKind: metadata?.storageKind ?? null, cachedMessagesCount: cachedSnapshot.messages.length, @@ -1317,7 +1357,16 @@ export function useAgentSession(options: UseAgentSessionOptions) { cachedTurnsCount: cachedSnapshot.threadTurns.length, topicId, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.cachedSnapshotApplied", + cachedSnapshotMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.cachedSnapshotApplied", + cachedSnapshotMetricContext, + ); return true; }, [ @@ -1352,6 +1401,223 @@ export function useAgentSession(options: UseAgentSessionOptions) { ], ); + const prefetchTopic = useCallback( + async (topicId: string): Promise => { + const resolvedTopicId = topicId.trim(); + const resolvedWorkspaceId = normalizeProjectId(workspaceId); + if ( + !resolvedTopicId || + !resolvedWorkspaceId || + isAuxiliaryAgentSessionId(resolvedTopicId) || + sessionIdRef.current === resolvedTopicId + ) { + return false; + } + + const selectedTopic = topics.find( + (topic) => topic.id === resolvedTopicId, + ); + const signature = buildSessionDetailPrefetchSignature( + resolvedTopicId, + selectedTopic, + ); + const prefetchKey = buildSessionDetailPrefetchKey( + resolvedWorkspaceId, + resolvedTopicId, + ); + const existingPrefetch = sessionDetailPrefetchRegistry.get(prefetchKey); + if (existingPrefetch?.signature === signature) { + return existingPrefetch.promise.then( + () => true, + () => false, + ); + } + + const cachedSnapshot = loadAgentSessionCachedSnapshot( + resolvedWorkspaceId, + resolvedTopicId, + { + topicUpdatedAt: selectedTopic?.updatedAt ?? null, + messagesCount: selectedTopic?.messagesCount ?? null, + }, + ); + if (cachedSnapshot?.cacheMetadata?.freshness === "fresh") { + return false; + } + + const prefetchStartedAt = Date.now(); + const prefetchMetricContext = { + cacheFreshness: cachedSnapshot?.cacheMetadata?.freshness ?? null, + sessionId: resolvedTopicId, + workspaceId: resolvedWorkspaceId, + }; + recordAgentUiPerformanceMetric( + "session.prefetch.start", + prefetchMetricContext, + ); + logAgentDebug( + "useAgentSession", + "sessionPrefetch.start", + prefetchMetricContext, + ); + + const promise = runtime + .getSession(resolvedTopicId, { + historyLimit: SESSION_DETAIL_PREFETCH_HISTORY_LIMIT, + }) + .then((detail) => { + const detailWorkspaceId = normalizeProjectId(detail.workspace_id); + if ( + detailWorkspaceId && + resolvedWorkspaceId && + detailWorkspaceId !== resolvedWorkspaceId + ) { + logAgentDebug("useAgentSession", "sessionPrefetch.skipped", { + detailWorkspaceId, + reason: "workspace_mismatch", + sessionId: resolvedTopicId, + workspaceId: resolvedWorkspaceId, + }); + return detail; + } + + const messages = hydrateSessionDetailMessages( + detail, + resolvedTopicId, + { + compactCompletedHistory: + shouldCompactCompletedSessionHistory(detail), + }, + ); + const threadTurns = detail.turns || []; + const threadItems = filterConversationThreadItems( + normalizeLegacyThreadItems(detail.items || []), + ); + saveAgentSessionCachedSnapshot( + resolvedWorkspaceId, + resolvedTopicId, + { + messages, + threadTurns, + threadItems, + currentTurnId: null, + }, + { + sessionUpdatedAt: detail.updated_at * 1000, + messagesCount: detail.messages_count ?? messages.length, + historyTruncated: + detail.history_truncated === true || + (typeof detail.messages_count === "number" && + detail.messages_count > messages.length), + }, + ); + if (detailWorkspaceId) { + savePersistedSessionWorkspaceId(resolvedTopicId, detailWorkspaceId); + } + const prefetchSuccessMetricContext = { + durationMs: Date.now() - prefetchStartedAt, + itemsCount: threadItems.length, + messagesCount: messages.length, + sessionId: resolvedTopicId, + turnsCount: threadTurns.length, + workspaceId: resolvedWorkspaceId, + }; + recordAgentUiPerformanceMetric( + "session.prefetch.success", + prefetchSuccessMetricContext, + ); + logAgentDebug( + "useAgentSession", + "sessionPrefetch.success", + prefetchSuccessMetricContext, + ); + return detail; + }) + .catch((error) => { + recordAgentUiPerformanceMetric("session.prefetch.error", { + durationMs: Date.now() - prefetchStartedAt, + sessionId: resolvedTopicId, + workspaceId: resolvedWorkspaceId, + }); + logAgentDebug( + "useAgentSession", + "sessionPrefetch.error", + { + error, + sessionId: resolvedTopicId, + workspaceId: resolvedWorkspaceId, + }, + { level: "warn", throttleMs: 1000 }, + ); + throw error; + }) + .finally(() => { + const current = sessionDetailPrefetchRegistry.get(prefetchKey); + if (current?.promise === promise) { + sessionDetailPrefetchRegistry.delete(prefetchKey); + } + }); + + sessionDetailPrefetchRegistry.set(prefetchKey, { + signature, + promise, + }); + + return promise.then( + () => true, + () => false, + ); + }, + [runtime, sessionIdRef, topics, workspaceId], + ); + + useEffect(() => { + if ( + import.meta.env?.MODE === "test" || + import.meta.env?.VITEST || + !topicsReady || + disableSessionRestore || + !workspaceId?.trim() + ) { + return; + } + + const candidates = topics + .filter( + (topic) => + topic.id !== sessionIdRef.current && + !isAuxiliaryAgentSessionId(topic.id), + ) + .slice(0, SESSION_DETAIL_PREFETCH_RECENT_LIMIT); + if (candidates.length === 0) { + return; + } + + return scheduleMinimumDelayIdleTask( + () => { + void (async () => { + for (const topic of candidates) { + if (sessionIdRef.current === topic.id) { + continue; + } + await prefetchTopic(topic.id); + } + })(); + }, + { + minimumDelayMs: SESSION_DETAIL_PREFETCH_DELAY_MS, + idleTimeoutMs: SESSION_DETAIL_PREFETCH_IDLE_TIMEOUT_MS, + }, + ); + }, [ + disableSessionRestore, + prefetchTopic, + sessionIdRef, + topics, + topicsReady, + workspaceId, + ]); + const loadRuntimeSessionDetail = useCallback( async (params: { topicId: string; @@ -1366,35 +1632,112 @@ export function useAgentSession(options: UseAgentSessionOptions) { resumeSessionStartHooks = false, } = params; const requestStartedAt = Date.now(); - logAgentDebug("useAgentSession", "switchTopic.fetchDetail.start", { + const fetchDetailStartMetricContext = { elapsedBeforeRequestMs: requestStartedAt - startedAt, mode, resumeSessionStartHooks, + sessionId: topicId, topicId, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.fetchDetail.start", + fetchDetailStartMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.fetchDetail.start", + fetchDetailStartMetricContext, + ); try { + const prefetchedDetail = + !resumeSessionStartHooks && + sessionDetailPrefetchRegistry.get( + buildSessionDetailPrefetchKey( + normalizeProjectId(workspaceId) || "", + topicId, + ), + ); + if (prefetchedDetail) { + try { + const detail = await prefetchedDetail.promise; + const fetchPrefetchMetricContext = { + itemsCount: detail.items?.length ?? 0, + messagesCount: detail.messages.length, + mode, + queuedTurnsCount: detail.queued_turns?.length ?? 0, + requestDurationMs: Date.now() - requestStartedAt, + sessionId: topicId, + topicId, + totalElapsedMs: Date.now() - startedAt, + turnsCount: detail.turns?.length ?? 0, + workspaceId, + }; + recordAgentUiPerformanceMetric( + "session.switch.fetchDetail.prefetch", + fetchPrefetchMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.fetchDetail.prefetch", + fetchPrefetchMetricContext, + ); + return detail; + } catch (prefetchError) { + logAgentDebug( + "useAgentSession", + "switchTopic.fetchDetail.prefetchFallback", + { + error: prefetchError, + mode, + topicId, + workspaceId, + }, + { level: "warn", throttleMs: 1000 }, + ); + } + } + const detail = resumeSessionStartHooks ? await runtime.getSession(topicId, { resumeSessionStartHooks: true, historyLimit: 40, }) : await runtime.getSession(topicId, { historyLimit: 40 }); - logAgentDebug("useAgentSession", "switchTopic.fetchDetail.success", { + const fetchSuccessMetricContext = { itemsCount: detail.items?.length ?? 0, messagesCount: detail.messages.length, mode, queuedTurnsCount: detail.queued_turns?.length ?? 0, requestDurationMs: Date.now() - requestStartedAt, resumeSessionStartHooks, + sessionId: topicId, topicId, totalElapsedMs: Date.now() - startedAt, turnsCount: detail.turns?.length ?? 0, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.fetchDetail.success", + fetchSuccessMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.fetchDetail.success", + fetchSuccessMetricContext, + ); return detail; } catch (error) { + recordAgentUiPerformanceMetric("session.switch.fetchDetail.error", { + mode, + requestDurationMs: Date.now() - requestStartedAt, + resumeSessionStartHooks, + sessionId: topicId, + topicId, + totalElapsedMs: Date.now() - startedAt, + workspaceId, + }); logAgentDebug( "useAgentSession", "switchTopic.fetchDetail.error", @@ -1566,7 +1909,12 @@ export function useAgentSession(options: UseAgentSessionOptions) { sessionMetadataPatch.accessMode = workspaceDefaultAccessMode; } - logAgentDebug("useAgentSession", "switchTopic.success", { + const switchSuccessMetricContext = { + accessModeSource: runtimeAccessMode + ? "execution_runtime" + : shadowAccessMode + ? "session_storage" + : "workspace_default", durationMs: Date.now() - startedAt, executionStrategySource: runtimeExecutionStrategy ? "session_detail" @@ -1582,16 +1930,21 @@ export function useAgentSession(options: UseAgentSessionOptions) { : topicPreference ? "session_storage" : null, - accessModeSource: runtimeAccessMode - ? "execution_runtime" - : shadowAccessMode - ? "session_storage" - : "workspace_default", queuedTurnsCount: detail.queued_turns?.length ?? 0, + sessionId: topicId, topicId, turnsCount: detail.turns?.length ?? 0, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.success", + switchSuccessMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.success", + switchSuccessMetricContext, + ); const persistedWorkspaceId = runtimeWorkspaceId || resolvedWorkspaceId; if (persistedWorkspaceId) { @@ -1841,6 +2194,31 @@ export function useAgentSession(options: UseAgentSessionOptions) { return; } + const canReuseActiveSwitch = + !options?.forceRefresh && + !options?.resumeSessionStartHooks && + !options?.allowDetachedSession && + options?.restoreSource !== "auto"; + const activeSwitch = activeSessionSwitchRef.current; + if (canReuseActiveSwitch && activeSwitch?.topicId === topicId) { + logAgentDebug("useAgentSession", "switchTopic.reuseInFlight", { + topicId, + workspaceId, + }); + return activeSwitch.promise; + } + + let resolveActiveSwitch: () => void = () => {}; + const activeSwitchPromise = new Promise((resolve) => { + resolveActiveSwitch = resolve; + }); + if (canReuseActiveSwitch) { + activeSessionSwitchRef.current = { + topicId, + promise: activeSwitchPromise, + }; + } + const currentSessionId = sessionIdRef.current; if (currentSessionId) { persistSessionModelPreference( @@ -1876,7 +2254,7 @@ export function useAgentSession(options: UseAgentSessionOptions) { cachedSnapshotMetadata?.freshness === "stale" || selectedTopic?.status === "running" || selectedTopic?.status === "waiting"; - logAgentDebug("useAgentSession", "switchTopic.start", { + const switchStartMetricContext = { cacheFreshness: cachedSnapshotMetadata?.freshness ?? null, cacheStorageKind: cachedSnapshotMetadata?.storageKind ?? null, cachedLocalMessagesCount: cachedTargetSnapshot?.messages.length ?? 0, @@ -1884,9 +2262,19 @@ export function useAgentSession(options: UseAgentSessionOptions) { messagesCount: messages.length, refreshCachedSnapshotImmediately: shouldRefreshCachedSnapshotImmediately, + sessionId: topicId, topicId, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.start", + switchStartMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.start", + switchStartMetricContext, + ); if (currentSessionId !== topicId) { applyCachedTopicSnapshot(topicId, cachedTargetSnapshot); } @@ -1903,16 +2291,26 @@ export function useAgentSession(options: UseAgentSessionOptions) { persistSessionRestoreCandidate(topicId); setIsAutoRestoringSession(false); setIsSessionHydrating(false); - logAgentDebug("useAgentSession", "switchTopic.deferHydration", { + const deferHydrationMetricContext = { cacheFreshness: cachedSnapshotMetadata?.freshness ?? null, cacheStorageKind: cachedSnapshotMetadata?.storageKind ?? null, cachedLocalMessagesCount: cachedTargetSnapshot?.messages.length ?? 0, currentSessionId, refreshImmediately: shouldRefreshCachedSnapshotImmediately, + sessionId: topicId, topicId, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.deferHydration", + deferHydrationMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.deferHydration", + deferHydrationMetricContext, + ); const hydrateCachedTopic = () => { deferredSessionHydrationCancelRef.current = null; void (async () => { @@ -1964,7 +2362,10 @@ export function useAgentSession(options: UseAgentSessionOptions) { } else { deferredSessionHydrationCancelRef.current = scheduleMinimumDelayIdleTask(hydrateCachedTopic, { - minimumDelayMs: 0, + minimumDelayMs: + cachedSnapshotMetadata?.freshness === "fresh" + ? SESSION_DETAIL_DEFERRED_HYDRATION_DELAY_MS + : 0, idleTimeoutMs: 1_500, }); } @@ -1986,11 +2387,21 @@ export function useAgentSession(options: UseAgentSessionOptions) { applyCachedTopicChromeState(topicId); persistSessionRestoreCandidate(topicId); setIsSessionHydrating(true); - logAgentDebug("useAgentSession", "switchTopic.pendingShellApplied", { + const pendingShellMetricContext = { currentSessionId, + sessionId: topicId, topicId, workspaceId, - }); + }; + recordAgentUiPerformanceMetric( + "session.switch.pendingShellApplied", + pendingShellMetricContext, + ); + logAgentDebug( + "useAgentSession", + "switchTopic.pendingShellApplied", + pendingShellMetricContext, + ); } const detail = await loadRuntimeSessionDetail({ @@ -2017,12 +2428,18 @@ export function useAgentSession(options: UseAgentSessionOptions) { } : null, switchRequestVersion, + useTransition: currentSessionId !== topicId, }); } catch (error) { if (sessionSwitchRequestVersionRef.current !== switchRequestVersion) { return; } handleSwitchTopicError(error, topicId); + } finally { + if (activeSessionSwitchRef.current?.promise === activeSwitchPromise) { + activeSessionSwitchRef.current = null; + } + resolveActiveSwitch(); } }, [ @@ -2116,6 +2533,9 @@ export function useAgentSession(options: UseAgentSessionOptions) { const incomingMessages = hydrateSessionDetailMessages( detail, targetSessionId, + { + compactCompletedHistory: shouldCompactCompletedSessionHistory(detail), + }, ); const mergedMessages = mergeHydratedMessagesWithLocalState( messagesRef.current, @@ -2835,6 +3255,7 @@ export function useAgentSession(options: UseAgentSessionOptions) { createFreshSession, ensureSession, switchTopic, + prefetchTopic, loadFullSessionHistory, deleteTopic, renameTopic, diff --git a/src/components/agent/chat/hooks/useAsterAgentChat.test.tsx b/src/components/agent/chat/hooks/useAsterAgentChat.test.tsx index 4111a5841..6341ab54e 100644 --- a/src/components/agent/chat/hooks/useAsterAgentChat.test.tsx +++ b/src/components/agent/chat/hooks/useAsterAgentChat.test.tsx @@ -462,7 +462,7 @@ describe("useAsterAgentChat 首页新会话", () => { expect(mockListAgentRuntimeSessions).toHaveBeenCalledTimes(1); expect(mockListAgentRuntimeSessions).toHaveBeenNthCalledWith(1, { workspaceId, - limit: 60, + limit: 21, }); expect(harness.getValue().topics.map((topic) => topic.id)).toEqual([ sessionId, @@ -6346,7 +6346,7 @@ describe("useAsterAgentChat 偏好持久化", () => { expect(value.messages).toHaveLength(2); expect(value.messages[1]?.content).toBe("内容已保存到项目目录。"); expect(value.messages[1]?.thinkingContent).toBeUndefined(); - expect(value.messages[1]?.toolCalls?.[0]?.id).toBe("tool-topic-a-1"); + expect(value.messages[1]?.toolCalls).toBeUndefined(); expect(value.messages[1]?.contentParts).toEqual([ { type: "text", @@ -6484,6 +6484,172 @@ describe("useAsterAgentChat 偏好持久化", () => { } }); + it("切换命中预取中的旧会话时应复用同一个详情请求", async () => { + const workspaceId = "ws-topic-prefetch-reuse"; + const createdAt = Math.floor(Date.now() / 1000); + const deferredTopicDetail = createDeferred<{ + id: string; + created_at: number; + updated_at: number; + messages_count: number; + messages: Array<{ + role: "assistant" | "user"; + timestamp: number; + content: Array<{ type: "text"; text: string }>; + }>; + turns: []; + items: []; + queued_turns: []; + execution_strategy: "react"; + }>(); + + mockListAgentRuntimeSessions.mockResolvedValue([]); + mockGetAgentRuntimeSession.mockReturnValue(deferredTopicDetail.promise); + + const harness = mountHook(workspaceId); + + try { + await flushEffects(); + await flushEffects(); + + let prefetchPromise: Promise | null = null; + await act(async () => { + prefetchPromise = harness.getValue().prefetchTopic("topic-a"); + await Promise.resolve(); + }); + + expect(mockGetAgentRuntimeSession).toHaveBeenCalledTimes(1); + expect(mockGetAgentRuntimeSession).toHaveBeenCalledWith("topic-a", { + historyLimit: 40, + }); + + let switchPromise: Promise | null = null; + await act(async () => { + switchPromise = harness.getValue().switchTopic("topic-a"); + await Promise.resolve(); + }); + + expect(mockGetAgentRuntimeSession).toHaveBeenCalledTimes(1); + + await act(async () => { + deferredTopicDetail.resolve({ + id: "topic-a", + created_at: createdAt, + updated_at: createdAt, + messages_count: 1, + messages: [ + { + role: "assistant", + timestamp: createdAt, + content: [{ type: "text", text: "这是预取复用的结果。" }], + }, + ], + turns: [], + items: [], + queued_turns: [], + execution_strategy: "react", + }); + await prefetchPromise; + await switchPromise; + }); + await flushEffects(); + + expect(harness.getValue().messages[0]?.content).toBe( + "这是预取复用的结果。", + ); + const snapshotMap = JSON.parse( + sessionStorage.getItem(`aster_session_snapshots_${workspaceId}`) || + "{}", + ); + expect(snapshotMap["topic-a"]?.messages?.[0]?.content).toBe( + "这是预取复用的结果。", + ); + } finally { + harness.unmount(); + } + }); + + it("预取中的旧会话跨工作区组件重挂载后仍应复用同一个详情请求", async () => { + const workspaceId = "ws-topic-prefetch-remount"; + const createdAt = Math.floor(Date.now() / 1000); + const deferredTopicDetail = createDeferred<{ + id: string; + created_at: number; + updated_at: number; + messages_count: number; + messages: Array<{ + role: "assistant" | "user"; + timestamp: number; + content: Array<{ type: "text"; text: string }>; + }>; + turns: []; + items: []; + queued_turns: []; + execution_strategy: "react"; + }>(); + + mockListAgentRuntimeSessions.mockResolvedValue([]); + mockGetAgentRuntimeSession.mockReturnValue(deferredTopicDetail.promise); + + const prefetchHarness = mountHook(workspaceId); + let switchHarness: ReturnType | null = null; + + try { + await flushEffects(); + await flushEffects(); + + let prefetchPromise: Promise | null = null; + await act(async () => { + prefetchPromise = prefetchHarness.getValue().prefetchTopic("topic-a"); + await Promise.resolve(); + }); + expect(mockGetAgentRuntimeSession).toHaveBeenCalledTimes(1); + + prefetchHarness.unmount(); + switchHarness = mountHook(workspaceId); + await flushEffects(); + + let switchPromise: Promise | null = null; + await act(async () => { + switchPromise = + switchHarness?.getValue().switchTopic("topic-a") ?? null; + await Promise.resolve(); + }); + + expect(mockGetAgentRuntimeSession).toHaveBeenCalledTimes(1); + + await act(async () => { + deferredTopicDetail.resolve({ + id: "topic-a", + created_at: createdAt, + updated_at: createdAt, + messages_count: 1, + messages: [ + { + role: "assistant", + timestamp: createdAt, + content: [{ type: "text", text: "这是跨重挂载复用的结果。" }], + }, + ], + turns: [], + items: [], + queued_turns: [], + execution_strategy: "react", + }); + await prefetchPromise; + await switchPromise; + }); + await flushEffects(); + + expect(switchHarness.getValue().messages[0]?.content).toBe( + "这是跨重挂载复用的结果。", + ); + } finally { + switchHarness?.unmount(); + prefetchHarness.unmount(); + } + }); + it("切换到 stale 快照话题时应先回放缓存，并立即后台刷新", async () => { const workspaceId = "ws-topic-history-stale-refresh"; const nowMs = Date.now(); diff --git a/src/components/agent/chat/hooks/useAsterAgentChat.ts b/src/components/agent/chat/hooks/useAsterAgentChat.ts index d5a5f8924..d38277876 100644 --- a/src/components/agent/chat/hooks/useAsterAgentChat.ts +++ b/src/components/agent/chat/hooks/useAsterAgentChat.ts @@ -302,7 +302,9 @@ export function useAsterAgentChat(options: UseAsterAgentChatRuntimeOptions) { useAgentTopicSnapshot({ sessionId: session.sessionId, hasActiveTopic, - suppressInactiveTopicWarning: session.isDetachedActiveSession === true, + suppressInactiveTopicWarning: + session.isDetachedActiveSession === true || + session.isSessionHydrating === true, messages: session.messages, isSending: stream.isSending, pendingActionCount: tools.pendingActions.length, @@ -411,7 +413,7 @@ export function useAsterAgentChat(options: UseAsterAgentChatRuntimeOptions) { ); titleApplied = true; } catch (error) { - console.warn("[AsterChat] 自动生成会话标题失败:", error); + console.debug("[AsterChat] 自动生成会话标题失败:", error); } finally { if (!cancelled && titleApplied) { autoTitleCompletedSessionIdsRef.current.add(activeSessionId); @@ -633,6 +635,7 @@ export function useAsterAgentChat(options: UseAsterAgentChatRuntimeOptions) { createFreshSession: session.createFreshSession, ensureSession: session.ensureSession, switchTopic: session.switchTopic, + prefetchTopic: session.prefetchTopic, loadFullSessionHistory: session.loadFullSessionHistory, deleteTopic: session.deleteTopic, renameTopic: session.renameTopic, diff --git a/src/components/agent/chat/index.test.tsx b/src/components/agent/chat/index.test.tsx index 8e7f9642c..27628d1b9 100644 --- a/src/components/agent/chat/index.test.tsx +++ b/src/components/agent/chat/index.test.tsx @@ -1858,9 +1858,10 @@ describe("AgentChatPage 任务中心初始会话标签", () => { }, ], }); - state.switchTopic = vi.fn(async (topicId: string) => { + const switchTopic = vi.fn(async (topicId: string) => { state.sessionId = topicId; }); + state.switchTopic = switchTopic; installMockAgentChatUnifiedState(state); const mounted = mountPage({ @@ -1877,6 +1878,9 @@ describe("AgentChatPage 任务中心初始会话标签", () => { source: "sidebar", }), ).toBe(true); + await flushEffects(); + expect(switchTopic).toHaveBeenCalledWith("topic-next"); + mounted.rerender({ initialSessionId: "topic-next", }); @@ -1884,6 +1888,7 @@ describe("AgentChatPage 任务中心初始会话标签", () => { mounted.rerender(); await flushEffects(); + expect(switchTopic).toHaveBeenCalledTimes(1); expect( mounted.container .querySelector('[data-testid="task-center-tab-topic-next"]') @@ -1901,6 +1906,49 @@ describe("AgentChatPage 任务中心初始会话标签", () => { ).toEqual(["topic-next", "topic-current"]); }); + it("new-task 首页收到外层侧栏打开历史会话时应立即切换会话", async () => { + const state: Record = createMockAgentChatUnifiedState({ + sessionId: null, + topics: [ + { + id: "topic-next", + title: "旧会话 B", + updatedAt: new Date(FIXED_TOPIC_UPDATED_AT), + workspaceId: "workspace-test", + }, + ], + }); + const switchTopic = vi.fn(async (topicId: string) => { + state.sessionId = topicId; + }); + state.switchTopic = switchTopic; + installMockAgentChatUnifiedState(state); + + const mounted = mountPage({ + agentEntry: "new-task", + projectId: "workspace-test", + }); + await flushEffects(); + + expect( + notifyTaskCenterTaskOpen({ + sessionId: "topic-next", + workspaceId: "workspace-test", + source: "sidebar", + }), + ).toBe(true); + await flushEffects(); + mounted.rerender(); + await flushEffects(); + + expect(switchTopic).toHaveBeenCalledWith("topic-next"); + expect( + mounted.container + .querySelector('[data-testid="task-center-tab-topic-next"]') + ?.getAttribute("data-active"), + ).toBe("true"); + }); + it("已在任务中心内切到归档导航会话时，不应继续显示旧的普通任务标签", async () => { localStorage.setItem( TASK_CENTER_OPEN_TAB_IDS_STORAGE_KEY, diff --git a/src/components/agent/chat/taskCenterDraftTaskEvents.ts b/src/components/agent/chat/taskCenterDraftTaskEvents.ts index b7ca55de8..740dceac4 100644 --- a/src/components/agent/chat/taskCenterDraftTaskEvents.ts +++ b/src/components/agent/chat/taskCenterDraftTaskEvents.ts @@ -1,6 +1,7 @@ export const TASK_CENTER_CREATE_DRAFT_TASK_EVENT = "lime:task-center:create-draft-task"; export const TASK_CENTER_OPEN_TASK_EVENT = "lime:task-center:open-task"; +export const TASK_CENTER_PREFETCH_TASK_EVENT = "lime:task-center:prefetch-task"; export interface TaskCenterCreateDraftTaskDetail { source?: "sidebar" | "conversation_shelf" | "tab_strip"; @@ -12,6 +13,8 @@ export interface TaskCenterOpenTaskDetail { source?: "sidebar" | "conversation_shelf" | "tab_strip"; } +export type TaskCenterPrefetchTaskDetail = TaskCenterOpenTaskDetail; + export function requestTaskCenterDraftTask( detail: TaskCenterCreateDraftTaskDetail = {}, ): boolean { @@ -60,16 +63,18 @@ export function notifyTaskCenterTaskOpen( return false; } - window.dispatchEvent( - new CustomEvent(TASK_CENTER_OPEN_TASK_EVENT, { + const event = new CustomEvent( + TASK_CENTER_OPEN_TASK_EVENT, + { + cancelable: true, detail: { ...detail, sessionId: normalizedSessionId, workspaceId: detail.workspaceId?.trim() || null, }, - }), + }, ); - return true; + return !window.dispatchEvent(event); } export function subscribeTaskCenterTaskOpenRequests( @@ -94,6 +99,7 @@ export function subscribeTaskCenterTaskOpenRequests( sessionId, workspaceId: detail?.workspaceId?.trim() || null, }); + event.preventDefault(); }; window.addEventListener(TASK_CENTER_OPEN_TASK_EVENT, listener); @@ -101,3 +107,60 @@ export function subscribeTaskCenterTaskOpenRequests( window.removeEventListener(TASK_CENTER_OPEN_TASK_EVENT, listener); }; } + +export function notifyTaskCenterTaskPrefetch( + detail: TaskCenterPrefetchTaskDetail, +): boolean { + if (typeof window === "undefined") { + return false; + } + + const normalizedSessionId = detail.sessionId.trim(); + if (!normalizedSessionId) { + return false; + } + + window.dispatchEvent( + new CustomEvent( + TASK_CENTER_PREFETCH_TASK_EVENT, + { + detail: { + ...detail, + sessionId: normalizedSessionId, + workspaceId: detail.workspaceId?.trim() || null, + }, + }, + ), + ); + return true; +} + +export function subscribeTaskCenterTaskPrefetchRequests( + handler: (detail: TaskCenterPrefetchTaskDetail) => void, +): () => void { + if (typeof window === "undefined") { + return () => undefined; + } + + const listener = (event: Event) => { + const detail = + event instanceof CustomEvent + ? (event.detail as TaskCenterPrefetchTaskDetail | undefined) + : undefined; + const sessionId = detail?.sessionId?.trim(); + if (!sessionId) { + return; + } + + handler({ + ...detail, + sessionId, + workspaceId: detail?.workspaceId?.trim() || null, + }); + }; + + window.addEventListener(TASK_CENTER_PREFETCH_TASK_EVENT, listener); + return () => { + window.removeEventListener(TASK_CENTER_PREFETCH_TASK_EVENT, listener); + }; +} diff --git a/src/components/agent/chat/types.ts b/src/components/agent/chat/types.ts index 8f5374de8..00f144508 100644 --- a/src/components/agent/chat/types.ts +++ b/src/components/agent/chat/types.ts @@ -104,8 +104,18 @@ export interface MessageTaskPreviewImageCandidate { name?: string; } +export interface MessageTranscriptSegment { + id: string; + index: number; + startMs?: number | null; + endMs?: number | null; + speaker?: string | null; + text: string; +} + export interface MessageGenericTaskPreview { kind: + | "audio_generate" | "broadcast_generate" | "modal_resource_search" | "transcription_generate" @@ -126,6 +136,21 @@ export interface MessageGenericTaskPreview { retryable?: boolean; metaItems?: string[]; imageCandidates?: MessageTaskPreviewImageCandidate[]; + taskFilePath?: string | null; + audioUrl?: string | null; + mimeType?: string | null; + durationMs?: number | null; + sourceText?: string | null; + voice?: string | null; + transcriptPath?: string | null; + sourcePath?: string | null; + sourceUrl?: string | null; + language?: string | null; + outputFormat?: string | null; + transcriptText?: string | null; + transcriptSegments?: MessageTranscriptSegment[]; + errorCode?: string | null; + errorMessage?: string | null; } export type MessageTaskPreview = diff --git a/src/components/agent/chat/utils/agentTaskRuntime.ts b/src/components/agent/chat/utils/agentTaskRuntime.ts index 5966d70ba..f14d783d3 100644 --- a/src/components/agent/chat/utils/agentTaskRuntime.ts +++ b/src/components/agent/chat/utils/agentTaskRuntime.ts @@ -113,32 +113,37 @@ function resolveLatestTurn( } function resolveLatestAssistantMessage(messages: Message[]): Message | null { - return ( - [...messages].reverse().find((message) => message.role === "assistant") || - null - ); + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (message?.role === "assistant") { + return message; + } + } + return null; } function resolveLatestUserMessage(messages: Message[]): Message | null { - return ( - [...messages] - .reverse() - .find( - (message) => - message.role === "user" && - (message.content.trim().length > 0 || - (Array.isArray(message.images) && message.images.length > 0)), - ) || null - ); + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if ( + message?.role === "user" && + (message.content.trim().length > 0 || + (Array.isArray(message.images) && message.images.length > 0)) + ) { + return message; + } + } + return null; } function resolveLatestUsage(messages: Message[]): AgentTokenUsage | undefined { - return [...messages] - .reverse() - .find( - (message) => - message.role === "assistant" && !message.isThinking && message.usage, - )?.usage; + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (message?.role === "assistant" && !message.isThinking && message.usage) { + return message.usage; + } + } + return undefined; } function resolveTaskStatus(params: { @@ -266,14 +271,18 @@ function resolveCompletedSummary( latestAssistant: Message | null, latestTurnItems: AgentThreadItem[], ): string | null { - const latestTurnSummary = [...latestTurnItems] - .reverse() - .find( - (item) => - item.type === "turn_summary" && - item.status === "completed" && - !isInternalRoutingTurnSummaryText(item.text), - ); + let latestTurnSummary: AgentThreadItem | null = null; + for (let index = latestTurnItems.length - 1; index >= 0; index -= 1) { + const item = latestTurnItems[index]; + if ( + item?.type === "turn_summary" && + item.status === "completed" && + !isInternalRoutingTurnSummaryText(item.text) + ) { + latestTurnSummary = item; + break; + } + } if (latestTurnSummary?.type === "turn_summary") { return shorten(firstMeaningfulLine(latestTurnSummary.text), 96) || null; } @@ -322,10 +331,19 @@ export function buildAgentTaskRuntimeCardModel({ const latestTurn = resolveLatestTurn(turns, currentTurnId); const latestAssistant = resolveLatestAssistantMessage(messages); const latestUser = resolveLatestUserMessage(messages); - const latestTurnItems = latestTurn - ? threadItems.filter((item) => item.turn_id === latestTurn.id) - : []; - const latestProcessItems = latestTurnItems.filter(isProcessItem); + const latestTurnItems: AgentThreadItem[] = []; + const latestProcessItems: AgentThreadItem[] = []; + if (latestTurn) { + for (const item of threadItems) { + if (item.turn_id !== latestTurn.id) { + continue; + } + latestTurnItems.push(item); + if (isProcessItem(item)) { + latestProcessItems.push(item); + } + } + } const visibleToolCalls = latestAssistant?.toolCalls?.filter( (toolCall) => toolCall.status !== "failed", @@ -364,12 +382,17 @@ export function buildAgentTaskRuntimeCardModel({ (childSubagentSessions.length > 0 ? "正在协调子任务" : "当前任务"); const title = shorten(firstMeaningfulLine(titleSource), 120) || "当前任务"; const subtaskStats = resolveSubtaskStats(childSubagentSessions); - const latestPreviewItem = [...latestProcessItems] - .reverse() - .find((item) => Boolean(resolveAgentThreadToolProcessPreview(item))); - const latestPreview = - (latestPreviewItem && - resolveAgentThreadToolProcessPreview(latestPreviewItem)) || + let latestPreview: string | null = null; + for (let index = latestProcessItems.length - 1; index >= 0; index -= 1) { + latestPreview = resolveAgentThreadToolProcessPreview( + latestProcessItems[index]!, + ); + if (latestPreview) { + break; + } + } + latestPreview = + latestPreview || shorten(firstMeaningfulLine(latestAssistant?.runtimeStatus?.detail), 96) || null; diff --git a/src/components/agent/chat/utils/chatLayoutVisibility.test.ts b/src/components/agent/chat/utils/chatLayoutVisibility.test.ts index ed6f2a51c..f7e90ab79 100644 --- a/src/components/agent/chat/utils/chatLayoutVisibility.test.ts +++ b/src/components/agent/chat/utils/chatLayoutVisibility.test.ts @@ -17,6 +17,22 @@ describe("chatLayoutVisibility", () => { ).toBe(true); }); + it("旧会话恢复 pending shell 阶段也应进入会话布局", () => { + expect( + shouldShowChatLayout({ + agentEntry: "new-task", + hasDisplayMessages: false, + hasPendingA2UIForm: false, + isThemeWorkbench: false, + hasUnconsumedInitialDispatch: false, + isPreparingSend: false, + isSending: false, + isSessionHydrating: true, + queuedTurnCount: 0, + }), + ).toBe(true); + }); + it("空白首页在无会话活动时仍应保留空态布局", () => { expect( shouldShowChatLayout({ diff --git a/src/components/agent/chat/utils/chatLayoutVisibility.ts b/src/components/agent/chat/utils/chatLayoutVisibility.ts index c3834f7f0..455916b03 100644 --- a/src/components/agent/chat/utils/chatLayoutVisibility.ts +++ b/src/components/agent/chat/utils/chatLayoutVisibility.ts @@ -7,6 +7,7 @@ interface ResolveChatLayoutVisibilityParams { hasUnconsumedInitialDispatch: boolean; isPreparingSend: boolean; isSending: boolean; + isSessionHydrating?: boolean; queuedTurnCount: number; } @@ -19,6 +20,7 @@ export function shouldShowChatLayout({ hasUnconsumedInitialDispatch, isPreparingSend, isSending, + isSessionHydrating = false, queuedTurnCount, }: ResolveChatLayoutVisibilityParams): boolean { return ( @@ -29,6 +31,7 @@ export function shouldShowChatLayout({ hasUnconsumedInitialDispatch || isPreparingSend || isSending || + isSessionHydrating || queuedTurnCount > 0 ); } diff --git a/src/components/agent/chat/utils/inputbarRuntimeStatusLine.ts b/src/components/agent/chat/utils/inputbarRuntimeStatusLine.ts index 0754b1aa1..7f8872de2 100644 --- a/src/components/agent/chat/utils/inputbarRuntimeStatusLine.ts +++ b/src/components/agent/chat/utils/inputbarRuntimeStatusLine.ts @@ -60,10 +60,13 @@ function resolveLatestTurn( } function resolveLatestAssistantMessage(messages: Message[]): Message | null { - return ( - [...messages].reverse().find((message) => message.role === "assistant") || - null - ); + for (let index = messages.length - 1; index >= 0; index -= 1) { + const message = messages[index]; + if (message?.role === "assistant") { + return message; + } + } + return null; } function resolveFallbackStatus(params: { @@ -169,11 +172,13 @@ function resolveLatestTurnBatchDescriptor( return null; } - return summarizeThreadProcessBatch( - threadItems.filter( - (item) => item.turn_id === latestTurn.id && isProcessThreadItem(item), - ), - ); + const latestProcessItems: AgentThreadItem[] = []; + for (const item of threadItems) { + if (item.turn_id === latestTurn.id && isProcessThreadItem(item)) { + latestProcessItems.push(item); + } + } + return summarizeThreadProcessBatch(latestProcessItems); } function resolveVisibleUsage( diff --git a/src/components/agent/chat/utils/taskPreviewFromToolResult.test.ts b/src/components/agent/chat/utils/taskPreviewFromToolResult.test.ts index de2115401..d6a640894 100644 --- a/src/components/agent/chat/utils/taskPreviewFromToolResult.test.ts +++ b/src/components/agent/chat/utils/taskPreviewFromToolResult.test.ts @@ -2,6 +2,7 @@ import { describe, expect, it } from "vitest"; import { buildImageTaskPreviewFromToolResult, buildTaskPreviewFromToolResult, + buildToolResultArtifactFromToolResult, } from "./taskPreviewFromToolResult"; describe("buildImageTaskPreviewFromToolResult", () => { @@ -212,3 +213,235 @@ describe("buildTaskPreviewFromToolResult video", () => { }); }); }); + +describe("buildTaskPreviewFromToolResult audio", () => { + it("配音任务应输出 audio_generate 预览并指向可打开的运行时文档", () => { + const preview = buildTaskPreviewFromToolResult({ + toolId: "tool-audio-1", + toolName: "lime_create_audio_generation_task", + toolArguments: JSON.stringify({ + sourceText: "欢迎来到 Lime 多模态工作台。", + voice: "warm_female", + voiceStyle: "温暖克制", + targetLanguage: "zh-CN", + durationMs: 8200, + }), + toolResult: { + metadata: { + task_id: "task-audio-1", + task_type: "audio_generate", + status: "pending_submit", + prompt: "欢迎来到 Lime 多模态工作台。", + artifact_path: ".lime/tasks/audio_generate/task-audio-1.json", + provider_id: "voice-runtime", + model: "voice-pro", + }, + }, + fallbackPrompt: "@配音 欢迎来到 Lime 多模态工作台。", + }); + + expect(preview).toMatchObject({ + kind: "audio_generate", + taskId: "task-audio-1", + taskType: "audio_generate", + prompt: "欢迎来到 Lime 多模态工作台。", + status: "running", + phase: "queued", + artifactPath: ".lime/runtime/audio-generate/task-audio-1.md", + taskFilePath: ".lime/tasks/audio_generate/task-audio-1.json", + providerId: "voice-runtime", + model: "voice-pro", + voice: "warm_female", + durationMs: 8200, + metaItems: ["warm_female", "温暖克制", "zh-CN", "8 秒"], + statusMessage: + "配音任务已写入 audio_task/audio_output，工作区会继续同步音频结果。", + }); + }); + + describe("buildTaskPreviewFromToolResult transcription", () => { + it("转写任务应输出 transcription_generate 预览并指向运行时文档", () => { + const preview = buildTaskPreviewFromToolResult({ + toolId: "tool-transcription-1", + toolName: "lime_create_transcription_task", + toolArguments: JSON.stringify({ + prompt: "请转写访谈音频", + sourcePath: "materials/interview.wav", + language: "zh-CN", + outputFormat: "txt", + }), + toolResult: { + metadata: { + task_id: "task-transcription-1", + task_type: "transcription_generate", + status: "pending_submit", + prompt: "请转写访谈音频", + artifact_path: + ".lime/tasks/transcription_generate/task-transcription-1.json", + provider_id: "openai-asr", + model: "gpt-4o-transcribe", + }, + }, + fallbackPrompt: "@转写 materials/interview.wav", + }); + + expect(preview).toMatchObject({ + kind: "transcription_generate", + taskId: "task-transcription-1", + taskType: "transcription_generate", + prompt: "请转写访谈音频", + status: "running", + phase: "queued", + artifactPath: + ".lime/runtime/transcription-generate/task-transcription-1.md", + taskFilePath: + ".lime/tasks/transcription_generate/task-transcription-1.json", + sourcePath: "materials/interview.wav", + language: "zh-CN", + outputFormat: "txt", + providerId: "openai-asr", + model: "gpt-4o-transcribe", + metaItems: ["materials/interview.wav", "zh-CN", "txt"], + statusMessage: "转写任务已提交，工作区会继续同步最新进度。", + }); + }); + + it("转写任务工具结果应生成 transcript viewer 文档，避免打开隐藏 task json", () => { + const artifact = buildToolResultArtifactFromToolResult({ + toolId: "tool-transcription-2", + toolName: "lime_create_transcription_task", + toolArguments: JSON.stringify({ + prompt: "请转写访谈音频", + sourcePath: "materials/interview.wav", + language: "zh-CN", + outputFormat: "txt", + }), + toolResult: { + metadata: { + task_id: "task-transcription-2", + task_type: "transcription_generate", + status: "succeeded", + artifact_path: + ".lime/tasks/transcription_generate/task-transcription-2.json", + transcript_path: + ".lime/runtime/transcripts/task-transcription-2.txt", + transcript_text: "欢迎来到 Lime 访谈节目。", + transcript_segments: [ + { + start: 1, + end: 3.2, + speaker: "主持人", + text: "欢迎来到 Lime 访谈节目。", + }, + ], + }, + }, + fallbackPrompt: "@转写 materials/interview.wav", + }); + + expect(artifact).toMatchObject({ + filePath: + ".lime/runtime/transcription-generate/task-transcription-2.md", + metadata: { + artifact_type: "document", + taskId: "task-transcription-2", + taskType: "transcription_generate", + taskFilePath: + ".lime/tasks/transcription_generate/task-transcription-2.json", + transcriptPath: ".lime/runtime/transcripts/task-transcription-2.txt", + transcriptText: "欢迎来到 Lime 访谈节目。", + modalityContractKey: "audio_transcription", + }, + }); + expect(artifact?.metadata.artifactDocument).toMatchObject({ + artifactId: "transcription-generate:task-transcription-2", + title: "内容转写任务", + metadata: { + taskFilePath: + ".lime/tasks/transcription_generate/task-transcription-2.json", + transcriptPath: ".lime/runtime/transcripts/task-transcription-2.txt", + transcriptText: "欢迎来到 Lime 访谈节目。", + transcriptCorrectionEnabled: true, + transcriptCorrectionStatus: "available", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + transcriptSegments: [ + { + id: "segment-1", + index: 1, + startMs: 1000, + endMs: 3200, + speaker: "主持人", + text: "欢迎来到 Lime 访谈节目。", + }, + ], + }, + }); + expect(artifact?.metadata.artifactDocument).toMatchObject({ + blocks: expect.arrayContaining([ + expect.objectContaining({ + id: "transcript-segments", + type: "table", + title: "转写时间轴（可逐段编辑校对）", + rows: [["00:01 - 00:03", "主持人", "欢迎来到 Lime 访谈节目。"]], + }), + expect.objectContaining({ + id: "transcript-text", + type: "code_block", + title: "转写文本（可编辑校对）", + code: "欢迎来到 Lime 访谈节目。", + }), + expect.objectContaining({ + id: "transcript-output", + type: "callout", + title: "Transcript 已同步，可校对保存", + body: expect.stringContaining("不改写原始 ASR 输出"), + }), + ]), + }); + }); + }); + + it("配音任务工具结果应生成轻量 artifact document，避免打开隐藏 task json", () => { + const artifact = buildToolResultArtifactFromToolResult({ + toolId: "tool-audio-2", + toolName: "lime_create_audio_generation_task", + toolArguments: JSON.stringify({ + sourceText: "请用轻快语气播报新品发布。", + voice: "brand_voice", + audioPath: "https://cdn.example/audio/task-audio-2.mp3", + }), + toolResult: { + metadata: { + task_id: "task-audio-2", + task_type: "audio_generate", + status: "succeeded", + artifact_path: ".lime/tasks/audio_generate/task-audio-2.json", + mime_type: "audio/mpeg", + }, + }, + fallbackPrompt: "@配音 请用轻快语气播报新品发布。", + }); + + expect(artifact).toMatchObject({ + filePath: ".lime/runtime/audio-generate/task-audio-2.md", + metadata: { + artifact_type: "document", + taskId: "task-audio-2", + taskType: "audio_generate", + taskFilePath: ".lime/tasks/audio_generate/task-audio-2.json", + audioUrl: "https://cdn.example/audio/task-audio-2.mp3", + modalityContractKey: "voice_generation", + }, + }); + expect(artifact?.metadata.artifactDocument).toMatchObject({ + artifactId: "audio-generate:task-audio-2", + title: "配音生成任务", + metadata: { + taskFilePath: ".lime/tasks/audio_generate/task-audio-2.json", + audioUrl: "https://cdn.example/audio/task-audio-2.mp3", + }, + }); + }); +}); diff --git a/src/components/agent/chat/utils/taskPreviewFromToolResult.ts b/src/components/agent/chat/utils/taskPreviewFromToolResult.ts index 30ec6d6c5..d81491e4d 100644 --- a/src/components/agent/chat/utils/taskPreviewFromToolResult.ts +++ b/src/components/agent/chat/utils/taskPreviewFromToolResult.ts @@ -8,6 +8,13 @@ import type { MessageTaskPreviewImageCandidate, MessageVideoTaskPreview, } from "../types"; +import { + countTranscriptSpeakers, + extractTranscriptSegmentsFromRecords, + formatTranscriptSegmentRange, + normalizeTranscriptSegments, + parseTranscriptContent, +} from "./transcriptSegments"; interface ToolResultPreviewParams { toolId?: string; @@ -25,6 +32,9 @@ const GENERIC_TASK_KINDS = new Set([ "typesetting", ]); const WEB_IMAGE_SEARCH_TOOL_NAMES = new Set(["lime_search_web_images"]); +const AUDIO_TASK_PREVIEW_ARTIFACT_ROOT = ".lime/runtime/audio-generate"; +const TRANSCRIPTION_TASK_PREVIEW_ARTIFACT_ROOT = + ".lime/runtime/transcription-generate"; function asRecord(value: unknown): Record | null { if (!value || typeof value !== "object" || Array.isArray(value)) { @@ -369,6 +379,15 @@ function extractGenericTaskArguments(toolArguments: string | undefined): { targetPlatform?: string; sourcePath?: string; sourceUrl?: string; + language?: string; + outputFormat?: string; + sourceText?: string; + voice?: string; + voiceStyle?: string; + targetLanguage?: string; + audioPath?: string; + mimeType?: string; + durationMs?: number; } { if (!toolArguments) { return {}; @@ -392,6 +411,33 @@ function extractGenericTaskArguments(toolArguments: string | undefined): { ), sourcePath: readMetadataString([parsed], ["source_path", "sourcePath"]), sourceUrl: readMetadataString([parsed], ["source_url", "sourceUrl"]), + language: readMetadataString( + [parsed], + ["language", "target_language", "targetLanguage"], + ), + outputFormat: readMetadataString( + [parsed], + ["output_format", "outputFormat", "format"], + ), + sourceText: readMetadataString( + [parsed], + ["source_text", "sourceText", "text"], + ), + voice: readMetadataString([parsed], ["voice"]), + voiceStyle: readMetadataString([parsed], ["voice_style", "voiceStyle"]), + targetLanguage: readMetadataString( + [parsed], + ["target_language", "targetLanguage"], + ), + audioPath: readMetadataString( + [parsed], + ["audio_path", "audioPath", "audio_url", "audioUrl"], + ), + mimeType: readMetadataString([parsed], ["mime_type", "mimeType"]), + durationMs: readMetadataPositiveNumber( + [parsed], + ["duration_ms", "durationMs"], + ), }; } catch { return {}; @@ -419,6 +465,36 @@ function buildWebImageSearchArtifactPath( return `.lime/runtime/resource-search/${identifier}.md`; } +function buildAudioTaskPreviewArtifactPath(taskId: string): string { + return `${AUDIO_TASK_PREVIEW_ARTIFACT_ROOT}/${buildPreviewId( + taskId, + "audio-task", + )}.md`; +} + +function buildTranscriptionTaskPreviewArtifactPath(taskId: string): string { + return `${TRANSCRIPTION_TASK_PREVIEW_ARTIFACT_ROOT}/${buildPreviewId( + taskId, + "transcription-task", + )}.md`; +} + +function formatDurationMsLabel(durationMs?: number): string | undefined { + if ( + typeof durationMs !== "number" || + !Number.isFinite(durationMs) || + durationMs <= 0 + ) { + return undefined; + } + if (durationMs < 60_000) { + return `${Math.max(1, Math.round(durationMs / 1000))} 秒`; + } + const minutes = Math.floor(durationMs / 60_000); + const seconds = Math.round((durationMs % 60_000) / 1000); + return seconds > 0 ? `${minutes} 分 ${seconds} 秒` : `${minutes} 分钟`; +} + function readWebImageSearchResult(params: ToolResultPreviewParams): { provider?: string; query?: string; @@ -610,10 +686,48 @@ function buildGenericTaskMetaItems( } } else if (kind === "transcription_generate") { push(taskArguments.sourcePath || taskArguments.sourceUrl); + push( + taskArguments.language || + taskArguments.targetLanguage || + readMetadataString(candidates, ["language", "target_language"]), + ); + push( + taskArguments.outputFormat || + readMetadataString(candidates, [ + "output_format", + "outputFormat", + "format", + ]), + ); } else if (kind === "url_parse") { push(taskArguments.sourceUrl || readMetadataString(candidates, ["url"])); } else if (kind === "typesetting") { push(taskArguments.targetPlatform); + } else if (kind === "audio_generate") { + push( + taskArguments.voice || + readMetadataString(candidates, [ + "voice", + "voice_preset", + "voicePreset", + ]), + ); + push( + taskArguments.voiceStyle || + readMetadataString(candidates, ["voice_style", "voiceStyle"]), + ); + push( + taskArguments.targetLanguage || + readMetadataString(candidates, ["target_language", "targetLanguage"]), + ); + push( + taskArguments.mimeType || + readMetadataString(candidates, ["mime_type", "mimeType"]), + ); + const durationMs = + taskArguments.durationMs || + readMetadataPositiveNumber(candidates, ["duration_ms", "durationMs"]); + push(formatDurationMsLabel(durationMs)); } return Array.from(items); @@ -625,6 +739,8 @@ function resolveGenericTaskStatusMessage( ): string { if (status === "complete" || status === "partial") { switch (kind) { + case "audio_generate": + return "音频结果已同步，打开查看即可继续预览与管理任务。"; case "broadcast_generate": return "播报整理结果已同步，打开查看即可继续审阅文稿。"; case "modal_resource_search": @@ -640,6 +756,8 @@ function resolveGenericTaskStatusMessage( if (status === "failed") { switch (kind) { + case "audio_generate": + return "配音生成失败，请调整文本、音色或模型后重试。"; case "broadcast_generate": return "播报整理失败，请调整输入内容后重试。"; case "modal_resource_search": @@ -658,6 +776,8 @@ function resolveGenericTaskStatusMessage( } switch (kind) { + case "audio_generate": + return "配音任务已写入 audio_task/audio_output，工作区会继续同步音频结果。"; case "broadcast_generate": return "播报整理任务已提交，工作区会继续同步最新进度。"; case "modal_resource_search": @@ -671,6 +791,417 @@ function resolveGenericTaskStatusMessage( } } +function buildAudioTaskPreviewFromToolResult( + params: ToolResultPreviewParams, +): MessageGenericTaskPreview | null { + const resultRecord = asRecord(params.toolResult); + const metadata = asRecord(resultRecord?.metadata); + const taskResult = asRecord(resultRecord?.result); + const taskId = readMetadataString( + [metadata, resultRecord, taskResult], + ["task_id", "taskId", "id"], + ); + const taskType = readMetadataString( + [metadata, resultRecord, taskResult], + ["task_type", "taskType"], + ); + if (!taskId || !taskType) { + return null; + } + + const normalizedTaskType = taskType.trim().toLowerCase(); + if ( + normalizedTaskType !== "audio_generate" && + normalizedTaskType !== "voice_generate" && + normalizedTaskType !== "voice" + ) { + return null; + } + + const parsedArguments = extractGenericTaskArguments(params.toolArguments); + const status = readMetadataString( + [metadata, resultRecord, taskResult], + ["status"], + ); + const previewStatus = resolveTaskPreviewStatus(status); + const candidates = [metadata, resultRecord, taskResult]; + const sourceText = + parsedArguments.sourceText || + readMetadataString(candidates, ["source_text", "sourceText", "prompt"]) || + params.fallbackPrompt.trim() || + "配音任务"; + const taskFilePath = + readMetadataString(candidates, ["artifact_path", "artifactPath"]) || + readMetadataString(candidates, ["path", "absolute_path", "absolutePath"]) || + null; + const audioUrl = + parsedArguments.audioPath || + readMetadataString(candidates, [ + "audio_path", + "audioPath", + "audio_url", + "audioUrl", + "url", + "result_url", + "resultUrl", + ]) || + null; + const durationMs = + parsedArguments.durationMs || + readMetadataPositiveNumber(candidates, ["duration_ms", "durationMs"]) || + null; + const mimeType = + parsedArguments.mimeType || + readMetadataString(candidates, ["mime_type", "mimeType"]) || + null; + const voice = + parsedArguments.voice || readMetadataString(candidates, ["voice"]) || null; + + return { + kind: "audio_generate", + taskId, + taskType: "audio_generate", + prompt: sourceText, + title: + parsedArguments.title || + readMetadataString(candidates, ["title"]) || + "配音生成任务", + status: previewStatus, + projectId: + readMetadataString(candidates, ["project_id", "projectId"]) || null, + contentId: + readMetadataString(candidates, ["content_id", "contentId"]) || null, + artifactPath: buildAudioTaskPreviewArtifactPath(taskId), + taskFilePath, + providerId: + readMetadataString(candidates, [ + "provider_id", + "providerId", + "provider", + ]) || null, + model: readMetadataString(candidates, ["model"]) || null, + phase: resolveTaskPreviewPhase(status), + statusMessage: resolveGenericTaskStatusMessage( + "audio_generate", + previewStatus, + ), + metaItems: buildGenericTaskMetaItems( + "audio_generate", + parsedArguments, + candidates, + ), + audioUrl, + mimeType, + durationMs, + sourceText, + voice, + }; +} + +export function buildAudioTaskArtifactDocument( + preview: MessageGenericTaskPreview, +) { + const taskFilePath = preview.taskFilePath?.trim(); + const audioUrl = preview.audioUrl?.trim(); + const errorCode = preview.errorCode?.trim(); + const errorMessage = preview.errorMessage?.trim(); + const highlights = [ + preview.status === "running" ? "状态：待执行" : `状态：${preview.status}`, + preview.voice?.trim() ? `音色：${preview.voice.trim()}` : null, + preview.model?.trim() ? `模型：${preview.model.trim()}` : null, + errorCode ? `错误码：${errorCode}` : null, + formatDurationMsLabel(preview.durationMs || undefined) + ? `时长：${formatDurationMsLabel(preview.durationMs || undefined)}` + : null, + ].filter((item): item is string => Boolean(item)); + const audioOutputTone = + preview.status === "failed" + ? ("danger" as const) + : audioUrl + ? ("success" as const) + : ("info" as const); + const audioOutputTitle = + preview.status === "failed" + ? "音频生成失败" + : audioUrl + ? "音频结果已同步" + : "等待音频执行器"; + const audioOutputBody = + preview.status === "failed" + ? [ + preview.statusMessage?.trim(), + errorCode ? `错误码：${errorCode}` : null, + errorMessage ? `原因：${errorMessage}` : null, + ] + .filter((item): item is string => Boolean(item)) + .join("\n") + : audioUrl + ? `音频路径：${audioUrl}` + : "当前步骤只创建标准任务产物，不生成真实音频、不伪造云端提交；后续执行器会回写 audio_output.audio_path。"; + + return { + schemaVersion: ARTIFACT_DOCUMENT_SCHEMA_VERSION, + artifactId: `audio-generate:${preview.taskId}`, + kind: "brief" as const, + title: preview.title?.trim() || "配音生成任务", + status: + preview.status === "failed" + ? ("failed" as const) + : preview.status === "complete" || preview.status === "partial" + ? ("ready" as const) + : ("streaming" as const), + language: "zh-CN", + summary: + preview.status === "running" + ? "配音任务已经写入标准 audio_task/audio_output 产物，等待执行器同步音频结果。" + : preview.statusMessage || "配音任务已经进入统一多模态运行合同主链。", + blocks: [ + { + id: "hero", + type: "hero_summary" as const, + eyebrow: "配音生成", + title: preview.prompt || "配音任务", + summary: + preview.statusMessage || + "配音任务已经写入标准 audio_task/audio_output 产物。", + highlights, + }, + { + id: "source-text", + type: "rich_text" as const, + contentFormat: "markdown" as const, + content: preview.sourceText || preview.prompt, + markdown: `### 待配音文本\n\n${preview.sourceText || preview.prompt}`, + text: preview.sourceText || preview.prompt, + }, + { + id: "audio-output", + type: "callout" as const, + tone: audioOutputTone, + title: audioOutputTitle, + body: audioOutputBody, + }, + ], + sources: taskFilePath + ? [ + { + id: "audio-task-file", + type: "file" as const, + label: "audio_generate task file", + locator: { + path: taskFilePath, + }, + reliability: "primary" as const, + }, + ] + : [], + metadata: { + generatedBy: "agent" as const, + rendererHints: { + density: "comfortable" as const, + }, + taskId: preview.taskId, + taskType: "audio_generate", + taskFilePath, + audioUrl: audioUrl || null, + mimeType: preview.mimeType || null, + durationMs: preview.durationMs || null, + voice: preview.voice || null, + errorCode: errorCode || null, + errorMessage: errorMessage || null, + modalityContractKey: "voice_generation", + }, + }; +} + +export function buildTranscriptionTaskArtifactDocument( + preview: MessageGenericTaskPreview, +) { + const taskFilePath = preview.taskFilePath?.trim(); + const transcriptPath = preview.transcriptPath?.trim(); + const transcriptText = + typeof preview.transcriptText === "string" && preview.transcriptText.trim() + ? preview.transcriptText + : null; + const sourcePath = preview.sourcePath?.trim(); + const sourceUrl = preview.sourceUrl?.trim(); + const errorCode = preview.errorCode?.trim(); + const errorMessage = preview.errorMessage?.trim(); + const transcriptSegments = normalizeTranscriptSegments( + preview.transcriptSegments || [], + ); + const speakerCount = countTranscriptSpeakers(transcriptSegments); + const highlights = [ + preview.status === "running" ? "状态：待转写" : `状态：${preview.status}`, + preview.language?.trim() ? `语言：${preview.language.trim()}` : null, + preview.outputFormat?.trim() + ? `格式：${preview.outputFormat.trim()}` + : null, + preview.model?.trim() ? `模型：${preview.model.trim()}` : null, + transcriptSegments.length > 0 ? `段落：${transcriptSegments.length}` : null, + speakerCount > 0 ? `说话人：${speakerCount}` : null, + transcriptText ? `字数：${transcriptText.trim().length}` : null, + errorCode ? `错误码：${errorCode}` : null, + ].filter((item): item is string => Boolean(item)); + const sourceLabel = sourcePath || sourceUrl || preview.prompt || "音频来源"; + const transcriptTone = + preview.status === "failed" + ? ("danger" as const) + : transcriptPath + ? ("success" as const) + : ("info" as const); + const transcriptTitle = + preview.status === "failed" + ? "转写失败" + : transcriptPath + ? "Transcript 已同步，可校对保存" + : "等待转写执行器"; + const transcriptBody = + preview.status === "failed" + ? [ + preview.statusMessage?.trim(), + errorCode ? `错误码：${errorCode}` : null, + errorMessage ? `原因：${errorMessage}` : null, + ] + .filter((item): item is string => Boolean(item)) + .join("\n") + : transcriptPath + ? transcriptText + ? `Transcript 已载入，可直接在下方编辑校对；保存后会作为同一运行时文档的新版本记录，不改写原始 ASR 输出。源文件路径：${transcriptPath}` + : `Transcript 路径：${transcriptPath}` + : "当前步骤只创建标准 transcription_generate 任务产物；lime-transcription-worker 会回写 transcript.completed 或 transcript.failed，不回退 frontend ASR。"; + + const transcriptBlocks = transcriptText + ? [ + { + id: "transcript-text", + type: "code_block" as const, + title: "转写文本（可编辑校对）", + language: "text", + code: transcriptText, + }, + ] + : []; + const segmentBlocks = + transcriptSegments.length > 0 + ? [ + { + id: "transcript-segments", + type: "table" as const, + title: "转写时间轴（可逐段编辑校对）", + columns: ["时间", "说话人", "内容"], + rows: transcriptSegments.map((segment) => [ + formatTranscriptSegmentRange(segment), + segment.speaker?.trim() || "未标注", + segment.text, + ]), + }, + ] + : []; + + return { + schemaVersion: ARTIFACT_DOCUMENT_SCHEMA_VERSION, + artifactId: `transcription-generate:${preview.taskId}`, + kind: "brief" as const, + title: preview.title?.trim() || "内容转写任务", + status: + preview.status === "failed" + ? ("failed" as const) + : preview.status === "complete" || preview.status === "partial" + ? ("ready" as const) + : ("streaming" as const), + language: "zh-CN", + summary: + preview.status === "running" + ? "转写任务已经写入标准 transcription_generate/transcript 产物，等待执行器同步结果。" + : preview.statusMessage || "转写任务已经进入统一多模态运行合同主链。", + blocks: [ + { + id: "hero", + type: "hero_summary" as const, + eyebrow: "内容转写", + title: preview.prompt || "转写任务", + summary: + preview.statusMessage || + "转写任务已经写入标准 transcription_generate/transcript 产物。", + highlights, + }, + { + id: "source", + type: "rich_text" as const, + contentFormat: "markdown" as const, + content: sourceLabel, + markdown: `### 转写来源\n\n${sourceLabel}`, + text: sourceLabel, + }, + ...segmentBlocks, + ...transcriptBlocks, + { + id: "transcript-output", + type: "callout" as const, + tone: transcriptTone, + title: transcriptTitle, + body: transcriptBody, + }, + ], + sources: [ + taskFilePath + ? { + id: "transcription-task-file", + type: "file" as const, + label: "transcription_generate task file", + locator: { + path: taskFilePath, + }, + reliability: "primary" as const, + } + : null, + transcriptPath + ? { + id: "transcript-file", + type: "file" as const, + label: "transcript output", + locator: { + path: transcriptPath, + }, + reliability: "primary" as const, + } + : null, + ].filter((item): item is NonNullable => Boolean(item)), + metadata: { + generatedBy: "agent" as const, + rendererHints: { + density: "comfortable" as const, + }, + taskId: preview.taskId, + taskType: "transcription_generate", + taskFilePath, + transcriptPath: transcriptPath || null, + sourcePath: sourcePath || null, + sourceUrl: sourceUrl || null, + language: preview.language || null, + outputFormat: preview.outputFormat || null, + transcriptText: transcriptText || null, + transcriptSegments, + transcriptCorrectionEnabled: Boolean( + transcriptText || transcriptSegments.length > 0, + ), + transcriptCorrectionStatus: + transcriptText || transcriptSegments.length > 0 + ? "available" + : "waiting_transcript", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + providerId: preview.providerId || null, + model: preview.model || null, + errorCode: errorCode || null, + errorMessage: errorMessage || null, + modalityContractKey: "audio_transcription", + }, + }; +} + export function buildImageTaskPreviewFromToolResult( params: ToolResultPreviewParams, ): MessageImageWorkbenchPreview | null { @@ -980,6 +1511,17 @@ function buildGenericTaskPreviewFromToolResult( extractArtifactProtocolPathsFromValue(taskResult)[0] || extractArtifactProtocolPathsFromValue(metadata)[0] || null; + const taskFilePath = + readMetadataString( + [metadata, resultRecord, taskResult], + ["artifact_path", "artifactPath"], + ) || + readMetadataString( + [metadata, resultRecord, taskResult], + ["path", "absolute_path", "absolutePath"], + ) || + artifactPath || + null; const prompt = parsedArguments.prompt || parsedArguments.query || @@ -991,6 +1533,34 @@ function buildGenericTaskPreviewFromToolResult( params.fallbackPrompt.trim() || "任务进行中"; const candidates = [metadata, resultRecord, taskResult]; + const transcriptRecordCandidates = [ + metadata, + resultRecord, + taskResult, + asRecord(metadata?.transcript), + asRecord(resultRecord?.transcript), + asRecord(taskResult?.transcript), + ]; + const rawTranscriptText = + kind === "transcription_generate" + ? readMetadataString(transcriptRecordCandidates, [ + "transcript_text", + "transcriptText", + "text", + ]) || null + : null; + const parsedTranscript = + kind === "transcription_generate" + ? parseTranscriptContent(rawTranscriptText) + : { text: null, segments: [] }; + const extractedTranscriptSegments = + kind === "transcription_generate" + ? extractTranscriptSegmentsFromRecords(transcriptRecordCandidates) + : []; + const transcriptSegments = + extractedTranscriptSegments.length > 0 + ? extractedTranscriptSegments + : parsedTranscript.segments; const metaItems = buildGenericTaskMetaItems( kind, parsedArguments, @@ -1016,7 +1586,10 @@ function buildGenericTaskPreviewFromToolResult( [metadata, resultRecord, taskResult], ["content_id", "contentId"], ) || null, - artifactPath, + artifactPath: + kind === "transcription_generate" + ? buildTranscriptionTaskPreviewArtifactPath(taskId) + : artifactPath, providerId: readMetadataString( [metadata, resultRecord, taskResult], @@ -1032,6 +1605,59 @@ function buildGenericTaskPreviewFromToolResult( ), metaItems, imageCandidates: kind === "modal_resource_search" ? [] : undefined, + taskFilePath: kind === "transcription_generate" ? taskFilePath : undefined, + sourcePath: + kind === "transcription_generate" + ? parsedArguments.sourcePath || + readMetadataString( + [metadata, resultRecord, taskResult], + ["source_path", "sourcePath"], + ) || + null + : undefined, + sourceUrl: + kind === "transcription_generate" + ? parsedArguments.sourceUrl || + readMetadataString( + [metadata, resultRecord, taskResult], + ["source_url", "sourceUrl"], + ) || + null + : undefined, + language: + kind === "transcription_generate" + ? parsedArguments.language || + parsedArguments.targetLanguage || + readMetadataString( + [metadata, resultRecord, taskResult], + ["language", "target_language", "targetLanguage"], + ) || + null + : undefined, + outputFormat: + kind === "transcription_generate" + ? parsedArguments.outputFormat || + readMetadataString( + [metadata, resultRecord, taskResult], + ["output_format", "outputFormat", "format"], + ) || + null + : undefined, + transcriptPath: + kind === "transcription_generate" + ? readMetadataString( + [metadata, resultRecord, taskResult], + ["transcript_path", "transcriptPath"], + ) || null + : undefined, + transcriptText: + kind === "transcription_generate" + ? parsedTranscript.text || rawTranscriptText || null + : undefined, + transcriptSegments: + kind === "transcription_generate" + ? normalizeTranscriptSegments(transcriptSegments) + : undefined, }; } @@ -1042,6 +1668,68 @@ export function buildToolResultArtifactFromToolResult( content: string; metadata: Record; } | null { + const audioPreview = buildAudioTaskPreviewFromToolResult(params); + if (audioPreview) { + const artifactPath = + audioPreview.artifactPath || + buildAudioTaskPreviewArtifactPath(audioPreview.taskId); + return { + filePath: artifactPath, + content: "", + metadata: { + artifactDocument: buildAudioTaskArtifactDocument(audioPreview), + artifact_type: "document", + previewText: + audioPreview.statusMessage || "配音任务已写入统一任务产物协议", + taskId: audioPreview.taskId, + taskType: "audio_generate", + taskFilePath: audioPreview.taskFilePath || null, + audioUrl: audioPreview.audioUrl || null, + modalityContractKey: "voice_generation", + }, + }; + } + + const taskPreview = buildGenericTaskPreviewFromToolResult(params); + if (taskPreview?.kind === "transcription_generate") { + const artifactPath = + taskPreview.artifactPath || + buildTranscriptionTaskPreviewArtifactPath(taskPreview.taskId); + return { + filePath: artifactPath, + content: "", + metadata: { + artifactDocument: buildTranscriptionTaskArtifactDocument(taskPreview), + artifact_type: "document", + previewText: + taskPreview.statusMessage || "转写任务已写入统一任务产物协议", + taskId: taskPreview.taskId, + taskType: "transcription_generate", + taskFilePath: taskPreview.taskFilePath || null, + transcriptPath: taskPreview.transcriptPath || null, + transcriptText: taskPreview.transcriptText || null, + transcriptSegments: taskPreview.transcriptSegments || [], + transcriptCorrectionEnabled: Boolean( + taskPreview.transcriptText || + (taskPreview.transcriptSegments || []).length > 0, + ), + transcriptCorrectionStatus: + taskPreview.transcriptText || + (taskPreview.transcriptSegments || []).length > 0 + ? "available" + : "waiting_transcript", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + sourcePath: taskPreview.sourcePath || null, + sourceUrl: taskPreview.sourceUrl || null, + language: taskPreview.language || null, + outputFormat: taskPreview.outputFormat || null, + modalityContractKey: "audio_transcription", + }, + }; + } + const webImageSearch = readWebImageSearchResult(params); if (!webImageSearch) { return null; @@ -1084,6 +1772,7 @@ export function buildTaskPreviewFromToolResult( ): MessageTaskPreview | null { return ( buildVideoTaskPreviewFromToolResult(params) || + buildAudioTaskPreviewFromToolResult(params) || buildWebImageSearchTaskPreviewFromToolResult(params) || buildGenericTaskPreviewFromToolResult(params) ); diff --git a/src/components/agent/chat/utils/transcriptSegments.test.ts b/src/components/agent/chat/utils/transcriptSegments.test.ts new file mode 100644 index 000000000..e0f8deca8 --- /dev/null +++ b/src/components/agent/chat/utils/transcriptSegments.test.ts @@ -0,0 +1,80 @@ +import { describe, expect, it } from "vitest"; +import { + formatTranscriptSegmentRange, + parseTranscriptContent, +} from "./transcriptSegments"; + +describe("transcriptSegments", () => { + it("应从 verbose JSON transcript 中解析时间轴和说话人", () => { + const parsed = parseTranscriptContent( + JSON.stringify({ + text: "主持人：欢迎来到 Lime。\n嘉宾：我们聊多模态运行合同。", + segments: [ + { + start: 0, + end: 2.4, + speaker: "host", + text: "欢迎来到 Lime。", + }, + { + start: 2.4, + end: 5.8, + speaker: "speaker_2", + text: "我们聊多模态运行合同。", + }, + ], + }), + ); + + expect(parsed.text).toBe( + "主持人：欢迎来到 Lime。\n嘉宾：我们聊多模态运行合同。", + ); + expect(parsed.segments).toMatchObject([ + { + index: 1, + startMs: 0, + endMs: 2400, + speaker: "host", + text: "欢迎来到 Lime。", + }, + { + index: 2, + startMs: 2400, + endMs: 5800, + speaker: "说话人 2", + text: "我们聊多模态运行合同。", + }, + ]); + expect(formatTranscriptSegmentRange(parsed.segments[1])).toBe( + "00:02 - 00:05", + ); + }); + + it("应从 VTT/SRT 内容中恢复逐段时间轴", () => { + const parsed = parseTranscriptContent(`WEBVTT + +00:00:01.000 --> 00:00:03.500 +欢迎来到 Lime 访谈。 + +00:00:04,000 --> 00:00:06,000 +嘉宾: 这次我们讲转写 viewer。 +`); + + expect(parsed.segments).toMatchObject([ + { + index: 1, + startMs: 1000, + endMs: 3500, + speaker: "主持人", + text: "欢迎来到 Lime 访谈。", + }, + { + index: 2, + startMs: 4000, + endMs: 6000, + speaker: "嘉宾", + text: "这次我们讲转写 viewer。", + }, + ]); + }); +}); diff --git a/src/components/agent/chat/utils/transcriptSegments.ts b/src/components/agent/chat/utils/transcriptSegments.ts new file mode 100644 index 000000000..6c9348dc7 --- /dev/null +++ b/src/components/agent/chat/utils/transcriptSegments.ts @@ -0,0 +1,428 @@ +import type { MessageTranscriptSegment } from "../types"; + +const MAX_TRANSCRIPT_SEGMENTS = 160; +const TIMECODE_PATTERN = /(?:(\d{1,2}):)?(\d{1,2}):(\d{2})([,.]\d{1,3})?/; + +interface TimedValue { + key: string; + value: unknown; +} + +interface TranscriptContentParseResult { + text: string | null; + segments: MessageTranscriptSegment[]; +} + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function readString( + candidates: Array | null | undefined>, + keys: string[], +): string | null { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + } + } + return null; +} + +function readTimedValue( + candidate: Record, + keys: string[], +): TimedValue | null { + for (const key of keys) { + if (candidate[key] !== undefined && candidate[key] !== null) { + return { key, value: candidate[key] }; + } + } + return null; +} + +export function parseTranscriptTimestampToMs(value: string): number | null { + const match = value.trim().match(TIMECODE_PATTERN); + if (!match) { + return null; + } + const hours = Number(match[1] || 0); + const minutes = Number(match[2] || 0); + const seconds = Number(match[3] || 0); + const fraction = match[4] + ? Number(`0${match[4].replace(",", ".")}`) * 1000 + : 0; + const totalMs = + hours * 60 * 60 * 1000 + minutes * 60 * 1000 + seconds * 1000 + fraction; + return Number.isFinite(totalMs) ? Math.round(totalMs) : null; +} + +function normalizeTimedMs(timedValue: TimedValue | null): number | null { + if (!timedValue) { + return null; + } + + if (typeof timedValue.value === "string" && timedValue.value.trim()) { + const parsedTimecode = parseTranscriptTimestampToMs(timedValue.value); + if (parsedTimecode !== null) { + return parsedTimecode; + } + const parsedNumber = Number(timedValue.value.trim()); + if (!Number.isFinite(parsedNumber)) { + return null; + } + return normalizeTimedMs({ key: timedValue.key, value: parsedNumber }); + } + + if ( + typeof timedValue.value !== "number" || + !Number.isFinite(timedValue.value) || + timedValue.value < 0 + ) { + return null; + } + + const key = timedValue.key.toLowerCase(); + if (key.includes("ms") || timedValue.value > 10_000) { + return Math.round(timedValue.value); + } + return Math.round(timedValue.value * 1000); +} + +function normalizeSpeakerLabel( + value: string | null | undefined, +): string | null { + if (!value) { + return null; + } + return value + .replace(/^speaker[_\s-]*/i, "说话人 ") + .replace(/^spk[_\s-]*/i, "说话人 ") + .trim(); +} + +function normalizeTranscriptSegment( + record: Record, + index: number, +): MessageTranscriptSegment | null { + const text = readString( + [record], + ["text", "transcript", "sentence", "content", "utterance", "word"], + ); + if (!text) { + return null; + } + + const startMs = normalizeTimedMs( + readTimedValue(record, [ + "start_ms", + "startMs", + "start_time_ms", + "startTimeMs", + "start_seconds", + "startSeconds", + "start_time", + "startTime", + "start", + "from", + "offset", + ]), + ); + const endMs = normalizeTimedMs( + readTimedValue(record, [ + "end_ms", + "endMs", + "end_time_ms", + "endTimeMs", + "end_seconds", + "endSeconds", + "end_time", + "endTime", + "end", + "to", + ]), + ); + const speaker = normalizeSpeakerLabel( + readString( + [record], + [ + "speaker", + "speaker_label", + "speakerLabel", + "speaker_id", + "speakerId", + "channel", + ], + ), + ); + + return { + id: + readString([record], ["id", "segment_id", "segmentId"]) || + `segment-${index + 1}`, + index: index + 1, + startMs, + endMs, + speaker, + text, + }; +} + +function readArrayRecords( + candidates: Array | null | undefined>, + keys: string[], +): Record[] { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (!Array.isArray(value) || value.length === 0) { + continue; + } + const records = value + .map((item) => asRecord(item)) + .filter((item): item is Record => Boolean(item)); + if (records.length > 0) { + return records; + } + } + } + return []; +} + +function extractSegmentsFromJsonRecord( + record: Record, +): MessageTranscriptSegment[] { + const nestedResult = asRecord(record.result); + const nestedTranscript = asRecord(record.transcript); + const nestedPayload = asRecord(record.payload); + const candidates = [record, nestedResult, nestedTranscript, nestedPayload]; + const records = readArrayRecords(candidates, [ + "segments", + "transcript_segments", + "transcriptSegments", + "utterances", + "speaker_labels", + "speakerLabels", + "speaker_segments", + "speakerSegments", + "timestamps", + ]); + + return normalizeTranscriptSegments( + records + .map((item, index) => normalizeTranscriptSegment(item, index)) + .filter((item): item is MessageTranscriptSegment => Boolean(item)), + ); +} + +function parseSrtOrVttSegments(content: string): MessageTranscriptSegment[] { + const normalized = content + .replace(/^\uFEFF/, "") + .replace(/^WEBVTT[^\n]*(\n|$)/i, "") + .trim(); + if (!normalized.includes("-->")) { + return []; + } + + const blocks = normalized.split(/\n\s*\n/); + const segments: MessageTranscriptSegment[] = []; + blocks.forEach((block) => { + const lines = block + .split(/\r?\n/) + .map((line) => line.trim()) + .filter(Boolean); + const timeIndex = lines.findIndex((line) => line.includes("-->")); + if (timeIndex < 0) { + return; + } + const [startLabel, endLabel] = lines[timeIndex].split("-->"); + const startMs = parseTranscriptTimestampToMs(startLabel || ""); + const endMs = parseTranscriptTimestampToMs( + (endLabel || "").trim().split(/\s+/)[0] || "", + ); + const textLines = lines.slice(timeIndex + 1); + if (textLines.length === 0) { + return; + } + const rawText = textLines + .join(" ") + .replace(/]+)>/gi, "$1: ") + .replace(/<\/v>/gi, "") + .trim(); + const speakerMatch = rawText.match(/^([^:：]{1,32})[:：]\s*(.+)$/); + const speaker = speakerMatch + ? normalizeSpeakerLabel(speakerMatch[1] || null) + : null; + const text = speakerMatch ? speakerMatch[2]?.trim() : rawText; + if (!text) { + return; + } + segments.push({ + id: `segment-${segments.length + 1}`, + index: segments.length + 1, + startMs, + endMs, + speaker, + text, + }); + }); + + return normalizeTranscriptSegments(segments); +} + +export function normalizeTranscriptSegments( + segments: MessageTranscriptSegment[] | undefined | null, +): MessageTranscriptSegment[] { + if (!segments || segments.length === 0) { + return []; + } + + return segments + .filter((segment) => segment.text.trim()) + .slice(0, MAX_TRANSCRIPT_SEGMENTS) + .sort((left, right) => { + const leftStart = left.startMs ?? Number.MAX_SAFE_INTEGER; + const rightStart = right.startMs ?? Number.MAX_SAFE_INTEGER; + if (leftStart !== rightStart) { + return leftStart - rightStart; + } + return left.index - right.index; + }) + .map((segment, index) => ({ + ...segment, + id: segment.id || `segment-${index + 1}`, + index: index + 1, + speaker: normalizeSpeakerLabel(segment.speaker) || null, + startMs: segment.startMs ?? null, + endMs: segment.endMs ?? null, + text: segment.text.trim(), + })); +} + +export function parseTranscriptContent( + content: string | null | undefined, +): TranscriptContentParseResult { + const text = typeof content === "string" && content.trim() ? content : null; + if (!text) { + return { text: null, segments: [] }; + } + + try { + const parsed = JSON.parse(text) as unknown; + const record = asRecord(parsed); + if (record) { + const jsonText = + readString( + [record, asRecord(record.result), asRecord(record.payload)], + [ + "text", + "transcript_text", + "transcriptText", + "transcript", + "content", + ], + ) || text; + return { + text: jsonText, + segments: extractSegmentsFromJsonRecord(record), + }; + } + if (Array.isArray(parsed)) { + return { + text, + segments: normalizeTranscriptSegments( + parsed + .map((item, index) => + asRecord(item) + ? normalizeTranscriptSegment(asRecord(item)!, index) + : null, + ) + .filter((item): item is MessageTranscriptSegment => Boolean(item)), + ), + }; + } + } catch { + // 非 JSON transcript 继续按 SRT/VTT 或纯文本处理。 + } + + return { + text, + segments: parseSrtOrVttSegments(text), + }; +} + +export function extractTranscriptSegmentsFromRecords( + candidates: Array | null | undefined>, +): MessageTranscriptSegment[] { + const records = readArrayRecords(candidates, [ + "segments", + "transcript_segments", + "transcriptSegments", + "utterances", + "speaker_labels", + "speakerLabels", + "speaker_segments", + "speakerSegments", + "timestamps", + ]); + return normalizeTranscriptSegments( + records + .map((item, index) => normalizeTranscriptSegment(item, index)) + .filter((item): item is MessageTranscriptSegment => Boolean(item)), + ); +} + +export function formatTranscriptTimestamp(ms?: number | null): string { + if (typeof ms !== "number" || !Number.isFinite(ms) || ms < 0) { + return "--:--"; + } + const totalSeconds = Math.floor(ms / 1000); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + const minuteLabel = String(minutes).padStart(2, "0"); + const secondLabel = String(seconds).padStart(2, "0"); + if (hours > 0) { + return `${String(hours).padStart(2, "0")}:${minuteLabel}:${secondLabel}`; + } + return `${minuteLabel}:${secondLabel}`; +} + +export function formatTranscriptSegmentRange( + segment: MessageTranscriptSegment, +): string { + const start = formatTranscriptTimestamp(segment.startMs); + const end = formatTranscriptTimestamp(segment.endMs); + if (start === "--:--" && end === "--:--") { + return `#${segment.index}`; + } + if (end === "--:--") { + return start; + } + return `${start} - ${end}`; +} + +export function countTranscriptSpeakers( + segments: MessageTranscriptSegment[] | undefined | null, +): number { + if (!segments || segments.length === 0) { + return 0; + } + return new Set( + segments + .map((segment) => segment.speaker?.trim()) + .filter((speaker): speaker is string => Boolean(speaker)), + ).size; +} diff --git a/src/components/agent/chat/workspace/ArtifactWorkbenchShell.test.tsx b/src/components/agent/chat/workspace/ArtifactWorkbenchShell.test.tsx index 70acd53be..0025a042b 100644 --- a/src/components/agent/chat/workspace/ArtifactWorkbenchShell.test.tsx +++ b/src/components/agent/chat/workspace/ArtifactWorkbenchShell.test.tsx @@ -256,6 +256,82 @@ function createStructuredEditableArtifact(): Artifact { }; } +function createTranscriptionDocumentArtifact(): Artifact { + const content = JSON.stringify({ + schemaVersion: "artifact_document.v1", + artifactId: "transcription-generate:task-transcription-1", + kind: "brief", + title: "内容转写任务", + status: "ready", + language: "zh-CN", + summary: "用于验证 transcript 校对稿保存。", + blocks: [ + { + id: "transcript-segments", + type: "table", + title: "转写时间轴（可逐段编辑校对）", + columns: ["时间", "说话人", "内容"], + rows: [["00:01 - 00:03", "主持人", "欢迎来到 Lime 访谈节目。"]], + }, + { + id: "transcript-text", + type: "code_block", + title: "转写文本（可编辑校对）", + language: "text", + code: "欢迎来到 Lime 访谈节目。", + }, + ], + sources: [ + { + id: "transcript-file", + type: "file", + label: "transcript output", + locator: { + path: ".lime/runtime/transcripts/task-transcription-1.txt", + }, + reliability: "primary", + }, + ], + metadata: { + taskId: "task-transcription-1", + taskType: "transcription_generate", + modalityContractKey: "audio_transcription", + transcriptPath: ".lime/runtime/transcripts/task-transcription-1.txt", + transcriptText: "欢迎来到 Lime 访谈节目。", + transcriptSegments: [ + { + id: "segment-1", + index: 1, + startMs: 1000, + endMs: 3000, + speaker: "主持人", + text: "欢迎来到 Lime 访谈节目。", + }, + ], + transcriptCorrectionEnabled: true, + transcriptCorrectionStatus: "available", + transcriptCorrectionSource: "artifact_document_version", + }, + }); + + return { + id: "artifact-transcription", + type: "document", + title: "task-transcription-1.artifact.json", + content, + status: "complete", + meta: { + filePath: + ".lime/runtime/transcription-generate/task-transcription-1.artifact.json", + filename: "task-transcription-1.artifact.json", + language: "json", + }, + position: { start: 0, end: content.length }, + createdAt: 1, + updatedAt: 1, + }; +} + function createAdvancedEditableArtifact(): Artifact { const content = JSON.stringify({ schemaVersion: "artifact_document.v1", @@ -1010,6 +1086,124 @@ describe("ArtifactWorkbenchShell", () => { ); }); + it("转写运行时文档保存时应记录校对稿 metadata", async () => { + const handleSaveArtifactDocument = vi.fn().mockResolvedValue(undefined); + const container = renderWorkbench(createTranscriptionDocumentArtifact(), { + onSaveArtifactDocument: handleSaveArtifactDocument, + }); + + await act(async () => { + await Promise.resolve(); + }); + + const editTrigger = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent?.includes("编辑"), + ); + expect(editTrigger).not.toBeUndefined(); + + await act(async () => { + editTrigger?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await Promise.resolve(); + }); + + const transcriptTextTrigger = Array.from( + container.querySelectorAll("button"), + ).find((button) => button.textContent?.includes("转写文本")); + expect(transcriptTextTrigger).not.toBeUndefined(); + + await act(async () => { + transcriptTextTrigger?.dispatchEvent( + new MouseEvent("click", { bubbles: true }), + ); + await Promise.resolve(); + }); + + const codeInput = container.querySelector( + '[data-testid="artifact-structured-edit-code"]', + ) as HTMLTextAreaElement | null; + expect(codeInput).not.toBeNull(); + + await act(async () => { + if (codeInput) { + setTextControlValue( + codeInput, + "欢迎来到 Lime 访谈节目。\n这里是人工校对后的补充。", + ); + } + await Promise.resolve(); + }); + + const saveButton = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent === "保存", + ); + expect(saveButton).not.toBeUndefined(); + + await act(async () => { + saveButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await Promise.resolve(); + }); + + expect(handleSaveArtifactDocument).toHaveBeenCalledTimes(1); + expect(handleSaveArtifactDocument).toHaveBeenCalledWith( + expect.objectContaining({ id: "artifact-transcription" }), + expect.objectContaining({ + blocks: expect.arrayContaining([ + expect.objectContaining({ + id: "transcript-text", + type: "code_block", + code: "欢迎来到 Lime 访谈节目。\n这里是人工校对后的补充。", + }), + expect.objectContaining({ + id: "transcript-correction-status", + type: "callout", + tone: "success", + title: "校对稿已保存", + body: expect.stringContaining("原始 ASR 输出文件保持不变"), + }), + ]), + metadata: expect.objectContaining({ + modalityContractKey: "audio_transcription", + transcriptCorrectionStatus: "saved", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + transcriptCorrectionEditedBlockId: "transcript-text", + transcriptCorrectionTextBlockId: "transcript-text", + transcriptCorrectionSegmentBlockId: "transcript-segments", + transcriptCorrectionSegmentCount: 1, + transcriptCorrectionSpeakerCount: 1, + transcriptCorrectionSourceTranscriptPath: + ".lime/runtime/transcripts/task-transcription-1.txt", + transcriptCorrectionDiffSummary: expect.objectContaining({ + textChanged: true, + originalSegmentCount: 1, + correctedSegmentCount: 1, + changedSegmentCount: 0, + originalSpeakerCount: 1, + correctedSpeakerCount: 1, + }), + transcriptSegmentsCorrected: [ + expect.objectContaining({ + id: "corrected-segment-1", + startMs: 1000, + endMs: 3000, + speaker: "主持人", + text: "欢迎来到 Lime 访谈节目。", + }), + ], + }), + }), + ); + expect( + ( + handleSaveArtifactDocument.mock.calls[0]?.[1].metadata as Record< + string, + unknown + > + ).transcriptCorrectionSavedAt, + ).toEqual(expect.any(String)); + }); + it("编辑态命中关联 timeline 时应支持跳回执行过程", async () => { const onJumpToTimelineItem = vi.fn(); const container = renderWorkbench(createArtifactDocumentArtifact(), { diff --git a/src/components/agent/chat/workspace/artifactWorkbenchDocument.tsx b/src/components/agent/chat/workspace/artifactWorkbenchDocument.tsx index 55c0e05d3..9043941ec 100644 --- a/src/components/agent/chat/workspace/artifactWorkbenchDocument.tsx +++ b/src/components/agent/chat/workspace/artifactWorkbenchDocument.tsx @@ -39,6 +39,7 @@ import { createArtifactDocumentNextVersion, updateArtifactDocumentStatus, } from "./artifactWorkbenchActions"; +import { applyTranscriptCorrectionVersionMetadata } from "./transcriptCorrectionArtifact"; import type { AgentThreadItem } from "../types"; import { buildArtifactTimelineLinkIndex, @@ -1389,7 +1390,7 @@ const EditInspectorPanel: React.FC<{ Workbench 编辑

- 当前支持章节头、摘要卡、正文块与提示块原位编辑，保存后会回写到同一份 + 当前支持章节头、摘要卡、正文块、表格、提示块与代码块原位编辑，保存后会回写到同一份 ArtifactDocument JSON，不再把正文打回聊天区。

@@ -2583,9 +2584,15 @@ export function useArtifactWorkbenchDocumentController({ selectedEditableBlock.blockId, nextDraft, ); + const correctedNextDocument = applyTranscriptCorrectionVersionMetadata( + nextDocument, + { + editedBlockId: selectedEditableBlock.blockId, + }, + ); const versionedDocument = createArtifactDocumentNextVersion( document, - nextDocument, + correctedNextDocument, { summary: `更新 ${selectedEditableBlock.label}`, createdBy: "user", diff --git a/src/components/agent/chat/workspace/browserControlLaunch.ts b/src/components/agent/chat/workspace/browserControlLaunch.ts index 99644ab6b..cbc2d98b1 100644 --- a/src/components/agent/chat/workspace/browserControlLaunch.ts +++ b/src/components/agent/chat/workspace/browserControlLaunch.ts @@ -1,5 +1,5 @@ import { - BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE, + resolveBrowserControlEntrySource, resolveBrowserControlRuntimeContractBinding, } from "@/lib/governance/modalityRuntimeContracts"; import type { ParsedBrowserWorkbenchCommand } from "../utils/browserWorkbenchCommand"; @@ -40,7 +40,7 @@ export function buildBrowserControlLaunchRequestMetadata( required_capabilities: runtimeContract.requiredCapabilities, routing_slot: runtimeContract.routingSlot, runtime_contract: runtimeContract.runtimeContract, - entry_source: BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE, + entry_source: resolveBrowserControlEntrySource(parsedCommand.trigger), requirement: parsedCommand.browserRequirement, requirement_reason: parsedCommand.browserRequirementReason, prompt: parsedCommand.prompt || parsedCommand.body, diff --git a/src/components/agent/chat/workspace/transcriptCorrectionArtifact.test.ts b/src/components/agent/chat/workspace/transcriptCorrectionArtifact.test.ts new file mode 100644 index 000000000..d62b75ccf --- /dev/null +++ b/src/components/agent/chat/workspace/transcriptCorrectionArtifact.test.ts @@ -0,0 +1,159 @@ +import { describe, expect, it } from "vitest"; +import type { ArtifactDocumentV1 } from "@/lib/artifact-document"; +import { + applyTranscriptCorrectionVersionMetadata, + extractCorrectedTranscriptSegments, +} from "./transcriptCorrectionArtifact"; + +function createTranscriptionDocument(): ArtifactDocumentV1 { + return { + schemaVersion: "artifact_document.v1", + artifactId: "transcription-generate:task-transcription-1", + kind: "brief", + title: "内容转写任务", + status: "ready", + language: "zh-CN", + blocks: [ + { + id: "transcript-segments", + type: "table", + title: "转写时间轴（可逐段编辑校对）", + columns: ["时间", "说话人", "内容"], + rows: [ + ["00:01 - 00:03", "主持人", "欢迎来到 Lime 访谈节目。"], + ["00:04 - 00:07", "嘉宾", "这里是修订后的回答。"], + ], + }, + { + id: "transcript-text", + type: "code_block", + title: "转写文本（可编辑校对）", + language: "text", + code: "欢迎来到 Lime 访谈节目。\n这里是修订后的回答。", + }, + ], + sources: [], + metadata: { + taskId: "task-transcription-1", + taskType: "transcription_generate", + modalityContractKey: "audio_transcription", + transcriptPath: ".lime/runtime/transcripts/task-transcription-1.txt", + transcriptText: "欢迎来到 Lime 访谈节目。\n这里是修订后的回答。", + transcriptSegments: [ + { + id: "segment-1", + index: 1, + startMs: 1000, + endMs: 3000, + speaker: "主持人", + text: "欢迎来到 Lime 访谈节目。", + }, + { + id: "segment-2", + index: 2, + startMs: 4000, + endMs: 7000, + speaker: "嘉宾", + text: "这里是修订后的回答。", + }, + ], + }, + }; +} + +describe("transcriptCorrectionArtifact", () => { + it("应从转写时间轴表格提取校对后的分段", () => { + expect( + extractCorrectedTranscriptSegments(createTranscriptionDocument()), + ).toEqual([ + { + id: "corrected-segment-1", + index: 1, + startMs: 1000, + endMs: 3000, + speaker: "主持人", + text: "欢迎来到 Lime 访谈节目。", + }, + { + id: "corrected-segment-2", + index: 2, + startMs: 4000, + endMs: 7000, + speaker: "嘉宾", + text: "这里是修订后的回答。", + }, + ]); + }); + + it("应只为 audio_transcription 文档补校对稿版本 metadata", () => { + const document = applyTranscriptCorrectionVersionMetadata( + createTranscriptionDocument(), + { + editedBlockId: "transcript-text", + savedAt: "2026-04-30T12:00:00.000Z", + }, + ); + + expect(document.metadata).toMatchObject({ + transcriptCorrectionEnabled: true, + transcriptCorrectionStatus: "saved", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + transcriptCorrectionSavedAt: "2026-04-30T12:00:00.000Z", + transcriptCorrectionEditedBlockId: "transcript-text", + transcriptCorrectionTextBlockId: "transcript-text", + transcriptCorrectionTextLength: 26, + transcriptCorrectionSegmentBlockId: "transcript-segments", + transcriptCorrectionSegmentCount: 2, + transcriptCorrectionSpeakerCount: 2, + transcriptCorrectionSourceTranscriptPath: + ".lime/runtime/transcripts/task-transcription-1.txt", + transcriptCorrectionDiffSummary: { + textChanged: false, + originalTextLength: 26, + correctedTextLength: 26, + textLengthDelta: 0, + originalSegmentCount: 2, + correctedSegmentCount: 2, + changedSegmentCount: 0, + originalSpeakerCount: 2, + correctedSpeakerCount: 2, + }, + transcriptSegmentsCorrected: [ + expect.objectContaining({ + id: "corrected-segment-1", + speaker: "主持人", + }), + expect.objectContaining({ + id: "corrected-segment-2", + speaker: "嘉宾", + }), + ], + }); + expect(document.blocks).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + id: "transcript-correction-status", + type: "callout", + tone: "success", + title: "校对稿已保存", + body: expect.stringContaining("原始 ASR 输出文件保持不变"), + }), + ]), + ); + }); + + it("非转写 ArtifactDocument 不应被补校对稿 metadata", () => { + const document: ArtifactDocumentV1 = { + ...createTranscriptionDocument(), + artifactId: "artifact-document:demo", + metadata: { + taskType: "report_generate", + modalityContractKey: "web_research", + }, + }; + + expect(applyTranscriptCorrectionVersionMetadata(document)).toBe(document); + }); +}); diff --git a/src/components/agent/chat/workspace/transcriptCorrectionArtifact.ts b/src/components/agent/chat/workspace/transcriptCorrectionArtifact.ts new file mode 100644 index 000000000..1bc46dfb0 --- /dev/null +++ b/src/components/agent/chat/workspace/transcriptCorrectionArtifact.ts @@ -0,0 +1,327 @@ +import type { + ArtifactDocumentBlock, + ArtifactDocumentV1, +} from "@/lib/artifact-document"; +import type { MessageTranscriptSegment } from "../types"; +import { + countTranscriptSpeakers, + normalizeTranscriptSegments, + parseTranscriptTimestampToMs, +} from "../utils/transcriptSegments"; + +const TRANSCRIPTION_TASK_TYPE = "transcription_generate"; +const AUDIO_TRANSCRIPTION_CONTRACT = "audio_transcription"; +const TRANSCRIPT_TEXT_BLOCK_ID = "transcript-text"; +const TRANSCRIPT_SEGMENTS_BLOCK_ID = "transcript-segments"; +const TRANSCRIPT_CORRECTION_STATUS_BLOCK_ID = "transcript-correction-status"; + +interface TranscriptCorrectionDiffSummary { + textChanged: boolean; + originalTextLength: number; + correctedTextLength: number; + textLengthDelta: number; + originalSegmentCount: number; + correctedSegmentCount: number; + changedSegmentCount: number; + originalSpeakerCount: number; + correctedSpeakerCount: number; +} + +function normalizeText(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function readMetadataText( + document: ArtifactDocumentV1, + key: string, +): string | null { + return normalizeText(document.metadata[key]); +} + +function isAudioTranscriptionDocument(document: ArtifactDocumentV1): boolean { + return ( + readMetadataText(document, "modalityContractKey") === + AUDIO_TRANSCRIPTION_CONTRACT || + readMetadataText(document, "taskType") === TRANSCRIPTION_TASK_TYPE || + document.artifactId.startsWith("transcription-generate:") + ); +} + +function findArtifactBlock( + document: ArtifactDocumentV1, + blockId: string, + blockType: T, +): Extract | null { + const block = document.blocks.find( + (item) => item.id === blockId && item.type === blockType, + ); + return ( + (block as Extract | undefined) || null + ); +} + +function resolveCorrectedTranscriptText(document: ArtifactDocumentV1): string { + const codeBlock = findArtifactBlock( + document, + TRANSCRIPT_TEXT_BLOCK_ID, + "code_block", + ); + return ( + normalizeText(codeBlock?.code) || + readMetadataText(document, "transcriptText") || + "" + ); +} + +function resolveOriginalTranscriptText(document: ArtifactDocumentV1): string { + return readMetadataText(document, "transcriptText") || ""; +} + +function parseSegmentRange(value: string): { + startMs: number | null; + endMs: number | null; +} { + const normalized = value.replace(/[–—]/g, "-").trim(); + const parts = normalized.includes("-->") + ? normalized.split("-->") + : normalized.split(/\s+-\s+/); + + return { + startMs: parts[0] ? parseTranscriptTimestampToMs(parts[0]) : null, + endMs: parts[1] ? parseTranscriptTimestampToMs(parts[1]) : null, + }; +} + +function normalizeSpeaker(value: string | undefined): string | null { + const speaker = normalizeText(value); + if (!speaker || speaker === "未标注") { + return null; + } + return speaker; +} + +function resolveOriginalTranscriptSegments( + document: ArtifactDocumentV1, +): MessageTranscriptSegment[] { + if (!Array.isArray(document.metadata.transcriptSegments)) { + return []; + } + + return normalizeTranscriptSegments( + document.metadata.transcriptSegments as MessageTranscriptSegment[], + ); +} + +function isSameSegment( + left: MessageTranscriptSegment | undefined, + right: MessageTranscriptSegment | undefined, +): boolean { + if (!left || !right) { + return false; + } + + return ( + left.text.trim() === right.text.trim() && + (left.speaker?.trim() || "") === (right.speaker?.trim() || "") && + (left.startMs ?? null) === (right.startMs ?? null) && + (left.endMs ?? null) === (right.endMs ?? null) + ); +} + +function countChangedSegments( + originalSegments: MessageTranscriptSegment[], + correctedSegments: MessageTranscriptSegment[], +): number { + const total = Math.max(originalSegments.length, correctedSegments.length); + let changedCount = 0; + + for (let index = 0; index < total; index += 1) { + if (!isSameSegment(originalSegments[index], correctedSegments[index])) { + changedCount += 1; + } + } + + return changedCount; +} + +function buildTranscriptCorrectionDiffSummary( + document: ArtifactDocumentV1, + correctedText: string, + correctedSegments: MessageTranscriptSegment[], +): TranscriptCorrectionDiffSummary { + const originalText = resolveOriginalTranscriptText(document); + const originalSegments = resolveOriginalTranscriptSegments(document); + const originalTextLength = originalText.trim().length; + const correctedTextLength = correctedText.trim().length; + + return { + textChanged: originalText.trim() !== correctedText.trim(), + originalTextLength, + correctedTextLength, + textLengthDelta: correctedTextLength - originalTextLength, + originalSegmentCount: originalSegments.length, + correctedSegmentCount: correctedSegments.length, + changedSegmentCount: countChangedSegments( + originalSegments, + correctedSegments, + ), + originalSpeakerCount: countTranscriptSpeakers(originalSegments), + correctedSpeakerCount: countTranscriptSpeakers(correctedSegments), + }; +} + +function formatDelta(value: number): string { + if (value > 0) { + return `+${value}`; + } + return String(value); +} + +function buildCorrectionStatusBody( + summary: TranscriptCorrectionDiffSummary, + sourceTranscriptPath: string | null, +): string { + const lines = [ + "校对稿已保存为当前运行时文档的新版本；原始 ASR 输出文件保持不变。", + `文本长度：${summary.originalTextLength} -> ${summary.correctedTextLength}（${formatDelta( + summary.textLengthDelta, + )}）`, + `时间轴段落：${summary.originalSegmentCount} -> ${summary.correctedSegmentCount}，变更段落 ${summary.changedSegmentCount} 个`, + `说话人数：${summary.originalSpeakerCount} -> ${summary.correctedSpeakerCount}`, + ]; + + if (sourceTranscriptPath) { + lines.push(`原始 transcript：${sourceTranscriptPath}`); + } + + return lines.join("\n"); +} + +function upsertCorrectionStatusBlock( + document: ArtifactDocumentV1, + diffSummary: TranscriptCorrectionDiffSummary, + sourceTranscriptPath: string | null, +): ArtifactDocumentBlock[] { + const statusBlock: ArtifactDocumentBlock = { + id: TRANSCRIPT_CORRECTION_STATUS_BLOCK_ID, + type: "callout", + tone: "success", + title: "校对稿已保存", + body: buildCorrectionStatusBody(diffSummary, sourceTranscriptPath), + }; + const existingIndex = document.blocks.findIndex( + (block) => block.id === TRANSCRIPT_CORRECTION_STATUS_BLOCK_ID, + ); + + if (existingIndex >= 0) { + return document.blocks.map((block, index) => + index === existingIndex ? statusBlock : block, + ); + } + + const transcriptOutputIndex = document.blocks.findIndex( + (block) => block.id === "transcript-output", + ); + if (transcriptOutputIndex < 0) { + return [...document.blocks, statusBlock]; + } + + return [ + ...document.blocks.slice(0, transcriptOutputIndex + 1), + statusBlock, + ...document.blocks.slice(transcriptOutputIndex + 1), + ]; +} + +export function extractCorrectedTranscriptSegments( + document: ArtifactDocumentV1, +): MessageTranscriptSegment[] { + const tableBlock = findArtifactBlock( + document, + TRANSCRIPT_SEGMENTS_BLOCK_ID, + "table", + ); + if (!tableBlock || !Array.isArray(tableBlock.rows)) { + return []; + } + + return tableBlock.rows + .map((row, index): MessageTranscriptSegment | null => { + const [rangeLabel = "", speakerLabel = "", ...textCells] = row; + const text = textCells.join(" | ").trim(); + if (!text) { + return null; + } + const { startMs, endMs } = parseSegmentRange(rangeLabel); + + return { + id: `corrected-segment-${index + 1}`, + index: index + 1, + startMs, + endMs, + speaker: normalizeSpeaker(speakerLabel), + text, + }; + }) + .filter((item): item is MessageTranscriptSegment => Boolean(item)); +} + +function countCorrectedSpeakers(segments: MessageTranscriptSegment[]): number { + return new Set( + segments + .map((segment) => segment.speaker?.trim()) + .filter((speaker): speaker is string => Boolean(speaker)), + ).size; +} + +export function applyTranscriptCorrectionVersionMetadata( + document: ArtifactDocumentV1, + options: { + editedBlockId?: string | null; + savedAt?: string; + } = {}, +): ArtifactDocumentV1 { + if (!isAudioTranscriptionDocument(document)) { + return document; + } + + const correctedText = resolveCorrectedTranscriptText(document); + const correctedSegments = extractCorrectedTranscriptSegments(document); + const sourceTranscriptPath = readMetadataText(document, "transcriptPath"); + const diffSummary = buildTranscriptCorrectionDiffSummary( + document, + correctedText, + correctedSegments, + ); + + return { + ...document, + blocks: upsertCorrectionStatusBlock( + document, + diffSummary, + sourceTranscriptPath, + ), + metadata: { + ...document.metadata, + transcriptCorrectionEnabled: true, + transcriptCorrectionStatus: "saved", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + transcriptCorrectionSavedAt: options.savedAt || new Date().toISOString(), + transcriptCorrectionEditedBlockId: options.editedBlockId || null, + transcriptCorrectionTextBlockId: correctedText + ? TRANSCRIPT_TEXT_BLOCK_ID + : null, + transcriptCorrectionTextLength: correctedText.trim().length, + transcriptCorrectionSegmentBlockId: + correctedSegments.length > 0 ? TRANSCRIPT_SEGMENTS_BLOCK_ID : null, + transcriptCorrectionSegmentCount: correctedSegments.length, + transcriptCorrectionSpeakerCount: + countCorrectedSpeakers(correctedSegments), + transcriptCorrectionSourceTranscriptPath: sourceTranscriptPath, + transcriptCorrectionDiffSummary: diffSummary, + transcriptSegmentsCorrected: correctedSegments, + }, + }; +} diff --git a/src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.test.tsx b/src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.test.tsx new file mode 100644 index 000000000..de08fe358 --- /dev/null +++ b/src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.test.tsx @@ -0,0 +1,496 @@ +import React, { useState } from "react"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getMediaTaskArtifact, + listMediaTaskArtifacts, + type ListMediaTaskArtifactsOutput, + type MediaTaskArtifactOutput, +} from "@/lib/api/mediaTasks"; +import { safeListen } from "@/lib/dev-bridge"; +import type { Message } from "../types"; +import { useWorkspaceAudioTaskPreviewRuntime } from "./useWorkspaceAudioTaskPreviewRuntime"; + +vi.mock("@/lib/api/mediaTasks", () => ({ + getMediaTaskArtifact: vi.fn(), + listMediaTaskArtifacts: vi.fn(), +})); + +vi.mock("@/lib/dev-bridge", () => ({ + safeListen: vi.fn(), +})); + +type HookProps = Parameters[0]; + +const mountedRoots: Array<{ container: HTMLDivElement; root: Root }> = []; + +function buildRunningAudioMessage(): Message { + return { + id: "assistant-audio-1", + role: "assistant", + content: "配音任务已创建。", + timestamp: new Date("2026-04-30T00:00:00.000Z"), + taskPreview: { + kind: "audio_generate", + taskId: "task-audio-1", + taskType: "audio_generate", + prompt: "请生成温暖旁白", + title: "配音生成任务", + status: "running", + artifactPath: ".lime/runtime/audio-generate/task-audio-1.md", + taskFilePath: ".lime/tasks/audio_generate/task-audio-1.json", + sourceText: "请生成温暖旁白", + voice: "warm_narrator", + audioUrl: null, + mimeType: null, + durationMs: null, + }, + }; +} + +function buildCompletedAudioArtifact(): MediaTaskArtifactOutput { + return { + success: true, + task_id: "task-audio-1", + task_type: "audio_generate", + task_family: "audio", + status: "succeeded", + normalized_status: "succeeded", + current_attempt_id: "attempt-1", + path: ".lime/tasks/audio_generate/task-audio-1.json", + absolute_path: "/workspace/.lime/tasks/audio_generate/task-audio-1.json", + artifact_path: ".lime/tasks/audio_generate/task-audio-1.json", + absolute_artifact_path: + "/workspace/.lime/tasks/audio_generate/task-audio-1.json", + reused_existing: false, + record: { + task_id: "task-audio-1", + task_type: "audio_generate", + task_family: "audio", + payload: { + source_text: "请生成温暖旁白", + voice: "warm_narrator", + provider_id: "limecore", + model: "voice-pro", + modality_contract_key: "voice_generation", + audio_path: ".lime/runtime/audio/task-audio-1.mp3", + mime_type: "audio/mpeg", + duration_ms: 1800, + audio_output: { + kind: "audio_output", + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-1.mp3", + mime_type: "audio/mpeg", + duration_ms: 1800, + source_text: "请生成温暖旁白", + voice: "warm_narrator", + provider_id: "limecore", + model: "voice-pro", + }, + }, + status: "succeeded", + normalized_status: "succeeded", + created_at: "2026-04-30T00:00:00.000Z", + result: { + kind: "audio_generation_result", + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-1.mp3", + audio_output: { + kind: "audio_output", + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-1.mp3", + mime_type: "audio/mpeg", + duration_ms: 1800, + }, + }, + }, + }; +} + +function buildFailedAudioArtifact(): MediaTaskArtifactOutput { + return { + success: true, + task_id: "task-audio-1", + task_type: "audio_generate", + task_family: "audio", + status: "failed", + normalized_status: "failed", + current_attempt_id: "attempt-1", + path: ".lime/tasks/audio_generate/task-audio-1.json", + absolute_path: "/workspace/.lime/tasks/audio_generate/task-audio-1.json", + artifact_path: ".lime/tasks/audio_generate/task-audio-1.json", + absolute_artifact_path: + "/workspace/.lime/tasks/audio_generate/task-audio-1.json", + reused_existing: false, + record: { + task_id: "task-audio-1", + task_type: "audio_generate", + task_family: "audio", + payload: { + source_text: "请生成温暖旁白", + voice: "warm_narrator", + provider_id: "missing-provider", + model: "voice-pro", + modality_contract_key: "voice_generation", + audio_output: { + kind: "audio_output", + status: "failed", + mime_type: "audio/mpeg", + source_text: "请生成温暖旁白", + voice: "warm_narrator", + provider_id: "missing-provider", + model: "voice-pro", + error_code: "audio_provider_unconfigured", + error_message: + "未找到可用的 voice_generation provider/API Key: missing-provider。", + retryable: true, + stage: "provider_config", + }, + }, + status: "failed", + normalized_status: "failed", + created_at: "2026-04-30T00:00:00.000Z", + last_error: { + code: "audio_provider_unconfigured", + message: + "未找到可用的 voice_generation provider/API Key: missing-provider。", + retryable: true, + stage: "provider_config", + }, + progress: { + phase: "failed", + message: + "未找到可用的 voice_generation provider/API Key: missing-provider。", + }, + }, + }; +} + +function buildEmptyAudioTaskIndex(): ListMediaTaskArtifactsOutput { + return { + success: true, + workspace_root: "/workspace", + artifact_root: "/workspace/.lime/tasks", + filters: { + task_family: "audio", + task_type: "audio_generate", + modality_contract_key: "voice_generation", + limit: 24, + }, + total: 0, + modality_runtime_contracts: { + snapshot_count: 0, + contract_keys: [], + blocked_count: 0, + routing_outcomes: [], + model_registry_assessment_count: 0, + audio_output_count: 0, + audio_output_statuses: [], + audio_output_error_codes: [], + transcript_count: 0, + transcript_statuses: [], + transcript_error_codes: [], + snapshots: [], + }, + tasks: [], + }; +} + +function buildCompletedAudioTaskIndex(): ListMediaTaskArtifactsOutput { + return { + ...buildEmptyAudioTaskIndex(), + total: 1, + modality_runtime_contracts: { + snapshot_count: 1, + contract_keys: ["voice_generation"], + blocked_count: 0, + routing_outcomes: [{ outcome: "accepted", count: 1 }], + model_registry_assessment_count: 0, + audio_output_count: 1, + audio_output_statuses: [{ status: "completed", count: 1 }], + audio_output_error_codes: [], + transcript_count: 0, + transcript_statuses: [], + transcript_error_codes: [], + snapshots: [ + { + task_id: "task-audio-1", + task_type: "audio_generate", + normalized_status: "succeeded", + contract_key: "voice_generation", + routing_slot: "voice_generation_model", + provider_id: "limecore", + model: "voice-pro", + routing_event: "task_created", + routing_outcome: "accepted", + failure_code: null, + audio_output_status: "completed", + audio_output_path: ".lime/runtime/audio/task-audio-1.mp3", + audio_output_mime_type: "audio/mpeg", + audio_output_duration_ms: 1800, + audio_output_error_code: null, + audio_output_retryable: null, + }, + ], + }, + }; +} + +function buildFailedAudioTaskIndex(): ListMediaTaskArtifactsOutput { + return { + ...buildEmptyAudioTaskIndex(), + total: 1, + modality_runtime_contracts: { + snapshot_count: 1, + contract_keys: ["voice_generation"], + blocked_count: 0, + routing_outcomes: [{ outcome: "failed", count: 1 }], + model_registry_assessment_count: 0, + audio_output_count: 1, + audio_output_statuses: [{ status: "failed", count: 1 }], + audio_output_error_codes: ["audio_provider_unconfigured"], + transcript_count: 0, + transcript_statuses: [], + transcript_error_codes: [], + snapshots: [ + { + task_id: "task-audio-1", + task_type: "audio_generate", + normalized_status: "failed", + contract_key: "voice_generation", + routing_slot: "voice_generation_model", + provider_id: "missing-provider", + model: "voice-pro", + routing_event: "task_created", + routing_outcome: "failed", + failure_code: "audio_provider_unconfigured", + audio_output_status: "failed", + audio_output_path: null, + audio_output_mime_type: null, + audio_output_duration_ms: null, + audio_output_error_code: "audio_provider_unconfigured", + audio_output_retryable: true, + }, + ], + }, + }; +} + +function renderHook(props?: Partial) { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + let latestMessages: Message[] = props?.messages ?? [ + buildRunningAudioMessage(), + ]; + + function Probe(currentProps: Partial) { + const [messages, setMessages] = useState( + currentProps.messages ?? [buildRunningAudioMessage()], + ); + latestMessages = messages; + useWorkspaceAudioTaskPreviewRuntime({ + projectRootPath: "/workspace", + ...currentProps, + messages, + setChatMessages: setMessages, + }); + return null; + } + + const render = async (nextProps?: Partial) => { + await act(async () => { + root.render(); + await Promise.resolve(); + }); + }; + + mountedRoots.push({ container, root }); + return { + render, + getMessages: () => latestMessages, + }; +} + +beforeEach(() => { + ( + globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; + } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.mocked(getMediaTaskArtifact).mockReset(); + vi.mocked(listMediaTaskArtifacts).mockReset(); + vi.mocked(listMediaTaskArtifacts).mockResolvedValue( + buildEmptyAudioTaskIndex(), + ); + vi.mocked(safeListen).mockReset(); + vi.mocked(safeListen).mockResolvedValue(vi.fn()); +}); + +afterEach(() => { + while (mountedRoots.length > 0) { + const mounted = mountedRoots.pop(); + if (!mounted) { + break; + } + act(() => { + mounted.root.unmount(); + }); + mounted.container.remove(); + } + vi.restoreAllMocks(); +}); + +describe("useWorkspaceAudioTaskPreviewRuntime", () => { + it("应优先从统一媒体任务索引恢复 audio_output 完成态而不读取隐藏 task JSON", async () => { + vi.mocked(listMediaTaskArtifacts).mockResolvedValueOnce( + buildCompletedAudioTaskIndex(), + ); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(listMediaTaskArtifacts).toHaveBeenCalledWith({ + projectRootPath: "/workspace", + taskFamily: "audio", + taskType: "audio_generate", + modalityContractKey: "voice_generation", + limit: 24, + }); + }); + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "audio_generate", + status: "complete", + audioUrl: ".lime/runtime/audio/task-audio-1.mp3", + mimeType: "audio/mpeg", + durationMs: 1800, + providerId: "limecore", + model: "voice-pro", + statusMessage: + "音频结果已同步，工作区已从 audio_output 读取可播放结果。", + }); + }); + expect(getMediaTaskArtifact).not.toHaveBeenCalled(); + }); + + it("应优先从统一媒体任务索引恢复 provider 失败且不保留旧音频路径", async () => { + vi.mocked(listMediaTaskArtifacts).mockResolvedValueOnce( + buildFailedAudioTaskIndex(), + ); + const runningWithStaleAudioPath = buildRunningAudioMessage(); + if (runningWithStaleAudioPath.taskPreview?.kind === "audio_generate") { + runningWithStaleAudioPath.taskPreview.audioUrl = + ".lime/runtime/audio/stale.mp3"; + } + const { render, getMessages } = renderHook({ + messages: [runningWithStaleAudioPath], + }); + + await render(); + + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "audio_generate", + status: "failed", + audioUrl: null, + providerId: "missing-provider", + model: "voice-pro", + errorCode: "audio_provider_unconfigured", + retryable: true, + statusMessage: + "配音 Provider 未配置，请先在语音生成设置中选择可用 Provider；任务保留在 audio_generate，不会回退 legacy TTS。", + }); + }); + expect(getMediaTaskArtifact).not.toHaveBeenCalled(); + }); + + it("应从完成态 audio_generate task artifact 恢复可播放音频预览", async () => { + vi.mocked(getMediaTaskArtifact).mockResolvedValueOnce( + buildCompletedAudioArtifact(), + ); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(getMediaTaskArtifact).toHaveBeenCalledWith({ + projectRootPath: "/workspace", + taskRef: ".lime/tasks/audio_generate/task-audio-1.json", + }); + }); + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "audio_generate", + status: "complete", + audioUrl: ".lime/runtime/audio/task-audio-1.mp3", + mimeType: "audio/mpeg", + durationMs: 1800, + providerId: "limecore", + model: "voice-pro", + statusMessage: + "音频结果已同步，工作区已从 audio_output 读取可播放结果。", + }); + }); + expect(getMessages()[0]?.artifacts?.[0]).toMatchObject({ + title: "task-audio-1.md", + status: "complete", + meta: { + taskId: "task-audio-1", + taskType: "audio_generate", + audioUrl: ".lime/runtime/audio/task-audio-1.mp3", + artifactDocument: { + status: "ready", + metadata: { + audioUrl: ".lime/runtime/audio/task-audio-1.mp3", + }, + }, + }, + }); + }); + + it("应从失败态 audio_generate task artifact 回流 provider 错误且不伪造音频路径", async () => { + vi.mocked(getMediaTaskArtifact).mockResolvedValueOnce( + buildFailedAudioArtifact(), + ); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "audio_generate", + status: "failed", + audioUrl: null, + errorCode: "audio_provider_unconfigured", + errorMessage: + "未找到可用的 voice_generation provider/API Key: missing-provider。", + retryable: true, + statusMessage: + "配音 Provider 未配置，请先在语音生成设置中选择可用 Provider；任务保留在 audio_generate，不会回退 legacy TTS。", + }); + }); + expect(getMessages()[0]?.artifacts?.[0]).toMatchObject({ + title: "task-audio-1.md", + status: "error", + meta: { + taskId: "task-audio-1", + taskType: "audio_generate", + audioUrl: null, + errorCode: "audio_provider_unconfigured", + artifactDocument: { + status: "failed", + metadata: { + audioUrl: null, + errorCode: "audio_provider_unconfigured", + }, + }, + }, + }); + expect( + JSON.stringify(getMessages()[0]?.artifacts?.[0]?.meta.artifactDocument), + ).toContain("不会回退 legacy TTS"); + }); +}); diff --git a/src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.ts b/src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.ts new file mode 100644 index 000000000..cc1b05fad --- /dev/null +++ b/src/components/agent/chat/workspace/useWorkspaceAudioTaskPreviewRuntime.ts @@ -0,0 +1,740 @@ +import { useEffect, useRef, type Dispatch, type SetStateAction } from "react"; +import { + getMediaTaskArtifact, + listMediaTaskArtifacts, + type ListMediaTaskArtifactsOutput, + type MediaTaskModalityRuntimeContractIndexEntry, + type MediaTaskArtifactOutput, + type MediaTaskLookupRequest, +} from "@/lib/api/mediaTasks"; +import { resolveArtifactProtocolFilePath } from "@/lib/artifact-protocol"; +import { safeListen } from "@/lib/dev-bridge"; +import type { Message, MessageGenericTaskPreview } from "../types"; +import { + buildArtifactFromWrite, + upsertMessageArtifact, +} from "../utils/messageArtifacts"; +import { buildAudioTaskArtifactDocument } from "../utils/taskPreviewFromToolResult"; +import { buildImageTaskLookupRequest } from "./imageTaskLocator"; +import { doesWorkspaceFileCandidateMatch } from "./workspaceFilePathMatch"; + +const AUDIO_TASK_EVENT_NAME = "lime://creation_task_submitted"; +const AUDIO_TASK_POLL_INTERVAL_MS = 3000; +const AUDIO_TASK_INDEX_RESTORE_LIMIT = 24; +const VOICE_GENERATION_CONTRACT_KEY = "voice_generation"; + +interface UseWorkspaceAudioTaskPreviewRuntimeParams { + projectRootPath?: string | null; + messages: Message[]; + setChatMessages: Dispatch>; +} + +interface CreationTaskSubmittedPayload { + task_id?: string; + task_type?: string; + task_family?: string; + status?: string; + path?: string; + absolute_path?: string; +} + +interface TrackedAudioTask { + taskId: string; + taskFilePath?: string | null; + artifactPath?: string | null; +} + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function readString( + candidates: Array | null | undefined>, + keys: string[], +): string | null { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + } + } + return null; +} + +function readPositiveNumber( + candidates: Array | null | undefined>, + keys: string[], +): number | null { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (typeof value === "number" && Number.isFinite(value) && value > 0) { + return value; + } + if (typeof value === "string" && value.trim()) { + const parsed = Number(value); + if (Number.isFinite(parsed) && parsed > 0) { + return parsed; + } + } + } + } + return null; +} + +function readBoolean( + candidates: Array | null | undefined>, + keys: string[], +): boolean | null { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (typeof value === "boolean") { + return value; + } + } + } + return null; +} + +function normalizePreviewStatus(status: string | undefined | null) { + switch ((status || "").trim().toLowerCase()) { + case "succeeded": + case "completed": + case "success": + return "complete" as const; + case "partial": + return "partial" as const; + case "failed": + case "error": + return "failed" as const; + case "cancelled": + case "canceled": + return "cancelled" as const; + case "running": + case "processing": + case "queued": + case "pending": + case "pending_submit": + default: + return "running" as const; + } +} + +function resolveAudioFailureStatusMessage( + errorCode?: string | null, + errorMessage?: string | null, +): string { + switch ((errorCode || "").trim()) { + case "audio_provider_unconfigured": + return "配音 Provider 未配置，请先在语音生成设置中选择可用 Provider；任务保留在 audio_generate，不会回退 legacy TTS。"; + case "audio_provider_model_unconfigured": + return "配音模型未配置，请先为语音生成设置默认模型；任务保留在 audio_generate，不会回退 legacy TTS。"; + case "audio_provider_resolver_unavailable": + return "当前无法解析 API Key Provider 凭证，配音任务已显式失败，没有伪造音频路径。"; + case "audio_provider_resolution_failed": + return "解析配音 Provider 凭证失败，请检查 API Key Provider 配置后重试。"; + case "audio_provider_client_missing": + return "当前 Provider 还没有 OpenAI-compatible speech adapter，配音任务不会回退旧 TTS。"; + case "audio_provider_request_failed": + return "调用配音 Provider 失败，请检查额度、网络或模型配置后重试。"; + case "audio_provider_response_read_failed": + return "读取配音 Provider 响应失败，请稍后重试或检查 Provider 网关。"; + case "audio_provider_empty_response": + return "配音 Provider 返回了空音频内容，已阻止写入空 audio_output。"; + case "audio_output_write_failed": + return "配音音频已返回但写入工作区失败，请检查项目目录权限后重试。"; + case "invalid_audio_task_payload": + return "配音任务缺少必要文本，已阻止调用音频 Provider。"; + case "voice_generation_contract_mismatch": + case "voice_generation_capability_gap": + case "voice_generation_routing_slot_mismatch": + case "voice_generation_task_type_mismatch": + return "配音任务运行合同不匹配，已阻止执行器误跑到非 voice_generation 主链。"; + default: + return ( + errorMessage?.trim() || "配音生成失败，请调整文本、音色或模型后重试。" + ); + } +} + +function resolveStatusMessage(preview: MessageGenericTaskPreview): string { + if (preview.status === "complete" || preview.status === "partial") { + return preview.audioUrl?.trim() + ? "音频结果已同步，工作区已从 audio_output 读取可播放结果。" + : "配音任务已完成，正在等待 audio_output 音频路径。"; + } + if (preview.status === "failed") { + return resolveAudioFailureStatusMessage( + preview.errorCode, + preview.errorMessage, + ); + } + if (preview.status === "cancelled") { + return "配音任务已经取消，当前不会继续生成音频。"; + } + return "配音任务已写入统一 audio_task/audio_output 协议，工作区会继续同步结果。"; +} + +function buildAudioPreviewFromArtifact( + artifact: MediaTaskArtifactOutput, + currentPreview: MessageGenericTaskPreview, +): MessageGenericTaskPreview | null { + if (artifact.task_type !== "audio_generate") { + return null; + } + + const payload = asRecord(artifact.record?.payload); + const result = asRecord(artifact.record?.result); + const lastError = asRecord(artifact.record?.last_error); + const progress = asRecord(artifact.record?.progress); + const audioOutput = + asRecord(payload?.audio_output) || asRecord(result?.audio_output); + const candidates = [audioOutput, result, payload]; + const errorCandidates = [audioOutput, lastError, progress, result, payload]; + const sourceText = + readString(candidates, ["source_text", "sourceText", "prompt"]) || + currentPreview.sourceText || + currentPreview.prompt; + const artifactAudioUrl = readString(candidates, [ + "audio_path", + "audioPath", + "audio_url", + "audioUrl", + "url", + "result_url", + "resultUrl", + ]); + const nextStatus = normalizePreviewStatus( + artifact.normalized_status || artifact.status || artifact.record?.status, + ); + const audioUrl = + nextStatus === "failed" + ? artifactAudioUrl + : artifactAudioUrl || currentPreview.audioUrl; + const errorCode = + nextStatus === "failed" + ? readString(errorCandidates, [ + "error_code", + "errorCode", + "failure_code", + "failureCode", + "code", + ]) || + currentPreview.errorCode || + null + : null; + const errorMessage = + nextStatus === "failed" + ? readString(errorCandidates, [ + "error_message", + "errorMessage", + "message", + "detail", + ]) || + currentPreview.errorMessage || + null + : null; + const retryable = + nextStatus === "failed" + ? (readBoolean(errorCandidates, ["retryable"]) ?? + currentPreview.retryable) + : currentPreview.retryable; + const nextPreview: MessageGenericTaskPreview = { + ...currentPreview, + kind: "audio_generate", + taskType: "audio_generate", + taskId: artifact.task_id || currentPreview.taskId, + prompt: sourceText, + status: nextStatus, + projectId: + readString(candidates, ["project_id", "projectId"]) ?? + currentPreview.projectId ?? + null, + contentId: + readString(candidates, ["content_id", "contentId"]) ?? + currentPreview.contentId ?? + null, + taskFilePath: + currentPreview.taskFilePath || + artifact.artifact_path || + artifact.path || + null, + providerId: + readString(candidates, ["provider_id", "providerId", "provider"]) ?? + currentPreview.providerId ?? + null, + model: readString(candidates, ["model"]) ?? currentPreview.model ?? null, + phase: + readString([asRecord(artifact.record?.progress)], ["phase"]) ?? + currentPreview.phase ?? + null, + audioUrl: audioUrl ?? null, + mimeType: + readString(candidates, ["mime_type", "mimeType"]) ?? + currentPreview.mimeType ?? + null, + durationMs: + readPositiveNumber(candidates, ["duration_ms", "durationMs"]) ?? + currentPreview.durationMs ?? + null, + sourceText, + voice: readString(candidates, ["voice"]) ?? currentPreview.voice ?? null, + errorCode, + errorMessage, + retryable, + }; + return { + ...nextPreview, + statusMessage: resolveStatusMessage(nextPreview), + }; +} + +function hasUsableAudioOutputIndex( + entry: MediaTaskModalityRuntimeContractIndexEntry, +): boolean { + if (entry.task_type !== "audio_generate") { + return false; + } + return Boolean(entry.audio_output_status); +} + +function buildAudioPreviewFromIndexEntry( + entry: MediaTaskModalityRuntimeContractIndexEntry, + currentPreview: MessageGenericTaskPreview, +): MessageGenericTaskPreview | null { + if (!hasUsableAudioOutputIndex(entry)) { + return null; + } + + const nextStatus = normalizePreviewStatus( + entry.audio_output_status || entry.normalized_status, + ); + const indexedAudioPath = entry.audio_output_path?.trim() || null; + const audioUrl = + nextStatus === "failed" + ? indexedAudioPath + : indexedAudioPath || currentPreview.audioUrl || null; + const errorCode = + nextStatus === "failed" + ? entry.audio_output_error_code?.trim() || + currentPreview.errorCode || + null + : null; + const errorMessage = + nextStatus === "failed" && errorCode === currentPreview.errorCode + ? currentPreview.errorMessage || null + : null; + const retryable = + nextStatus === "failed" + ? (entry.audio_output_retryable ?? currentPreview.retryable) + : currentPreview.retryable; + const nextPreview: MessageGenericTaskPreview = { + ...currentPreview, + kind: "audio_generate", + taskType: "audio_generate", + taskId: entry.task_id || currentPreview.taskId, + status: nextStatus, + providerId: entry.provider_id?.trim() || currentPreview.providerId || null, + model: entry.model?.trim() || currentPreview.model || null, + audioUrl, + mimeType: + entry.audio_output_mime_type?.trim() || currentPreview.mimeType || null, + durationMs: + typeof entry.audio_output_duration_ms === "number" && + Number.isFinite(entry.audio_output_duration_ms) && + entry.audio_output_duration_ms > 0 + ? entry.audio_output_duration_ms + : currentPreview.durationMs || null, + errorCode, + errorMessage, + retryable, + }; + return { + ...nextPreview, + statusMessage: resolveStatusMessage(nextPreview), + }; +} + +function areAudioPreviewsEqual( + left: MessageGenericTaskPreview, + right: MessageGenericTaskPreview, +): boolean { + return ( + left.status === right.status && + left.audioUrl === right.audioUrl && + left.mimeType === right.mimeType && + left.durationMs === right.durationMs && + left.sourceText === right.sourceText && + left.voice === right.voice && + left.providerId === right.providerId && + left.model === right.model && + left.phase === right.phase && + left.statusMessage === right.statusMessage && + left.errorCode === right.errorCode && + left.errorMessage === right.errorMessage && + left.retryable === right.retryable + ); +} + +function findAudioTaskViewerArtifact( + message: Message, + preview: MessageGenericTaskPreview, +) { + const artifacts = message.artifacts || []; + const artifactPath = preview.artifactPath?.trim(); + return artifacts.find((artifact) => { + const meta = artifact.meta || {}; + if ( + typeof meta.taskId === "string" && + meta.taskId === preview.taskId && + meta.taskType === "audio_generate" + ) { + return true; + } + return artifactPath + ? doesWorkspaceFileCandidateMatch( + resolveArtifactProtocolFilePath(artifact), + artifactPath, + ) + : false; + }); +} + +function syncAudioTaskViewerArtifact( + message: Message, + preview: MessageGenericTaskPreview, +): Message { + const artifactPath = preview.artifactPath?.trim(); + if (!artifactPath) { + return message; + } + + const artifactDocument = buildAudioTaskArtifactDocument(preview); + const existingArtifact = findAudioTaskViewerArtifact(message, preview); + const nextArtifact = buildArtifactFromWrite({ + filePath: artifactPath, + content: JSON.stringify(artifactDocument, null, 2), + context: { + artifact: existingArtifact, + artifactId: + existingArtifact?.id || `artifact:${message.id}:${artifactPath}`, + source: "tool_result", + sourceMessageId: message.id, + status: + preview.status === "failed" + ? "error" + : preview.status === "running" + ? "streaming" + : "complete", + metadata: { + artifactDocument, + artifact_type: "document", + previewText: preview.statusMessage || "配音任务已写入统一任务产物协议", + taskId: preview.taskId, + taskType: "audio_generate", + taskFilePath: preview.taskFilePath || null, + audioUrl: preview.audioUrl || null, + mimeType: preview.mimeType || null, + durationMs: preview.durationMs || null, + voice: preview.voice || null, + providerId: preview.providerId || null, + model: preview.model || null, + errorCode: preview.errorCode || null, + errorMessage: preview.errorMessage || null, + modalityContractKey: "voice_generation", + }, + }, + }); + + return upsertMessageArtifact(message, nextArtifact); +} + +function shouldTrackAudioPreview(preview: MessageGenericTaskPreview): boolean { + if (preview.kind !== "audio_generate") { + return false; + } + if (preview.status === "running") { + return true; + } + return ( + (preview.status === "complete" || preview.status === "partial") && + !preview.audioUrl?.trim() + ); +} + +function collectTrackedAudioTasks(messages: Message[]): TrackedAudioTask[] { + const tasks = new Map(); + messages.forEach((message) => { + const preview = message.taskPreview; + if (!preview || preview.kind !== "audio_generate") { + return; + } + if (!shouldTrackAudioPreview(preview)) { + return; + } + const taskId = preview.taskId.trim(); + if (!taskId || tasks.has(taskId)) { + return; + } + tasks.set(taskId, { + taskId, + taskFilePath: preview.taskFilePath, + artifactPath: preview.artifactPath, + }); + }); + return Array.from(tasks.values()); +} + +function buildLookupRequest(params: { + task: TrackedAudioTask; + projectRootPath?: string | null; +}): MediaTaskLookupRequest | null { + return buildImageTaskLookupRequest({ + taskId: params.task.taskId, + taskFilePath: params.task.taskFilePath, + artifactPath: params.task.artifactPath, + projectRootPath: params.projectRootPath, + }); +} + +function updateAudioPreviewMessages(params: { + previous: Message[]; + artifact: MediaTaskArtifactOutput; +}): Message[] { + let changed = false; + const nextMessages = params.previous.map((message) => { + const currentPreview = message.taskPreview; + if ( + !currentPreview || + currentPreview.kind !== "audio_generate" || + currentPreview.taskId !== params.artifact.task_id + ) { + return message; + } + const nextPreview = buildAudioPreviewFromArtifact( + params.artifact, + currentPreview, + ); + if (!nextPreview || areAudioPreviewsEqual(currentPreview, nextPreview)) { + return message; + } + changed = true; + return syncAudioTaskViewerArtifact( + { + ...message, + taskPreview: nextPreview, + }, + nextPreview, + ); + }); + return changed ? nextMessages : params.previous; +} + +function updateAudioPreviewMessagesFromIndex(params: { + previous: Message[]; + snapshotsByTaskId: Map; +}): Message[] { + let changed = false; + const nextMessages = params.previous.map((message) => { + const currentPreview = message.taskPreview; + if (!currentPreview || currentPreview.kind !== "audio_generate") { + return message; + } + const snapshot = params.snapshotsByTaskId.get(currentPreview.taskId); + if (!snapshot) { + return message; + } + const nextPreview = buildAudioPreviewFromIndexEntry( + snapshot, + currentPreview, + ); + if (!nextPreview || areAudioPreviewsEqual(currentPreview, nextPreview)) { + return message; + } + changed = true; + return syncAudioTaskViewerArtifact( + { + ...message, + taskPreview: nextPreview, + }, + nextPreview, + ); + }); + return changed ? nextMessages : params.previous; +} + +export function useWorkspaceAudioTaskPreviewRuntime({ + projectRootPath, + messages, + setChatMessages, +}: UseWorkspaceAudioTaskPreviewRuntimeParams) { + const contextRef = useRef({ projectRootPath, messages }); + + useEffect(() => { + contextRef.current = { projectRootPath, messages }; + }, [projectRootPath, messages]); + + useEffect(() => { + let disposed = false; + let polling = false; + + const syncFromTaskIndex = async ( + tasks: TrackedAudioTask[], + ): Promise> => { + const workspaceRoot = contextRef.current.projectRootPath?.trim(); + if (!workspaceRoot || tasks.length === 0) { + return new Set(); + } + let output: ListMediaTaskArtifactsOutput; + try { + output = await listMediaTaskArtifacts({ + projectRootPath: workspaceRoot, + taskFamily: "audio", + taskType: "audio_generate", + modalityContractKey: VOICE_GENERATION_CONTRACT_KEY, + limit: Math.max(AUDIO_TASK_INDEX_RESTORE_LIMIT, tasks.length), + }); + } catch (error) { + console.warn( + "[AudioTaskPreviewRuntime] 读取音频任务索引失败，回退 task artifact:", + error, + ); + return new Set(); + } + if (disposed) { + return new Set(); + } + const trackedTaskIds = new Set(tasks.map((task) => task.taskId)); + const snapshotsByTaskId = new Map< + string, + MediaTaskModalityRuntimeContractIndexEntry + >(); + output.modality_runtime_contracts.snapshots.forEach((snapshot) => { + if ( + trackedTaskIds.has(snapshot.task_id) && + hasUsableAudioOutputIndex(snapshot) + ) { + snapshotsByTaskId.set(snapshot.task_id, snapshot); + } + }); + if (snapshotsByTaskId.size === 0) { + return new Set(); + } + setChatMessages((previous) => + updateAudioPreviewMessagesFromIndex({ + previous, + snapshotsByTaskId, + }), + ); + return new Set(snapshotsByTaskId.keys()); + }; + + const syncTask = async (task: TrackedAudioTask) => { + const request = buildLookupRequest({ + task, + projectRootPath: contextRef.current.projectRootPath, + }); + if (!request) { + return; + } + const artifact = await getMediaTaskArtifact(request); + if (disposed || artifact.task_type !== "audio_generate") { + return; + } + setChatMessages((previous) => + updateAudioPreviewMessages({ previous, artifact }), + ); + }; + + const syncOnce = async () => { + if (disposed || polling) { + return; + } + const tasks = collectTrackedAudioTasks(contextRef.current.messages); + if (tasks.length === 0) { + return; + } + polling = true; + try { + const indexedTaskIds = await syncFromTaskIndex(tasks); + await Promise.all( + tasks + .filter((task) => !indexedTaskIds.has(task.taskId)) + .map((task) => syncTask(task)), + ); + } catch (error) { + console.warn("[AudioTaskPreviewRuntime] 同步音频任务状态失败:", error); + } finally { + polling = false; + } + }; + + void syncOnce(); + const timerId = window.setInterval(() => { + void syncOnce(); + }, AUDIO_TASK_POLL_INTERVAL_MS); + + let unlisten: (() => void) | null = null; + safeListen(AUDIO_TASK_EVENT_NAME, (event) => { + if (disposed) { + return; + } + const payload = event.payload || {}; + const taskId = payload.task_id?.trim(); + const taskType = payload.task_type?.trim(); + const taskFamily = payload.task_family?.trim(); + if ( + !taskId || + (taskType !== "audio_generate" && taskFamily !== "audio") + ) { + return; + } + const tracked = collectTrackedAudioTasks( + contextRef.current.messages, + ).find((task) => task.taskId === taskId); + if (!tracked) { + return; + } + void syncTask({ + ...tracked, + taskFilePath: + payload.absolute_path?.trim() || + payload.path?.trim() || + tracked.taskFilePath, + }); + }) + .then((dispose) => { + if (disposed) { + void dispose(); + return; + } + unlisten = dispose; + }) + .catch((error) => { + console.warn("[AudioTaskPreviewRuntime] 监听音频任务事件失败:", error); + }); + + return () => { + disposed = true; + window.clearInterval(timerId); + if (unlisten) { + unlisten(); + } + }; + }, [setChatMessages]); +} diff --git a/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts b/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts index b62a3cd12..2693bb144 100644 --- a/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts +++ b/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts @@ -216,6 +216,38 @@ function getRenderedSceneProps(params: ReturnType) { return (getValue().mainAreaNode as any).props; } +function buildHeavySessionRuntimeFixture(sessionId = "session") { + const messages = Array.from({ length: 24 }, (_, index) => ({ + id: `${sessionId}-msg-${index}`, + role: index % 2 === 0 ? "user" : "assistant", + content: `${sessionId} 消息 ${index}`, + timestamp: new Date(2026, 3, 30, 10, index), + })); + const turns = Array.from({ length: 6 }, (_, index) => ({ + id: `${sessionId}-turn-${index}`, + thread_id: `${sessionId}-thread`, + prompt_text: `${sessionId} 任务 ${index}`, + status: "completed", + started_at: `2026-04-30T10:0${index}:00.000Z`, + created_at: `2026-04-30T10:0${index}:00.000Z`, + updated_at: `2026-04-30T10:0${index}:01.000Z`, + })); + const threadItems = Array.from({ length: 28 }, (_, index) => ({ + id: `${sessionId}-item-${index}`, + thread_id: `${sessionId}-thread`, + turn_id: `${sessionId}-turn-${Math.min(5, Math.floor(index / 5))}`, + sequence: index + 1, + status: "completed", + started_at: `2026-04-30T10:00:${String(index).padStart(2, "0")}.000Z`, + updated_at: `2026-04-30T10:00:${String(index).padStart(2, "0")}.500Z`, + type: "tool_call", + tool_name: "Read", + arguments: { index }, + })); + + return { messages, turns, threadItems }; +} + beforeEach(() => { ( globalThis as typeof globalThis & { @@ -388,6 +420,240 @@ describe("useWorkspaceConversationSceneRuntime", () => { expect(typeof sessionView?.renderPanel).toBe("function"); }); + it("恢复旧会话首帧应先透传消息，并延迟运行轨迹投影", () => { + vi.useFakeTimers(); + const { messages, turns, threadItems } = + buildHeavySessionRuntimeFixture("restore"); + const params = createBaseParams({ + displayMessages: messages, + turns, + currentTurnId: "restore-turn-5", + effectiveThreadItems: threadItems, + pendingActions: [ + { + requestId: "req-1", + actionType: "elicitation", + prompt: "补充信息", + status: "pending", + }, + ], + queuedTurns: [ + { + queued_turn_id: "queued-1", + message_preview: "继续处理", + message_text: "继续处理", + created_at: 1_777_520_000, + image_count: 0, + position: 1, + }, + ], + isAutoRestoringSession: true, + }); + + const harness = renderHook(params); + let sceneProps = (harness.getValue().mainAreaNode as any).props; + + expect(sceneProps.messageListProps.messages).toBe(messages); + expect(sceneProps.messageListProps.turns).toEqual([]); + expect(sceneProps.messageListProps.threadItems).toEqual([]); + expect(sceneProps.messageListProps.currentTurnId).toBeNull(); + expect(sceneProps.messageListProps.pendingActions).toEqual([]); + expect(sceneProps.messageListProps.queuedTurns).toEqual([]); + expect( + sceneProps.canvasWorkbenchLayoutProps.sessionView.summaryStats, + ).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + key: "session-runtime-items", + value: "轨迹 0", + }), + ]), + ); + + act(() => { + vi.advanceTimersByTime(700); + }); + + sceneProps = (harness.getValue().mainAreaNode as any).props; + expect(sceneProps.messageListProps.turns).toBe(turns); + expect(sceneProps.messageListProps.threadItems).toBe(threadItems); + expect(sceneProps.messageListProps.currentTurnId).toBe("restore-turn-5"); + expect(sceneProps.messageListProps.pendingActions).toHaveLength(1); + expect(sceneProps.messageListProps.queuedTurns).toHaveLength(1); + vi.useRealTimers(); + }); + + it("历史窗口 hydrate 完成后仍应先透传消息，并延迟运行轨迹投影", () => { + vi.useFakeTimers(); + const { messages, turns, threadItems } = + buildHeavySessionRuntimeFixture("history-window"); + const params = createBaseParams({ + displayMessages: messages, + turns, + currentTurnId: "history-window-turn-5", + effectiveThreadItems: threadItems, + isAutoRestoringSession: false, + sessionHistoryWindow: { + loadedMessages: 40, + totalMessages: 320, + isLoadingFull: false, + error: null, + }, + }); + + const harness = renderHook(params); + let sceneProps = (harness.getValue().mainAreaNode as any).props; + + expect(sceneProps.messageListProps.messages).toBe(messages); + expect(sceneProps.messageListProps.turns).toEqual([]); + expect(sceneProps.messageListProps.threadItems).toEqual([]); + expect(sceneProps.messageListProps.currentTurnId).toBeNull(); + + act(() => { + vi.advanceTimersByTime(700); + }); + + sceneProps = (harness.getValue().mainAreaNode as any).props; + expect(sceneProps.messageListProps.turns).toBe(turns); + expect(sceneProps.messageListProps.threadItems).toBe(threadItems); + expect(sceneProps.messageListProps.currentTurnId).toBe( + "history-window-turn-5", + ); + vi.useRealTimers(); + }); + + it("发送中会话不应延迟运行轨迹投影", () => { + vi.useFakeTimers(); + const { messages, turns, threadItems } = + buildHeavySessionRuntimeFixture("sending"); + const params = createBaseParams({ + displayMessages: messages, + turns, + currentTurnId: "sending-turn-5", + effectiveThreadItems: threadItems, + isAutoRestoringSession: true, + isSending: true, + }); + + const sceneProps = getRenderedSceneProps(params); + expect(sceneProps.messageListProps.turns).toBe(turns); + expect(sceneProps.messageListProps.threadItems).toBe(threadItems); + expect(sceneProps.messageListProps.currentTurnId).toBe("sending-turn-5"); + vi.useRealTimers(); + }); + + it("聚焦 timeline 或存在 A2UI 表单时不应延迟运行轨迹投影", () => { + const { messages, turns, threadItems } = + buildHeavySessionRuntimeFixture("interactive"); + const focusedSceneProps = getRenderedSceneProps( + createBaseParams({ + displayMessages: messages, + turns, + currentTurnId: "interactive-turn-5", + effectiveThreadItems: threadItems, + isAutoRestoringSession: true, + focusedTimelineItemId: "interactive-item-1", + }), + ); + expect(focusedSceneProps.messageListProps.turns).toBe(turns); + expect(focusedSceneProps.messageListProps.threadItems).toBe(threadItems); + + const pendingA2UISceneProps = getRenderedSceneProps( + createBaseParams({ + displayMessages: messages, + turns, + currentTurnId: "interactive-turn-5", + effectiveThreadItems: threadItems, + isAutoRestoringSession: true, + pendingA2UIForm: { + id: "form-1", + title: "补充信息", + schema: {}, + }, + }), + ); + expect(pendingA2UISceneProps.messageListProps.turns).toBe(turns); + expect(pendingA2UISceneProps.messageListProps.threadItems).toBe( + threadItems, + ); + }); + + it("切换到另一条同长度旧会话时应重新延迟运行轨迹投影", () => { + vi.useFakeTimers(); + const buildSession = (sessionId: string) => { + const messages = Array.from({ length: 24 }, (_, index) => ({ + id: `${sessionId}-msg-${index}`, + role: index % 2 === 0 ? "user" : "assistant", + content: `${sessionId} 消息 ${index}`, + timestamp: new Date(2026, 3, 30, 11, index), + })); + const turns = Array.from({ length: 6 }, (_, index) => ({ + id: `${sessionId}-turn-${index}`, + thread_id: `${sessionId}-thread`, + prompt_text: `${sessionId} 任务 ${index}`, + status: "completed", + started_at: `2026-04-30T11:0${index}:00.000Z`, + created_at: `2026-04-30T11:0${index}:00.000Z`, + updated_at: `2026-04-30T11:0${index}:01.000Z`, + })); + const threadItems = Array.from({ length: 28 }, (_, index) => ({ + id: `${sessionId}-item-${index}`, + thread_id: `${sessionId}-thread`, + turn_id: `${sessionId}-turn-${Math.min(5, Math.floor(index / 5))}`, + sequence: index + 1, + status: "completed", + started_at: `2026-04-30T11:00:${String(index).padStart(2, "0")}.000Z`, + updated_at: `2026-04-30T11:00:${String(index).padStart(2, "0")}.500Z`, + type: "tool_call", + tool_name: "Read", + arguments: { index }, + })); + + return { messages, turns, threadItems }; + }; + const sessionA = buildSession("session-a"); + const sessionB = buildSession("session-b"); + const buildParams = ( + sessionId: string, + session: ReturnType, + ) => + createBaseParams({ + sessionId, + displayMessages: session.messages, + turns: session.turns, + currentTurnId: session.turns.at(-1)?.id ?? null, + effectiveThreadItems: session.threadItems, + isAutoRestoringSession: true, + }); + + const harness = renderHook(buildParams("session-a", sessionA)); + act(() => { + vi.advanceTimersByTime(700); + }); + + let sceneProps = (harness.getValue().mainAreaNode as any).props; + expect(sceneProps.messageListProps.turns).toBe(sessionA.turns); + expect(sceneProps.messageListProps.threadItems).toBe(sessionA.threadItems); + + harness.render(buildParams("session-b", sessionB)); + sceneProps = (harness.getValue().mainAreaNode as any).props; + + expect(sceneProps.messageListProps.messages).toBe(sessionB.messages); + expect(sceneProps.messageListProps.turns).toEqual([]); + expect(sceneProps.messageListProps.threadItems).toEqual([]); + expect(sceneProps.messageListProps.currentTurnId).toBeNull(); + + act(() => { + vi.advanceTimersByTime(700); + }); + + sceneProps = (harness.getValue().mainAreaNode as any).props; + expect(sceneProps.messageListProps.turns).toBe(sessionB.turns); + expect(sceneProps.messageListProps.threadItems).toBe(sessionB.threadItems); + expect(sceneProps.messageListProps.currentTurnId).toBe("session-b-turn-5"); + vi.useRealTimers(); + }); + it("应向画布壳透传 workspaceView 头部语义", () => { const params = createBaseParams({ settledWorkbenchArtifacts: [{ id: "artifact-1" }, { id: "artifact-2" }], diff --git a/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.tsx b/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.tsx index a24ba4a43..058b8d39c 100644 --- a/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.tsx +++ b/src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.tsx @@ -4,11 +4,12 @@ import type { ReactNode, SetStateAction, } from "react"; -import { useMemo } from "react"; +import { useEffect, useMemo, useState } from "react"; import { StepProgress } from "@/lib/workspace/workbenchUi"; import { useWorkspaceNavigationActions } from "./useWorkspaceNavigationActions"; import { useWorkspaceInputbarSceneRuntime } from "./useWorkspaceInputbarSceneRuntime"; import { useWorkspaceCanvasSceneRuntime } from "./useWorkspaceCanvasSceneRuntime"; +import { scheduleMinimumDelayIdleTask } from "@/lib/utils/scheduleMinimumDelayIdleTask"; import { CanvasSessionOverviewPanel } from "../components/CanvasSessionOverviewPanel"; import { MessageList } from "../components/MessageList"; import { TeamWorkspaceDock } from "../components/TeamWorkspaceDock"; @@ -214,6 +215,73 @@ function resolvePathLeaf(value?: string | null): string { return segments.at(-1) || normalized; } +const SESSION_RUNTIME_PROJECTION_DEFER_MESSAGE_THRESHOLD = 20; +const SESSION_RUNTIME_PROJECTION_DEFER_TURN_THRESHOLD = 6; +const SESSION_RUNTIME_PROJECTION_DEFER_ITEM_THRESHOLD = 24; +const SESSION_RUNTIME_PROJECTION_DEFER_DELAY_MS = 700; +const SESSION_RUNTIME_PROJECTION_DEFER_IDLE_TIMEOUT_MS = 1_800; +const EMPTY_PROJECTED_TURNS: NonNullable< + ConversationScenePresentationParams["messageList"]["turns"] +> = []; +const EMPTY_PROJECTED_THREAD_ITEMS: NonNullable< + ConversationScenePresentationParams["messageList"]["threadItems"] +> = []; +const EMPTY_PROJECTED_PENDING_ACTIONS: NonNullable< + ConversationScenePresentationParams["messageList"]["pendingActions"] +> = []; +const EMPTY_PROJECTED_SUBMITTED_ACTIONS: NonNullable< + ConversationScenePresentationParams["messageList"]["submittedActionsInFlight"] +> = []; +const EMPTY_PROJECTED_QUEUED_TURNS: NonNullable< + ConversationScenePresentationParams["messageList"]["queuedTurns"] +> = []; +const EMPTY_PROJECTED_CHILD_SUBAGENT_SESSIONS: NonNullable< + ConversationScenePresentationParams["messageList"]["childSubagentSessions"] +> = []; + +interface SessionRuntimeProjectionState { + key: string; + sessionId: string; + firstMessageId: string; + lastMessageId: string; + ready: boolean; +} + +function buildSessionRuntimeProjectionState(params: { + key: string; + sessionId: string; + firstMessageId: string; + lastMessageId: string; + ready: boolean; +}): SessionRuntimeProjectionState { + return params; +} + +function resolveNextSessionRuntimeProjectionState( + current: SessionRuntimeProjectionState, + next: SessionRuntimeProjectionState, +): SessionRuntimeProjectionState { + return current.key === next.key && + current.sessionId === next.sessionId && + current.firstMessageId === next.firstMessageId && + current.lastMessageId === next.lastMessageId && + current.ready === next.ready + ? current + : next; +} + +function hasPersistedHiddenSessionHistory( + sessionHistoryWindow: + | ConversationScenePresentationParams["messageList"]["sessionHistoryWindow"] + | null + | undefined, +): boolean { + return Boolean( + sessionHistoryWindow && + sessionHistoryWindow.totalMessages > sessionHistoryWindow.loadedMessages, + ); +} + interface UseWorkspaceConversationSceneRuntimeParams { messageListEmptyStateVariant?: "default" | "task-center"; navbarContextVariant?: "default" | "task-center"; @@ -474,14 +542,14 @@ export function useWorkspaceConversationSceneRuntime({ currentStepIndex, goToStep, displayMessages, - turns = [], - effectiveThreadItems = [], + turns = EMPTY_PROJECTED_TURNS, + effectiveThreadItems = EMPTY_PROJECTED_THREAD_ITEMS, currentTurnId, threadRead, - pendingActions = [], - submittedActionsInFlight, - queuedTurns = [], - childSubagentSessions = [], + pendingActions = EMPTY_PROJECTED_PENDING_ACTIONS, + submittedActionsInFlight = EMPTY_PROJECTED_SUBMITTED_ACTIONS, + queuedTurns = EMPTY_PROJECTED_QUEUED_TURNS, + childSubagentSessions = EMPTY_PROJECTED_CHILD_SUBAGENT_SESSIONS, sessionHistoryWindow = null, loadFullSessionHistory, isPreparingSend, @@ -522,6 +590,157 @@ export function useWorkspaceConversationSceneRuntime({ focusedTimelineItemId, timelineFocusRequestKey, }: UseWorkspaceConversationSceneRuntimeParams) { + const sessionRuntimeProjectionSessionId = sessionId ?? "no-session"; + const sessionRuntimeProjectionFirstMessageId = + displayMessages[0]?.id ?? "no-first-message"; + const sessionRuntimeProjectionLastMessageId = + displayMessages[displayMessages.length - 1]?.id ?? "no-last-message"; + const sessionRuntimeProjectionLastTurnId = + turns[turns.length - 1]?.id ?? "no-last-turn"; + const sessionRuntimeProjectionLastItemId = + effectiveThreadItems[effectiveThreadItems.length - 1]?.id ?? "no-last-item"; + const sessionRuntimeProjectionKey = [ + sessionRuntimeProjectionSessionId, + sessionRuntimeProjectionFirstMessageId, + sessionRuntimeProjectionLastMessageId, + sessionRuntimeProjectionLastTurnId, + sessionRuntimeProjectionLastItemId, + ].join("|"); + const shouldTreatAsRestoredHistoryWindow = + isAutoRestoringSession || + hasPersistedHiddenSessionHistory(sessionHistoryWindow); + const hasHeavySessionRuntimeProjection = + displayMessages.length >= + SESSION_RUNTIME_PROJECTION_DEFER_MESSAGE_THRESHOLD || + turns.length >= SESSION_RUNTIME_PROJECTION_DEFER_TURN_THRESHOLD || + effectiveThreadItems.length >= + SESSION_RUNTIME_PROJECTION_DEFER_ITEM_THRESHOLD; + const shouldConsiderDeferringSessionRuntimeProjection = + shouldTreatAsRestoredHistoryWindow && + !isSending && + !focusedTimelineItemId && + !pendingA2UIForm && + hasHeavySessionRuntimeProjection; + const [sessionRuntimeProjectionState, setSessionRuntimeProjectionState] = + useState(() => + buildSessionRuntimeProjectionState({ + key: sessionRuntimeProjectionKey, + sessionId: sessionRuntimeProjectionSessionId, + firstMessageId: sessionRuntimeProjectionFirstMessageId, + lastMessageId: sessionRuntimeProjectionLastMessageId, + ready: !shouldConsiderDeferringSessionRuntimeProjection, + }), + ); + const sessionRuntimeProjectionAlreadyReady = + sessionRuntimeProjectionState.key === sessionRuntimeProjectionKey && + sessionRuntimeProjectionState.ready; + const isAppendOnlyMessageProjectionUpdate = + sessionRuntimeProjectionState.key !== sessionRuntimeProjectionKey && + sessionRuntimeProjectionState.sessionId === + sessionRuntimeProjectionSessionId && + sessionRuntimeProjectionState.firstMessageId === + sessionRuntimeProjectionFirstMessageId && + sessionRuntimeProjectionState.lastMessageId !== + sessionRuntimeProjectionLastMessageId; + const shouldDeferSessionRuntimeProjection = + shouldConsiderDeferringSessionRuntimeProjection && + !sessionRuntimeProjectionAlreadyReady && + !isAppendOnlyMessageProjectionUpdate; + const sessionRuntimeProjectionReady = + sessionRuntimeProjectionState.key === sessionRuntimeProjectionKey + ? sessionRuntimeProjectionState.ready + : !shouldDeferSessionRuntimeProjection; + + useEffect(() => { + if (!shouldDeferSessionRuntimeProjection) { + const nextState = buildSessionRuntimeProjectionState({ + key: sessionRuntimeProjectionKey, + sessionId: sessionRuntimeProjectionSessionId, + firstMessageId: sessionRuntimeProjectionFirstMessageId, + lastMessageId: sessionRuntimeProjectionLastMessageId, + ready: true, + }); + setSessionRuntimeProjectionState((current) => + resolveNextSessionRuntimeProjectionState(current, nextState), + ); + return; + } + + const pendingState = buildSessionRuntimeProjectionState({ + key: sessionRuntimeProjectionKey, + sessionId: sessionRuntimeProjectionSessionId, + firstMessageId: sessionRuntimeProjectionFirstMessageId, + lastMessageId: sessionRuntimeProjectionLastMessageId, + ready: false, + }); + setSessionRuntimeProjectionState((current) => + resolveNextSessionRuntimeProjectionState(current, pendingState), + ); + return scheduleMinimumDelayIdleTask( + () => { + const readyState = buildSessionRuntimeProjectionState({ + key: sessionRuntimeProjectionKey, + sessionId: sessionRuntimeProjectionSessionId, + firstMessageId: sessionRuntimeProjectionFirstMessageId, + lastMessageId: sessionRuntimeProjectionLastMessageId, + ready: true, + }); + setSessionRuntimeProjectionState((current) => + current.key === sessionRuntimeProjectionKey + ? resolveNextSessionRuntimeProjectionState(current, readyState) + : current, + ); + }, + { + minimumDelayMs: SESSION_RUNTIME_PROJECTION_DEFER_DELAY_MS, + idleTimeoutMs: SESSION_RUNTIME_PROJECTION_DEFER_IDLE_TIMEOUT_MS, + }, + ); + }, [ + displayMessages.length, + effectiveThreadItems.length, + focusedTimelineItemId, + hasHeavySessionRuntimeProjection, + isSending, + pendingA2UIForm, + sessionRuntimeProjectionFirstMessageId, + sessionRuntimeProjectionKey, + sessionRuntimeProjectionLastMessageId, + sessionRuntimeProjectionSessionId, + shouldConsiderDeferringSessionRuntimeProjection, + shouldDeferSessionRuntimeProjection, + shouldTreatAsRestoredHistoryWindow, + turns.length, + ]); + + const shouldUseDeferredSessionRuntimeProjection = + shouldDeferSessionRuntimeProjection && !sessionRuntimeProjectionReady; + const projectedTurns = shouldUseDeferredSessionRuntimeProjection + ? EMPTY_PROJECTED_TURNS + : turns; + const projectedThreadItems = shouldUseDeferredSessionRuntimeProjection + ? EMPTY_PROJECTED_THREAD_ITEMS + : effectiveThreadItems; + const projectedCurrentTurnId = shouldUseDeferredSessionRuntimeProjection + ? null + : currentTurnId; + const projectedThreadRead = shouldUseDeferredSessionRuntimeProjection + ? null + : threadRead; + const projectedPendingActions = shouldUseDeferredSessionRuntimeProjection + ? EMPTY_PROJECTED_PENDING_ACTIONS + : pendingActions; + const projectedSubmittedActionsInFlight = + shouldUseDeferredSessionRuntimeProjection + ? EMPTY_PROJECTED_SUBMITTED_ACTIONS + : submittedActionsInFlight; + const projectedQueuedTurns = shouldUseDeferredSessionRuntimeProjection + ? EMPTY_PROJECTED_QUEUED_TURNS + : queuedTurns; + const projectedChildSubagentSessions = + shouldUseDeferredSessionRuntimeProjection + ? EMPTY_PROJECTED_CHILD_SUBAGENT_SESSIONS + : childSubagentSessions; const handleQuoteMessage = (content: string) => { const normalized = content.trim(); if (!normalized) { @@ -549,7 +768,9 @@ export function useWorkspaceConversationSceneRuntime({ activeTheme === "general" && layoutMode === "chat-canvas"; const currentSessionTurn = - turns.find((turn) => turn.id === currentTurnId) || turns.at(-1) || null; + projectedTurns.find((turn) => turn.id === projectedCurrentTurnId) || + projectedTurns.at(-1) || + null; const currentSessionStatus = resolveSessionStatusBadge( isSending ? "running" : currentSessionTurn?.status, ); @@ -557,31 +778,31 @@ export function useWorkspaceConversationSceneRuntime({ () => buildAgentTaskRuntimeCardModel({ messages: displayMessages, - turns, - threadItems: effectiveThreadItems, - currentTurnId, - threadRead, - pendingActions, - queuedTurns, - childSubagentSessions, + turns: projectedTurns, + threadItems: projectedThreadItems, + currentTurnId: projectedCurrentTurnId, + threadRead: projectedThreadRead, + pendingActions: projectedPendingActions, + queuedTurns: projectedQueuedTurns, + childSubagentSessions: projectedChildSubagentSessions, isSending, }), [ - childSubagentSessions, - currentTurnId, displayMessages, - effectiveThreadItems, isSending, - pendingActions, - queuedTurns, - threadRead, - turns, + projectedChildSubagentSessions, + projectedCurrentTurnId, + projectedPendingActions, + projectedQueuedTurns, + projectedThreadItems, + projectedThreadRead, + projectedTurns, ], ); - const runtimeItemCount = effectiveThreadItems.filter( + const runtimeItemCount = projectedThreadItems.filter( (item) => item.type !== "user_message" && item.type !== "agent_message", ).length; - const inProgressItemCount = effectiveThreadItems.filter( + const inProgressItemCount = projectedThreadItems.filter( (item) => item.status === "in_progress", ).length; const sessionSummaryStats: CanvasWorkbenchSummaryStat[] = [ @@ -604,23 +825,23 @@ export function useWorkspaceConversationSceneRuntime({ }, { key: "session-follow-up", - label: pendingActions.length > 0 ? "待补信息" : "排队消息", + label: projectedPendingActions.length > 0 ? "待补信息" : "排队消息", value: - pendingActions.length > 0 - ? `待补信息 ${pendingActions.length}` - : queuedTurns.length > 0 - ? `排队 ${queuedTurns.length}` + projectedPendingActions.length > 0 + ? `待补信息 ${projectedPendingActions.length}` + : projectedQueuedTurns.length > 0 + ? `排队 ${projectedQueuedTurns.length}` : "无需跟进", detail: - pendingActions.length > 0 + projectedPendingActions.length > 0 ? "仍在等待用户补充或确认的信息。" - : queuedTurns.length > 0 - ? `另有 ${queuedTurns.length} 条消息正在排队。` + : projectedQueuedTurns.length > 0 + ? `另有 ${projectedQueuedTurns.length} 条消息正在排队。` : "当前没有待处理的补充或排队消息。", tone: - pendingActions.length > 0 + projectedPendingActions.length > 0 ? "accent" - : queuedTurns.length > 0 + : projectedQueuedTurns.length > 0 ? "default" : "default", }, @@ -632,8 +853,8 @@ export function useWorkspaceConversationSceneRuntime({ tabBadge: inProgressItemCount > 0 ? `进行中 ${inProgressItemCount}` - : queuedTurns.length > 0 - ? `排队 ${queuedTurns.length}` + : projectedQueuedTurns.length > 0 + ? `排队 ${projectedQueuedTurns.length}` : undefined, tabBadgeTone: inProgressItemCount > 0 ? "sky" : "slate", subtitle: currentSessionTurn @@ -654,20 +875,20 @@ export function useWorkspaceConversationSceneRuntime({ : `轨迹 ${runtimeItemCount}`, tone: inProgressItemCount > 0 ? "accent" : "default", }, - ...(pendingActions.length > 0 + ...(projectedPendingActions.length > 0 ? [ { key: "session-pending-actions", - label: `待补信息 ${pendingActions.length}`, + label: `待补信息 ${projectedPendingActions.length}`, tone: "accent" as const, }, ] : []), - ...(queuedTurns.length > 0 + ...(projectedQueuedTurns.length > 0 ? [ { key: "session-queued-turns", - label: `排队 ${queuedTurns.length}`, + label: `排队 ${projectedQueuedTurns.length}`, tone: "default" as const, }, ] @@ -675,11 +896,11 @@ export function useWorkspaceConversationSceneRuntime({ ], renderPanel: () => ( @@ -945,17 +1166,18 @@ export function useWorkspaceConversationSceneRuntime({ onStepClick: goToStep, }, messageList: { + sessionId, messages: displayMessages, emptyStateVariant: messageListEmptyStateVariant, providerType, - turns, - threadItems: effectiveThreadItems, - currentTurnId, - threadRead, - pendingActions, - submittedActionsInFlight, - queuedTurns, - childSubagentSessions, + turns: projectedTurns, + threadItems: projectedThreadItems, + currentTurnId: projectedCurrentTurnId, + threadRead: projectedThreadRead, + pendingActions: projectedPendingActions, + submittedActionsInFlight: projectedSubmittedActionsInFlight, + queuedTurns: projectedQueuedTurns, + childSubagentSessions: projectedChildSubagentSessions, sessionHistoryWindow, onLoadFullHistory: loadFullSessionHistory, isRestoringSession: isAutoRestoringSession, diff --git a/src/components/agent/chat/workspace/useWorkspaceImageTaskPreviewRuntime.test.tsx b/src/components/agent/chat/workspace/useWorkspaceImageTaskPreviewRuntime.test.tsx index c08c9cace..85f660ed0 100644 --- a/src/components/agent/chat/workspace/useWorkspaceImageTaskPreviewRuntime.test.tsx +++ b/src/components/agent/chat/workspace/useWorkspaceImageTaskPreviewRuntime.test.tsx @@ -69,6 +69,12 @@ const EMPTY_MODALITY_RUNTIME_CONTRACT_INDEX = { blocked_count: 0, routing_outcomes: [], model_registry_assessment_count: 0, + audio_output_count: 0, + audio_output_statuses: [], + audio_output_error_codes: [], + transcript_count: 0, + transcript_statuses: [], + transcript_error_codes: [], snapshots: [], }; diff --git a/src/components/agent/chat/workspace/useWorkspaceInitialSessionNavigation.ts b/src/components/agent/chat/workspace/useWorkspaceInitialSessionNavigation.ts index 9ad3d4247..8bdae93e0 100644 --- a/src/components/agent/chat/workspace/useWorkspaceInitialSessionNavigation.ts +++ b/src/components/agent/chat/workspace/useWorkspaceInitialSessionNavigation.ts @@ -43,6 +43,15 @@ export function resetInitialSessionNavigationDeduplicationForTests() { recentInitialSessionNavigationStarts.clear(); } +export function rememberInitialSessionNavigationStart(sessionId: string) { + const normalizedSessionId = normalizeSessionId(sessionId); + if (!normalizedSessionId) { + return; + } + + recentInitialSessionNavigationStarts.set(normalizedSessionId, Date.now()); +} + export function useWorkspaceInitialSessionNavigation({ initialSessionId, currentSessionId, diff --git a/src/components/agent/chat/workspace/useWorkspaceSendActions.test.tsx b/src/components/agent/chat/workspace/useWorkspaceSendActions.test.tsx index 3887bf6db..0f6546515 100644 --- a/src/components/agent/chat/workspace/useWorkspaceSendActions.test.tsx +++ b/src/components/agent/chat/workspace/useWorkspaceSendActions.test.tsx @@ -3798,6 +3798,20 @@ describe("useWorkspaceSendActions", () => { timestamps: true, speaker_labels: true, entry_source: "at_transcription_command", + modality_contract_key: "audio_transcription", + modality: "audio", + required_capabilities: [ + "text_generation", + "audio_transcription", + ], + routing_slot: "audio_transcription_model", + runtime_contract: expect.objectContaining({ + contract_key: "audio_transcription", + routing_slot: "audio_transcription_model", + executor_binding: expect.objectContaining({ + binding_key: "transcription_generate", + }), + }), }, }, }, @@ -5810,6 +5824,7 @@ describe("useWorkspaceSendActions", () => { runtime_contract: expect.objectContaining({ contract_key: "browser_control", }), + entry_source: "at_browser_agent_command", }), }, }, @@ -5863,6 +5878,7 @@ describe("useWorkspaceSendActions", () => { runtime_contract: expect.objectContaining({ contract_key: "browser_control", }), + entry_source: "at_mini_tester_command", }), }, }, diff --git a/src/components/agent/chat/workspace/useWorkspaceSendActions.ts b/src/components/agent/chat/workspace/useWorkspaceSendActions.ts index 0754c958d..7a60b3d9c 100644 --- a/src/components/agent/chat/workspace/useWorkspaceSendActions.ts +++ b/src/components/agent/chat/workspace/useWorkspaceSendActions.ts @@ -109,9 +109,11 @@ import { import { buildBrowserControlLaunchRequestMetadata } from "./browserControlLaunch"; import { PDF_EXTRACT_DEFAULT_ENTRY_SOURCE, + AUDIO_TRANSCRIPTION_DEFAULT_ENTRY_SOURCE, TEXT_TRANSFORM_DEFAULT_ENTRY_SOURCE, VOICE_GENERATION_DEFAULT_ENTRY_SOURCE, WEB_RESEARCH_DEFAULT_ENTRY_SOURCE, + resolveAudioTranscriptionRuntimeContractBinding, resolvePdfExtractRuntimeContractBinding, resolveTextTransformRuntimeContractBinding, resolveVoiceGenerationRuntimeContractBinding, @@ -2035,6 +2037,12 @@ function buildTranscriptionSkillLaunchRequestContext(params: { contentId?: string | null; sessionId?: string | null; }): Record { + const runtimeContract = resolveAudioTranscriptionRuntimeContractBinding(); + const entrySource = resolveContractEntrySource( + runtimeContract.boundEntrySources, + AUDIO_TRANSCRIPTION_DEFAULT_ENTRY_SOURCE, + ); + return { kind: "transcription_task", transcription_task: { @@ -2049,7 +2057,12 @@ function buildTranscriptionSkillLaunchRequestContext(params: { project_id: params.projectId || undefined, content_id: params.contentId || undefined, session_id: params.sessionId || undefined, - entry_source: "at_transcription_command", + entry_source: entrySource, + modality_contract_key: runtimeContract.contractKey, + modality: runtimeContract.modality, + required_capabilities: runtimeContract.requiredCapabilities, + routing_slot: runtimeContract.routingSlot, + runtime_contract: runtimeContract.runtimeContract, }, }; } diff --git a/src/components/agent/chat/workspace/useWorkspaceTopicSwitch.ts b/src/components/agent/chat/workspace/useWorkspaceTopicSwitch.ts index 5b5a920af..d27905773 100644 --- a/src/components/agent/chat/workspace/useWorkspaceTopicSwitch.ts +++ b/src/components/agent/chat/workspace/useWorkspaceTopicSwitch.ts @@ -138,6 +138,15 @@ export function useWorkspaceTopicSwitch({ return "busy" as const; } + let resolutionActive = true; + const finishResolutionIfNeeded = () => { + if (!resolutionActive) { + return; + } + resolutionActive = false; + finishTopicProjectResolution(); + }; + try { const currentProjectId = normalizeProjectId(projectId); const topicBoundProjectId = normalizeProjectId( @@ -151,18 +160,16 @@ export function useWorkspaceTopicSwitch({ topicId, }); - if ( - !externalProjectId && - currentProjectId && - topicBoundProjectId === currentProjectId - ) { + if (currentProjectId && topicBoundProjectId === currentProjectId) { rememberProjectId(currentProjectId); logAgentDebug("AgentChatPage", "switchTopic.fastPathCurrentProject", { allowDetachedSession: options?.allowDetachedSession === true, currentProjectId, + externalProjectId: externalProjectId ?? null, forceRefresh: options?.forceRefresh === true, topicId, }); + finishResolutionIfNeeded(); await runTopicSwitch(topicId, options); return "success" as const; } @@ -226,6 +233,7 @@ export function useWorkspaceTopicSwitch({ } rememberProjectId(targetProjectId); + finishResolutionIfNeeded(); await runTopicSwitch(topicId, options); return "success" as const; } catch (error) { @@ -245,7 +253,7 @@ export function useWorkspaceTopicSwitch({ toast.error("切换会话失败，请稍后重试"); return "error" as const; } finally { - finishTopicProjectResolution(); + finishResolutionIfNeeded(); } }, [ diff --git a/src/components/agent/chat/workspace/useWorkspaceTranscriptionTaskPreviewRuntime.test.tsx b/src/components/agent/chat/workspace/useWorkspaceTranscriptionTaskPreviewRuntime.test.tsx new file mode 100644 index 000000000..82700af8c --- /dev/null +++ b/src/components/agent/chat/workspace/useWorkspaceTranscriptionTaskPreviewRuntime.test.tsx @@ -0,0 +1,614 @@ +import React, { useState } from "react"; +import { act } from "react"; +import { createRoot, type Root } from "react-dom/client"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { + getMediaTaskArtifact, + listMediaTaskArtifacts, + type ListMediaTaskArtifactsOutput, + type MediaTaskArtifactOutput, +} from "@/lib/api/mediaTasks"; +import { readFilePreview } from "@/lib/api/fileBrowser"; +import { safeListen } from "@/lib/dev-bridge"; +import type { Message } from "../types"; +import { useWorkspaceTranscriptionTaskPreviewRuntime } from "./useWorkspaceTranscriptionTaskPreviewRuntime"; + +vi.mock("@/lib/api/mediaTasks", () => ({ + getMediaTaskArtifact: vi.fn(), + listMediaTaskArtifacts: vi.fn(), +})); + +vi.mock("@/lib/api/fileBrowser", () => ({ + readFilePreview: vi.fn(), +})); + +vi.mock("@/lib/dev-bridge", () => ({ + safeListen: vi.fn(), +})); + +type HookProps = Parameters< + typeof useWorkspaceTranscriptionTaskPreviewRuntime +>[0]; + +const mountedRoots: Array<{ container: HTMLDivElement; root: Root }> = []; + +function buildRunningTranscriptionMessage(): Message { + return { + id: "assistant-transcription-1", + role: "assistant", + content: "转写任务已创建。", + timestamp: new Date("2026-04-30T00:00:00.000Z"), + taskPreview: { + kind: "transcription_generate", + taskId: "task-transcription-1", + taskType: "transcription_generate", + prompt: "请转写访谈音频", + title: "内容转写任务", + status: "running", + artifactPath: + ".lime/runtime/transcription-generate/task-transcription-1.md", + taskFilePath: + ".lime/tasks/transcription_generate/task-transcription-1.json", + sourcePath: "materials/interview.wav", + sourceUrl: null, + transcriptPath: null, + language: "zh-CN", + outputFormat: "txt", + }, + }; +} + +function buildCompletedTranscriptionArtifact(): MediaTaskArtifactOutput { + return { + success: true, + task_id: "task-transcription-1", + task_type: "transcription_generate", + task_family: "audio", + status: "succeeded", + normalized_status: "succeeded", + current_attempt_id: "attempt-1", + path: ".lime/tasks/transcription_generate/task-transcription-1.json", + absolute_path: + "/workspace/.lime/tasks/transcription_generate/task-transcription-1.json", + artifact_path: + ".lime/tasks/transcription_generate/task-transcription-1.json", + absolute_artifact_path: + "/workspace/.lime/tasks/transcription_generate/task-transcription-1.json", + reused_existing: false, + record: { + task_id: "task-transcription-1", + task_type: "transcription_generate", + task_family: "audio", + payload: { + prompt: "请转写访谈音频", + source_path: "materials/interview.wav", + provider_id: "openai-asr", + model: "gpt-4o-transcribe", + modality_contract_key: "audio_transcription", + transcript: { + kind: "transcript", + status: "completed", + transcript_path: ".lime/runtime/transcripts/task-transcription-1.txt", + source_path: "materials/interview.wav", + language: "zh-CN", + output_format: "txt", + provider_id: "openai-asr", + model: "gpt-4o-transcribe", + }, + }, + status: "succeeded", + normalized_status: "succeeded", + created_at: "2026-04-30T00:00:00.000Z", + result: { + kind: "transcription_result", + status: "completed", + transcript_path: ".lime/runtime/transcripts/task-transcription-1.txt", + transcript: { + kind: "transcript", + status: "completed", + transcript_path: ".lime/runtime/transcripts/task-transcription-1.txt", + }, + }, + }, + }; +} + +function buildFailedTranscriptionArtifact(): MediaTaskArtifactOutput { + return { + success: true, + task_id: "task-transcription-1", + task_type: "transcription_generate", + task_family: "audio", + status: "failed", + normalized_status: "failed", + current_attempt_id: "attempt-1", + path: ".lime/tasks/transcription_generate/task-transcription-1.json", + absolute_path: + "/workspace/.lime/tasks/transcription_generate/task-transcription-1.json", + artifact_path: + ".lime/tasks/transcription_generate/task-transcription-1.json", + absolute_artifact_path: + "/workspace/.lime/tasks/transcription_generate/task-transcription-1.json", + reused_existing: false, + record: { + task_id: "task-transcription-1", + task_type: "transcription_generate", + task_family: "audio", + payload: { + prompt: "请转写访谈音频", + source_path: "materials/interview.wav", + provider_id: "missing-provider", + model: "gpt-4o-transcribe", + modality_contract_key: "audio_transcription", + transcript: { + kind: "transcript", + status: "failed", + source_path: "materials/interview.wav", + provider_id: "missing-provider", + model: "gpt-4o-transcribe", + error_code: "transcription_provider_unconfigured", + error_message: + "未找到可用的 audio_transcription provider/API Key: missing-provider。", + retryable: true, + stage: "provider_config", + }, + }, + status: "failed", + normalized_status: "failed", + created_at: "2026-04-30T00:00:00.000Z", + last_error: { + code: "transcription_provider_unconfigured", + message: + "未找到可用的 audio_transcription provider/API Key: missing-provider。", + retryable: true, + stage: "provider_config", + }, + progress: { + phase: "failed", + message: + "未找到可用的 audio_transcription provider/API Key: missing-provider。", + }, + }, + }; +} + +function buildEmptyTranscriptionTaskIndex(): ListMediaTaskArtifactsOutput { + return { + success: true, + workspace_root: "/workspace", + artifact_root: "/workspace/.lime/tasks", + filters: { + task_family: "audio", + task_type: "transcription_generate", + modality_contract_key: "audio_transcription", + limit: 24, + }, + total: 0, + modality_runtime_contracts: { + snapshot_count: 0, + contract_keys: [], + blocked_count: 0, + routing_outcomes: [], + model_registry_assessment_count: 0, + audio_output_count: 0, + audio_output_statuses: [], + audio_output_error_codes: [], + transcript_count: 0, + transcript_statuses: [], + transcript_error_codes: [], + snapshots: [], + }, + tasks: [], + }; +} + +function buildCompletedTranscriptionTaskIndex(): ListMediaTaskArtifactsOutput { + return { + ...buildEmptyTranscriptionTaskIndex(), + total: 1, + modality_runtime_contracts: { + snapshot_count: 1, + contract_keys: ["audio_transcription"], + blocked_count: 0, + routing_outcomes: [{ outcome: "accepted", count: 1 }], + model_registry_assessment_count: 0, + audio_output_count: 0, + audio_output_statuses: [], + audio_output_error_codes: [], + transcript_count: 1, + transcript_statuses: [{ status: "completed", count: 1 }], + transcript_error_codes: [], + snapshots: [ + { + task_id: "task-transcription-1", + task_type: "transcription_generate", + normalized_status: "succeeded", + contract_key: "audio_transcription", + routing_slot: "audio_transcription_model", + provider_id: "openai-asr", + model: "gpt-4o-transcribe", + routing_event: "task_created", + routing_outcome: "accepted", + failure_code: null, + transcript_status: "completed", + transcript_path: ".lime/runtime/transcripts/task-transcription-1.txt", + transcript_source_path: "materials/interview.wav", + transcript_source_url: null, + transcript_language: "zh-CN", + transcript_output_format: "txt", + transcript_error_code: null, + transcript_retryable: null, + }, + ], + }, + }; +} + +function buildFailedTranscriptionTaskIndex(): ListMediaTaskArtifactsOutput { + return { + ...buildEmptyTranscriptionTaskIndex(), + total: 1, + modality_runtime_contracts: { + snapshot_count: 1, + contract_keys: ["audio_transcription"], + blocked_count: 0, + routing_outcomes: [{ outcome: "failed", count: 1 }], + model_registry_assessment_count: 0, + audio_output_count: 0, + audio_output_statuses: [], + audio_output_error_codes: [], + transcript_count: 1, + transcript_statuses: [{ status: "failed", count: 1 }], + transcript_error_codes: ["transcription_provider_unconfigured"], + snapshots: [ + { + task_id: "task-transcription-1", + task_type: "transcription_generate", + normalized_status: "failed", + contract_key: "audio_transcription", + routing_slot: "audio_transcription_model", + provider_id: "missing-provider", + model: "gpt-4o-transcribe", + routing_event: "task_created", + routing_outcome: "failed", + failure_code: "transcription_provider_unconfigured", + transcript_status: "failed", + transcript_path: null, + transcript_source_path: "materials/interview.wav", + transcript_source_url: null, + transcript_language: "zh-CN", + transcript_output_format: "txt", + transcript_error_code: "transcription_provider_unconfigured", + transcript_retryable: true, + }, + ], + }, + }; +} + +function renderHook(props?: Partial) { + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + let latestMessages: Message[] = props?.messages ?? [ + buildRunningTranscriptionMessage(), + ]; + + function Probe(currentProps: Partial) { + const [messages, setMessages] = useState( + currentProps.messages ?? [buildRunningTranscriptionMessage()], + ); + latestMessages = messages; + useWorkspaceTranscriptionTaskPreviewRuntime({ + projectRootPath: "/workspace", + ...currentProps, + messages, + setChatMessages: setMessages, + }); + return null; + } + + const render = async (nextProps?: Partial) => { + await act(async () => { + root.render(); + await Promise.resolve(); + }); + }; + + mountedRoots.push({ container, root }); + return { + render, + getMessages: () => latestMessages, + }; +} + +beforeEach(() => { + ( + globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; + } + ).IS_REACT_ACT_ENVIRONMENT = true; + vi.mocked(getMediaTaskArtifact).mockReset(); + vi.mocked(listMediaTaskArtifacts).mockReset(); + vi.mocked(readFilePreview).mockReset(); + vi.mocked(listMediaTaskArtifacts).mockResolvedValue( + buildEmptyTranscriptionTaskIndex(), + ); + vi.mocked(readFilePreview).mockResolvedValue({ + path: "/workspace/.lime/runtime/transcripts/task-transcription-1.txt", + content: null, + isBinary: false, + size: 0, + error: "not configured", + }); + vi.mocked(safeListen).mockReset(); + vi.mocked(safeListen).mockResolvedValue(vi.fn()); +}); + +afterEach(() => { + while (mountedRoots.length > 0) { + const mounted = mountedRoots.pop(); + if (!mounted) { + break; + } + act(() => { + mounted.root.unmount(); + }); + mounted.container.remove(); + } + vi.restoreAllMocks(); +}); + +describe("useWorkspaceTranscriptionTaskPreviewRuntime", () => { + it("应优先从统一媒体任务索引恢复 transcript 完成态而不读取隐藏 task JSON", async () => { + vi.mocked(listMediaTaskArtifacts).mockResolvedValueOnce( + buildCompletedTranscriptionTaskIndex(), + ); + vi.mocked(readFilePreview).mockResolvedValueOnce({ + path: "/workspace/.lime/runtime/transcripts/task-transcription-1.txt", + content: "欢迎来到 Lime 访谈节目。\n今天我们讨论多模态工作流。", + isBinary: false, + size: 31, + error: null, + }); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(listMediaTaskArtifacts).toHaveBeenCalledWith({ + projectRootPath: "/workspace", + taskFamily: "audio", + taskType: "transcription_generate", + modalityContractKey: "audio_transcription", + limit: 24, + }); + }); + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "transcription_generate", + status: "complete", + transcriptPath: ".lime/runtime/transcripts/task-transcription-1.txt", + sourcePath: "materials/interview.wav", + language: "zh-CN", + outputFormat: "txt", + providerId: "openai-asr", + model: "gpt-4o-transcribe", + transcriptText: "欢迎来到 Lime 访谈节目。\n今天我们讨论多模态工作流。", + statusMessage: + "转写结果已同步，工作区已载入 transcript 文本，可直接复制校对。", + }); + }); + expect(readFilePreview).toHaveBeenCalledWith( + "/workspace/.lime/runtime/transcripts/task-transcription-1.txt", + 256 * 1024, + ); + expect(getMediaTaskArtifact).not.toHaveBeenCalled(); + }); + + it("应从 transcript 文件解析时间轴和说话人并写入 viewer 文档", async () => { + vi.mocked(listMediaTaskArtifacts).mockResolvedValueOnce( + buildCompletedTranscriptionTaskIndex(), + ); + vi.mocked(readFilePreview).mockResolvedValueOnce({ + path: "/workspace/.lime/runtime/transcripts/task-transcription-1.vtt", + content: `WEBVTT + +00:00:01.000 --> 00:00:03.500 +欢迎来到 Lime 访谈。 + +00:00:04.000 --> 00:00:06.000 +嘉宾: 这次我们讲转写 viewer。 +`, + isBinary: false, + size: 120, + error: null, + }); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "transcription_generate", + status: "complete", + transcriptSegments: [ + { + index: 1, + startMs: 1000, + endMs: 3500, + speaker: "主持人", + text: "欢迎来到 Lime 访谈。", + }, + { + index: 2, + startMs: 4000, + endMs: 6000, + speaker: "嘉宾", + text: "这次我们讲转写 viewer。", + }, + ], + }); + }); + expect(getMessages()[0]?.artifacts?.[0]).toMatchObject({ + meta: { + artifactDocument: { + blocks: expect.arrayContaining([ + expect.objectContaining({ + id: "transcript-segments", + type: "table", + title: "转写时间轴（可逐段编辑校对）", + columns: ["时间", "说话人", "内容"], + rows: [ + ["00:01 - 00:03", "主持人", "欢迎来到 Lime 访谈。"], + ["00:04 - 00:06", "嘉宾", "这次我们讲转写 viewer。"], + ], + }), + ]), + metadata: { + transcriptSegments: expect.arrayContaining([ + expect.objectContaining({ + speaker: "主持人", + text: "欢迎来到 Lime 访谈。", + }), + ]), + }, + }, + }, + }); + expect(getMediaTaskArtifact).not.toHaveBeenCalled(); + }); + + it("应优先从统一媒体任务索引恢复 provider 失败且不保留旧 transcript_path", async () => { + vi.mocked(listMediaTaskArtifacts).mockResolvedValueOnce( + buildFailedTranscriptionTaskIndex(), + ); + const runningWithStaleTranscriptPath = buildRunningTranscriptionMessage(); + if ( + runningWithStaleTranscriptPath.taskPreview?.kind === + "transcription_generate" + ) { + runningWithStaleTranscriptPath.taskPreview.transcriptPath = + ".lime/runtime/transcripts/stale.txt"; + } + const { render, getMessages } = renderHook({ + messages: [runningWithStaleTranscriptPath], + }); + + await render(); + + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "transcription_generate", + status: "failed", + transcriptPath: null, + providerId: "missing-provider", + model: "gpt-4o-transcribe", + errorCode: "transcription_provider_unconfigured", + retryable: true, + statusMessage: + "转写 Provider 未配置，请先在转写设置中选择可用 Provider；任务保留在 transcription_generate，不会回退 frontend ASR。", + }); + }); + expect(getMediaTaskArtifact).not.toHaveBeenCalled(); + }); + + it("应从完成态 transcription_generate task artifact 恢复 transcript viewer", async () => { + vi.mocked(getMediaTaskArtifact).mockResolvedValueOnce( + buildCompletedTranscriptionArtifact(), + ); + vi.mocked(readFilePreview).mockResolvedValueOnce({ + path: "/workspace/.lime/runtime/transcripts/task-transcription-1.txt", + content: "欢迎来到 Lime 访谈节目。\n今天我们讨论多模态工作流。", + isBinary: false, + size: 31, + error: null, + }); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(getMediaTaskArtifact).toHaveBeenCalledWith({ + projectRootPath: "/workspace", + taskRef: ".lime/tasks/transcription_generate/task-transcription-1.json", + }); + }); + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "transcription_generate", + status: "complete", + transcriptPath: ".lime/runtime/transcripts/task-transcription-1.txt", + providerId: "openai-asr", + model: "gpt-4o-transcribe", + transcriptText: "欢迎来到 Lime 访谈节目。\n今天我们讨论多模态工作流。", + statusMessage: + "转写结果已同步，工作区已载入 transcript 文本，可直接复制校对。", + }); + }); + expect(getMessages()[0]?.artifacts?.[0]).toMatchObject({ + title: "task-transcription-1.md", + status: "complete", + meta: { + taskId: "task-transcription-1", + taskType: "transcription_generate", + transcriptPath: ".lime/runtime/transcripts/task-transcription-1.txt", + artifactDocument: { + status: "ready", + metadata: { + transcriptPath: + ".lime/runtime/transcripts/task-transcription-1.txt", + transcriptText: + "欢迎来到 Lime 访谈节目。\n今天我们讨论多模态工作流。", + }, + }, + }, + }); + expect( + JSON.stringify(getMessages()[0]?.artifacts?.[0]?.meta.artifactDocument), + ).toContain("转写文本（可编辑校对）"); + }); + + it("应从失败态 transcription_generate task artifact 回流 provider 错误且不伪造 transcript_path", async () => { + vi.mocked(getMediaTaskArtifact).mockResolvedValueOnce( + buildFailedTranscriptionArtifact(), + ); + const { render, getMessages } = renderHook(); + + await render(); + + await vi.waitFor(() => { + expect(getMessages()[0]?.taskPreview).toMatchObject({ + kind: "transcription_generate", + status: "failed", + transcriptPath: null, + errorCode: "transcription_provider_unconfigured", + errorMessage: + "未找到可用的 audio_transcription provider/API Key: missing-provider。", + retryable: true, + statusMessage: + "转写 Provider 未配置，请先在转写设置中选择可用 Provider；任务保留在 transcription_generate，不会回退 frontend ASR。", + }); + }); + expect(getMessages()[0]?.artifacts?.[0]).toMatchObject({ + title: "task-transcription-1.md", + status: "error", + meta: { + taskId: "task-transcription-1", + taskType: "transcription_generate", + transcriptPath: null, + errorCode: "transcription_provider_unconfigured", + artifactDocument: { + status: "failed", + metadata: { + transcriptPath: null, + errorCode: "transcription_provider_unconfigured", + }, + }, + }, + }); + expect( + JSON.stringify(getMessages()[0]?.artifacts?.[0]?.meta.artifactDocument), + ).toContain("不会回退 frontend ASR"); + }); +}); diff --git a/src/components/agent/chat/workspace/useWorkspaceTranscriptionTaskPreviewRuntime.ts b/src/components/agent/chat/workspace/useWorkspaceTranscriptionTaskPreviewRuntime.ts new file mode 100644 index 000000000..f319d99c7 --- /dev/null +++ b/src/components/agent/chat/workspace/useWorkspaceTranscriptionTaskPreviewRuntime.ts @@ -0,0 +1,942 @@ +import { useEffect, useRef, type Dispatch, type SetStateAction } from "react"; +import { + getMediaTaskArtifact, + listMediaTaskArtifacts, + type ListMediaTaskArtifactsOutput, + type MediaTaskModalityRuntimeContractIndexEntry, + type MediaTaskArtifactOutput, + type MediaTaskLookupRequest, +} from "@/lib/api/mediaTasks"; +import { readFilePreview } from "@/lib/api/fileBrowser"; +import { resolveArtifactProtocolFilePath } from "@/lib/artifact-protocol"; +import { safeListen } from "@/lib/dev-bridge"; +import type { Message, MessageGenericTaskPreview } from "../types"; +import { + buildArtifactFromWrite, + upsertMessageArtifact, +} from "../utils/messageArtifacts"; +import { buildTranscriptionTaskArtifactDocument } from "../utils/taskPreviewFromToolResult"; +import { + extractTranscriptSegmentsFromRecords, + normalizeTranscriptSegments, + parseTranscriptContent, +} from "../utils/transcriptSegments"; +import { buildImageTaskLookupRequest } from "./imageTaskLocator"; +import { doesWorkspaceFileCandidateMatch } from "./workspaceFilePathMatch"; +import { resolveAbsoluteWorkspacePath } from "./workspacePath"; + +const TRANSCRIPTION_TASK_EVENT_NAME = "lime://creation_task_submitted"; +const TRANSCRIPTION_TASK_POLL_INTERVAL_MS = 3000; +const TRANSCRIPTION_TASK_INDEX_RESTORE_LIMIT = 24; +const TRANSCRIPTION_TEXT_PREVIEW_MAX_SIZE = 256 * 1024; +const AUDIO_TRANSCRIPTION_CONTRACT_KEY = "audio_transcription"; + +interface LoadedTranscriptPreview { + text: string | null; + segments: MessageGenericTaskPreview["transcriptSegments"]; +} + +interface UseWorkspaceTranscriptionTaskPreviewRuntimeParams { + projectRootPath?: string | null; + messages: Message[]; + setChatMessages: Dispatch>; +} + +interface CreationTaskSubmittedPayload { + task_id?: string; + task_type?: string; + task_family?: string; + status?: string; + path?: string; + absolute_path?: string; +} + +interface TrackedTranscriptionTask { + taskId: string; + taskFilePath?: string | null; + artifactPath?: string | null; +} + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + +function readString( + candidates: Array | null | undefined>, + keys: string[], +): string | null { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (typeof value === "string" && value.trim()) { + return value.trim(); + } + } + } + return null; +} + +function readBoolean( + candidates: Array | null | undefined>, + keys: string[], +): boolean | null { + for (const candidate of candidates) { + if (!candidate) { + continue; + } + for (const key of keys) { + const value = candidate[key]; + if (typeof value === "boolean") { + return value; + } + } + } + return null; +} + +function normalizePreviewStatus(status: string | undefined | null) { + switch ((status || "").trim().toLowerCase()) { + case "succeeded": + case "completed": + case "success": + return "complete" as const; + case "partial": + return "partial" as const; + case "failed": + case "error": + return "failed" as const; + case "cancelled": + case "canceled": + return "cancelled" as const; + case "running": + case "processing": + case "queued": + case "pending": + case "pending_submit": + default: + return "running" as const; + } +} + +function resolveTranscriptionFailureStatusMessage( + errorCode?: string | null, + errorMessage?: string | null, +): string { + switch ((errorCode || "").trim()) { + case "transcription_provider_unconfigured": + return "转写 Provider 未配置，请先在转写设置中选择可用 Provider；任务保留在 transcription_generate，不会回退 frontend ASR。"; + case "transcription_provider_model_unconfigured": + return "转写模型未配置，请先为 audio_transcription 设置默认模型；任务保留在 transcription_generate，不会回退 generic_file transcript。"; + case "transcription_provider_resolver_unavailable": + return "当前无法解析 API Key Provider 凭证，转写任务已显式失败，没有伪造 transcript_path。"; + case "transcription_provider_resolution_failed": + return "解析转写 Provider 凭证失败，请检查 API Key Provider 配置后重试。"; + case "transcription_provider_client_missing": + return "当前 Provider 还没有 OpenAI-compatible ASR adapter，转写任务不会回退前端直连 ASR。"; + case "transcription_provider_request_failed": + return "调用转写 Provider 失败，请检查额度、网络或模型配置后重试。"; + case "transcription_provider_response_read_failed": + return "读取转写 Provider 响应失败，请稍后重试或检查 Provider 网关。"; + case "transcription_provider_empty_response": + return "转写 Provider 返回了空 transcript，已阻止写入空转写结果。"; + case "transcription_source_unavailable": + return "音频来源文件不可读，请检查 source_path 是否仍在工作区内。"; + case "transcription_source_download_failed": + return "下载音频来源失败，请检查 source_url、网络或访问权限后重试。"; + case "transcription_source_empty": + return "音频来源为空，已阻止调用转写 Provider。"; + case "transcription_source_mime_invalid": + return "音频来源 MIME 类型不可用于转写，请换用支持的音频或视频文件。"; + case "transcript_output_write_failed": + return "转写结果已返回但写入工作区失败，请检查项目目录权限后重试。"; + case "invalid_transcription_task_payload": + return "转写任务缺少 source_path/source_url，已阻止调用转写 Provider。"; + case "audio_transcription_task_type_mismatch": + case "audio_transcription_contract_mismatch": + case "audio_transcription_capability_gap": + case "audio_transcription_routing_slot_mismatch": + return "转写任务运行合同不匹配，已阻止执行器误跑到非 audio_transcription 主链。"; + default: + return ( + errorMessage?.trim() || + "转写失败，请检查音频来源、模型或 Provider 配置后重试。" + ); + } +} + +function resolveStatusMessage(preview: MessageGenericTaskPreview): string { + if (preview.status === "complete" || preview.status === "partial") { + return preview.transcriptText?.trim() + ? "转写结果已同步，工作区已载入 transcript 文本，可直接复制校对。" + : preview.transcriptPath?.trim() + ? "转写结果已同步，工作区已从 transcript 读取可校对文本。" + : "转写任务已完成，正在等待 transcript 输出路径。"; + } + if (preview.status === "failed") { + return resolveTranscriptionFailureStatusMessage( + preview.errorCode, + preview.errorMessage, + ); + } + if (preview.status === "cancelled") { + return "转写任务已经取消，当前不会继续生成 transcript。"; + } + return "转写任务已写入统一 transcription_generate/transcript 协议，工作区会继续同步结果。"; +} + +function shouldLoadTranscriptText(preview: MessageGenericTaskPreview): boolean { + if (preview.kind !== "transcription_generate") { + return false; + } + if (preview.status !== "complete" && preview.status !== "partial") { + return false; + } + return ( + Boolean(preview.transcriptPath?.trim()) && !preview.transcriptText?.trim() + ); +} + +async function loadTranscriptText(params: { + projectRootPath?: string | null; + transcriptPath?: string | null; +}): Promise { + const absolutePath = resolveAbsoluteWorkspacePath( + params.projectRootPath, + params.transcriptPath, + ); + if (!absolutePath) { + return { text: null, segments: [] }; + } + try { + const preview = await readFilePreview( + absolutePath, + TRANSCRIPTION_TEXT_PREVIEW_MAX_SIZE, + ); + if ( + preview.error || + preview.isBinary || + typeof preview.content !== "string" + ) { + return { text: null, segments: [] }; + } + return parseTranscriptContent(preview.content); + } catch (error) { + console.warn( + "[TranscriptionTaskPreviewRuntime] 读取 transcript 文本失败:", + error, + ); + return { text: null, segments: [] }; + } +} + +async function loadTranscriptTextsFromIndex(params: { + projectRootPath?: string | null; + snapshotsByTaskId: Map; +}): Promise> { + const loaded = new Map(); + await Promise.all( + Array.from(params.snapshotsByTaskId.values()).map(async (snapshot) => { + const status = normalizePreviewStatus( + snapshot.transcript_status || snapshot.normalized_status, + ); + if (status !== "complete" && status !== "partial") { + return; + } + const transcriptPreview = await loadTranscriptText({ + projectRootPath: params.projectRootPath, + transcriptPath: snapshot.transcript_path, + }); + if ( + transcriptPreview.text || + (transcriptPreview.segments && transcriptPreview.segments.length > 0) + ) { + loaded.set(snapshot.task_id, transcriptPreview); + } + }), + ); + return loaded; +} + +function resolveTranscriptPathFromArtifact( + artifact: MediaTaskArtifactOutput, +): string | null { + const payload = asRecord(artifact.record?.payload); + const result = asRecord(artifact.record?.result); + const transcript = + asRecord(payload?.transcript) || asRecord(result?.transcript); + return readString( + [transcript, result, payload], + ["transcript_path", "transcriptPath", "path"], + ); +} + +function buildTranscriptionPreviewFromArtifact( + artifact: MediaTaskArtifactOutput, + currentPreview: MessageGenericTaskPreview, + loadedTranscript?: LoadedTranscriptPreview | null, +): MessageGenericTaskPreview | null { + if (artifact.task_type !== "transcription_generate") { + return null; + } + + const payload = asRecord(artifact.record?.payload); + const result = asRecord(artifact.record?.result); + const lastError = asRecord(artifact.record?.last_error); + const progress = asRecord(artifact.record?.progress); + const transcript = + asRecord(payload?.transcript) || asRecord(result?.transcript); + const candidates = [transcript, result, payload]; + const errorCandidates = [transcript, lastError, progress, result, payload]; + const sourcePath = + readString(candidates, ["source_path", "sourcePath"]) || + currentPreview.sourcePath || + null; + const sourceUrl = + readString(candidates, ["source_url", "sourceUrl"]) || + currentPreview.sourceUrl || + null; + const transcriptStatus = readString([transcript], ["status"]); + const nextStatus = normalizePreviewStatus( + transcriptStatus || + artifact.normalized_status || + artifact.status || + artifact.record?.status, + ); + const indexedTranscriptPath = readString(candidates, [ + "transcript_path", + "transcriptPath", + "path", + ]); + const transcriptPath = + nextStatus === "failed" + ? indexedTranscriptPath + : indexedTranscriptPath || currentPreview.transcriptPath || null; + const errorCode = + nextStatus === "failed" + ? readString(errorCandidates, [ + "error_code", + "errorCode", + "failure_code", + "failureCode", + "code", + ]) || + currentPreview.errorCode || + null + : null; + const errorMessage = + nextStatus === "failed" + ? readString(errorCandidates, [ + "error_message", + "errorMessage", + "message", + "detail", + ]) || + currentPreview.errorMessage || + null + : null; + const retryable = + nextStatus === "failed" + ? (readBoolean(errorCandidates, ["retryable"]) ?? + currentPreview.retryable) + : currentPreview.retryable; + const payloadSegments = normalizeTranscriptSegments([ + ...extractTranscriptSegmentsFromRecords(candidates), + ...extractTranscriptSegmentsFromRecords(errorCandidates), + ]); + const nextSegments = + nextStatus === "failed" + ? [] + : loadedTranscript?.segments && loadedTranscript.segments.length > 0 + ? loadedTranscript.segments + : payloadSegments.length > 0 + ? payloadSegments + : currentPreview.transcriptSegments || []; + const prompt = + currentPreview.prompt || sourcePath || sourceUrl || "内容转写任务"; + const nextPreview: MessageGenericTaskPreview = { + ...currentPreview, + kind: "transcription_generate", + taskType: "transcription_generate", + taskId: artifact.task_id || currentPreview.taskId, + prompt, + status: nextStatus, + projectId: + readString(candidates, ["project_id", "projectId"]) ?? + currentPreview.projectId ?? + null, + contentId: + readString(candidates, ["content_id", "contentId"]) ?? + currentPreview.contentId ?? + null, + taskFilePath: + currentPreview.taskFilePath || + artifact.artifact_path || + artifact.path || + null, + providerId: + readString(candidates, ["provider_id", "providerId", "provider"]) ?? + currentPreview.providerId ?? + null, + model: readString(candidates, ["model"]) ?? currentPreview.model ?? null, + phase: readString([progress], ["phase"]) ?? currentPreview.phase ?? null, + transcriptPath, + sourcePath, + sourceUrl, + language: + readString(candidates, [ + "language", + "target_language", + "targetLanguage", + ]) ?? + currentPreview.language ?? + null, + outputFormat: + readString(candidates, ["output_format", "outputFormat", "format"]) ?? + currentPreview.outputFormat ?? + null, + transcriptText: + nextStatus === "failed" + ? null + : loadedTranscript?.text || currentPreview.transcriptText || null, + transcriptSegments: nextSegments, + errorCode, + errorMessage, + retryable, + }; + return { + ...nextPreview, + statusMessage: resolveStatusMessage(nextPreview), + }; +} + +function hasUsableTranscriptIndex( + entry: MediaTaskModalityRuntimeContractIndexEntry, +): boolean { + if (entry.task_type !== "transcription_generate") { + return false; + } + return Boolean(entry.transcript_status); +} + +function buildTranscriptionPreviewFromIndexEntry( + entry: MediaTaskModalityRuntimeContractIndexEntry, + currentPreview: MessageGenericTaskPreview, + loadedTranscript?: LoadedTranscriptPreview | null, +): MessageGenericTaskPreview | null { + if (!hasUsableTranscriptIndex(entry)) { + return null; + } + + const nextStatus = normalizePreviewStatus( + entry.transcript_status || entry.normalized_status, + ); + const indexedTranscriptPath = entry.transcript_path?.trim() || null; + const transcriptPath = + nextStatus === "failed" + ? indexedTranscriptPath + : indexedTranscriptPath || currentPreview.transcriptPath || null; + const errorCode = + nextStatus === "failed" + ? entry.transcript_error_code?.trim() || currentPreview.errorCode || null + : null; + const errorMessage = + nextStatus === "failed" && errorCode === currentPreview.errorCode + ? currentPreview.errorMessage || null + : null; + const retryable = + nextStatus === "failed" + ? (entry.transcript_retryable ?? currentPreview.retryable) + : currentPreview.retryable; + const nextSegments = + nextStatus === "failed" + ? [] + : loadedTranscript?.segments && loadedTranscript.segments.length > 0 + ? loadedTranscript.segments + : currentPreview.transcriptSegments || []; + const nextPreview: MessageGenericTaskPreview = { + ...currentPreview, + kind: "transcription_generate", + taskType: "transcription_generate", + taskId: entry.task_id || currentPreview.taskId, + status: nextStatus, + providerId: entry.provider_id?.trim() || currentPreview.providerId || null, + model: entry.model?.trim() || currentPreview.model || null, + transcriptPath, + sourcePath: + entry.transcript_source_path?.trim() || currentPreview.sourcePath || null, + sourceUrl: + entry.transcript_source_url?.trim() || currentPreview.sourceUrl || null, + language: + entry.transcript_language?.trim() || currentPreview.language || null, + outputFormat: + entry.transcript_output_format?.trim() || + currentPreview.outputFormat || + null, + transcriptText: + nextStatus === "failed" + ? null + : loadedTranscript?.text || currentPreview.transcriptText || null, + transcriptSegments: nextSegments, + errorCode, + errorMessage, + retryable, + }; + return { + ...nextPreview, + statusMessage: resolveStatusMessage(nextPreview), + }; +} + +function areTranscriptionPreviewsEqual( + left: MessageGenericTaskPreview, + right: MessageGenericTaskPreview, +): boolean { + const leftSegments = left.transcriptSegments || []; + const rightSegments = right.transcriptSegments || []; + return ( + left.status === right.status && + left.transcriptPath === right.transcriptPath && + left.sourcePath === right.sourcePath && + left.sourceUrl === right.sourceUrl && + left.language === right.language && + left.outputFormat === right.outputFormat && + left.transcriptText === right.transcriptText && + JSON.stringify(leftSegments) === JSON.stringify(rightSegments) && + left.providerId === right.providerId && + left.model === right.model && + left.phase === right.phase && + left.statusMessage === right.statusMessage && + left.errorCode === right.errorCode && + left.errorMessage === right.errorMessage && + left.retryable === right.retryable + ); +} + +function findTranscriptionTaskViewerArtifact( + message: Message, + preview: MessageGenericTaskPreview, +) { + const artifacts = message.artifacts || []; + const artifactPath = preview.artifactPath?.trim(); + return artifacts.find((artifact) => { + const meta = artifact.meta || {}; + if ( + typeof meta.taskId === "string" && + meta.taskId === preview.taskId && + meta.taskType === "transcription_generate" + ) { + return true; + } + return artifactPath + ? doesWorkspaceFileCandidateMatch( + resolveArtifactProtocolFilePath(artifact), + artifactPath, + ) + : false; + }); +} + +function syncTranscriptionTaskViewerArtifact( + message: Message, + preview: MessageGenericTaskPreview, +): Message { + const artifactPath = preview.artifactPath?.trim(); + if (!artifactPath) { + return message; + } + + const artifactDocument = buildTranscriptionTaskArtifactDocument(preview); + const existingArtifact = findTranscriptionTaskViewerArtifact( + message, + preview, + ); + const nextArtifact = buildArtifactFromWrite({ + filePath: artifactPath, + content: JSON.stringify(artifactDocument, null, 2), + context: { + artifact: existingArtifact, + artifactId: + existingArtifact?.id || `artifact:${message.id}:${artifactPath}`, + source: "tool_result", + sourceMessageId: message.id, + status: + preview.status === "failed" + ? "error" + : preview.status === "running" + ? "streaming" + : "complete", + metadata: { + artifactDocument, + artifact_type: "document", + previewText: preview.statusMessage || "转写任务已写入统一任务产物协议", + taskId: preview.taskId, + taskType: "transcription_generate", + taskFilePath: preview.taskFilePath || null, + transcriptPath: preview.transcriptPath || null, + transcriptText: preview.transcriptText || null, + transcriptSegments: preview.transcriptSegments || [], + transcriptCorrectionEnabled: Boolean( + preview.transcriptText || + (preview.transcriptSegments || []).length > 0, + ), + transcriptCorrectionStatus: + preview.transcriptText || + (preview.transcriptSegments || []).length > 0 + ? "available" + : "waiting_transcript", + transcriptCorrectionSource: "artifact_document_version", + transcriptCorrectionPatchKind: "artifact_document_version", + transcriptCorrectionOriginalImmutable: true, + sourcePath: preview.sourcePath || null, + sourceUrl: preview.sourceUrl || null, + language: preview.language || null, + outputFormat: preview.outputFormat || null, + providerId: preview.providerId || null, + model: preview.model || null, + errorCode: preview.errorCode || null, + errorMessage: preview.errorMessage || null, + modalityContractKey: AUDIO_TRANSCRIPTION_CONTRACT_KEY, + }, + }, + }); + + return upsertMessageArtifact(message, nextArtifact); +} + +function shouldTrackTranscriptionPreview( + preview: MessageGenericTaskPreview, +): boolean { + if (preview.kind !== "transcription_generate") { + return false; + } + if (preview.status === "running") { + return true; + } + return ( + (preview.status === "complete" || preview.status === "partial") && + (!preview.transcriptPath?.trim() || shouldLoadTranscriptText(preview)) + ); +} + +function collectTrackedTranscriptionTasks( + messages: Message[], +): TrackedTranscriptionTask[] { + const tasks = new Map(); + messages.forEach((message) => { + const preview = message.taskPreview; + if (!preview || preview.kind !== "transcription_generate") { + return; + } + if (!shouldTrackTranscriptionPreview(preview)) { + return; + } + const taskId = preview.taskId.trim(); + if (!taskId || tasks.has(taskId)) { + return; + } + tasks.set(taskId, { + taskId, + taskFilePath: preview.taskFilePath, + artifactPath: preview.artifactPath, + }); + }); + return Array.from(tasks.values()); +} + +function buildLookupRequest(params: { + task: TrackedTranscriptionTask; + projectRootPath?: string | null; +}): MediaTaskLookupRequest | null { + return buildImageTaskLookupRequest({ + taskId: params.task.taskId, + taskFilePath: params.task.taskFilePath, + artifactPath: params.task.artifactPath, + projectRootPath: params.projectRootPath, + }); +} + +function updateTranscriptionPreviewMessages(params: { + previous: Message[]; + artifact: MediaTaskArtifactOutput; + transcriptPreviewsByTaskId?: Map; +}): Message[] { + let changed = false; + const nextMessages = params.previous.map((message) => { + const currentPreview = message.taskPreview; + if ( + !currentPreview || + currentPreview.kind !== "transcription_generate" || + currentPreview.taskId !== params.artifact.task_id + ) { + return message; + } + const nextPreview = buildTranscriptionPreviewFromArtifact( + params.artifact, + currentPreview, + params.transcriptPreviewsByTaskId?.get(params.artifact.task_id), + ); + if ( + !nextPreview || + areTranscriptionPreviewsEqual(currentPreview, nextPreview) + ) { + return message; + } + changed = true; + return syncTranscriptionTaskViewerArtifact( + { + ...message, + taskPreview: nextPreview, + }, + nextPreview, + ); + }); + return changed ? nextMessages : params.previous; +} + +function updateTranscriptionPreviewMessagesFromIndex(params: { + previous: Message[]; + snapshotsByTaskId: Map; + transcriptPreviewsByTaskId?: Map; +}): Message[] { + let changed = false; + const nextMessages = params.previous.map((message) => { + const currentPreview = message.taskPreview; + if (!currentPreview || currentPreview.kind !== "transcription_generate") { + return message; + } + const snapshot = params.snapshotsByTaskId.get(currentPreview.taskId); + if (!snapshot) { + return message; + } + const nextPreview = buildTranscriptionPreviewFromIndexEntry( + snapshot, + currentPreview, + params.transcriptPreviewsByTaskId?.get(currentPreview.taskId), + ); + if ( + !nextPreview || + areTranscriptionPreviewsEqual(currentPreview, nextPreview) + ) { + return message; + } + changed = true; + return syncTranscriptionTaskViewerArtifact( + { + ...message, + taskPreview: nextPreview, + }, + nextPreview, + ); + }); + return changed ? nextMessages : params.previous; +} + +export function useWorkspaceTranscriptionTaskPreviewRuntime({ + projectRootPath, + messages, + setChatMessages, +}: UseWorkspaceTranscriptionTaskPreviewRuntimeParams) { + const contextRef = useRef({ projectRootPath, messages }); + + useEffect(() => { + contextRef.current = { projectRootPath, messages }; + }, [projectRootPath, messages]); + + useEffect(() => { + let disposed = false; + let polling = false; + + const syncFromTaskIndex = async ( + tasks: TrackedTranscriptionTask[], + ): Promise> => { + const workspaceRoot = contextRef.current.projectRootPath?.trim(); + if (!workspaceRoot || tasks.length === 0) { + return new Set(); + } + let output: ListMediaTaskArtifactsOutput; + try { + output = await listMediaTaskArtifacts({ + projectRootPath: workspaceRoot, + taskFamily: "audio", + taskType: "transcription_generate", + modalityContractKey: AUDIO_TRANSCRIPTION_CONTRACT_KEY, + limit: Math.max(TRANSCRIPTION_TASK_INDEX_RESTORE_LIMIT, tasks.length), + }); + } catch (error) { + console.warn( + "[TranscriptionTaskPreviewRuntime] 读取转写任务索引失败，回退 task artifact:", + error, + ); + return new Set(); + } + if (disposed) { + return new Set(); + } + const trackedTaskIds = new Set(tasks.map((task) => task.taskId)); + const snapshotsByTaskId = new Map< + string, + MediaTaskModalityRuntimeContractIndexEntry + >(); + output.modality_runtime_contracts.snapshots.forEach((snapshot) => { + if ( + trackedTaskIds.has(snapshot.task_id) && + hasUsableTranscriptIndex(snapshot) + ) { + snapshotsByTaskId.set(snapshot.task_id, snapshot); + } + }); + if (snapshotsByTaskId.size === 0) { + return new Set(); + } + const transcriptPreviewsByTaskId = await loadTranscriptTextsFromIndex({ + projectRootPath: workspaceRoot, + snapshotsByTaskId, + }); + if (disposed) { + return new Set(); + } + setChatMessages((previous) => + updateTranscriptionPreviewMessagesFromIndex({ + previous, + snapshotsByTaskId, + transcriptPreviewsByTaskId, + }), + ); + return new Set(snapshotsByTaskId.keys()); + }; + + const syncTask = async (task: TrackedTranscriptionTask) => { + const request = buildLookupRequest({ + task, + projectRootPath: contextRef.current.projectRootPath, + }); + if (!request) { + return; + } + const artifact = await getMediaTaskArtifact(request); + if (disposed || artifact.task_type !== "transcription_generate") { + return; + } + const transcriptPath = resolveTranscriptPathFromArtifact(artifact); + const transcriptPreview = await loadTranscriptText({ + projectRootPath: contextRef.current.projectRootPath, + transcriptPath, + }); + if (disposed) { + return; + } + const transcriptPreviewsByTaskId = new Map< + string, + LoadedTranscriptPreview + >(); + if ( + transcriptPreview.text || + (transcriptPreview.segments && transcriptPreview.segments.length > 0) + ) { + transcriptPreviewsByTaskId.set(artifact.task_id, transcriptPreview); + } + setChatMessages((previous) => + updateTranscriptionPreviewMessages({ + previous, + artifact, + transcriptPreviewsByTaskId, + }), + ); + }; + + const syncOnce = async () => { + if (disposed || polling) { + return; + } + const tasks = collectTrackedTranscriptionTasks( + contextRef.current.messages, + ); + if (tasks.length === 0) { + return; + } + polling = true; + try { + const indexedTaskIds = await syncFromTaskIndex(tasks); + await Promise.all( + tasks + .filter((task) => !indexedTaskIds.has(task.taskId)) + .map((task) => syncTask(task)), + ); + } catch (error) { + console.warn( + "[TranscriptionTaskPreviewRuntime] 同步转写任务状态失败:", + error, + ); + } finally { + polling = false; + } + }; + + void syncOnce(); + const timerId = window.setInterval(() => { + void syncOnce(); + }, TRANSCRIPTION_TASK_POLL_INTERVAL_MS); + + let unlisten: (() => void) | null = null; + safeListen( + TRANSCRIPTION_TASK_EVENT_NAME, + (event) => { + if (disposed) { + return; + } + const payload = event.payload || {}; + const taskId = payload.task_id?.trim(); + const taskType = payload.task_type?.trim(); + const taskFamily = payload.task_family?.trim(); + const taskPath = `${payload.absolute_path || ""} ${payload.path || ""}`; + const isTranscriptionEvent = + taskType === "transcription_generate" || + taskPath.includes("transcription_generate") || + taskFamily === "audio"; + if (!taskId || !isTranscriptionEvent) { + return; + } + const tracked = collectTrackedTranscriptionTasks( + contextRef.current.messages, + ).find((task) => task.taskId === taskId); + if (!tracked) { + return; + } + void syncTask({ + ...tracked, + taskFilePath: + payload.absolute_path?.trim() || + payload.path?.trim() || + tracked.taskFilePath, + }); + }, + ) + .then((dispose) => { + if (disposed) { + void dispose(); + return; + } + unlisten = dispose; + }) + .catch((error) => { + console.warn( + "[TranscriptionTaskPreviewRuntime] 监听转写任务事件失败:", + error, + ); + }); + + return () => { + disposed = true; + window.clearInterval(timerId); + if (unlisten) { + unlisten(); + } + }; + }, [setChatMessages]); +} diff --git a/src/components/api-key-provider/ApiKeyProviderSection.ui.test.tsx b/src/components/api-key-provider/ApiKeyProviderSection.ui.test.tsx index fb548395f..4000aa297 100644 --- a/src/components/api-key-provider/ApiKeyProviderSection.ui.test.tsx +++ b/src/components/api-key-provider/ApiKeyProviderSection.ui.test.tsx @@ -359,6 +359,47 @@ describe("ApiKeyProviderSection 模型管理布局", () => { ); }); + it("国内分类里的 SenseNova 应使用 v2 OpenAI 兼容接口", async () => { + mockGetSystemProviderCatalog.mockResolvedValueOnce([ + { + id: "sensenova", + name: "SenseNova", + type: "openai", + api_host: "https://api.sensenova.cn/compatible-mode/v2", + group: "chinese", + sort_order: 29, + legacy_ids: [], + }, + ]); + createHookState(); + const container = renderSection(); + + await act(async () => { + findByTestId("add-model-button").click(); + await Promise.resolve(); + await Promise.resolve(); + }); + + await act(async () => { + findByTestId("model-catalog-category-cn").click(); + await Promise.resolve(); + }); + + await act(async () => { + container + .querySelector( + '[data-template-id="catalog-sensenova"]', + ) + ?.click(); + await Promise.resolve(); + }); + + expect(container.textContent ?? "").toContain( + "https://api.sensenova.cn/compatible-mode/v2", + ); + expect(container.textContent ?? "").toContain("SenseChat-5"); + }); + it("海外分类应展示国内厂商的国际订阅入口", async () => { createHookState(); const container = renderSection(); @@ -481,4 +522,119 @@ describe("ApiKeyProviderSection 模型管理布局", () => { expect(mockTestConnection).toHaveBeenCalledWith("custom-1", "my-model"); expect(hookState.selectProvider).toHaveBeenCalledWith("custom-1"); }); + + it("添加流程应把 SenseNova 文档页修正为真实 API Base URL", async () => { + const hookState = createHookState(); + renderSection(); + + await act(async () => { + findByTestId("add-model-button").click(); + await Promise.resolve(); + }); + + await act(async () => { + findByTestId("custom-provider-template-card").click(); + await Promise.resolve(); + }); + + await act(async () => { + setInputValue( + findByTestId("model-provider-name-input"), + "SenseNova", + ); + setInputValue( + findByTestId("model-api-host-input"), + "https://platform.sensenova.cn/docs", + ); + setInputValue( + findByTestId("model-api-key-input"), + "sk-test", + ); + setInputValue( + findByTestId("model-draft-input"), + "sensenova-test-model", + ); + }); + + await act(async () => { + findByTestId("model-draft-add-button").click(); + await Promise.resolve(); + }); + + await act(async () => { + findByTestId("model-activate-button").click(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(hookState.addCustomProvider).toHaveBeenCalledWith( + expect.objectContaining({ + api_host: "https://api.sensenova.cn/compatible-mode/v2", + }), + ); + expect(hookState.selectProvider).toHaveBeenCalledWith("custom-1"); + }); + + it("添加流程在保存成功但连接测试失败时仍应进入 Provider 配置页", async () => { + mockTestConnection.mockResolvedValueOnce({ + success: false, + error: "模型无权限", + }); + const hookState = createHookState(); + renderSection(); + + await act(async () => { + findByTestId("add-model-button").click(); + await Promise.resolve(); + }); + + await act(async () => { + findByTestId("custom-provider-template-card").click(); + await Promise.resolve(); + }); + + await act(async () => { + setInputValue( + findByTestId("model-provider-name-input"), + "My API", + ); + setInputValue( + findByTestId("model-api-host-input"), + "https://api.example.com/v1", + ); + setInputValue( + findByTestId("model-api-key-input"), + "sk-test", + ); + setInputValue( + findByTestId("model-draft-input"), + "my-model", + ); + }); + + await act(async () => { + findByTestId("model-draft-add-button").click(); + await Promise.resolve(); + }); + + await act(async () => { + findByTestId("model-activate-button").click(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + }); + + expect(hookState.addCustomProvider).toHaveBeenCalled(); + expect(hookState.updateProvider).toHaveBeenCalledWith( + "custom-1", + expect.objectContaining({ + enabled: true, + custom_models: ["my-model"], + }), + ); + expect(mockTestConnection).toHaveBeenCalledWith("custom-1", "my-model"); + expect(hookState.selectProvider).toHaveBeenCalledWith("custom-1"); + expect(document.body.textContent ?? "").not.toContain("模型无权限"); + }); }); diff --git a/src/components/api-key-provider/ModelAddPanel.tsx b/src/components/api-key-provider/ModelAddPanel.tsx index ce6e376a5..b36f6ef39 100644 --- a/src/components/api-key-provider/ModelAddPanel.tsx +++ b/src/components/api-key-provider/ModelAddPanel.tsx @@ -28,7 +28,9 @@ import { dedupeModelIds, getProviderTypeLabel, isSupportedProviderType, + normalizeKnownProviderApiHost, resolvePromptCacheModeRequestValue, + SENSENOVA_OPENAI_COMPATIBLE_API_HOST, } from "./providerConfigUtils"; import { ArrowLeft, @@ -387,6 +389,7 @@ const RESOURCE_PROVIDER_API_HOSTS: Record = { "moonshotai-cn": "https://api.moonshot.cn", openai: "https://api.openai.com", openrouter: "https://openrouter.ai/api/v1/", + sensenova: SENSENOVA_OPENAI_COMPATIBLE_API_HOST, siliconflow: "https://api.siliconflow.cn", "siliconflow-cn": "https://api.siliconflow.cn", xiaomi: "https://token-plan-cn.xiaomimimo.com/anthropic", @@ -394,6 +397,10 @@ const RESOURCE_PROVIDER_API_HOSTS: Record = { "zhipuai-coding-plan": "https://open.bigmodel.cn/api/anthropic", }; +const RESOURCE_PROVIDER_DEFAULT_MODELS: Record = { + sensenova: ["SenseChat-5"], +}; + const ANTHROPIC_COMPATIBLE_REGISTRY_PROVIDER_IDS = new Set([ "kimi-for-coding", "minimax", @@ -445,7 +452,7 @@ function buildCatalogTemplates( category: resolveProviderCategory(item.id, item.group), type: providerType, apiHost: item.api_host, - defaultModels: [], + defaultModels: RESOURCE_PROVIDER_DEFAULT_MODELS[item.id] ?? [], iconProviderId: item.id, systemProviderId: item.id, }; @@ -471,6 +478,9 @@ function buildRegistryTemplates( const firstModel = models[0]; const apiHost = RESOURCE_PROVIDER_API_HOSTS[providerId] ?? ""; + const defaultModels = + RESOURCE_PROVIDER_DEFAULT_MODELS[providerId] ?? + models.map((model) => model.id).slice(0, 3); templates.push({ id: `registry-${providerId}`, name: firstModel.provider_name || providerId, @@ -482,7 +492,7 @@ function buildRegistryTemplates( ? "anthropic-compatible" : "openai", apiHost, - defaultModels: [], + defaultModels, iconProviderId: providerId, providerResourceId: providerId, }); @@ -531,6 +541,9 @@ function validateForm(state: FormState): string | null { if (!state.apiHost.trim()) { return "请填写 API Base URL。"; } + if (state.apiHost.trim() !== normalizeKnownProviderApiHost(state.apiHost)) { + return "检测到你填写的是文档页或旧接口地址，已自动修正 API Base URL，请确认后再激活。"; + } try { new URL(state.apiHost.trim()); } catch { @@ -735,7 +748,18 @@ export const ModelAddPanel: React.FC = ({ }, []); const activateProvider = useCallback(async () => { - const validationError = validateForm(formState); + const normalizedFormState = { + ...formState, + apiHost: normalizeKnownProviderApiHost(formState.apiHost), + }; + if (normalizedFormState.apiHost !== formState.apiHost) { + setFormState((previous) => ({ + ...previous, + apiHost: normalizedFormState.apiHost, + })); + } + + const validationError = validateForm(normalizedFormState); if (validationError) { setSubmitError(validationError); return; @@ -747,13 +771,13 @@ export const ModelAddPanel: React.FC = ({ try { const request: AddCustomProviderRequest = { - name: formState.name.trim(), - type: formState.type, - api_host: formState.apiHost.trim(), + name: normalizedFormState.name.trim(), + type: normalizedFormState.type, + api_host: normalizedFormState.apiHost, prompt_cache_mode: resolvePromptCacheModeRequestValue( - formState.type, - formState.promptCacheMode, - formState.apiHost, + normalizedFormState.type, + normalizedFormState.promptCacheMode, + normalizedFormState.apiHost, ), }; @@ -768,31 +792,29 @@ export const ModelAddPanel: React.FC = ({ api_host: request.api_host, enabled: true, prompt_cache_mode: request.prompt_cache_mode, - custom_models: formState.models, + custom_models: normalizedFormState.models, }); } else { const created = await onAddProvider(request); providerId = created.id; await onUpdateProvider(providerId, { enabled: true, - custom_models: formState.models, + custom_models: normalizedFormState.models, }); } - if (formState.apiKey.trim()) { - await onAddApiKey(providerId, formState.apiKey.trim()); + if (normalizedFormState.apiKey.trim()) { + await onAddApiKey(providerId, normalizedFormState.apiKey.trim()); } const testResult = await apiKeyProviderApi.testConnection( providerId, - formState.models[0], + normalizedFormState.models[0], ); if (!testResult.success) { - throw new Error( - testResult.error || - "已保存配置，但连接测试未通过，请检查密钥或模型 ID。", - ); + onActivated(providerId); + return; } onActivated(providerId); @@ -1004,6 +1026,12 @@ export const ModelAddPanel: React.FC = ({ apiHost: event.target.value, })) } + onBlur={() => + setFormState((previous) => ({ + ...previous, + apiHost: normalizeKnownProviderApiHost(previous.apiHost), + })) + } placeholder="https://api.example.com/v1" className="h-12 rounded-[18px] border-slate-200 bg-white px-4" disabled={submitting} @@ -1074,6 +1102,14 @@ export const ModelAddPanel: React.FC = ({ apiHost: event.target.value, })) } + onBlur={() => + setFormState((previous) => ({ + ...previous, + apiHost: normalizeKnownProviderApiHost( + previous.apiHost, + ), + })) + } placeholder="https://api.example.com/v1" className="h-12 rounded-[18px] border-slate-200 bg-white px-4" disabled={submitting} @@ -1228,7 +1264,7 @@ export const ModelAddPanel: React.FC = ({ ) : ( <> - 测试连接并激活 + 保存并测试 )} diff --git a/src/components/api-key-provider/providerConfigUtils.test.ts b/src/components/api-key-provider/providerConfigUtils.test.ts index 76f7c46cd..c68271f63 100644 --- a/src/components/api-key-provider/providerConfigUtils.test.ts +++ b/src/components/api-key-provider/providerConfigUtils.test.ts @@ -15,11 +15,13 @@ import { getLatestSelectableModel, getFieldsForProviderType, getSpecialProtocolHint, + normalizeKnownProviderApiHost, parseCustomModelsValue, providerTypeRequiresField, PROVIDER_TYPE_FIELDS, PROVIDER_TYPE_VALUES, serializeCustomModels, + SENSENOVA_OPENAI_COMPATIBLE_API_HOST, sortSelectableModels, } from "./providerConfigUtils"; import type { EnhancedModelMetadata } from "@/lib/types/modelRegistry"; @@ -355,3 +357,27 @@ describe("模型辅助函数", () => { expect(getLatestSelectableModel([])).toBeNull(); }); }); + +describe("已知 Provider API Host 归一化", () => { + test("应把 SenseNova 文档页和旧 v1 地址修正为 v2 OpenAI 兼容 Base URL", () => { + expect( + normalizeKnownProviderApiHost("https://platform.sensenova.cn/docs"), + ).toBe(SENSENOVA_OPENAI_COMPATIBLE_API_HOST); + expect( + normalizeKnownProviderApiHost( + "https://www.sensecore.cn/help/docs/model-as-a-service/nova/overview/compatible-mode", + ), + ).toBe(SENSENOVA_OPENAI_COMPATIBLE_API_HOST); + expect( + normalizeKnownProviderApiHost( + "https://api.sensenova.cn/compatible-mode/v1", + ), + ).toBe(SENSENOVA_OPENAI_COMPATIBLE_API_HOST); + }); + + test("未知地址只做 trim，不改写用户输入", () => { + expect(normalizeKnownProviderApiHost(" https://api.example.com/v1 ")).toBe( + "https://api.example.com/v1", + ); + }); +}); diff --git a/src/components/api-key-provider/providerConfigUtils.ts b/src/components/api-key-provider/providerConfigUtils.ts index 9b8f1fe7c..b2b4c625f 100644 --- a/src/components/api-key-provider/providerConfigUtils.ts +++ b/src/components/api-key-provider/providerConfigUtils.ts @@ -12,6 +12,9 @@ import { canonicalizeKnownProviderModelId } from "@/lib/model/xiaomiModelNormali import { getProviderPromptCacheMode } from "@/lib/model/providerPromptCacheSupport"; import type { EnhancedModelMetadata } from "@/lib/types/modelRegistry"; +export const SENSENOVA_OPENAI_COMPATIBLE_API_HOST = + "https://api.sensenova.cn/compatible-mode/v2"; + /** 支持的 Provider 类型列表 */ export const PROVIDER_TYPE_OPTIONS: { value: ProviderType; label: string }[] = [ { value: "openai", label: "OpenAI 兼容" }, @@ -267,3 +270,38 @@ export function providerTypeRequiresField( const extraFields = PROVIDER_TYPE_FIELDS[type] || []; return extraFields.includes(field); } + +export function normalizeKnownProviderApiHost(value: string): string { + const trimmed = value.trim(); + if (!trimmed) { + return ""; + } + + try { + const url = new URL(trimmed); + const hostname = url.hostname.toLowerCase(); + const pathname = url.pathname.toLowerCase(); + + if (hostname === "platform.sensenova.cn" && pathname.startsWith("/docs")) { + return SENSENOVA_OPENAI_COMPATIBLE_API_HOST; + } + + if ( + hostname === "www.sensecore.cn" && + pathname.includes("/model-as-a-service/nova/overview/compatible-mode") + ) { + return SENSENOVA_OPENAI_COMPATIBLE_API_HOST; + } + + if ( + hostname === "api.sensenova.cn" && + pathname.replace(/\/+$/, "") === "/compatible-mode/v1" + ) { + return SENSENOVA_OPENAI_COMPATIBLE_API_HOST; + } + } catch { + return trimmed; + } + + return trimmed; +} diff --git a/src/components/app-sidebar/AppSidebarConversationShelf.tsx b/src/components/app-sidebar/AppSidebarConversationShelf.tsx index 2de9993b6..fcd177fb5 100644 --- a/src/components/app-sidebar/AppSidebarConversationShelf.tsx +++ b/src/components/app-sidebar/AppSidebarConversationShelf.tsx @@ -1,12 +1,30 @@ +import { + useCallback, + useEffect, + useRef, + useState, + type MouseEvent, +} from "react"; +import { createPortal } from "react-dom"; import styled from "styled-components"; import { Archive, + Check, ChevronDown, Clock3, MessageSquarePlus, + MoreHorizontal, + Pencil, + Pin, + Trash2, Undo2, } from "lucide-react"; import type { AsterSessionInfo } from "@/lib/api/agentRuntime"; +import { recordAgentUiPerformanceMetric } from "@/lib/agentUiPerformanceMetrics"; +import { + formatSidebarSessionMeta, + resolveSidebarSessionTitle, +} from "@/components/app-sidebar/sidebarSessionFormatting"; interface AppSidebarConversationShelfProps { recentSessions: AsterSessionInfo[]; @@ -20,12 +38,56 @@ interface AppSidebarConversationShelfProps { actionSessionId: string | null; onCreateConversation: () => void; onNavigateToConversation: (session: AsterSessionInfo) => void; + onPrefetchConversation?: (session: AsterSessionInfo) => void; + onRenameConversation?: (session: AsterSessionInfo) => void; + onDeleteConversation?: (session: AsterSessionInfo) => void; onToggleArchive: (session: AsterSessionInfo, archived: boolean) => void; onShowMoreRecent: () => void; onShowMoreArchived: () => void; onToggleArchivedCollapsed: () => void; } +const CONVERSATION_HOVER_PREFETCH_DELAY_MS = 900; +const FAVORITE_SESSION_IDS_STORAGE_KEY = "lime.app-sidebar.favorite-session-ids"; +const CONVERSATION_MENU_WIDTH = 188; +const CONVERSATION_MENU_APPROX_HEIGHT = 252; +const CONVERSATION_MENU_VIEWPORT_MARGIN = 12; + +type ConversationMenuState = { + session: AsterSessionInfo; + archived: boolean; + top: number; + left: number; +} | null; + +function loadFavoriteSessionIds(): string[] { + if (typeof window === "undefined") { + return []; + } + + try { + const parsed = JSON.parse( + window.localStorage.getItem(FAVORITE_SESSION_IDS_STORAGE_KEY) ?? "[]", + ); + return Array.isArray(parsed) + ? parsed.filter((value): value is string => typeof value === "string") + : []; + } catch { + return []; + } +} + +function persistFavoriteSessionIds(sessionIds: string[]) { + if (typeof window === "undefined") { + return; + } + + window.localStorage.setItem( + FAVORITE_SESSION_IDS_STORAGE_KEY, + JSON.stringify(sessionIds), + ); +} + const ConversationShelf = styled.div` display: flex; flex-direction: column; @@ -33,6 +95,42 @@ const ConversationShelf = styled.div` margin: 2px 0 12px; `; +const ConversationMultiSelectToolbar = styled.div` + min-height: 38px; + border-radius: 16px; + border: 1px solid var(--sidebar-card-border, var(--sidebar-border)); + background: var(--lime-surface, #ffffff); + color: var(--lime-text, #1a3b2b); + box-shadow: var(--sidebar-card-shadow); + display: flex; + align-items: center; + justify-content: space-between; + gap: 10px; + padding: 0 8px 0 12px; + font-size: 12px; + font-weight: 750; +`; + +const ConversationMultiSelectDoneButton = styled.button` + min-height: 28px; + border: 1px solid var(--lime-card-subtle-border, #d9eadf); + border-radius: 11px; + background: var(--lime-surface-soft, #f8fcf9); + color: var(--lime-brand-strong, #166534); + cursor: pointer; + padding: 0 10px; + font-size: 12px; + font-weight: 800; + transition: + background-color 0.16s ease, + border-color 0.16s ease; + + &:hover { + border-color: var(--lime-brand-soft-border, #bbf7d0); + background: var(--lime-brand-soft, #ecfdf5); + } +`; + const ConversationSection = styled.div<{ $compact?: boolean }>` display: flex; flex-direction: column; @@ -210,6 +308,27 @@ const ConversationItemDot = styled.span<{ $active?: boolean }>` $active ? "var(--sidebar-active-foreground)" : "rgba(148, 163, 184, 0.72)"}; `; +const ConversationSelectionMark = styled.span<{ $selected?: boolean }>` + width: 16px; + height: 16px; + flex-shrink: 0; + border-radius: 6px; + border: 1px solid + ${({ $selected }) => + $selected ? "var(--sidebar-active-foreground)" : "var(--sidebar-border)"}; + background: ${({ $selected }) => + $selected ? "var(--sidebar-active-foreground)" : "transparent"}; + color: var(--sidebar-active); + display: inline-flex; + align-items: center; + justify-content: center; + + svg { + width: 11px; + height: 11px; + } +`; + const ConversationItemLabel = styled.span` flex: 1; min-width: 0; @@ -221,6 +340,21 @@ const ConversationItemLabel = styled.span` font-weight: 500; `; +const ConversationFavoriteBadge = styled.span` + width: 16px; + height: 16px; + flex-shrink: 0; + display: inline-flex; + align-items: center; + justify-content: center; + color: var(--sidebar-muted); + + svg { + width: 13px; + height: 13px; + } +`; + const ConversationItemMeta = styled.span` flex-shrink: 0; font-size: 11px; @@ -268,6 +402,56 @@ const ConversationItemActionButton = styled.button` } `; +const ConversationMenuSurface = styled.div` + position: fixed; + z-index: 110; + width: ${CONVERSATION_MENU_WIDTH}px; + padding: 14px 10px; + border-radius: 24px; + border: 1px solid var(--lime-card-subtle-border, rgba(226, 240, 226, 0.9)); + background: var(--lime-surface, #ffffff); + color: var(--lime-text-strong, #0f172a); + box-shadow: + 0 22px 64px rgba(15, 23, 42, 0.18), + 0 1px 0 rgba(255, 255, 255, 0.76) inset; +`; + +const ConversationMenuItem = styled.button<{ $danger?: boolean }>` + width: 100%; + min-height: 42px; + border: none; + border-radius: 14px; + background: transparent; + color: ${({ $danger }) => + $danger ? "var(--lime-danger, #b91c1c)" : "var(--lime-text-strong, #0f172a)"}; + display: flex; + align-items: center; + gap: 16px; + padding: 0 16px; + cursor: pointer; + text-align: left; + font-size: 16px; + font-weight: 680; + transition: + background-color 0.16s ease, + color 0.16s ease; + + &:hover { + background: ${({ $danger }) => + $danger + ? "var(--lime-danger-soft, #fff1f2)" + : "var(--lime-surface-hover, #f4fdf4)"}; + } + + svg { + width: 19px; + height: 19px; + flex-shrink: 0; + color: ${({ $danger }) => + $danger ? "var(--lime-danger, #b91c1c)" : "var(--sidebar-muted)"}; + } +`; + const ConversationEmptyState = styled.div` display: flex; align-items: center; @@ -287,42 +471,6 @@ const ConversationEmptyState = styled.div` text-align: center; `; -function formatSidebarSessionTime(updatedAt: number): string { - const diffMs = Date.now() - updatedAt * 1000; - const diffMinutes = Math.max(1, Math.floor(diffMs / (1000 * 60))); - - if (diffMinutes < 60) { - return `${diffMinutes}分`; - } - - const diffHours = Math.floor(diffMinutes / 60); - if (diffHours < 24) { - return `${diffHours}时`; - } - - const diffDays = Math.floor(diffHours / 24); - if (diffDays < 30) { - return `${diffDays}天`; - } - - return new Date(updatedAt * 1000).toLocaleDateString("zh-CN", { - month: "numeric", - day: "numeric", - }); -} - -function formatSidebarSessionMeta(session: AsterSessionInfo): string { - if (typeof session.archived_at === "number" && session.archived_at > 0) { - return `归档 ${formatSidebarSessionTime(session.archived_at)}`; - } - - return formatSidebarSessionTime(session.updated_at); -} - -function resolveSidebarSessionTitle(session: AsterSessionInfo): string { - return session.name?.trim() || "未命名对话"; -} - function renderEmptyState(text: string) { return ( @@ -344,13 +492,276 @@ export function AppSidebarConversationShelf({ actionSessionId, onCreateConversation, onNavigateToConversation, + onPrefetchConversation, + onRenameConversation, + onDeleteConversation, onToggleArchive, onShowMoreRecent, onShowMoreArchived, onToggleArchivedCollapsed, }: AppSidebarConversationShelfProps) { + const [menuState, setMenuState] = useState(null); + const [favoriteSessionIds, setFavoriteSessionIds] = useState( + loadFavoriteSessionIds, + ); + const [multiSelectMode, setMultiSelectMode] = useState(false); + const [selectedSessionIds, setSelectedSessionIds] = useState>( + () => new Set(), + ); + const hoverPrefetchTimerRef = useRef | null>( + null, + ); + const hoverPrefetchSessionRef = useRef(null); + const clearHoverPrefetch = useCallback(() => { + if (hoverPrefetchTimerRef.current !== null) { + const session = hoverPrefetchSessionRef.current; + clearTimeout(hoverPrefetchTimerRef.current); + hoverPrefetchTimerRef.current = null; + hoverPrefetchSessionRef.current = null; + if (session) { + recordAgentUiPerformanceMetric( + "sidebar.conversation.prefetchCancelled", + { + sessionId: session.id, + source: "conversation_shelf", + workspaceId: session.workspace_id ?? null, + }, + ); + } + } + }, []); + const scheduleHoverPrefetch = useCallback( + (session: AsterSessionInfo) => { + if ( + hoverPrefetchTimerRef.current !== null && + hoverPrefetchSessionRef.current?.id === session.id + ) { + return; + } + + clearHoverPrefetch(); + recordAgentUiPerformanceMetric("sidebar.conversation.prefetchScheduled", { + sessionId: session.id, + source: "conversation_shelf", + workspaceId: session.workspace_id ?? null, + }); + hoverPrefetchSessionRef.current = session; + hoverPrefetchTimerRef.current = setTimeout(() => { + hoverPrefetchTimerRef.current = null; + hoverPrefetchSessionRef.current = null; + recordAgentUiPerformanceMetric("sidebar.conversation.prefetchFired", { + sessionId: session.id, + source: "conversation_shelf", + workspaceId: session.workspace_id ?? null, + }); + onPrefetchConversation?.(session); + }, CONVERSATION_HOVER_PREFETCH_DELAY_MS); + }, + [clearHoverPrefetch, onPrefetchConversation], + ); + + useEffect(() => clearHoverPrefetch, [clearHoverPrefetch]); + + useEffect(() => { + if (!menuState) { + return; + } + + const closeMenu = () => setMenuState(null); + window.addEventListener("click", closeMenu); + window.addEventListener("resize", closeMenu); + window.addEventListener("scroll", closeMenu, true); + + return () => { + window.removeEventListener("click", closeMenu); + window.removeEventListener("resize", closeMenu); + window.removeEventListener("scroll", closeMenu, true); + }; + }, [menuState]); + + const openConversationMenu = useCallback( + ( + event: MouseEvent, + session: AsterSessionInfo, + archived: boolean, + ) => { + event.stopPropagation(); + clearHoverPrefetch(); + const rect = event.currentTarget.getBoundingClientRect(); + setMenuState({ + session, + archived, + top: Math.max( + CONVERSATION_MENU_VIEWPORT_MARGIN, + Math.min( + rect.bottom + 8, + window.innerHeight - + CONVERSATION_MENU_APPROX_HEIGHT - + CONVERSATION_MENU_VIEWPORT_MARGIN, + ), + ), + left: Math.max( + CONVERSATION_MENU_VIEWPORT_MARGIN, + Math.min( + rect.right - CONVERSATION_MENU_WIDTH, + window.innerWidth - + CONVERSATION_MENU_WIDTH - + CONVERSATION_MENU_VIEWPORT_MARGIN, + ), + ), + }); + }, + [clearHoverPrefetch], + ); + + const toggleFavoriteSession = useCallback((session: AsterSessionInfo) => { + setFavoriteSessionIds((current) => { + const exists = current.includes(session.id); + const next = exists + ? current.filter((sessionId) => sessionId !== session.id) + : [session.id, ...current]; + persistFavoriteSessionIds(next); + return next; + }); + }, []); + + const enterMultiSelectMode = useCallback((session: AsterSessionInfo) => { + setMultiSelectMode(true); + setSelectedSessionIds(new Set([session.id])); + }, []); + + const exitMultiSelectMode = useCallback(() => { + setMultiSelectMode(false); + setSelectedSessionIds(new Set()); + }, []); + + const toggleSelectedSession = useCallback((session: AsterSessionInfo) => { + setSelectedSessionIds((current) => { + const next = new Set(current); + if (next.has(session.id)) { + next.delete(session.id); + } else { + next.add(session.id); + } + return next; + }); + }, []); + + const runMenuAction = useCallback( + (action: () => void) => { + setMenuState(null); + action(); + }, + [], + ); + + useEffect(() => { + if (!multiSelectMode || typeof window === "undefined") { + return; + } + + const handleKeyDown = (event: KeyboardEvent) => { + if (event.key === "Escape") { + exitMultiSelectMode(); + } + }; + + window.addEventListener("keydown", handleKeyDown); + return () => { + window.removeEventListener("keydown", handleKeyDown); + }; + }, [exitMultiSelectMode, multiSelectMode]); + + const renderConversationMenu = () => { + if (!menuState || typeof document === "undefined") { + return null; + } + + const { session, archived, top, left } = menuState; + const title = resolveSidebarSessionTitle(session); + const favorite = favoriteSessionIds.includes(session.id); + const archiveLabel = archived ? "恢复" : "归档"; + const ArchiveIcon = archived ? Undo2 : Archive; + + return createPortal( + event.stopPropagation()} + > + {onRenameConversation ? ( + runMenuAction(() => onRenameConversation(session))} + > + + 重命名 + + ) : null} + runMenuAction(() => toggleFavoriteSession(session))} + > + + {favorite ? "取消收藏" : "收藏"} + + + runMenuAction(() => onToggleArchive(session, !archived)) + } + > + + {archiveLabel} + + runMenuAction(() => enterMultiSelectMode(session))} + > + + 多选 + + {onDeleteConversation ? ( + runMenuAction(() => onDeleteConversation(session))} + > + + 删除 + + ) : null} + , + document.body, + ); + }; + return ( + {multiSelectMode ? ( + + 已选择 {selectedSessionIds.size} 个对话 + + 完成 + + + ) : null} 最近对话 @@ -369,6 +780,9 @@ export function AppSidebarConversationShelf({ : recentSessions.length > 0 ? recentSessions.map((session) => { const isCurrentConversation = currentSessionId === session.id; + const title = resolveSidebarSessionTitle(session); + const favorite = favoriteSessionIds.includes(session.id); + const selected = selectedSessionIds.has(session.id); return ( onNavigateToConversation(session)} - title={resolveSidebarSessionTitle(session)} + onClick={() => { + clearHoverPrefetch(); + recordAgentUiPerformanceMetric( + "sidebar.conversation.click", + { + sessionId: session.id, + source: "conversation_shelf", + workspaceId: session.workspace_id ?? null, + }, + ); + if (multiSelectMode) { + toggleSelectedSession(session); + return; + } + onNavigateToConversation(session); + }} + onBlur={clearHoverPrefetch} + onFocus={() => scheduleHoverPrefetch(session)} + onPointerEnter={() => scheduleHoverPrefetch(session)} + onPointerLeave={clearHoverPrefetch} + title={title} > - + {multiSelectMode ? ( + + {selected ? : null} + + ) : ( + + )} - {resolveSidebarSessionTitle(session)} + {title} + {favorite ? ( + + + + ) : null} {formatSidebarSessionMeta(session)} { - event.stopPropagation(); - onToggleArchive(session, true); - }} + onClick={(event) => + openConversationMenu(event, session, false) + } > - + ); @@ -439,6 +887,9 @@ export function AppSidebarConversationShelf({ ? archivedSessions.map((session) => { const isCurrentConversation = currentSessionId === session.id; + const title = resolveSidebarSessionTitle(session); + const favorite = favoriteSessionIds.includes(session.id); + const selected = selectedSessionIds.has(session.id); return ( onNavigateToConversation(session)} - title={resolveSidebarSessionTitle(session)} + onClick={() => { + clearHoverPrefetch(); + recordAgentUiPerformanceMetric( + "sidebar.conversation.click", + { + sessionId: session.id, + source: "conversation_shelf", + workspaceId: session.workspace_id ?? null, + }, + ); + if (multiSelectMode) { + toggleSelectedSession(session); + return; + } + onNavigateToConversation(session); + }} + onBlur={clearHoverPrefetch} + onFocus={() => scheduleHoverPrefetch(session)} + onPointerEnter={() => scheduleHoverPrefetch(session)} + onPointerLeave={clearHoverPrefetch} + title={title} > - + {multiSelectMode ? ( + + {selected ? : null} + + ) : ( + + )} - {resolveSidebarSessionTitle(session)} + {title} + {favorite ? ( + + + + ) : null} {formatSidebarSessionMeta(session)} { - event.stopPropagation(); - onToggleArchive(session, false); - }} + onClick={(event) => + openConversationMenu(event, session, true) + } > - + ); @@ -491,6 +974,7 @@ export function AppSidebarConversationShelf({ ) : null} + {renderConversationMenu()} ); } diff --git a/src/components/app-sidebar/sidebarSessionFormatting.ts b/src/components/app-sidebar/sidebarSessionFormatting.ts new file mode 100644 index 000000000..8ec274c61 --- /dev/null +++ b/src/components/app-sidebar/sidebarSessionFormatting.ts @@ -0,0 +1,37 @@ +import type { AsterSessionInfo } from "@/lib/api/agentRuntime"; + +function formatSidebarSessionTime(updatedAt: number): string { + const diffMs = Date.now() - updatedAt * 1000; + const diffMinutes = Math.max(1, Math.floor(diffMs / (1000 * 60))); + + if (diffMinutes < 60) { + return `${diffMinutes}分`; + } + + const diffHours = Math.floor(diffMinutes / 60); + if (diffHours < 24) { + return `${diffHours}时`; + } + + const diffDays = Math.floor(diffHours / 24); + if (diffDays < 30) { + return `${diffDays}天`; + } + + return new Date(updatedAt * 1000).toLocaleDateString("zh-CN", { + month: "numeric", + day: "numeric", + }); +} + +export function formatSidebarSessionMeta(session: AsterSessionInfo): string { + if (typeof session.archived_at === "number" && session.archived_at > 0) { + return `归档 ${formatSidebarSessionTime(session.archived_at)}`; + } + + return formatSidebarSessionTime(session.updated_at); +} + +export function resolveSidebarSessionTitle(session: AsterSessionInfo): string { + return session.name?.trim() || "未命名对话"; +} diff --git a/src/components/artifact/renderers/BrowserAssistRenderer.test.tsx b/src/components/artifact/renderers/BrowserAssistRenderer.test.tsx index da9513775..5fcc3f1a9 100644 --- a/src/components/artifact/renderers/BrowserAssistRenderer.test.tsx +++ b/src/components/artifact/renderers/BrowserAssistRenderer.test.tsx @@ -97,4 +97,55 @@ describe("BrowserAssistRenderer", () => { container.querySelector('[data-testid="browser-runtime-workspace"]'), ).toBeNull(); }); + + it("带 browserActionIndex 的 Artifact 应展示 Browser Assist 复盘", async () => { + const container = await renderArtifact( + createArtifact({ + meta: { + browserActionIndex: { + actionCount: 2, + sessionCount: 1, + observationCount: 1, + screenshotCount: 1, + lastUrl: "https://example.com/", + sessionIds: ["browser-session-1"], + targetIds: ["target-1"], + profileKeys: ["general_browser_assist"], + items: [ + { + artifactKind: "browser_session", + action: "navigate", + status: "completed", + success: true, + sessionId: "browser-session-1", + targetId: "target-1", + backend: "cdp_direct", + lastUrl: "https://example.com/", + }, + { + artifactKind: "browser_snapshot", + action: "get_page_info", + status: "completed", + success: true, + sessionId: "browser-session-1", + targetId: "target-1", + entrySource: "at_browser_agent_command", + backend: "lime_extension_bridge", + lastUrl: "https://example.com/", + observationAvailable: true, + screenshotAvailable: true, + }, + ], + }, + }, + }), + ); + + expect(container.textContent).toContain("browser_replay_viewer"); + expect(container.textContent).toContain("Browser Assist 复盘"); + expect(container.textContent).toContain("get_page_info"); + expect(container.textContent).toContain("browser_snapshot"); + expect(container.textContent).toContain("https://example.com/"); + expect(container.textContent).toContain("observation / screenshot"); + }); }); diff --git a/src/components/artifact/renderers/BrowserAssistRenderer.tsx b/src/components/artifact/renderers/BrowserAssistRenderer.tsx index cabed0f35..c8a941ca6 100644 --- a/src/components/artifact/renderers/BrowserAssistRenderer.tsx +++ b/src/components/artifact/renderers/BrowserAssistRenderer.tsx @@ -5,9 +5,52 @@ */ import React, { memo } from "react"; -import { AlertCircle, Loader2 } from "lucide-react"; +import { + AlertCircle, + Camera, + Eye, + Loader2, + MousePointerClick, +} from "lucide-react"; import type { ArtifactRendererProps } from "@/lib/artifact/types"; +interface BrowserActionReplayItem { + artifactKind?: string; + toolName?: string; + action?: string; + status?: string; + success?: boolean; + sessionId?: string; + targetId?: string; + profileKey?: string; + backend?: string; + requestId?: string; + lastUrl?: string; + title?: string; + entrySource?: string; + observationAvailable?: boolean; + screenshotAvailable?: boolean; +} + +interface BrowserActionReplayIndex { + actionCount: number; + sessionCount: number; + observationCount: number; + screenshotCount: number; + lastUrl?: string; + sessionIds: string[]; + targetIds: string[]; + profileKeys: string[]; + items: BrowserActionReplayItem[]; +} + +function asRecord(value: unknown): Record | null { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + return value as Record; +} + function readMetaString( meta: Record, ...keys: string[] @@ -21,6 +64,365 @@ function readMetaString( return undefined; } +function readString( + record: Record, + ...keys: string[] +): string | undefined { + return readMetaString(record, ...keys); +} + +function readNumber( + record: Record, + ...keys: string[] +): number { + for (const key of keys) { + const value = record[key]; + if (typeof value === "number" && Number.isFinite(value)) { + return value; + } + } + return 0; +} + +function readBoolean( + record: Record, + ...keys: string[] +): boolean | undefined { + for (const key of keys) { + const value = record[key]; + if (typeof value === "boolean") { + return value; + } + } + return undefined; +} + +function readStringList( + record: Record, + ...keys: string[] +): string[] { + for (const key of keys) { + const value = record[key]; + if (Array.isArray(value)) { + return value.filter((item): item is string => typeof item === "string"); + } + } + return []; +} + +function normalizeReplayItem(value: unknown): BrowserActionReplayItem | null { + const record = asRecord(value); + if (!record) { + return null; + } + + const item: BrowserActionReplayItem = { + artifactKind: readString(record, "artifactKind", "artifact_kind"), + toolName: readString(record, "toolName", "tool_name"), + action: readString(record, "action"), + status: readString(record, "status"), + success: readBoolean(record, "success"), + sessionId: readString(record, "sessionId", "session_id"), + targetId: readString(record, "targetId", "target_id"), + profileKey: readString(record, "profileKey", "profile_key"), + backend: readString(record, "backend"), + requestId: readString(record, "requestId", "request_id"), + lastUrl: readString(record, "lastUrl", "last_url"), + title: readString(record, "title"), + entrySource: readString(record, "entrySource", "entry_source"), + observationAvailable: readBoolean( + record, + "observationAvailable", + "observation_available", + ), + screenshotAvailable: readBoolean( + record, + "screenshotAvailable", + "screenshot_available", + ), + }; + + return Object.values(item).some( + (field) => field !== undefined && field !== "", + ) + ? item + : null; +} + +function findBrowserActionIndexRecord( + meta: Record, + content: string, +): Record | null { + const direct = + asRecord(meta.browserActionIndex) || asRecord(meta.browser_action_index); + if (direct) { + return direct; + } + + const modalityRuntimeContracts = + asRecord(meta.modalityRuntimeContracts) || + asRecord(meta.modality_runtime_contracts); + const snapshotIndex = + asRecord(modalityRuntimeContracts?.snapshotIndex) || + asRecord(modalityRuntimeContracts?.snapshot_index); + const nested = + asRecord(snapshotIndex?.browserActionIndex) || + asRecord(snapshotIndex?.browser_action_index); + if (nested) { + return nested; + } + + if (!content.trim()) { + return null; + } + + try { + const parsed = asRecord(JSON.parse(content)); + if (!parsed) { + return null; + } + return ( + asRecord(parsed.browserActionIndex) || + asRecord(parsed.browser_action_index) || + null + ); + } catch { + return null; + } +} + +function normalizeBrowserActionIndex( + meta: Record, + content: string, +): BrowserActionReplayIndex | null { + const record = findBrowserActionIndexRecord(meta, content); + if (!record) { + return null; + } + + const rawItems = Array.isArray(record.items) ? record.items : []; + const items = rawItems + .map((item) => normalizeReplayItem(item)) + .filter((item): item is BrowserActionReplayItem => item !== null); + const actionCount = readNumber(record, "actionCount", "action_count"); + const sessionCount = readNumber(record, "sessionCount", "session_count"); + const observationCount = readNumber( + record, + "observationCount", + "observation_count", + ); + const screenshotCount = readNumber( + record, + "screenshotCount", + "screenshot_count", + ); + + if ( + actionCount === 0 && + sessionCount === 0 && + observationCount === 0 && + screenshotCount === 0 && + items.length === 0 + ) { + return null; + } + + return { + actionCount, + sessionCount, + observationCount, + screenshotCount, + lastUrl: readString(record, "lastUrl", "last_url"), + sessionIds: readStringList(record, "sessionIds", "session_ids"), + targetIds: readStringList(record, "targetIds", "target_ids"), + profileKeys: readStringList(record, "profileKeys", "profile_keys"), + items, + }; +} + +function formatReplayStatus(item: BrowserActionReplayItem): string { + if (item.success === true && !item.status) { + return "成功"; + } + if (item.success === false && !item.status) { + return "失败"; + } + + switch (item.status) { + case "completed": + case "success": + case "succeeded": + return "成功"; + case "failed": + case "error": + return "失败"; + case "running": + return "执行中"; + case "pending": + return "待处理"; + default: + return item.status || "未知状态"; + } +} + +function BrowserReplayStat({ + label, + value, + hint, +}: { + label: string; + value: string; + hint: string; +}) { + return ( +
+
{label}
+
+ {value} +
+
{hint}
+
+ ); +} + +function BrowserReplayView({ index }: { index: BrowserActionReplayIndex }) { + const recentItems = index.items.slice(-5).reverse(); + const latestItem = recentItems.find((item) => item.lastUrl) || recentItems[0]; + const latestUrl = index.lastUrl || latestItem?.lastUrl || "暂无 URL"; + + return ( +
+
+
+
+
+
+ + browser_replay_viewer +
+

+ Browser Assist 复盘 +

+

+ 从 browserActionIndex 恢复浏览器动作、会话与观察证据。 +

+
+
+ + browser_control + + + {index.actionCount} actions + +
+
+
+ +
+ + + + +
+ + {recentItems.length > 0 ? ( +
+
+ + 最近浏览器动作 +
+
+ {recentItems.map((item, itemIndex) => ( +
+
+ + {item.action || item.toolName || "browser action"} + + + {item.artifactKind || "browser_session"} + + + {formatReplayStatus(item)} + + {item.backend ? ( + + {item.backend} + + ) : null} + {item.screenshotAvailable ? ( + + + screenshot + + ) : null} +
+
+ {item.lastUrl ? ( +
+ URL： + + {item.lastUrl} + +
+ ) : null} +
+ {item.sessionId ? ( + + session： + + {item.sessionId} + + + ) : null} + {item.targetId ? ( + + target： + + {item.targetId} + + + ) : null} + {item.entrySource ? ( + + entry： + + {item.entrySource} + + + ) : null} +
+
+
+ ))} +
+
+ ) : null} +
+
+ ); +} + export const BrowserAssistRenderer: React.FC = memo( ({ artifact }) => { const initialSessionId = readMetaString( @@ -52,6 +454,10 @@ export const BrowserAssistRenderer: React.FC = memo( const launchError = artifact.error || readMetaString(artifact.meta, "launchError", "launch_error"); + const replayIndex = normalizeBrowserActionIndex( + artifact.meta, + artifact.content, + ); if (artifact.status === "pending" || launchState === "launching") { return ( @@ -100,6 +506,10 @@ export const BrowserAssistRenderer: React.FC = memo( ); } + if (replayIndex) { + return ; + } + if (!initialSessionId && !initialProfileKey) { return (
diff --git a/src/components/onboarding/steps/VoiceShortcutTestStep.test.tsx b/src/components/onboarding/steps/VoiceShortcutTestStep.test.tsx index d50703a60..c508e7d61 100644 --- a/src/components/onboarding/steps/VoiceShortcutTestStep.test.tsx +++ b/src/components/onboarding/steps/VoiceShortcutTestStep.test.tsx @@ -14,6 +14,10 @@ const listeners = vi.hoisted(() => ({ registered_shortcut: "CommandOrControl+Shift+V", translate_shortcut_registered: false, registered_translate_shortcut: null, + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", })), })); @@ -117,6 +121,10 @@ describe("VoiceShortcutTestStep", () => { registered_shortcut: null, translate_shortcut_registered: false, registered_translate_shortcut: null, + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }); const container = await renderStep(); diff --git a/src/components/settings-v2/agent/voice/index.test.tsx b/src/components/settings-v2/agent/voice/index.test.tsx index 8106967a0..c2713c686 100644 --- a/src/components/settings-v2/agent/voice/index.test.tsx +++ b/src/components/settings-v2/agent/voice/index.test.tsx @@ -9,6 +9,13 @@ const { mockSaveVoiceInputConfig, mockGetAsrCredentials, mockGetVoiceShortcutRuntimeStatus, + mockListVoiceModelCatalog, + mockGetVoiceModelInstallState, + mockDownloadVoiceModel, + mockDeleteVoiceModel, + mockSetDefaultVoiceModel, + mockTestTranscribeVoiceModelFile, + mockOpenDialog, mockValidateShortcut, } = vi.hoisted(() => ({ mockGetConfig: vi.fn(), @@ -17,6 +24,13 @@ const { mockSaveVoiceInputConfig: vi.fn(), mockGetAsrCredentials: vi.fn(), mockGetVoiceShortcutRuntimeStatus: vi.fn(), + mockListVoiceModelCatalog: vi.fn(), + mockGetVoiceModelInstallState: vi.fn(), + mockDownloadVoiceModel: vi.fn(), + mockDeleteVoiceModel: vi.fn(), + mockSetDefaultVoiceModel: vi.fn(), + mockTestTranscribeVoiceModelFile: vi.fn(), + mockOpenDialog: vi.fn(), mockValidateShortcut: vi.fn(), })); @@ -35,10 +49,23 @@ vi.mock("@/lib/api/hotkeys", () => ({ getVoiceShortcutRuntimeStatus: mockGetVoiceShortcutRuntimeStatus, })); +vi.mock("@/lib/api/voiceModels", () => ({ + listVoiceModelCatalog: mockListVoiceModelCatalog, + getVoiceModelInstallState: mockGetVoiceModelInstallState, + downloadVoiceModel: mockDownloadVoiceModel, + deleteVoiceModel: mockDeleteVoiceModel, + setDefaultVoiceModel: mockSetDefaultVoiceModel, + testTranscribeVoiceModelFile: mockTestTranscribeVoiceModelFile, +})); + vi.mock("@/lib/api/experimentalFeatures", () => ({ validateShortcut: mockValidateShortcut, })); +vi.mock("@tauri-apps/plugin-dialog", () => ({ + open: mockOpenDialog, +})); + vi.mock("@/hooks/useConfiguredProviders", () => ({ useConfiguredProviders: () => ({ providers: [ @@ -205,28 +232,18 @@ async function flushEffects(times = 4) { }); } -beforeEach(() => { - ( - globalThis as typeof globalThis & { - IS_REACT_ACT_ENVIRONMENT?: boolean; - } - ).IS_REACT_ACT_ENVIRONMENT = true; - - vi.clearAllMocks(); - - mockGetConfig.mockResolvedValue({ - workspace_preferences: { - media_defaults: { - voice: { - preferredProviderId: "openai", - preferredModelId: "gpt-4o-mini-tts", - allowFallback: false, - }, - }, - }, +function createDeferred() { + let resolve!: (value: T) => void; + let reject!: (reason?: unknown) => void; + const promise = new Promise((resolvePromise, rejectPromise) => { + resolve = resolvePromise; + reject = rejectPromise; }); + return { promise, resolve, reject }; +} - mockGetVoiceInputConfig.mockResolvedValue({ +function createVoiceInputConfig(overrides: Record = {}) { + return { enabled: true, shortcut: "CommandOrControl+Shift+V", translate_shortcut: "CommandOrControl+Shift+T", @@ -263,8 +280,33 @@ beforeEach(() => { selected_device_id: undefined, sound_enabled: true, translate_instruction_id: "translate_en", + ...overrides, + }; +} + +beforeEach(() => { + ( + globalThis as typeof globalThis & { + IS_REACT_ACT_ENVIRONMENT?: boolean; + } + ).IS_REACT_ACT_ENVIRONMENT = true; + + vi.clearAllMocks(); + + mockGetConfig.mockResolvedValue({ + workspace_preferences: { + media_defaults: { + voice: { + preferredProviderId: "openai", + preferredModelId: "gpt-4o-mini-tts", + allowFallback: false, + }, + }, + }, }); + mockGetVoiceInputConfig.mockResolvedValue(createVoiceInputConfig()); + mockGetAsrCredentials.mockResolvedValue([ { id: "openai-default", @@ -281,8 +323,85 @@ beforeEach(() => { registered_shortcut: "CommandOrControl+Shift+V", translate_shortcut_registered: true, registered_translate_shortcut: "CommandOrControl+Shift+T", + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }); + mockListVoiceModelCatalog.mockResolvedValue([ + { + id: "sensevoice-small-int8-2024-07-17", + name: "SenseVoice Small INT8", + provider: "FunAudioLLM / sherpa-onnx", + description: "本地离线 ASR", + version: "2024-07-17", + languages: ["zh", "en", "ja", "ko", "yue"], + size_bytes: 262144000, + download_url: "https://example.test/sensevoice.tar.bz2", + vad_model_id: "silero-vad-onnx", + vad_download_url: "https://example.test/silero_vad.onnx", + runtime: "sherpa-onnx", + bundled: false, + checksum_sha256: null, + }, + ]); + mockGetVoiceModelInstallState.mockResolvedValue({ + model_id: "sensevoice-small-int8-2024-07-17", + installed: false, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: null, + tokens_file: null, + vad_file: null, + installed_bytes: 0, + last_verified_at: 1, + missing_files: ["model.int8.onnx", "tokens.txt", "silero_vad.onnx"], + default_credential_id: null, + }); + mockDownloadVoiceModel.mockResolvedValue({ + state: { + model_id: "sensevoice-small-int8-2024-07-17", + installed: true, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: "/mock/model.int8.onnx", + tokens_file: "/mock/tokens.txt", + vad_file: "/mock/silero_vad.onnx", + installed_bytes: 262144000, + last_verified_at: 2, + missing_files: [], + default_credential_id: null, + }, + }); + mockDeleteVoiceModel.mockResolvedValue({ + model_id: "sensevoice-small-int8-2024-07-17", + installed: false, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: null, + tokens_file: null, + vad_file: null, + installed_bytes: 0, + last_verified_at: 3, + missing_files: ["model.int8.onnx", "tokens.txt", "silero_vad.onnx"], + default_credential_id: null, + }); + mockSetDefaultVoiceModel.mockResolvedValue({ + id: "sensevoice-local-sensevoice-small-int8-2024-07-17", + provider: "sensevoice_local", + name: "SenseVoice Small 本地", + is_default: true, + disabled: false, + language: "auto", + }); + mockTestTranscribeVoiceModelFile.mockResolvedValue({ + text: "这是测试音频的本地转写结果。", + duration_secs: 2.5, + sample_rate: 16000, + language: "auto", + }); + mockOpenDialog.mockResolvedValue("/tmp/interview.wav"); mockValidateShortcut.mockResolvedValue(true); mockSaveConfig.mockResolvedValue(undefined); mockSaveVoiceInputConfig.mockResolvedValue(undefined); @@ -302,12 +421,22 @@ afterEach(() => { }); describe("VoiceSettings", () => { - it("应同时渲染语音输入、语音处理和语音服务模型设置", async () => { + it("应同时渲染语音输入、语音模型、语音处理和语音服务模型设置", async () => { const container = renderComponent(); await flushEffects(6); const text = container.textContent ?? ""; expect(text).toContain("语音输入"); + expect(text).toContain("语音模型"); + expect(text).toContain("语音输入快捷键"); + expect(text).toContain("按住录音，松开识别"); + expect(text).toContain("🌐 Fn"); + expect(text).toContain("SenseVoice Small"); + expect(text).toContain("本地"); + expect(text).toContain("未安装（ONNX int8 量化"); + expect(text).toContain("下载并设为默认后可离线转写"); + expect(text).toContain("下载模型"); + expect(text).toContain("当前平台不支持 Fn，已使用快捷键回退"); expect(text).toContain("语音处理"); expect(text).toContain("语音服务模型"); expect(text).toContain("OpenAI Whisper 默认凭证"); @@ -317,6 +446,187 @@ describe("VoiceSettings", () => { expect(text).toContain("翻译模式快捷键已注册"); }); + it("关闭语音输入时应展示 Fn 快捷键未开启状态", async () => { + mockGetVoiceInputConfig.mockResolvedValueOnce( + createVoiceInputConfig({ enabled: false }), + ); + + const container = renderComponent(); + await flushEffects(6); + + const text = container.textContent ?? ""; + expect(text).toContain("语音输入未开启，不会注册 Fn 或全局快捷键"); + expect(text).toContain("未启用，不会注册全局快捷键"); + }); + + it("点击下载模型时应调用本地模型下载命令", async () => { + const container = renderComponent(); + await flushEffects(6); + + const downloadButton = Array.from( + container.querySelectorAll("button"), + ).find((element) => element.textContent?.includes("下载模型")); + expect(downloadButton).toBeInstanceOf(HTMLButtonElement); + + await act(async () => { + downloadButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await flushEffects(4); + }); + + expect(mockDownloadVoiceModel).toHaveBeenCalledWith( + "sensevoice-small-int8-2024-07-17", + ); + expect(container.textContent ?? "").toContain("已安装"); + }); + + it("模型下载中应展示下载状态占位进度", async () => { + const pendingDownload = createDeferred<{ + state: { + model_id: string; + installed: boolean; + installing: boolean; + install_dir: string; + model_file: string; + tokens_file: string; + vad_file: string; + installed_bytes: number; + last_verified_at: number; + missing_files: string[]; + default_credential_id: null; + }; + }>(); + mockDownloadVoiceModel.mockReturnValueOnce(pendingDownload.promise); + + const container = renderComponent(); + await flushEffects(6); + + const downloadButton = Array.from( + container.querySelectorAll("button"), + ).find((element) => element.textContent?.includes("下载模型")); + expect(downloadButton).toBeInstanceOf(HTMLButtonElement); + + await act(async () => { + downloadButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await Promise.resolve(); + }); + + expect(container.textContent ?? "").toContain( + "正在下载 model.int8.onnx (1/2)", + ); + expect(container.textContent ?? "").toContain("完成后自动校验并安装"); + + pendingDownload.resolve({ + state: { + model_id: "sensevoice-small-int8-2024-07-17", + installed: true, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: "/mock/model.int8.onnx", + tokens_file: "/mock/tokens.txt", + vad_file: "/mock/silero_vad.onnx", + installed_bytes: 262144000, + last_verified_at: 2, + missing_files: [], + default_credential_id: null, + }, + }); + await flushEffects(6); + }); + + it("模型已安装时应支持设为默认", async () => { + mockGetVoiceModelInstallState.mockResolvedValue({ + model_id: "sensevoice-small-int8-2024-07-17", + installed: true, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: "/mock/model.int8.onnx", + tokens_file: "/mock/tokens.txt", + vad_file: "/mock/silero_vad.onnx", + installed_bytes: 262144000, + last_verified_at: 4, + missing_files: [], + default_credential_id: null, + }); + + const container = renderComponent(); + await flushEffects(6); + + const defaultButton = Array.from(container.querySelectorAll("button")).find( + (element) => element.textContent?.includes("设为默认"), + ); + expect(defaultButton).toBeInstanceOf(HTMLButtonElement); + + await act(async () => { + defaultButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await flushEffects(6); + }); + + expect(mockSetDefaultVoiceModel).toHaveBeenCalledWith( + "sensevoice-small-int8-2024-07-17", + ); + }); + + it("模型已安装时应支持选择 WAV 文件并测试转写", async () => { + mockGetVoiceModelInstallState.mockResolvedValue({ + model_id: "sensevoice-small-int8-2024-07-17", + installed: true, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: "/mock/model.int8.onnx", + tokens_file: "/mock/tokens.txt", + vad_file: "/mock/silero_vad.onnx", + installed_bytes: 262144000, + last_verified_at: 4, + missing_files: [], + default_credential_id: null, + }); + + const container = renderComponent(); + await flushEffects(6); + + const input = container.querySelector( + "input[aria-label='WAV 文件路径']", + ) as HTMLInputElement | null; + expect(input).toBeInstanceOf(HTMLInputElement); + + const selectButton = Array.from(container.querySelectorAll("button")).find( + (element) => element.textContent?.includes("选择 WAV"), + ); + expect(selectButton).toBeInstanceOf(HTMLButtonElement); + + await act(async () => { + selectButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await flushEffects(4); + }); + + expect(mockOpenDialog).toHaveBeenCalledWith({ + title: "选择 WAV 测试音频", + multiple: false, + directory: false, + filters: [{ name: "WAV 音频", extensions: ["wav"] }], + }); + expect(input?.value).toBe("/tmp/interview.wav"); + + const testButton = Array.from(container.querySelectorAll("button")).find( + (element) => element.textContent?.includes("测试转写"), + ); + expect(testButton).toBeInstanceOf(HTMLButtonElement); + expect((testButton as HTMLButtonElement).disabled).toBe(false); + + await act(async () => { + testButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await flushEffects(6); + }); + + expect(mockTestTranscribeVoiceModelFile).toHaveBeenCalledWith( + "sensevoice-small-int8-2024-07-17", + "/tmp/interview.wav", + ); + expect(container.textContent ?? "").toContain( + "这是测试音频的本地转写结果。", + ); + }); + it("切换语音输入开关时应保存 voice_input 配置", async () => { const container = renderComponent(); await flushEffects(6); diff --git a/src/components/settings-v2/agent/voice/index.tsx b/src/components/settings-v2/agent/voice/index.tsx index 962188f2d..6f46074de 100644 --- a/src/components/settings-v2/agent/voice/index.tsx +++ b/src/components/settings-v2/agent/voice/index.tsx @@ -6,13 +6,20 @@ import { type ChangeEvent, type ReactNode, } from "react"; +import { open as openDialog } from "@tauri-apps/plugin-dialog"; import { AlertCircle, CheckCircle2, + Download, + FolderOpen, + HardDrive, + Loader2, Mic, + Trash2, Wand2, type LucideIcon, } from "lucide-react"; +import { Button } from "@/components/ui/button"; import { WorkbenchInfoTip } from "@/components/media/WorkbenchInfoTip"; import { ShortcutSettings } from "@/components/smart-input/ShortcutSettings"; import { MicrophoneTest } from "@/components/voice/MicrophoneTest"; @@ -33,6 +40,17 @@ import { getVoiceShortcutRuntimeStatus, type VoiceShortcutRuntimeStatus, } from "@/lib/api/hotkeys"; +import { + deleteVoiceModel, + downloadVoiceModel, + getVoiceModelInstallState, + listVoiceModelCatalog, + setDefaultVoiceModel, + testTranscribeVoiceModelFile, + type VoiceModelCatalogEntry, + type VoiceModelInstallState, + type VoiceModelTestTranscribeResult, +} from "@/lib/api/voiceModels"; import { buildPersistedMediaGenerationPreference, getTtsModelsForProvider, @@ -54,6 +72,9 @@ const DEFAULT_MEDIA_PREFERENCE: MediaGenerationPreference = { }; type PillTone = "neutral" | "success" | "warning"; +type VoiceModelAction = "download" | "delete" | "default" | "test"; + +const PRIMARY_MODEL_FILE_NAME = "model.int8.onnx"; function normalizeOptionalText(value?: string | null): string | undefined { const normalized = value?.trim(); @@ -194,6 +215,52 @@ function LoadingSkeleton() { ); } +function formatBytes(value: number): string { + if (!Number.isFinite(value) || value <= 0) { + return "0 MB"; + } + + const mb = value / 1024 / 1024; + if (mb < 1024) { + return `${mb.toFixed(0)} MB`; + } + return `${(mb / 1024).toFixed(1)} GB`; +} + +function getVoiceModelDisplayName(entry: VoiceModelCatalogEntry): string { + return entry.name.replace(/\s+INT8$/i, "").trim() || entry.name; +} + +function getVoiceModelInstallStatusText( + entry: VoiceModelCatalogEntry, + state: VoiceModelInstallState | null, + action: VoiceModelAction | null, +): string { + if (action === "download") { + return `正在下载 ${PRIMARY_MODEL_FILE_NAME} (1/2)`; + } + + const modelSize = entry.size_bytes + ? `约 ${formatBytes(entry.size_bytes)}` + : "大小待目录返回"; + + if (state?.installed) { + return `已安装（ONNX int8 量化，${modelSize}）`; + } + + return `未安装（ONNX int8 量化，${modelSize}）`; +} + +function getErrorMessage(error: unknown, fallback: string): string { + if (error instanceof Error && error.message.trim()) { + return error.message; + } + if (typeof error === "string" && error.trim()) { + return error; + } + return fallback; +} + function buildPrimaryShortcutStatus( voiceConfig: VoiceInputConfig | null, runtimeStatus: VoiceShortcutRuntimeStatus | null, @@ -250,6 +317,24 @@ function buildTranslateShortcutStatus( return { text: "翻译模式配置已保存，但运行时尚未注册", tone: "warning" }; } +function buildFnShortcutStatus( + runtimeStatus: VoiceShortcutRuntimeStatus | null, +): { text: string; tone: PillTone } { + if (!runtimeStatus) { + return { text: "Fn 状态加载中", tone: "neutral" }; + } + + if (runtimeStatus.fn_registered) { + return { text: "Fn 按住录音已注册", tone: "success" }; + } + + if (runtimeStatus.fn_supported) { + return { text: "Fn 支持可用，等待运行时注册", tone: "warning" }; + } + + return { text: "当前平台不支持 Fn，已使用快捷键回退", tone: "warning" }; +} + export function VoiceSettings() { const [config, setConfig] = useState(null); const [voiceConfig, setVoiceConfig] = useState(null); @@ -258,6 +343,19 @@ export function VoiceSettings() { const [asrCredentials, setAsrCredentials] = useState( [], ); + const [voiceModelCatalog, setVoiceModelCatalog] = useState< + VoiceModelCatalogEntry[] + >([]); + const [voiceModelState, setVoiceModelState] = + useState(null); + const [voiceModelAction, setVoiceModelAction] = + useState(null); + const [voiceModelTestPath, setVoiceModelTestPath] = useState(""); + const [voiceModelTestResult, setVoiceModelTestResult] = + useState(null); + const [voiceModelTestError, setVoiceModelTestError] = useState( + null, + ); const [message, setMessage] = useState<{ type: "success" | "error"; text: string; @@ -271,13 +369,25 @@ export function VoiceSettings() { setLoading(true); try { - const [nextConfig, nextVoiceConfig, nextVoiceShortcutStatus, nextAsr] = - await Promise.all([ - getConfig(), - getVoiceInputConfig(), - getVoiceShortcutRuntimeStatus().catch(() => null), - getAsrCredentials().catch(() => []), - ]); + const [ + nextConfig, + nextVoiceConfig, + nextVoiceShortcutStatus, + nextAsr, + nextVoiceModelCatalog, + ] = await Promise.all([ + getConfig(), + getVoiceInputConfig(), + getVoiceShortcutRuntimeStatus().catch(() => null), + getAsrCredentials().catch(() => []), + listVoiceModelCatalog().catch(() => []), + ]); + const primaryVoiceModel = nextVoiceModelCatalog[0] ?? null; + const nextVoiceModelState = primaryVoiceModel + ? await getVoiceModelInstallState(primaryVoiceModel.id).catch( + () => null, + ) + : null; const normalizedVoiceConfig = ensureValidVoiceInstructionSelection(nextVoiceConfig); @@ -286,6 +396,8 @@ export function VoiceSettings() { setVoiceConfig(normalizedVoiceConfig); setVoiceShortcutStatus(nextVoiceShortcutStatus); setAsrCredentials(nextAsr); + setVoiceModelCatalog(nextVoiceModelCatalog); + setVoiceModelState(nextVoiceModelState); setGlobalVoicePreference( nextConfig.workspace_preferences?.media_defaults?.voice ?? DEFAULT_MEDIA_PREFERENCE, @@ -323,13 +435,16 @@ export function VoiceSettings() { () => null, ); setVoiceShortcutStatus(nextRuntimeStatus); + if (nextVoiceConfig.enabled !== voiceConfig.enabled) { + await loadVoiceSettings(); + } showMessage("success", "语音设置已保存"); } catch (error) { console.error("保存语音设置失败:", error); showMessage("error", "保存语音设置失败"); } }, - [showMessage, voiceConfig], + [loadVoiceSettings, showMessage, voiceConfig], ); const persistGlobalVoicePreference = useCallback( @@ -415,6 +530,17 @@ export function VoiceSettings() { [voiceConfig, voiceShortcutStatus], ); + const fnShortcutStatus = useMemo( + () => buildFnShortcutStatus(voiceShortcutStatus), + [voiceShortcutStatus], + ); + + const primaryVoiceModel = voiceModelCatalog[0] ?? null; + const isVoiceModelDefault = Boolean( + voiceModelState?.default_credential_id || + defaultAsrCredential?.provider === "sensevoice_local", + ); + const voiceInstructions = voiceConfig?.instructions ?? []; const defaultInstructionId = voiceConfig?.processor.default_instruction_id ?? ""; @@ -599,6 +725,122 @@ export function VoiceSettings() { void persistGlobalVoicePreference(DEFAULT_MEDIA_PREFERENCE); }; + const handleDownloadVoiceModel = async () => { + if (!primaryVoiceModel) { + return; + } + + setVoiceModelAction("download"); + try { + const result = await downloadVoiceModel(primaryVoiceModel.id); + setVoiceModelState(result.state); + setVoiceModelTestError(null); + setVoiceModelTestResult(null); + showMessage("success", "SenseVoice Small 模型已下载"); + } catch (error) { + console.error("下载 SenseVoice Small 模型失败:", error); + showMessage("error", "下载 SenseVoice Small 模型失败"); + } finally { + setVoiceModelAction(null); + } + }; + + const handleDeleteVoiceModel = async () => { + if (!primaryVoiceModel) { + return; + } + + setVoiceModelAction("delete"); + try { + const state = await deleteVoiceModel(primaryVoiceModel.id); + setVoiceModelState(state); + setVoiceModelTestError(null); + setVoiceModelTestResult(null); + await loadVoiceSettings(); + showMessage("success", "SenseVoice Small 模型已删除"); + } catch (error) { + console.error("删除 SenseVoice Small 模型失败:", error); + showMessage("error", "删除 SenseVoice Small 模型失败"); + } finally { + setVoiceModelAction(null); + } + }; + + const handleSetDefaultVoiceModel = async () => { + if (!primaryVoiceModel) { + return; + } + + setVoiceModelAction("default"); + try { + await setDefaultVoiceModel(primaryVoiceModel.id); + await loadVoiceSettings(); + showMessage("success", "SenseVoice Small 已设为默认识别服务"); + } catch (error) { + console.error("设置 SenseVoice Small 默认模型失败:", error); + showMessage("error", "设置 SenseVoice Small 默认模型失败"); + } finally { + setVoiceModelAction(null); + } + }; + + const handleSelectVoiceModelTestFile = async () => { + if (!voiceModelState?.installed || voiceModelAction !== null) { + return; + } + + try { + const selected = await openDialog({ + title: "选择 WAV 测试音频", + multiple: false, + directory: false, + filters: [{ name: "WAV 音频", extensions: ["wav"] }], + }); + const filePath = Array.isArray(selected) ? selected[0] : selected; + if (!filePath) { + return; + } + setVoiceModelTestPath(filePath); + setVoiceModelTestError(null); + setVoiceModelTestResult(null); + } catch (error) { + console.error("选择 SenseVoice Small 测试文件失败:", error); + setVoiceModelTestError(getErrorMessage(error, "选择 WAV 文件失败")); + showMessage("error", "选择 WAV 文件失败"); + } + }; + + const handleTestVoiceModel = async () => { + if (!primaryVoiceModel || !voiceModelState?.installed) { + return; + } + + const filePath = voiceModelTestPath.trim(); + if (!filePath) { + setVoiceModelTestError("请先输入本机 WAV 文件路径"); + return; + } + + setVoiceModelAction("test"); + setVoiceModelTestError(null); + setVoiceModelTestResult(null); + try { + const result = await testTranscribeVoiceModelFile( + primaryVoiceModel.id, + filePath, + ); + setVoiceModelTestResult(result); + showMessage("success", "SenseVoice Small 测试转写完成"); + } catch (error) { + console.error("SenseVoice Small 测试转写失败:", error); + const errorMessage = getErrorMessage(error, "测试转写失败"); + setVoiceModelTestError(errorMessage); + showMessage("error", "测试转写失败"); + } finally { + setVoiceModelAction(null); + } + }; + if (loading) { return ; } @@ -618,26 +860,63 @@ export function VoiceSettings() { icon={Mic} > -
-
-

- 当前默认识别服务 -

-

- {defaultAsrCredential - ? `${defaultAsrCredential.name || defaultAsrCredential.provider}（已设为默认）` - : "尚未配置默认语音识别凭证"} -

+
+
+
+

+ 按住录音，松开识别 +

+

+ {defaultAsrCredential + ? `${defaultAsrCredential.name || defaultAsrCredential.provider}（默认识别服务）` + : "尚未配置默认语音识别凭证"} +

+
+
+ {voiceConfig?.enabled ? ( + + 🌐 Fn + + ) : null} + +
- + {voiceConfig?.enabled ? ( +
+ + Fn + + {voiceShortcutStatus?.fn_note ?? + "按住 Fn 开始录音，松开后停止并识别。"} + {voiceShortcutStatus?.fn_fallback_shortcut + ? ` 回退快捷键：${voiceShortcutStatus.fn_fallback_shortcut}` + : ""} +
+ ) : ( +

+ 语音输入未开启，不会注册 Fn 或全局快捷键。 +

+ )}
@@ -677,6 +956,28 @@ export function VoiceSettings() {
+ +
+
+ + Fn + + + {fnShortcutStatus.text} + +
+

+ {voiceShortcutStatus?.fn_note ?? "正在读取 Fn 快捷键运行时状态。"} + {voiceShortcutStatus?.fn_fallback_shortcut + ? ` 回退快捷键：${voiceShortcutStatus.fn_fallback_shortcut}` + : ""} +

+
+ + + + + {primaryVoiceModel ? ( +
+
+
+
+ +
+
+
+

+ {getVoiceModelDisplayName(primaryVoiceModel)} +

+ + 本地 + + {isVoiceModelDefault ? ( + 默认识别服务 + ) : null} +
+

+ {primaryVoiceModel.description || + "本地离线 ASR，模型按需下载到用户数据目录。"} +

+
+ + {getVoiceModelInstallStatusText( + primaryVoiceModel, + voiceModelState, + voiceModelAction, + )} + + · + {primaryVoiceModel.runtime} + {primaryVoiceModel.version ? ( + <> + · + {primaryVoiceModel.version} + + ) : null} + {primaryVoiceModel.languages.length ? ( + <> + · + {primaryVoiceModel.languages.join(" / ")} + + ) : null} +
+ {voiceModelAction === "download" ? ( +
+
+
+
+
+ 正在下载模型包与 VAD 文件 + 完成后自动校验并安装 +
+
+ ) : null} +

+ {voiceModelState?.installed + ? voiceModelState.install_dir + : "模型未内置，点击下载后写入 Lime 应用数据目录。"} +

+ {!voiceModelState?.installed && + voiceModelState?.missing_files.length && + voiceModelAction !== "download" ? ( +

+ 缺失文件：{voiceModelState.missing_files.join("、")} +

+ ) : null} + {voiceModelState?.installed ? ( +

+ 已安装大小：{formatBytes(voiceModelState.installed_bytes)} +

+ ) : null} +
+
+ +
+ {voiceModelState?.installed ? ( + <> + + + + ) : ( + + )} +
+
+ +
+
+

测试转写

+

+ 选择或输入本机 16-bit PCM WAV 文件路径，直接验证当前 + SenseVoice Small + 安装与本地推理链路；多声道音频仅使用第一声道。 +

+
+
+ + + setVoiceModelTestPath(event.target.value) + } + placeholder="选择或输入 /Users/me/audio.wav" + disabled={ + !voiceModelState?.installed || voiceModelAction !== null + } + /> + +
+ {!voiceModelState?.installed ? ( +

+ 请先下载并安装模型后再测试转写。 +

+ ) : null} + {voiceModelTestError ? ( +
+ {voiceModelTestError} +
+ ) : null} + {voiceModelTestResult ? ( +
+
+ 转写完成 + + {voiceModelTestResult.sample_rate} Hz ·{" "} + {voiceModelTestResult.duration_secs.toFixed(2)} 秒 ·{" "} + {voiceModelTestResult.language || "auto"} + +
+

+ {voiceModelTestResult.text || "未识别到文本"} +

+
+ ) : null} +
+
+ ) : ( +
+ 当前没有可用的本地语音模型清单。 +
+ )} + + + { expect(text).toContain("外观"); expect(text).toContain("管理主题、语言、提示音效、推荐行为和底部入口。"); expect(text).toContain("主题：跟随系统"); - expect(text).toContain("配色：Lime 经典"); + expect(text).toContain("配色：墨绿"); expect(text).toContain("语言：中文"); expect(text).toContain("提示音效：已开启"); expect(text).toContain("基础外观"); expect(text).toContain("主题模式"); expect(text).toContain("色彩方案"); - expect(text).toContain("Lime 经典"); - expect(text).toContain("森林"); - expect(text).toContain("海雾"); - expect(text).toContain("砂岩"); + expect(text).toContain("随机"); + expect(text).toContain("墨绿"); + expect(text).toContain("自然"); + expect(text).toContain("海洋"); + expect(text).toContain("复古"); + expect(text).toContain("霓虹"); + expect(text).toContain("青柠"); + expect(text).toContain("黄昏"); + expect(text).toContain("极简"); + expect(text).toContain("活力"); + expect(text).toContain("文艺"); + expect(text).toContain("奢华"); expect(text).toContain("界面语言"); expect(text).toContain("可选系统入口"); expect(text).not.toContain("持续流程"); @@ -246,27 +254,55 @@ describe("AppearanceSettings", () => { it("切换色彩方案时应持久化并立即应用到根节点", async () => { const { container } = await renderPage(); - const forestButton = Array.from(container.querySelectorAll("button")).find( - (button) => button.textContent?.includes("森林"), + const luxuryButton = Array.from(container.querySelectorAll("button")).find( + (button) => button.textContent?.includes("奢华"), ); - expect(forestButton).toBeInstanceOf(HTMLButtonElement); + expect(luxuryButton).toBeInstanceOf(HTMLButtonElement); await act(async () => { - forestButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + luxuryButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); await Promise.resolve(); }); expect(localStorage.getItem(LIME_COLOR_SCHEME_STORAGE_KEY)).toBe( - "lime-forest", + "lime-luxury", ); expect(document.documentElement.dataset.limeColorScheme).toBe( - "lime-forest", + "lime-luxury", ); expect( document.documentElement.style.getPropertyValue("--lime-chrome-rail"), - ).toBe("#f4f7f1"); - expect(container.textContent ?? "").toContain("配色：森林"); + ).toBe("#f4efe2"); + expect(container.textContent ?? "").toContain("配色：奢华"); + }); + + it("点击随机配色时应落到真实预设并持久化", async () => { + const randomSpy = vi.spyOn(Math, "random").mockReturnValue(0); + + try { + const { container } = await renderPage(); + const randomButton = Array.from( + container.querySelectorAll("button"), + ).find((button) => button.textContent?.includes("随机")); + + expect(randomButton).toBeInstanceOf(HTMLButtonElement); + + await act(async () => { + randomButton?.dispatchEvent(new MouseEvent("click", { bubbles: true })); + await Promise.resolve(); + }); + + expect(localStorage.getItem(LIME_COLOR_SCHEME_STORAGE_KEY)).toBe( + "lime-forest", + ); + expect(document.documentElement.dataset.limeColorScheme).toBe( + "lime-forest", + ); + expect(container.textContent ?? "").toContain("配色：自然"); + } finally { + randomSpy.mockRestore(); + } }); it("切换主题模式时应持久化并立即应用到整个应用根节点", async () => { diff --git a/src/components/settings-v2/general/appearance/index.tsx b/src/components/settings-v2/general/appearance/index.tsx index 356fbd12b..a770e3783 100644 --- a/src/components/settings-v2/general/appearance/index.tsx +++ b/src/components/settings-v2/general/appearance/index.tsx @@ -23,6 +23,7 @@ import { Volume2, Waypoints, Bot, + Shuffle, type LucideIcon, } from "lucide-react"; import { WorkbenchInfoTip } from "@/components/media/WorkbenchInfoTip"; @@ -339,6 +340,16 @@ export function AppearanceSettings() { [], ); + const handleRandomColorScheme = useCallback(() => { + const candidates = LIME_COLOR_SCHEMES.filter( + (scheme) => scheme.id !== colorSchemeId, + ); + const nextScheme = + candidates[Math.floor(Math.random() * candidates.length)] ?? + LIME_COLOR_SCHEMES[0]; + handleColorSchemeChange(nextScheme.id); + }, [colorSchemeId, handleColorSchemeChange]); + const handleLanguageChange = useCallback( async (nextLanguage: Language) => { if (!config) { @@ -600,6 +611,24 @@ export function AppearanceSettings() {
+ {LIME_COLOR_SCHEMES.map((option) => { const active = colorSchemeId === option.id; return ( diff --git a/src/components/settings-v2/general/hotkeys/hotkeyCatalog.test.ts b/src/components/settings-v2/general/hotkeys/hotkeyCatalog.test.ts index 7e5b6f570..4acad814f 100644 --- a/src/components/settings-v2/general/hotkeys/hotkeyCatalog.test.ts +++ b/src/components/settings-v2/general/hotkeys/hotkeyCatalog.test.ts @@ -30,6 +30,10 @@ describe("hotkey catalog", () => { registered_shortcut: "CommandOrControl+Shift+V", translate_shortcut_registered: true, registered_translate_shortcut: "CommandOrControl+Shift+T", + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }, }, }); @@ -76,6 +80,10 @@ describe("hotkey catalog", () => { registered_shortcut: null, translate_shortcut_registered: false, registered_translate_shortcut: null, + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }, }, }); diff --git a/src/components/voice/AddAsrCredentialModal.tsx b/src/components/voice/AddAsrCredentialModal.tsx index ca151f0e3..4c300cfe4 100644 --- a/src/components/voice/AddAsrCredentialModal.tsx +++ b/src/components/voice/AddAsrCredentialModal.tsx @@ -23,6 +23,7 @@ interface AddAsrCredentialModalProps { const ProviderIcon = ({ type }: { type: AsrProviderType }) => { switch (type) { case "whisper_local": + case "sensevoice_local": return ; case "openai": return ; @@ -96,6 +97,15 @@ export function AddAsrCredentialModal({ selectedProvider === "whisper_local" ? { model: whisperModel } : undefined, + sensevoice_config: + selectedProvider === "sensevoice_local" + ? { + model_id: "sensevoice-small-int8-2024-07-17", + use_itn: true, + num_threads: 4, + vad_model_id: "silero-vad-onnx", + } + : undefined, xunfei_config: selectedProvider === "xunfei" ? { @@ -133,6 +143,7 @@ export function AddAsrCredentialModal({ if (!selectedProvider) return false; switch (selectedProvider) { case "whisper_local": + case "sensevoice_local": return true; case "xunfei": return xunfeiAppId && xunfeiApiKey && xunfeiApiSecret; diff --git a/src/components/voice/AsrProviderSection.tsx b/src/components/voice/AsrProviderSection.tsx index dca182776..a942dc460 100644 --- a/src/components/voice/AsrProviderSection.tsx +++ b/src/components/voice/AsrProviderSection.tsx @@ -22,6 +22,7 @@ import { const ProviderIcon = ({ type }: { type: AsrProviderType }) => { switch (type) { case "whisper_local": + case "sensevoice_local": return ; case "openai": return ; diff --git a/src/components/voice/types.ts b/src/components/voice/types.ts index d0f87d1b7..440330973 100644 --- a/src/components/voice/types.ts +++ b/src/components/voice/types.ts @@ -9,6 +9,7 @@ export type { AsrProviderType, WhisperModelSize, WhisperLocalConfig, + SenseVoiceLocalConfig, XunfeiConfig, BaiduConfig, OpenAIAsrConfig, @@ -53,6 +54,13 @@ export const ASR_PROVIDERS: AsrProviderInfo[] = [ icon: "cpu", requiresCredentials: false, }, + { + type: "sensevoice_local", + label: "SenseVoice Small 本地", + description: "按需下载的离线语音识别模型", + icon: "cpu", + requiresCredentials: false, + }, { type: "xunfei", label: "讯飞语音", diff --git a/src/hooks/useOemLimeHubProviderSync.test.tsx b/src/hooks/useOemLimeHubProviderSync.test.tsx index c2c9ba4b7..ae5adf321 100644 --- a/src/hooks/useOemLimeHubProviderSync.test.tsx +++ b/src/hooks/useOemLimeHubProviderSync.test.tsx @@ -11,6 +11,9 @@ import { const apiKeyProviderMocks = vi.hoisted(() => ({ addApiKey: vi.fn(), getProviders: vi.fn(), + getUiState: vi.fn(), + setUiState: vi.fn(), + toggleApiKey: vi.fn(), updateProvider: vi.fn(), })); @@ -23,6 +26,9 @@ vi.mock("@/lib/api/apiKeyProvider", () => ({ apiKeyProviderApi: { addApiKey: apiKeyProviderMocks.addApiKey, getProviders: apiKeyProviderMocks.getProviders, + getUiState: apiKeyProviderMocks.getUiState, + setUiState: apiKeyProviderMocks.setUiState, + toggleApiKey: apiKeyProviderMocks.toggleApiKey, updateProvider: apiKeyProviderMocks.updateProvider, }, })); @@ -54,6 +60,9 @@ async function flushEffects() { await Promise.resolve(); await Promise.resolve(); await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); + await Promise.resolve(); }); } @@ -91,6 +100,9 @@ describe("useOemLimeHubProviderSync", () => { }, ]); apiKeyProviderMocks.addApiKey.mockResolvedValue({ id: "local-key-001" }); + apiKeyProviderMocks.getUiState.mockResolvedValue(null); + apiKeyProviderMocks.setUiState.mockResolvedValue(undefined); + apiKeyProviderMocks.toggleApiKey.mockResolvedValue({ id: "old-key-001" }); apiKeyProviderMocks.updateProvider.mockResolvedValue(undefined); controlPlaneMocks.createClientAccessToken.mockResolvedValue({ token: { id: "cloud-token-001" }, @@ -313,6 +325,12 @@ describe("useOemLimeHubProviderSync", () => { }); it("已有本地云端 Key 时不应重复创建桌面 Key", async () => { + apiKeyProviderMocks.getUiState.mockResolvedValue( + JSON.stringify({ + tenantId: "tenant-0001", + models: ["gpt-5.2-pro", "gpt-5.2-fast"], + }), + ); setStoredOemCloudSessionState({ token: "session-token-001", tenant: { @@ -369,6 +387,111 @@ describe("useOemLimeHubProviderSync", () => { expect(apiKeyProviderMocks.addApiKey).not.toHaveBeenCalled(); }); + it("云端模型目录扩展时应重签本地托管 Key 并停用旧托管 Key", async () => { + apiKeyProviderMocks.getUiState.mockResolvedValue( + JSON.stringify({ + tenantId: "tenant-0001", + models: ["gpt-5.2-pro", "gpt-5.2-fast"], + }), + ); + controlPlaneMocks.listClientProviderOfferModels.mockResolvedValue([ + { + id: "model-001", + modelId: "gpt-5.2-fast", + }, + { + id: "model-002", + modelId: "claude-opus-4-7", + }, + ]); + setStoredOemCloudSessionState({ + token: "session-token-001", + tenant: { + id: "tenant-0001", + }, + user: { + id: "user-001", + }, + session: { + id: "session-001", + }, + }); + setOemCloudBootstrapSnapshot({ + providerPreference: { + providerSource: "oem_cloud", + providerKey: "offer-main", + defaultModel: "gpt-5.2-pro", + }, + providerOffersSummary: [ + { + source: "oem_cloud", + providerKey: "offer-main", + defaultModel: "gpt-5.2-pro", + }, + ], + }); + + apiKeyProviderMocks.getProviders.mockResolvedValue([ + { + id: "lime-hub", + name: "Acme Hub", + api_host: "https://gateway-api.limeai.run/root", + type: "openai", + enabled: true, + sort_order: 0, + custom_models: ["gpt-5.2-pro", "gpt-5.2-fast"], + api_key_count: 1, + api_keys: [ + { + id: "old-managed-key", + alias: "Lime 云端模型", + enabled: true, + }, + ], + }, + ]); + + const container = document.createElement("div"); + document.body.appendChild(container); + const root = createRoot(container); + mountedHarness = { container, root }; + + act(() => { + root.render(); + }); + + await flushEffects(); + + expect(controlPlaneMocks.createClientAccessToken).toHaveBeenCalledWith( + "tenant-0001", + { + name: "Lime Desktop Cloud Model Key", + scopes: ["llm:invoke"], + allowedModels: [ + "gpt-5.2-pro", + "gpt-5.2-fast", + "claude-opus-4-7", + ], + }, + ); + expect(apiKeyProviderMocks.addApiKey).toHaveBeenCalledWith({ + provider_id: "lime-hub", + api_key: "sk-lime-desktop", + alias: "Lime 云端模型", + }); + expect(apiKeyProviderMocks.toggleApiKey).toHaveBeenCalledWith( + "old-managed-key", + false, + ); + expect(apiKeyProviderMocks.setUiState).toHaveBeenCalledWith( + "oem_lime_hub_provider_sync:managed_key_models", + JSON.stringify({ + tenantId: "tenant-0001", + models: ["gpt-5.2-pro", "gpt-5.2-fast", "claude-opus-4-7"], + }), + ); + }); + it("本地云端 Key 已禁用时应重新创建桌面 Key", async () => { setStoredOemCloudSessionState({ token: "session-token-001", diff --git a/src/hooks/useOemLimeHubProviderSync.ts b/src/hooks/useOemLimeHubProviderSync.ts index 428475d49..fb48482f5 100644 --- a/src/hooks/useOemLimeHubProviderSync.ts +++ b/src/hooks/useOemLimeHubProviderSync.ts @@ -21,6 +21,10 @@ import { } from "@/lib/oemLimeHubProvider"; import { hasTauriInvokeCapability } from "@/lib/tauri-runtime"; +const MANAGED_LIME_HUB_KEY_ALIAS = "Lime 云端模型"; +const MANAGED_LIME_HUB_KEY_MODELS_STATE = + "oem_lime_hub_provider_sync:managed_key_models"; + function buildSyncSignature( runtime: ReturnType, customModels: string[], @@ -107,6 +111,16 @@ async function resolveSyncedCustomModels( ); } +function buildManagedKeyModelsState(input: { + tenantId: string; + customModels: string[]; +}): string { + return JSON.stringify({ + tenantId: input.tenantId, + models: input.customModels, + }); +} + function hasUsableLocalApiKey(provider: { api_key_count?: number; api_keys?: Array<{ enabled?: boolean }>; @@ -119,16 +133,44 @@ function hasUsableLocalApiKey(provider: { ); } +function isManagedLimeHubKey(key: { + alias?: string | null; + enabled?: boolean; +}): boolean { + return ( + key.enabled !== false && + (key.alias ?? "").trim() === MANAGED_LIME_HUB_KEY_ALIAS + ); +} + async function ensureLocalLimeHubApiKey(input: { runtime: NonNullable>; customModels: string[]; localApiKeyReady: boolean; + apiKeys: Array<{ id?: string; alias?: string | null; enabled?: boolean }>; }): Promise { - const { runtime, customModels, localApiKeyReady } = input; - if (localApiKeyReady || !runtime.sessionToken || customModels.length === 0) { + const { runtime, customModels, localApiKeyReady, apiKeys } = input; + if (!runtime.sessionToken || customModels.length === 0) { return localApiKeyReady; } + if (localApiKeyReady) { + const expectedState = buildManagedKeyModelsState({ + tenantId: runtime.tenantId, + customModels, + }); + try { + const currentState = await apiKeyProviderApi.getUiState( + MANAGED_LIME_HUB_KEY_MODELS_STATE, + ); + if (currentState === expectedState) { + return true; + } + } catch { + // UI state 只用于判断本地托管 key 是否覆盖最新模型；读取失败时重新签发更安全。 + } + } + const response = await createClientAccessToken(runtime.tenantId, { name: "Lime Desktop Cloud Model Key", scopes: ["llm:invoke"], @@ -139,11 +181,34 @@ async function ensureLocalLimeHubApiKey(input: { return false; } - await apiKeyProviderApi.addApiKey({ + const addedKey = await apiKeyProviderApi.addApiKey({ provider_id: OEM_LIME_HUB_PROVIDER_ID, api_key: apiKey, - alias: "Lime 云端模型", + alias: MANAGED_LIME_HUB_KEY_ALIAS, }); + + await Promise.all( + apiKeys + .filter(isManagedLimeHubKey) + .filter((key) => key.id && key.id !== addedKey.id) + .map((key) => + apiKeyProviderApi.toggleApiKey(key.id as string, false).catch(() => { + // 新 key 已写入；旧托管 key 禁用失败不阻塞本次同步，下次同步会继续尝试。 + }), + ), + ); + + try { + await apiKeyProviderApi.setUiState( + MANAGED_LIME_HUB_KEY_MODELS_STATE, + buildManagedKeyModelsState({ + tenantId: runtime.tenantId, + customModels, + }), + ); + } catch { + // 状态写入失败不影响新 key 生效；后续同步会再补写。 + } return true; } @@ -251,6 +316,7 @@ export function useOemLimeHubProviderSync() { runtime, customModels: nextCustomModels, localApiKeyReady, + apiKeys: limeHubProvider.api_keys ?? [], }); lastAppliedSignatureRef.current = buildSyncSignature( diff --git a/src/hooks/useProviderModels.test.ts b/src/hooks/useProviderModels.test.ts index 22da877f7..cf07d1e8b 100644 --- a/src/hooks/useProviderModels.test.ts +++ b/src/hooks/useProviderModels.test.ts @@ -287,9 +287,7 @@ describe("loadProviderModels", () => { hasApiKey: true, }); - expect(models.map((model) => model.id)).toEqual( - expect.arrayContaining(["gpt-5.5", "deepseek-v4-flash"]), - ); + expect(models.map((model) => model.id)).toEqual(["gpt-5.5", "gpt-5.4"]); expect(models.every((model) => model.provider_id === "lime-hub")).toBe( true, ); diff --git a/src/lib/agentUiPerformanceMetrics.test.ts b/src/lib/agentUiPerformanceMetrics.test.ts new file mode 100644 index 000000000..4f1d5c600 --- /dev/null +++ b/src/lib/agentUiPerformanceMetrics.test.ts @@ -0,0 +1,99 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + clearAgentUiPerformanceMetrics, + getAgentUiPerformanceMetrics, + recordAgentUiPerformanceMetric, + summarizeAgentUiPerformanceMetrics, +} from "./agentUiPerformanceMetrics"; + +describe("agentUiPerformanceMetrics", () => { + afterEach(() => { + clearAgentUiPerformanceMetrics(); + }); + + it("应按会话汇总旧会话打开链路的关键耗时", () => { + clearAgentUiPerformanceMetrics(); + + recordAgentUiPerformanceMetric("sidebar.conversation.click", { + sessionId: "session-a", + source: "conversation_shelf", + workspaceId: "workspace-a", + }); + recordAgentUiPerformanceMetric("session.switch.start", { + sessionId: "session-a", + workspaceId: "workspace-a", + }); + recordAgentUiPerformanceMetric("session.switch.fetchDetail.start", { + sessionId: "session-a", + workspaceId: "workspace-a", + }); + recordAgentUiPerformanceMetric("session.switch.fetchDetail.success", { + requestDurationMs: 174, + sessionId: "session-a", + workspaceId: "workspace-a", + }); + recordAgentUiPerformanceMetric("agentRuntime.getSession.start", { + sessionId: "session-a", + }); + recordAgentUiPerformanceMetric("agentRuntime.getSession.success", { + durationMs: 181, + sessionId: "session-a", + }); + recordAgentUiPerformanceMetric("session.switch.success", { + durationMs: 220, + sessionId: "session-a", + }); + recordAgentUiPerformanceMetric("messageList.paint", { + historicalContentPartsDeferredCount: 2, + historicalMarkdownDeferredCount: 3, + hiddenHistoryCount: 120, + messagesCount: 40, + persistedHiddenHistoryCount: 120, + renderedMessagesCount: 10, + sessionId: "session-a", + threadItemsScanDeferred: true, + threadItemsCount: 0, + }); + + const summary = summarizeAgentUiPerformanceMetrics(); + expect(summary.sessions).toEqual([ + expect.objectContaining({ + sessionId: "session-a", + workspaceId: "workspace-a", + fetchDetailDurationMs: 174, + runtimeGetSessionDurationMs: 181, + switchStartCount: 1, + fetchDetailStartCount: 1, + fetchDetailErrorCount: 0, + runtimeGetSessionStartCount: 1, + runtimeGetSessionErrorCount: 0, + messageListPaintCount: 1, + finalMessagesCount: 40, + finalRenderedMessagesCount: 10, + hiddenHistoryCount: 120, + historicalContentPartsDeferredMax: 2, + historicalMarkdownDeferredMax: 3, + persistedHiddenHistoryCount: 120, + threadItemsScanDeferredCount: 1, + }), + ]); + expect(summary.sessions[0]?.clickToSwitchStartMs).toBeGreaterThanOrEqual(0); + expect( + summary.sessions[0]?.clickToMessageListPaintMs, + ).toBeGreaterThanOrEqual(0); + }); + + it("应在 window 上暴露给 Playwright 读取的 API", () => { + recordAgentUiPerformanceMetric("sidebar.conversation.click", { + sessionId: "session-window-api", + }); + + expect(window.__LIME_AGENTUI_PERF__?.entries()).toHaveLength(1); + expect(window.__LIME_AGENTUI_PERF__?.summary().sessions[0]?.sessionId).toBe( + "session-window-api", + ); + + window.__LIME_AGENTUI_PERF__?.clear(); + expect(getAgentUiPerformanceMetrics()).toHaveLength(0); + }); +}); diff --git a/src/lib/agentUiPerformanceMetrics.ts b/src/lib/agentUiPerformanceMetrics.ts new file mode 100644 index 000000000..babc15cde --- /dev/null +++ b/src/lib/agentUiPerformanceMetrics.ts @@ -0,0 +1,378 @@ +export interface AgentUiPerformanceEntry { + id: number; + phase: string; + at: number; + wallTime: number; + sessionId?: string | null; + workspaceId?: string | null; + source?: string | null; + metrics: Record; +} + +export interface AgentUiPerformanceSessionSummary { + sessionId: string; + workspaceId?: string | null; + clickToSwitchStartMs?: number; + clickToCachedSnapshotMs?: number; + clickToPendingShellMs?: number; + clickToFetchStartMs?: number; + fetchDetailDurationMs?: number; + runtimeGetSessionDurationMs?: number; + clickToSwitchSuccessMs?: number; + clickToMessageListPaintMs?: number; + switchStartCount?: number; + fetchDetailStartCount?: number; + fetchDetailErrorCount?: number; + runtimeGetSessionStartCount?: number; + runtimeGetSessionErrorCount?: number; + messageListPaintCount?: number; + finalMessagesCount?: number; + finalRenderedMessagesCount?: number; + finalThreadItemsCount?: number; + hiddenHistoryCount?: number; + persistedHiddenHistoryCount?: number; + historicalContentPartsDeferredMax?: number; + historicalMarkdownDeferredMax?: number; + threadItemsScanDeferredCount?: number; + maxUsedJSHeapSize?: number; + phases: string[]; +} + +export interface AgentUiPerformanceSnapshot { + entries: AgentUiPerformanceEntry[]; + sessions: AgentUiPerformanceSessionSummary[]; +} + +export interface AgentUiPerformanceApi { + entries: () => AgentUiPerformanceEntry[]; + clear: () => void; + summary: () => AgentUiPerformanceSnapshot; +} + +type MetricValue = string | number | boolean | null; + +const MAX_AGENT_UI_PERFORMANCE_ENTRIES = 500; +const entries: AgentUiPerformanceEntry[] = []; +let nextEntryId = 1; + +declare global { + interface Window { + __LIME_AGENTUI_PERF__?: AgentUiPerformanceApi; + } +} + +function now(): number { + if (typeof performance !== "undefined" && performance.now) { + return performance.now(); + } + return Date.now(); +} + +function normalizeString(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + const normalized = value.trim(); + return normalized ? normalized : null; +} + +function normalizeMetricValue(value: unknown): MetricValue | undefined { + if (value == null) { + return null; + } + + if (typeof value === "boolean") { + return value; + } + + if (typeof value === "number") { + return Number.isFinite(value) ? value : null; + } + + if (typeof value === "string") { + return value.length > 200 ? `${value.slice(0, 200)}...` : value; + } + + return undefined; +} + +function normalizeMetrics( + context: Record | undefined, +): Record { + const metrics: Record = {}; + for (const [key, value] of Object.entries(context ?? {})) { + if (key === "sessionId" || key === "topicId" || key === "workspaceId") { + continue; + } + + const normalized = normalizeMetricValue(value); + if (normalized !== undefined) { + metrics[key] = normalized; + } + } + + const memory = + typeof window !== "undefined" + ? ( + window.performance as Performance & { + memory?: { + usedJSHeapSize?: number; + totalJSHeapSize?: number; + }; + } + )?.memory + : undefined; + if (typeof memory?.usedJSHeapSize === "number") { + metrics.usedJSHeapSize = memory.usedJSHeapSize; + } + if (typeof memory?.totalJSHeapSize === "number") { + metrics.totalJSHeapSize = memory.totalJSHeapSize; + } + + return metrics; +} + +function normalizeSessionId(context?: Record): string | null { + return ( + normalizeString(context?.sessionId) ?? + normalizeString(context?.topicId) ?? + null + ); +} + +function pushEntry(entry: AgentUiPerformanceEntry): void { + entries.push(entry); + if (entries.length > MAX_AGENT_UI_PERFORMANCE_ENTRIES) { + entries.splice(0, entries.length - MAX_AGENT_UI_PERFORMANCE_ENTRIES); + } +} + +function firstEntry( + sessionEntries: AgentUiPerformanceEntry[], + phase: string, +): AgentUiPerformanceEntry | null { + return sessionEntries.find((entry) => entry.phase === phase) ?? null; +} + +function lastEntry( + sessionEntries: AgentUiPerformanceEntry[], + phase: string, +): AgentUiPerformanceEntry | null { + for (let index = sessionEntries.length - 1; index >= 0; index -= 1) { + const entry = sessionEntries[index]; + if (entry?.phase === phase) { + return entry; + } + } + return null; +} + +function deltaMs( + start: AgentUiPerformanceEntry | null, + end: AgentUiPerformanceEntry | null, +): number | undefined { + if (!start || !end) { + return undefined; + } + return Math.max(0, Math.round(end.at - start.at)); +} + +function metricNumber( + entry: AgentUiPerformanceEntry | null, + key: string, +): number | undefined { + const value = entry?.metrics[key]; + return typeof value === "number" && Number.isFinite(value) + ? Math.round(value) + : undefined; +} + +function maxMetric( + sessionEntries: AgentUiPerformanceEntry[], + key: string, +): number | undefined { + let max: number | undefined; + for (const entry of sessionEntries) { + const value = entry.metrics[key]; + if (typeof value !== "number" || !Number.isFinite(value)) { + continue; + } + max = max === undefined ? value : Math.max(max, value); + } + return max === undefined ? undefined : Math.round(max); +} + +function countEntries( + sessionEntries: AgentUiPerformanceEntry[], + phase: string, +): number { + return sessionEntries.reduce( + (count, entry) => count + (entry.phase === phase ? 1 : 0), + 0, + ); +} + +function countMetricTrue( + sessionEntries: AgentUiPerformanceEntry[], + key: string, +): number { + return sessionEntries.reduce( + (count, entry) => count + (entry.metrics[key] === true ? 1 : 0), + 0, + ); +} + +export function summarizeAgentUiPerformanceMetrics(): AgentUiPerformanceSnapshot { + const grouped = new Map(); + for (const entry of entries) { + const sessionId = entry.sessionId?.trim(); + if (!sessionId) { + continue; + } + const sessionEntries = grouped.get(sessionId) ?? []; + sessionEntries.push(entry); + grouped.set(sessionId, sessionEntries); + } + + const sessions: AgentUiPerformanceSessionSummary[] = Array.from( + grouped.entries(), + ).map(([sessionId, sessionEntries]) => { + const click = firstEntry(sessionEntries, "sidebar.conversation.click"); + const switchStart = firstEntry(sessionEntries, "session.switch.start"); + const cachedSnapshot = firstEntry( + sessionEntries, + "session.switch.cachedSnapshotApplied", + ); + const pendingShell = firstEntry( + sessionEntries, + "session.switch.pendingShellApplied", + ); + const fetchStart = firstEntry( + sessionEntries, + "session.switch.fetchDetail.start", + ); + const fetchSuccess = lastEntry( + sessionEntries, + "session.switch.fetchDetail.success", + ); + const runtimeGetSessionSuccess = lastEntry( + sessionEntries, + "agentRuntime.getSession.success", + ); + const switchSuccess = lastEntry(sessionEntries, "session.switch.success"); + const messageListPaint = lastEntry(sessionEntries, "messageList.paint"); + const finalMessageList = + messageListPaint ?? lastEntry(sessionEntries, "messageList.commit"); + + return { + sessionId, + workspaceId: + sessionEntries.find((entry) => entry.workspaceId)?.workspaceId ?? null, + clickToSwitchStartMs: deltaMs(click, switchStart), + clickToCachedSnapshotMs: deltaMs(click, cachedSnapshot), + clickToPendingShellMs: deltaMs(click, pendingShell), + clickToFetchStartMs: deltaMs(click, fetchStart), + fetchDetailDurationMs: metricNumber(fetchSuccess, "requestDurationMs"), + runtimeGetSessionDurationMs: metricNumber( + runtimeGetSessionSuccess, + "durationMs", + ), + clickToSwitchSuccessMs: deltaMs(click, switchSuccess), + clickToMessageListPaintMs: deltaMs(click, messageListPaint), + switchStartCount: countEntries(sessionEntries, "session.switch.start"), + fetchDetailStartCount: countEntries( + sessionEntries, + "session.switch.fetchDetail.start", + ), + fetchDetailErrorCount: countEntries( + sessionEntries, + "session.switch.fetchDetail.error", + ), + runtimeGetSessionStartCount: countEntries( + sessionEntries, + "agentRuntime.getSession.start", + ), + runtimeGetSessionErrorCount: countEntries( + sessionEntries, + "agentRuntime.getSession.error", + ), + messageListPaintCount: countEntries(sessionEntries, "messageList.paint"), + finalMessagesCount: metricNumber(finalMessageList, "messagesCount"), + finalRenderedMessagesCount: metricNumber( + finalMessageList, + "renderedMessagesCount", + ), + finalThreadItemsCount: metricNumber(finalMessageList, "threadItemsCount"), + hiddenHistoryCount: metricNumber(finalMessageList, "hiddenHistoryCount"), + persistedHiddenHistoryCount: metricNumber( + finalMessageList, + "persistedHiddenHistoryCount", + ), + historicalContentPartsDeferredMax: maxMetric( + sessionEntries, + "historicalContentPartsDeferredCount", + ), + historicalMarkdownDeferredMax: maxMetric( + sessionEntries, + "historicalMarkdownDeferredCount", + ), + threadItemsScanDeferredCount: countMetricTrue( + sessionEntries, + "threadItemsScanDeferred", + ), + maxUsedJSHeapSize: maxMetric(sessionEntries, "usedJSHeapSize"), + phases: sessionEntries.map((entry) => entry.phase), + }; + }); + + sessions.sort((left, right) => left.sessionId.localeCompare(right.sessionId)); + return { + entries: entries.slice(), + sessions, + }; +} + +export function clearAgentUiPerformanceMetrics(): void { + entries.splice(0, entries.length); + nextEntryId = 1; +} + +export function getAgentUiPerformanceMetrics(): AgentUiPerformanceEntry[] { + return entries.slice(); +} + +export function recordAgentUiPerformanceMetric( + phase: string, + context?: Record, +): AgentUiPerformanceEntry { + const entry: AgentUiPerformanceEntry = { + id: nextEntryId, + phase, + at: now(), + wallTime: Date.now(), + sessionId: normalizeSessionId(context), + workspaceId: normalizeString(context?.workspaceId), + source: normalizeString(context?.source), + metrics: normalizeMetrics(context), + }; + nextEntryId += 1; + pushEntry(entry); + installAgentUiPerformanceApi(); + return entry; +} + +export function installAgentUiPerformanceApi(): AgentUiPerformanceApi | null { + if (typeof window === "undefined") { + return null; + } + + const api: AgentUiPerformanceApi = { + entries: getAgentUiPerformanceMetrics, + clear: clearAgentUiPerformanceMetrics, + summary: summarizeAgentUiPerformanceMetrics, + }; + window.__LIME_AGENTUI_PERF__ = api; + return api; +} + +installAgentUiPerformanceApi(); diff --git a/src/lib/api/agent.test.ts b/src/lib/api/agent.test.ts index ac574c71d..203719dc6 100644 --- a/src/lib/api/agent.test.ts +++ b/src/lib/api/agent.test.ts @@ -954,6 +954,46 @@ describe("Agent API 治理护栏", () => { "Artifact 校验已恢复 1 个产物，fallback 0 次。", ], }, + modalityRuntimeContracts: { + snapshotCount: 2, + snapshotIndex: { + browserActionIndex: { + actionCount: 2, + sessionCount: 1, + observationCount: 1, + screenshotCount: 1, + lastUrl: "https://example.com/", + sessionIds: ["browser-session-1"], + targetIds: ["target-1"], + profileKeys: ["general_browser_assist"], + statusCounts: [{ status: "completed", count: 2 }], + artifactKindCounts: [ + { artifactKind: "browser_session", count: 1 }, + { artifactKind: "browser_snapshot", count: 1 }, + ], + actionCounts: [ + { action: "navigate", count: 1 }, + { action: "get_page_info", count: 1 }, + ], + backendCounts: [{ backend: "lime_extension_bridge", count: 1 }], + items: [ + { + artifactKind: "browser_snapshot", + action: "get_page_info", + status: "completed", + success: true, + sessionId: "browser-session-1", + targetId: "target-1", + entrySource: "at_browser_agent_command", + backend: "lime_extension_bridge", + lastUrl: "https://example.com/", + observationAvailable: true, + screenshotAvailable: true, + }, + ], + }, + }, + }, }, artifacts: [ { @@ -994,6 +1034,26 @@ describe("Agent API 治理护栏", () => { "Artifact 校验已恢复 1 个产物，fallback 0 次。", ], }), + modality_runtime_contracts: expect.objectContaining({ + snapshot_count: 2, + snapshot_index: expect.objectContaining({ + browser_action_index: expect.objectContaining({ + action_count: 2, + last_url: "https://example.com/", + observation_count: 1, + screenshot_count: 1, + items: [ + expect.objectContaining({ + artifact_kind: "browser_snapshot", + action: "get_page_info", + backend: "lime_extension_bridge", + observation_available: true, + screenshot_available: true, + }), + ], + }), + }), + }), }), artifacts: [ expect.objectContaining({ diff --git a/src/lib/api/agentRuntime/index.ts b/src/lib/api/agentRuntime/index.ts index bf9ecbfd9..329cddcbb 100644 --- a/src/lib/api/agentRuntime/index.ts +++ b/src/lib/api/agentRuntime/index.ts @@ -30,6 +30,7 @@ export { } from "./inventoryClient"; export { cancelMediaTaskArtifact, + completeAudioGenerationTaskArtifact, createAudioGenerationTaskArtifact, createMediaClient, createImageGenerationTaskArtifact, diff --git a/src/lib/api/agentRuntime/mediaClient.ts b/src/lib/api/agentRuntime/mediaClient.ts index 3b459a7c7..c052993ff 100644 --- a/src/lib/api/agentRuntime/mediaClient.ts +++ b/src/lib/api/agentRuntime/mediaClient.ts @@ -1,4 +1,5 @@ import type { + CompleteAudioGenerationTaskArtifactRequest, CreateAudioGenerationTaskArtifactRequest, CreateImageGenerationTaskArtifactRequest, ListMediaTaskArtifactsRequest, @@ -34,6 +35,14 @@ export function createMediaClient({ }); } + async function completeAudioGenerationTaskArtifact( + request: CompleteAudioGenerationTaskArtifactRequest, + ): Promise { + return await bridgeInvoke("complete_audio_generation_task_artifact", { + request, + }); + } + async function getMediaTaskArtifact( request: MediaTaskLookupRequest, ): Promise { @@ -58,6 +67,7 @@ export function createMediaClient({ return { cancelMediaTaskArtifact, + completeAudioGenerationTaskArtifact, createAudioGenerationTaskArtifact, createImageGenerationTaskArtifact, getMediaTaskArtifact, @@ -67,6 +77,7 @@ export function createMediaClient({ export const { cancelMediaTaskArtifact, + completeAudioGenerationTaskArtifact, createAudioGenerationTaskArtifact, createImageGenerationTaskArtifact, getMediaTaskArtifact, diff --git a/src/lib/api/agentRuntime/normalizers.ts b/src/lib/api/agentRuntime/normalizers.ts index 8dacec3d1..5681ff935 100644 --- a/src/lib/api/agentRuntime/normalizers.ts +++ b/src/lib/api/agentRuntime/normalizers.ts @@ -1,8 +1,14 @@ import { normalizeLegacyToolSurfaceName } from "../agentTextNormalization"; import { normalizeQueuedTurnSnapshots } from "../queuedTurn"; import type { + AgentRuntimeEvidenceActionCount, AgentRuntimeEvidenceArtifact, + AgentRuntimeEvidenceArtifactKindCount, + AgentRuntimeEvidenceBackendCount, + AgentRuntimeEvidenceBrowserActionItem, + AgentRuntimeEvidenceBrowserActionIndex, AgentRuntimeEvidencePack, + AgentRuntimeEvidenceStatusCount, AgentRuntimeHandoffArtifact, AgentRuntimeHandoffBundle, AgentRuntimeAnalysisArtifact, @@ -88,6 +94,15 @@ function readRecordField( return isRecord(value) ? value : undefined; } +function readArrayField( + record: Record, + camelKey: string, + snakeKey?: string, +): unknown[] { + const value = record[camelKey] ?? (snakeKey ? record[snakeKey] : undefined); + return Array.isArray(value) ? value : []; +} + function normalizeEvidenceVerificationOutcome( value?: string, ): @@ -120,6 +135,250 @@ function normalizeEvidenceSignalCoverageEntry(value: unknown) { }; } +function normalizeEvidenceStatusCount( + value: unknown, +): AgentRuntimeEvidenceStatusCount | null { + if (!isRecord(value)) { + return null; + } + + const status = readStringField(value, "status"); + if (!status) { + return null; + } + + return { + status, + count: readNumberField(value, "count"), + }; +} + +function normalizeEvidenceArtifactKindCount( + value: unknown, +): AgentRuntimeEvidenceArtifactKindCount | null { + if (!isRecord(value)) { + return null; + } + + const artifactKind = readStringField(value, "artifactKind", "artifact_kind"); + if (!artifactKind) { + return null; + } + + return { + artifact_kind: artifactKind, + count: readNumberField(value, "count"), + }; +} + +function normalizeEvidenceActionCount( + value: unknown, +): AgentRuntimeEvidenceActionCount | null { + if (!isRecord(value)) { + return null; + } + + const action = readStringField(value, "action"); + if (!action) { + return null; + } + + return { + action, + count: readNumberField(value, "count"), + }; +} + +function normalizeEvidenceBackendCount( + value: unknown, +): AgentRuntimeEvidenceBackendCount | null { + if (!isRecord(value)) { + return null; + } + + const backend = readStringField(value, "backend"); + if (!backend) { + return null; + } + + return { + backend, + count: readNumberField(value, "count"), + }; +} + +function normalizeBrowserActionItem( + value: unknown, +): AgentRuntimeEvidenceBrowserActionItem | null { + if (!isRecord(value)) { + return null; + } + + const item: AgentRuntimeEvidenceBrowserActionItem = { + artifact_path: readOptionalStringField( + value, + "artifactPath", + "artifact_path", + ), + contract_key: readOptionalStringField(value, "contractKey", "contract_key"), + source: readOptionalStringField(value, "source"), + entry_source: readOptionalStringField(value, "entrySource", "entry_source"), + artifact_kind: readOptionalStringField( + value, + "artifactKind", + "artifact_kind", + ), + tool_name: readOptionalStringField(value, "toolName", "tool_name"), + action: readOptionalStringField(value, "action"), + status: readOptionalStringField(value, "status"), + success: readOptionalBooleanField(value, "success"), + session_id: readOptionalStringField(value, "sessionId", "session_id"), + target_id: readOptionalStringField(value, "targetId", "target_id"), + profile_key: readOptionalStringField(value, "profileKey", "profile_key"), + backend: readOptionalStringField(value, "backend"), + request_id: readOptionalStringField(value, "requestId", "request_id"), + last_url: readOptionalStringField(value, "lastUrl", "last_url"), + title: readOptionalStringField(value, "title"), + attempt_count: readOptionalNumberField( + value, + "attemptCount", + "attempt_count", + ), + observation_available: readOptionalBooleanField( + value, + "observationAvailable", + "observation_available", + ), + screenshot_available: readOptionalBooleanField( + value, + "screenshotAvailable", + "screenshot_available", + ), + }; + + const hasReadableField = Object.values(item).some( + (field) => field !== undefined && field !== "", + ); + + return hasReadableField ? item : null; +} + +function normalizeBrowserActionIndex( + value: unknown, +): AgentRuntimeEvidenceBrowserActionIndex | undefined { + if (!isRecord(value)) { + return undefined; + } + + const rawStatusCounts = readArrayField( + value, + "statusCounts", + "status_counts", + ); + const rawArtifactKindCounts = readArrayField( + value, + "artifactKindCounts", + "artifact_kind_counts", + ); + const rawActionCounts = readArrayField( + value, + "actionCounts", + "action_counts", + ); + const rawBackendCounts = readArrayField( + value, + "backendCounts", + "backend_counts", + ); + const rawItems = readArrayField(value, "items"); + + const index: AgentRuntimeEvidenceBrowserActionIndex = { + action_count: readNumberField(value, "actionCount", "action_count"), + session_count: readNumberField(value, "sessionCount", "session_count"), + observation_count: readNumberField( + value, + "observationCount", + "observation_count", + ), + screenshot_count: readNumberField( + value, + "screenshotCount", + "screenshot_count", + ), + last_url: readOptionalStringField(value, "lastUrl", "last_url"), + session_ids: readStringListField(value, "sessionIds", "session_ids"), + target_ids: readStringListField(value, "targetIds", "target_ids"), + profile_keys: readStringListField(value, "profileKeys", "profile_keys"), + status_counts: rawStatusCounts + .map((entry: unknown) => normalizeEvidenceStatusCount(entry)) + .filter(Boolean) as AgentRuntimeEvidenceStatusCount[], + artifact_kind_counts: rawArtifactKindCounts + .map((entry: unknown) => normalizeEvidenceArtifactKindCount(entry)) + .filter(Boolean) as AgentRuntimeEvidenceArtifactKindCount[], + action_counts: rawActionCounts + .map((entry: unknown) => normalizeEvidenceActionCount(entry)) + .filter(Boolean) as AgentRuntimeEvidenceActionCount[], + backend_counts: rawBackendCounts + .map((entry: unknown) => normalizeEvidenceBackendCount(entry)) + .filter(Boolean) as AgentRuntimeEvidenceBackendCount[], + items: rawItems + .map((entry: unknown) => normalizeBrowserActionItem(entry)) + .filter(Boolean) as AgentRuntimeEvidenceBrowserActionItem[], + }; + + if ( + index.action_count === 0 && + index.session_count === 0 && + index.observation_count === 0 && + index.screenshot_count === 0 && + !index.last_url && + index.items.length === 0 + ) { + return undefined; + } + + return index; +} + +function normalizeEvidenceModalityRuntimeContracts(value: unknown) { + if (!isRecord(value)) { + return undefined; + } + + const snapshotIndexRecord = readRecordField( + value, + "snapshotIndex", + "snapshot_index", + ); + const browserActionIndex = normalizeBrowserActionIndex( + snapshotIndexRecord + ? readRecordField( + snapshotIndexRecord, + "browserActionIndex", + "browser_action_index", + ) + : undefined, + ); + const snapshotCount = readNumberField( + value, + "snapshotCount", + "snapshot_count", + ); + + if (snapshotCount === 0 && !browserActionIndex) { + return undefined; + } + + return { + snapshot_count: snapshotCount, + snapshot_index: browserActionIndex + ? { + browser_action_index: browserActionIndex, + } + : undefined, + }; +} + function normalizeArtifactValidatorVerificationSummary(value: unknown) { if (!isRecord(value)) { return undefined; @@ -266,12 +525,20 @@ function normalizeEvidenceObservabilitySummary(value: unknown) { "schema_version", ); const knownGaps = readStringListField(value, "knownGaps", "known_gaps"); + const modalityRuntimeContracts = normalizeEvidenceModalityRuntimeContracts( + readRecordField( + value, + "modalityRuntimeContracts", + "modality_runtime_contracts", + ), + ); if ( !schemaVersion && signalCoverage.length === 0 && knownGaps.length === 0 && - !verificationSummary + !verificationSummary && + !modalityRuntimeContracts ) { return undefined; } @@ -281,6 +548,7 @@ function normalizeEvidenceObservabilitySummary(value: unknown) { known_gaps: knownGaps, signal_coverage: signalCoverage, verification_summary: verificationSummary, + modality_runtime_contracts: modalityRuntimeContracts, }; } diff --git a/src/lib/api/agentRuntime/sessionClient.ts b/src/lib/api/agentRuntime/sessionClient.ts index 7df7965dc..35f0013ad 100644 --- a/src/lib/api/agentRuntime/sessionClient.ts +++ b/src/lib/api/agentRuntime/sessionClient.ts @@ -1,4 +1,5 @@ import { logAgentDebug } from "@/lib/agentDebug"; +import { recordAgentUiPerformanceMetric } from "@/lib/agentUiPerformanceMetrics"; import { normalizeLegacyThreadItem } from "../agentTextNormalization"; import type { AgentThreadItem } from "../agentProtocol"; import { normalizeQueuedTurnSnapshots } from "../queuedTurn"; @@ -87,7 +88,17 @@ export function createSessionClient({ }, 1000) : null; - logAgentDebug("AgentApi", "runtimeListSessions.start"); + const listMetricContext = { + archivedOnly, + includeArchived, + limit: limit ?? null, + workspaceId: workspaceId ?? null, + }; + recordAgentUiPerformanceMetric( + "agentRuntime.listSessions.start", + listMetricContext, + ); + logAgentDebug("AgentApi", "runtimeListSessions.start", listMetricContext); try { const request = { @@ -105,6 +116,11 @@ export function createSessionClient({ : undefined, ); settled = true; + recordAgentUiPerformanceMetric("agentRuntime.listSessions.success", { + ...listMetricContext, + durationMs: Date.now() - startedAt, + sessionsCount: sessions.length, + }); logAgentDebug("AgentApi", "runtimeListSessions.success", { archivedOnly, durationMs: Date.now() - startedAt, @@ -116,6 +132,10 @@ export function createSessionClient({ return sessions; } catch (error) { settled = true; + recordAgentUiPerformanceMetric("agentRuntime.listSessions.error", { + ...listMetricContext, + durationMs: Date.now() - startedAt, + }); logAgentDebug( "AgentApi", "runtimeListSessions.error", @@ -188,13 +208,22 @@ export function createSessionClient({ }, 1000) : null; - logAgentDebug("AgentApi", "runtimeGetSession.start", { + const getSessionMetricContext = { historyLimit: historyLimit ?? null, historyOffset: historyOffset ?? null, historyBeforeMessageId: historyBeforeMessageId ?? null, resumeSessionStartHooks, sessionId, - }); + }; + recordAgentUiPerformanceMetric( + "agentRuntime.getSession.start", + getSessionMetricContext, + ); + logAgentDebug( + "AgentApi", + "runtimeGetSession.start", + getSessionMetricContext, + ); try { const detail = await invokeCommand( @@ -233,6 +262,16 @@ export function createSessionClient({ thread_read: normalizeThreadReadModel(normalizedDetail?.thread_read), }; settled = true; + recordAgentUiPerformanceMetric("agentRuntime.getSession.success", { + ...getSessionMetricContext, + childSubagentSessionsCount: + normalizedSessionDetail.child_subagent_sessions?.length ?? 0, + durationMs: Date.now() - startedAt, + itemsCount: normalizedSessionDetail.items?.length ?? 0, + messagesCount: normalizedSessionDetail.messages?.length ?? 0, + queuedTurnsCount: normalizedSessionDetail.queued_turns?.length ?? 0, + turnsCount: normalizedSessionDetail.turns?.length ?? 0, + }); logAgentDebug("AgentApi", "runtimeGetSession.success", { childSubagentSessionsCount: normalizedSessionDetail.child_subagent_sessions?.length ?? 0, @@ -250,6 +289,10 @@ export function createSessionClient({ return normalizedSessionDetail; } catch (error) { settled = true; + recordAgentUiPerformanceMetric("agentRuntime.getSession.error", { + ...getSessionMetricContext, + durationMs: Date.now() - startedAt, + }); logAgentDebug( "AgentApi", "runtimeGetSession.error", diff --git a/src/lib/api/agentRuntime/types.ts b/src/lib/api/agentRuntime/types.ts index 52bf94fc8..203711e62 100644 --- a/src/lib/api/agentRuntime/types.ts +++ b/src/lib/api/agentRuntime/types.ts @@ -545,6 +545,73 @@ export interface AgentRuntimeEvidenceSignalCoverageEntry { detail: string; } +export interface AgentRuntimeEvidenceCountEntry { + count: number; +} + +export interface AgentRuntimeEvidenceStatusCount extends AgentRuntimeEvidenceCountEntry { + status: string; +} + +export interface AgentRuntimeEvidenceArtifactKindCount extends AgentRuntimeEvidenceCountEntry { + artifact_kind: string; +} + +export interface AgentRuntimeEvidenceActionCount extends AgentRuntimeEvidenceCountEntry { + action: string; +} + +export interface AgentRuntimeEvidenceBackendCount extends AgentRuntimeEvidenceCountEntry { + backend: string; +} + +export interface AgentRuntimeEvidenceBrowserActionItem { + artifact_path?: string; + contract_key?: string; + source?: string; + entry_source?: string; + artifact_kind?: string; + tool_name?: string; + action?: string; + status?: string; + success?: boolean; + session_id?: string; + target_id?: string; + profile_key?: string; + backend?: string; + request_id?: string; + last_url?: string; + title?: string; + attempt_count?: number; + observation_available?: boolean; + screenshot_available?: boolean; +} + +export interface AgentRuntimeEvidenceBrowserActionIndex { + action_count: number; + session_count: number; + observation_count: number; + screenshot_count: number; + last_url?: string; + session_ids: string[]; + target_ids: string[]; + profile_keys: string[]; + status_counts: AgentRuntimeEvidenceStatusCount[]; + artifact_kind_counts: AgentRuntimeEvidenceArtifactKindCount[]; + action_counts: AgentRuntimeEvidenceActionCount[]; + backend_counts: AgentRuntimeEvidenceBackendCount[]; + items: AgentRuntimeEvidenceBrowserActionItem[]; +} + +export interface AgentRuntimeEvidenceSnapshotIndex { + browser_action_index?: AgentRuntimeEvidenceBrowserActionIndex; +} + +export interface AgentRuntimeEvidenceModalityRuntimeContracts { + snapshot_count: number; + snapshot_index?: AgentRuntimeEvidenceSnapshotIndex; +} + export interface AgentRuntimeArtifactValidatorVerificationSummary { applicable: boolean; record_count: number; @@ -592,6 +659,7 @@ export interface AgentRuntimeEvidenceObservabilitySummary { known_gaps: string[]; signal_coverage: AgentRuntimeEvidenceSignalCoverageEntry[]; verification_summary?: AgentRuntimeEvidenceVerificationSummary; + modality_runtime_contracts?: AgentRuntimeEvidenceModalityRuntimeContracts; } export interface AgentRuntimeEvidencePack { @@ -1096,6 +1164,16 @@ export interface CreateAudioGenerationTaskArtifactRequest { outputPath?: string; } +export interface CompleteAudioGenerationTaskArtifactRequest { + projectRootPath: string; + taskRef: string; + audioPath: string; + mimeType?: string; + durationMs?: number; + providerId?: string; + model?: string; +} + export interface MediaTaskArtifactRecord { task_id: string; task_type: string; @@ -1172,6 +1250,20 @@ export interface MediaTaskModalityRuntimeContractIndexEntry { failure_code?: string | null; model_capability_assessment_source?: string | null; model_supports_image_generation?: boolean | null; + audio_output_status?: string | null; + audio_output_path?: string | null; + audio_output_mime_type?: string | null; + audio_output_duration_ms?: number | null; + audio_output_error_code?: string | null; + audio_output_retryable?: boolean | null; + transcript_status?: string | null; + transcript_path?: string | null; + transcript_source_url?: string | null; + transcript_source_path?: string | null; + transcript_language?: string | null; + transcript_output_format?: string | null; + transcript_error_code?: string | null; + transcript_retryable?: boolean | null; } export interface MediaTaskRoutingOutcomeCount { @@ -1179,12 +1271,28 @@ export interface MediaTaskRoutingOutcomeCount { count: number; } +export interface MediaTaskAudioOutputStatusCount { + status: string; + count: number; +} + +export interface MediaTaskTranscriptStatusCount { + status: string; + count: number; +} + export interface MediaTaskModalityRuntimeContractIndex { snapshot_count: number; contract_keys: string[]; blocked_count: number; routing_outcomes: MediaTaskRoutingOutcomeCount[]; model_registry_assessment_count: number; + audio_output_count: number; + audio_output_statuses: MediaTaskAudioOutputStatusCount[]; + audio_output_error_codes: string[]; + transcript_count: number; + transcript_statuses: MediaTaskTranscriptStatusCount[]; + transcript_error_codes: string[]; snapshots: MediaTaskModalityRuntimeContractIndexEntry[]; } diff --git a/src/lib/api/apiKeyProvider.test.ts b/src/lib/api/apiKeyProvider.test.ts index e873ebc42..4a081b925 100644 --- a/src/lib/api/apiKeyProvider.test.ts +++ b/src/lib/api/apiKeyProvider.test.ts @@ -109,10 +109,7 @@ describe("apiKeyProvider API", () => { name: "Lime Hub", type: "openai", api_key_count: 1, - custom_models: expect.arrayContaining([ - "gpt-5.5", - "deepseek-v4-flash", - ]), + custom_models: ["gpt-5.5", "gpt-5.4"], }), ]); }); diff --git a/src/lib/api/asrProvider.ts b/src/lib/api/asrProvider.ts index 7c95c07a6..fd3bfa732 100644 --- a/src/lib/api/asrProvider.ts +++ b/src/lib/api/asrProvider.ts @@ -9,7 +9,12 @@ import { safeInvoke } from "@/lib/dev-bridge"; // ============ ASR Provider 类型 ============ /** ASR Provider 类型 */ -export type AsrProviderType = "whisper_local" | "xunfei" | "baidu" | "openai"; +export type AsrProviderType = + | "whisper_local" + | "sensevoice_local" + | "xunfei" + | "baidu" + | "openai"; /** Whisper 模型大小 */ export type WhisperModelSize = "tiny" | "base" | "small" | "medium"; @@ -20,6 +25,15 @@ export interface WhisperLocalConfig { model_path?: string; } +/** SenseVoice 本地配置 */ +export interface SenseVoiceLocalConfig { + model_id: string; + model_dir?: string; + use_itn: boolean; + num_threads: number; + vad_model_id?: string; +} + /** 讯飞配置 */ export interface XunfeiConfig { app_id: string; @@ -49,6 +63,7 @@ export interface AsrCredentialEntry { disabled: boolean; language: string; whisper_config?: WhisperLocalConfig; + sensevoice_config?: SenseVoiceLocalConfig; xunfei_config?: XunfeiConfig; baidu_config?: BaiduConfig; openai_config?: OpenAIAsrConfig; diff --git a/src/lib/api/hotkeys.test.ts b/src/lib/api/hotkeys.test.ts index 31e2e764d..df8201485 100644 --- a/src/lib/api/hotkeys.test.ts +++ b/src/lib/api/hotkeys.test.ts @@ -26,6 +26,10 @@ describe("hotkeys API", () => { registered_shortcut: "CommandOrControl+Shift+V", translate_shortcut_registered: false, registered_translate_shortcut: null, + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }); await expect(getScreenshotShortcutRuntimeStatus()).resolves.toEqual( @@ -47,12 +51,17 @@ describe("hotkeys API", () => { registered_shortcut: "CommandOrControl+Shift+V", translate_shortcut_registered: true, registered_translate_shortcut: "CommandOrControl+Shift+T", + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }); await expect(getHotkeyRuntimeStatus()).resolves.toEqual({ screenshot: expect.objectContaining({ shortcut_registered: false }), voice: expect.objectContaining({ translate_shortcut_registered: true, + fn_supported: false, }), }); diff --git a/src/lib/api/hotkeys.ts b/src/lib/api/hotkeys.ts index af01073b3..d188d13ba 100644 --- a/src/lib/api/hotkeys.ts +++ b/src/lib/api/hotkeys.ts @@ -15,6 +15,10 @@ export interface VoiceShortcutRuntimeStatus { registered_shortcut?: string | null; translate_shortcut_registered: boolean; registered_translate_shortcut?: string | null; + fn_supported: boolean; + fn_registered: boolean; + fn_fallback_shortcut?: string | null; + fn_note: string; } export interface HotkeyRuntimeStatus { diff --git a/src/lib/api/mediaTasks.test.ts b/src/lib/api/mediaTasks.test.ts index 328e39bef..6c14f2d81 100644 --- a/src/lib/api/mediaTasks.test.ts +++ b/src/lib/api/mediaTasks.test.ts @@ -2,6 +2,7 @@ import { beforeEach, describe, expect, it, vi } from "vitest"; import { safeInvoke } from "@/lib/dev-bridge"; import { cancelMediaTaskArtifact, + completeAudioGenerationTaskArtifact, createAudioGenerationTaskArtifact, createImageGenerationTaskArtifact, getMediaTaskArtifact, @@ -149,6 +150,80 @@ describe("mediaTasks API", () => { ); }); + it("应通过统一网关完成音频任务并回写 audio_output", async () => { + vi.mocked(safeInvoke).mockResolvedValueOnce({ + success: true, + task_id: "task-audio-2", + task_type: "audio_generate", + task_family: "audio", + status: "succeeded", + normalized_status: "succeeded", + path: ".lime/tasks/audio_generate/task-audio-2.json", + absolute_path: "/workspace/.lime/tasks/audio_generate/task-audio-2.json", + artifact_path: ".lime/tasks/audio_generate/task-audio-2.json", + absolute_artifact_path: + "/workspace/.lime/tasks/audio_generate/task-audio-2.json", + reused_existing: false, + record: { + task_id: "task-audio-2", + task_type: "audio_generate", + task_family: "audio", + payload: { + source_text: "请生成温暖旁白", + modality_contract_key: "voice_generation", + audio_path: ".lime/runtime/audio/task-audio-2.mp3", + audio_output: { + kind: "audio_output", + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-2.mp3", + mime_type: "audio/mpeg", + duration_ms: 2400, + }, + }, + status: "succeeded", + normalized_status: "succeeded", + created_at: "2026-04-04T12:00:00Z", + result: { + kind: "audio_generation_result", + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-2.mp3", + }, + }, + }); + + await expect( + completeAudioGenerationTaskArtifact({ + projectRootPath: "/workspace", + taskRef: "task-audio-2", + audioPath: ".lime/runtime/audio/task-audio-2.mp3", + mimeType: "audio/mpeg", + durationMs: 2400, + providerId: "limecore", + model: "voice-pro", + }), + ).resolves.toEqual( + expect.objectContaining({ + task_id: "task-audio-2", + normalized_status: "succeeded", + }), + ); + + expect(vi.mocked(safeInvoke)).toHaveBeenCalledWith( + "complete_audio_generation_task_artifact", + { + request: { + projectRootPath: "/workspace", + taskRef: "task-audio-2", + audioPath: ".lime/runtime/audio/task-audio-2.mp3", + mimeType: "audio/mpeg", + durationMs: 2400, + providerId: "limecore", + model: "voice-pro", + }, + }, + ); + }); + it("应通过统一网关读取、列出和取消媒体任务 artifact", async () => { vi.mocked(safeInvoke) .mockResolvedValueOnce({ diff --git a/src/lib/api/mediaTasks.ts b/src/lib/api/mediaTasks.ts index 7cf2a141b..b32ae4715 100644 --- a/src/lib/api/mediaTasks.ts +++ b/src/lib/api/mediaTasks.ts @@ -1,4 +1,5 @@ export type { + CompleteAudioGenerationTaskArtifactRequest, CreateAudioGenerationTaskArtifactRequest, CreateImageGenerationTaskArtifactRequest, ListMediaTaskArtifactsOutput, @@ -7,13 +8,16 @@ export type { MediaTaskModalityRuntimeContractIndexEntry, MediaTaskArtifactOutput, MediaTaskArtifactRecord, + MediaTaskAudioOutputStatusCount, MediaTaskListFilters, MediaTaskRoutingOutcomeCount, + MediaTaskTranscriptStatusCount, MediaTaskLookupRequest, } from "./agentRuntime/types"; export { cancelMediaTaskArtifact, + completeAudioGenerationTaskArtifact, createAudioGenerationTaskArtifact, createImageGenerationTaskArtifact, getMediaTaskArtifact, diff --git a/src/lib/api/skillCatalog.test.ts b/src/lib/api/skillCatalog.test.ts index 569865ec8..1c9d32ff3 100644 --- a/src/lib/api/skillCatalog.test.ts +++ b/src/lib/api/skillCatalog.test.ts @@ -386,6 +386,50 @@ describe("skillCatalog", () => { ); }); + it("应解析服务端下发的首页展示协议并允许 home-only command 无触发词", () => { + const seeded = getSeededSkillCatalog(); + const catalog = saveSkillCatalog( + { + ...seeded, + version: "tenant-home-presentation", + entries: [ + { + id: "home:input-suggestion:email", + kind: "command", + title: "帮我写一封工作邮件", + summary: "输入框 Tab 起手建议。", + commandKey: "home_input_email", + surfaceScopes: ["home"], + homePresentation: { + slot: "input_suggestion", + label: "帮我写一封工作邮件", + order: 10, + prompt: "请帮我写一封工作邮件。", + }, + }, + ], + }, + "bootstrap_sync", + ); + + const entry = listSkillCatalogCommandEntries(catalog).find( + (candidate) => candidate.commandKey === "home_input_email", + ); + + expect(entry).toEqual( + expect.objectContaining({ + id: "home:input-suggestion:email", + triggers: [], + surfaceScopes: ["home"], + homePresentation: expect.objectContaining({ + slot: "input_suggestion", + label: "帮我写一封工作邮件", + prompt: "请帮我写一封工作邮件。", + }), + }), + ); + }); + it("读取旧版 raw skill catalog 时应把 cloud_scene 正规化为本地 agent_turn", async () => { saveSkillCatalog(buildLegacyCloudSceneCatalog(), "bootstrap_sync"); diff --git a/src/lib/api/skillCatalog.ts b/src/lib/api/skillCatalog.ts index 41570a9e1..a299f4788 100644 --- a/src/lib/api/skillCatalog.ts +++ b/src/lib/api/skillCatalog.ts @@ -70,6 +70,22 @@ export interface SkillCatalogSceneTemplate { prompt: string; } +export type SkillCatalogHomePresentationSlot = + | "input_suggestion" + | "starter_chip" + | "guide_card"; + +export interface SkillCatalogHomePresentation { + slot: SkillCatalogHomePresentationSlot; + label?: string; + title?: string; + summary?: string; + iconToken?: string; + order?: number; + groupKey?: string; + prompt?: string; +} + export type SkillCatalogCommandTriggerMode = "mention" | "slash"; export interface SkillCatalogCommandTrigger { @@ -86,6 +102,7 @@ export interface SkillCatalogSkillEntry { groupKey: string; aliases?: string[]; surfaceScopes?: ServiceSkillSurfaceScope[]; + homePresentation?: SkillCatalogHomePresentation; execution: SkillCatalogExecution; renderContract?: SkillCatalogRenderContract; } @@ -98,6 +115,7 @@ export interface SkillCatalogCommandEntry { commandKey: string; aliases?: string[]; surfaceScopes?: ServiceSkillSurfaceScope[]; + homePresentation?: SkillCatalogHomePresentation; triggers: SkillCatalogCommandTrigger[]; binding?: { skillId?: string; @@ -120,6 +138,7 @@ export interface SkillCatalogSceneEntry { linkedEntryId?: string; aliases?: string[]; surfaceScopes?: ServiceSkillSurfaceScope[]; + homePresentation?: SkillCatalogHomePresentation; linkedSkillId?: string; executionKind?: SkillCatalogExecutionKind | "scene"; placeholder?: string; @@ -991,6 +1010,39 @@ function parseSkillCatalogSceneTemplates( return templates.length > 0 ? templates : undefined; } +function parseSkillCatalogHomePresentation( + value: unknown, +): SkillCatalogHomePresentation | undefined { + if (!isPlainRecord(value)) { + return undefined; + } + + const slot = normalizeText(value.slot); + if ( + slot !== "input_suggestion" && + slot !== "starter_chip" && + slot !== "guide_card" + ) { + return undefined; + } + + const order = + typeof value.order === "number" && Number.isFinite(value.order) + ? value.order + : undefined; + + return { + slot, + label: normalizeText(value.label) ?? undefined, + title: normalizeText(value.title) ?? undefined, + summary: normalizeText(value.summary) ?? undefined, + iconToken: normalizeText(value.iconToken) ?? undefined, + order, + groupKey: normalizeText(value.groupKey) ?? undefined, + prompt: normalizeText(value.prompt) ?? undefined, + }; +} + function parseSkillCatalogEntry(value: unknown): SkillCatalogEntry | null { if (!isPlainRecord(value)) { return null; @@ -1003,6 +1055,9 @@ function parseSkillCatalogEntry(value: unknown): SkillCatalogEntry | null { if (!id || !kind || !title || !summary) { return null; } + const homePresentation = parseSkillCatalogHomePresentation( + value.homePresentation, + ); if (kind === "command") { const commandKey = normalizeText(value.commandKey); @@ -1014,7 +1069,7 @@ function parseSkillCatalogEntry(value: unknown): SkillCatalogEntry | null { ) : []; - if (!commandKey || triggers.length === 0) { + if (!commandKey || (triggers.length === 0 && !homePresentation)) { return null; } @@ -1035,6 +1090,7 @@ function parseSkillCatalogEntry(value: unknown): SkillCatalogEntry | null { commandKey, aliases: normalizeSearchAliases(value.aliases), surfaceScopes: normalizeSurfaceScopes(value.surfaceScopes), + homePresentation, triggers, binding, renderContract: parseRenderContract(value.renderContract), @@ -1058,6 +1114,7 @@ function parseSkillCatalogEntry(value: unknown): SkillCatalogEntry | null { linkedEntryId: normalizeText(value.linkedEntryId) ?? undefined, aliases: normalizeSearchAliases(value.aliases), surfaceScopes: normalizeSurfaceScopes(value.surfaceScopes), + homePresentation, linkedSkillId: normalizeText(value.linkedSkillId) ?? undefined, executionKind: parseSceneExecutionKind(value.executionKind), placeholder: normalizeText(value.placeholder) ?? undefined, @@ -1083,6 +1140,7 @@ function parseSkillCatalogEntry(value: unknown): SkillCatalogEntry | null { groupKey, aliases: normalizeSearchAliases(value.aliases), surfaceScopes: normalizeSurfaceScopes(value.surfaceScopes), + homePresentation, execution, renderContract: parseRenderContract(value.renderContract), }; diff --git a/src/lib/api/voiceModels.test.ts b/src/lib/api/voiceModels.test.ts new file mode 100644 index 000000000..1eab5591e --- /dev/null +++ b/src/lib/api/voiceModels.test.ts @@ -0,0 +1,169 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { safeInvoke } from "@/lib/dev-bridge"; +import { resolveOemCloudRuntimeContext } from "./oemCloudRuntime"; +import { + deleteVoiceModel, + downloadVoiceModel, + getVoiceModelInstallState, + listVoiceModelCatalog, + setDefaultVoiceModel, + testTranscribeVoiceModelFile, +} from "./voiceModels"; + +vi.mock("@/lib/dev-bridge", () => ({ + safeInvoke: vi.fn(), +})); + +vi.mock("./oemCloudRuntime", () => ({ + resolveOemCloudRuntimeContext: vi.fn(), +})); + +describe("voiceModels API", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(resolveOemCloudRuntimeContext).mockReturnValue(null); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("应代理本地语音模型管理命令", async () => { + vi.mocked(safeInvoke) + .mockResolvedValueOnce([{ id: "sensevoice-small-int8-2024-07-17" }]) + .mockResolvedValueOnce({ installed: false }) + .mockResolvedValueOnce({ state: { installed: true } }) + .mockResolvedValueOnce({ installed: false }) + .mockResolvedValueOnce({ id: "sensevoice-local", is_default: true }) + .mockResolvedValueOnce({ + text: "这是一段测试转写结果。", + duration_secs: 3.2, + sample_rate: 16000, + language: "auto", + }); + + await expect(listVoiceModelCatalog()).resolves.toEqual([ + expect.objectContaining({ id: "sensevoice-small-int8-2024-07-17" }), + ]); + await expect( + getVoiceModelInstallState("sensevoice-small-int8-2024-07-17"), + ).resolves.toEqual(expect.objectContaining({ installed: false })); + await expect( + downloadVoiceModel("sensevoice-small-int8-2024-07-17"), + ).resolves.toEqual(expect.objectContaining({ state: { installed: true } })); + await expect( + deleteVoiceModel("sensevoice-small-int8-2024-07-17"), + ).resolves.toEqual(expect.objectContaining({ installed: false })); + await expect( + setDefaultVoiceModel("sensevoice-small-int8-2024-07-17"), + ).resolves.toEqual(expect.objectContaining({ is_default: true })); + await expect( + testTranscribeVoiceModelFile( + "sensevoice-small-int8-2024-07-17", + "/tmp/interview.wav", + ), + ).resolves.toEqual( + expect.objectContaining({ text: "这是一段测试转写结果。" }), + ); + + expect(safeInvoke).toHaveBeenNthCalledWith(1, "voice_models_list_catalog"); + expect(safeInvoke).toHaveBeenNthCalledWith( + 2, + "voice_models_get_install_state", + { modelId: "sensevoice-small-int8-2024-07-17" }, + ); + expect(safeInvoke).toHaveBeenNthCalledWith(3, "voice_models_download", { + modelId: "sensevoice-small-int8-2024-07-17", + }); + expect(safeInvoke).toHaveBeenNthCalledWith(4, "voice_models_delete", { + modelId: "sensevoice-small-int8-2024-07-17", + }); + expect(safeInvoke).toHaveBeenNthCalledWith(5, "voice_models_set_default", { + modelId: "sensevoice-small-int8-2024-07-17", + }); + expect(safeInvoke).toHaveBeenNthCalledWith( + 6, + "voice_models_test_transcribe_file", + { + modelId: "sensevoice-small-int8-2024-07-17", + filePath: "/tmp/interview.wav", + }, + ); + }); + + it("应优先使用 limecore 下发的语音模型目录并传给下载命令", async () => { + vi.mocked(resolveOemCloudRuntimeContext).mockReturnValue({ + baseUrl: "https://cloud.example.com", + controlPlaneBaseUrl: "https://cloud.example.com/api", + sceneBaseUrl: "https://cloud.example.com/scene-api", + gatewayBaseUrl: "https://cloud.example.com/gateway-api", + tenantId: "tenant-0001", + sessionToken: null, + hubProviderName: null, + loginPath: "/login", + desktopClientId: "desktop-client", + desktopOauthRedirectUrl: "lime://oauth/callback", + desktopOauthNextPath: "/welcome", + }); + const fetchMock = vi.fn().mockResolvedValue({ + ok: true, + json: async () => ({ + code: 200, + message: "success", + data: { + items: [ + { + id: "sensevoice-small-int8-2024-07-17", + name: "SenseVoice Small INT8", + provider: "FunAudioLLM / sherpa-onnx", + description: "后端下发的离线语音模型", + version: "2024-07-17", + languages: ["zh", "en"], + runtime: "sherpa-onnx", + bundled: false, + sizeBytes: 262144000, + download: { + archive: { + downloadUrl: + "https://models.example.com/voice/sensevoice.tar.bz2", + sha256: "abc123", + }, + vad: { + modelId: "silero-vad-onnx", + downloadUrl: + "https://models.example.com/voice/silero_vad.onnx", + }, + }, + }, + ], + }, + }), + }); + vi.stubGlobal("fetch", fetchMock); + vi.mocked(safeInvoke).mockResolvedValueOnce({ + state: { installed: true }, + }); + + await expect( + downloadVoiceModel("sensevoice-small-int8-2024-07-17"), + ).resolves.toEqual(expect.objectContaining({ state: { installed: true } })); + + expect(fetchMock).toHaveBeenCalledWith( + "https://cloud.example.com/api/v1/public/tenants/tenant-0001/client/voice-model-catalog", + { + headers: { + Accept: "application/json", + }, + }, + ); + expect(safeInvoke).toHaveBeenCalledWith("voice_models_download", { + modelId: "sensevoice-small-int8-2024-07-17", + catalogEntry: expect.objectContaining({ + id: "sensevoice-small-int8-2024-07-17", + download_url: "https://models.example.com/voice/sensevoice.tar.bz2", + vad_download_url: "https://models.example.com/voice/silero_vad.onnx", + checksum_sha256: "abc123", + }), + }); + }); +}); diff --git a/src/lib/api/voiceModels.ts b/src/lib/api/voiceModels.ts new file mode 100644 index 000000000..9dba18c95 --- /dev/null +++ b/src/lib/api/voiceModels.ts @@ -0,0 +1,220 @@ +/** + * @file voiceModels.ts + * @description 本地语音模型管理 API + */ + +import { safeInvoke } from "@/lib/dev-bridge"; +import { resolveOemCloudRuntimeContext } from "./oemCloudRuntime"; +import type { AsrCredentialEntry } from "./asrProvider"; + +export interface VoiceModelCatalogEntry { + id: string; + name: string; + provider: string; + description: string; + version: string; + languages: string[]; + size_bytes: number; + download_url: string; + vad_model_id?: string | null; + vad_download_url?: string | null; + runtime: string; + bundled: boolean; + checksum_sha256?: string | null; +} + +export interface VoiceModelInstallState { + model_id: string; + installed: boolean; + installing: boolean; + install_dir: string; + model_file?: string | null; + tokens_file?: string | null; + vad_file?: string | null; + installed_bytes: number; + last_verified_at?: number | null; + missing_files: string[]; + default_credential_id?: string | null; +} + +export interface VoiceModelDownloadResult { + state: VoiceModelInstallState; +} + +export interface VoiceModelTestTranscribeResult { + text: string; + duration_secs: number; + sample_rate: number; + language?: string | null; +} + +interface OemVoiceModelCatalogResponse { + items?: OemVoiceModelCatalogItem[]; +} + +interface OemVoiceModelCatalogItem { + id?: string; + name?: string; + provider?: string; + description?: string; + version?: string; + languages?: string[]; + runtime?: string; + bundled?: boolean; + sizeBytes?: number; + checksumSha256?: string | null; + download?: { + archive?: OemVoiceModelDownloadAsset; + vad?: OemVoiceModelDownloadAsset | null; + }; +} + +interface OemVoiceModelDownloadAsset { + modelId?: string; + downloadUrl?: string; + sha256?: string | null; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object"; +} + +function normalizeText(value: unknown): string | null { + if (typeof value !== "string") { + return null; + } + + const normalized = value.trim(); + return normalized ? normalized : null; +} + +function unwrapEnvelope(payload: unknown): T { + if (isRecord(payload) && "data" in payload) { + return payload.data as T; + } + + return payload as T; +} + +function mapOemVoiceModelCatalogItem( + item: OemVoiceModelCatalogItem, +): VoiceModelCatalogEntry | null { + const id = normalizeText(item.id); + if (!id) { + return null; + } + + const archive = item.download?.archive; + const vad = item.download?.vad ?? null; + + return { + id, + name: normalizeText(item.name) ?? id, + provider: normalizeText(item.provider) ?? "FunAudioLLM / sherpa-onnx", + description: + normalizeText(item.description) ?? + "本地离线 ASR，模型按需下载到用户数据目录。", + version: normalizeText(item.version) ?? "", + languages: Array.isArray(item.languages) ? item.languages : [], + size_bytes: typeof item.sizeBytes === "number" ? item.sizeBytes : 0, + download_url: normalizeText(archive?.downloadUrl) ?? "", + vad_model_id: normalizeText(vad?.modelId), + vad_download_url: normalizeText(vad?.downloadUrl), + runtime: normalizeText(item.runtime) ?? "sherpa-onnx", + bundled: item.bundled === true, + checksum_sha256: + normalizeText(archive?.sha256) ?? normalizeText(item.checksumSha256), + }; +} + +async function fetchOemVoiceModelCatalog(): Promise< + VoiceModelCatalogEntry[] | null +> { + const runtime = resolveOemCloudRuntimeContext(); + if (!runtime) { + return null; + } + + const response = await fetch( + `${runtime.controlPlaneBaseUrl}/v1/public/tenants/${encodeURIComponent( + runtime.tenantId, + )}/client/voice-model-catalog`, + { + headers: { + Accept: "application/json", + }, + }, + ); + const payload = (await response.json().catch(() => null)) as unknown; + if (!response.ok) { + const message = + isRecord(payload) && typeof payload.message === "string" + ? payload.message + : `拉取语音模型目录失败 (${response.status})`; + throw new Error(message); + } + + const data = unwrapEnvelope(payload); + return (data.items ?? []) + .map(mapOemVoiceModelCatalogItem) + .filter((item): item is VoiceModelCatalogEntry => Boolean(item)); +} + +export async function listVoiceModelCatalog(): Promise< + VoiceModelCatalogEntry[] +> { + const oemCatalog = await fetchOemVoiceModelCatalog(); + if (oemCatalog) { + return oemCatalog; + } + + return safeInvoke("voice_models_list_catalog"); +} + +export async function getVoiceModelInstallState( + modelId: string, +): Promise { + return safeInvoke("voice_models_get_install_state", { + modelId, + }); +} + +export async function downloadVoiceModel( + modelId: string, +): Promise { + const oemCatalog = await fetchOemVoiceModelCatalog(); + const catalogEntry = oemCatalog?.find((item) => item.id === modelId); + return safeInvoke("voice_models_download", { + modelId, + ...(catalogEntry ? { catalogEntry } : {}), + }); +} + +export async function deleteVoiceModel( + modelId: string, +): Promise { + return safeInvoke("voice_models_delete", { + modelId, + }); +} + +export async function setDefaultVoiceModel( + modelId: string, +): Promise { + return safeInvoke("voice_models_set_default", { + modelId, + }); +} + +export async function testTranscribeVoiceModelFile( + modelId: string, + filePath: string, +): Promise { + return safeInvoke( + "voice_models_test_transcribe_file", + { + modelId, + filePath, + }, + ); +} diff --git a/src/lib/appearance/colorSchemes.test.ts b/src/lib/appearance/colorSchemes.test.ts index d6b42c27d..661f8d390 100644 --- a/src/lib/appearance/colorSchemes.test.ts +++ b/src/lib/appearance/colorSchemes.test.ts @@ -2,6 +2,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { DEFAULT_LIME_COLOR_SCHEME_ID, LIME_COLOR_SCHEME_CHANGED_EVENT, + LIME_COLOR_SCHEMES, LIME_COLOR_SCHEME_STORAGE_KEY, applyLimeColorScheme, persistLimeColorScheme, @@ -24,6 +25,25 @@ describe("colorSchemes", () => { expect(resolveLimeColorSchemeId(null)).toBe(DEFAULT_LIME_COLOR_SCHEME_ID); }); + it("应提供参考图中的完整预设配色矩阵", () => { + expect(LIME_COLOR_SCHEMES.map((scheme) => scheme.label)).toEqual([ + "墨绿", + "自然", + "海洋", + "复古", + "霓虹", + "青柠", + "黄昏", + "极简", + "活力", + "文艺", + "奢华", + ]); + expect( + LIME_COLOR_SCHEMES.every((scheme) => scheme.swatches.length === 3), + ).toBe(true); + }); + it("应用配色时应写入根节点 dataset 与 CSS 变量", () => { const resolvedId = applyLimeColorScheme("lime-sand"); @@ -73,15 +93,15 @@ describe("colorSchemes", () => { const listener = vi.fn(); window.addEventListener(LIME_COLOR_SCHEME_CHANGED_EVENT, listener); - const resolvedId = persistLimeColorScheme("lime-forest"); + const resolvedId = persistLimeColorScheme("lime-luxury"); - expect(resolvedId).toBe("lime-forest"); + expect(resolvedId).toBe("lime-luxury"); expect(localStorage.getItem(LIME_COLOR_SCHEME_STORAGE_KEY)).toBe( - "lime-forest", + "lime-luxury", ); expect(listener).toHaveBeenCalledTimes(1); expect(listener.mock.calls[0]?.[0]).toMatchObject({ - detail: { colorSchemeId: "lime-forest" }, + detail: { colorSchemeId: "lime-luxury" }, }); window.removeEventListener(LIME_COLOR_SCHEME_CHANGED_EVENT, listener); diff --git a/src/lib/appearance/colorSchemes.ts b/src/lib/appearance/colorSchemes.ts index 5c1771b7d..f5b666b15 100644 --- a/src/lib/appearance/colorSchemes.ts +++ b/src/lib/appearance/colorSchemes.ts @@ -7,7 +7,14 @@ export type LimeColorSchemeId = | "lime-classic" | "lime-forest" | "lime-ocean" - | "lime-sand"; + | "lime-sand" + | "lime-neon" + | "lime-citron" + | "lime-dusk" + | "lime-minimal" + | "lime-vivid" + | "lime-literary" + | "lime-luxury"; export interface LimeColorScheme { id: LimeColorSchemeId; @@ -246,15 +253,15 @@ function withPalette(overrides: Partial) { export const LIME_COLOR_SCHEMES: readonly LimeColorScheme[] = [ { id: "lime-classic", - label: "Lime 经典", - description: "保留当前高识别度的 Lime 绿色、浅青与清亮界面。", + label: "墨绿", + description: "经典深绿，温暖米色背景。", swatches: ["#f8fcf7", "#10b981", "#0ea5e9"], variables: classicVariables, }, { id: "lime-forest", - label: "森林", - description: "降低饱和度，保留绿色主轴，适合长时间创作。", + label: "自然", + description: "舒适放松的清新自然风。", swatches: ["#f4f7f1", "#2f6f46", "#8aa16e"], variables: withPalette({ "--lime-text": "#233c31", @@ -356,8 +363,8 @@ export const LIME_COLOR_SCHEMES: readonly LimeColorScheme[] = [ }, { id: "lime-ocean", - label: "海雾", - description: "偏冷静的蓝绿灰，适合信息密集和工程型工作流。", + label: "海洋", + description: "沉静专业的蓝色调。", swatches: ["#f3f8fa", "#0f766e", "#2563eb"], variables: withPalette({ "--lime-text": "#173346", @@ -459,8 +466,8 @@ export const LIME_COLOR_SCHEMES: readonly LimeColorScheme[] = [ }, { id: "lime-sand", - label: "砂岩", - description: "暖灰纸面与稳重墨绿，作为柔和低饱和备选。", + label: "复古", + description: "温暖怀旧的琥珀色调。", swatches: ["#f7f3e8", "#2f4638", "#c9a46a"], variables: withPalette({ "--lime-text": "#2f4638", @@ -566,6 +573,715 @@ export const LIME_COLOR_SCHEMES: readonly LimeColorScheme[] = [ "linear-gradient(135deg,#233c31 0%,#2f6f46 100%)", }), }, + { + id: "lime-neon", + label: "霓虹", + description: "赛博明亮的粉紫色调。", + swatches: ["#fdf4ff", "#b026c6", "#22c55e"], + variables: withPalette({ + "--lime-text": "#2f1b45", + "--lime-text-muted": "#7c6a8a", + "--lime-surface": "#fffaff", + "--lime-surface-subtle": "#fef7ff", + "--lime-surface-soft": "#fbf0ff", + "--lime-surface-muted": "#f5e7fb", + "--lime-surface-hover": "#f0ddfb", + "--lime-surface-border": "#ead2f5", + "--lime-surface-border-strong": "#dbb5eb", + "--lime-brand-strong": "#86198f", + "--lime-brand": "#c026d3", + "--lime-brand-muted": "#22c55e", + "--lime-brand-soft": "#fae8ff", + "--lime-info": "#0e7490", + "--lime-info-soft": "#ecfeff", + "--lime-info-border": "#bae6fd", + "--lime-focus-ring": "rgba(192, 38, 211, 0.16)", + "--lime-app-bg": "#f8f2fb", + "--lime-shell-surface": + "linear-gradient(180deg, #f4e8fb 0%, #fffaff 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #fffaff 0%, #f8f2fb 56%, #fdf8ff 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,250,255,0.96) 0%, rgba(248,242,251,0.92) 100%)", + "--lime-stage-surface-top": "#fffaff", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,255,255,0.96) 0%, rgba(251,240,255,0.94) 100%)", + "--lime-card-subtle-border": "rgba(234, 210, 245, 0.76)", + "--lime-divider-subtle": "rgba(134, 25, 143, 0.14)", + "--lime-chrome-rail": "#fbf0ff", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #f4e8fb 0%, #fbf0ff 100%)", + "--lime-chrome-surface": "#fef7ff", + "--lime-chrome-active-tab": "#fffaff", + "--lime-chrome-tab-hover": "#f0ddfb", + "--lime-chrome-tab-active-surface": "#fffaff", + "--lime-chrome-border": "rgba(234, 210, 245, 0.76)", + "--lime-chrome-divider": "rgba(234, 210, 245, 0.66)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(192, 38, 211, 0.034), transparent 42%), radial-gradient(circle at 78% 115%, rgba(34, 197, 94, 0.04), transparent 46%), linear-gradient(180deg, #fffaff 0%, #fdf8ff 58%, #fffaff 100%)", + "--lime-chrome-stage-seam": "rgba(134, 25, 143, 0.07)", + "--lime-chrome-text": "#2f1b45", + "--lime-chrome-muted": "#7c6a8a", + "--lime-sidebar-surface": + "linear-gradient(180deg, #f4e8fb 0%, #fbf4ff 48%, #fffaff 100%)", + "--lime-sidebar-surface-top": "#f4e8fb", + "--lime-sidebar-surface-middle": "#fbf4ff", + "--lime-sidebar-surface-bottom": "#fffaff", + "--lime-sidebar-border": "rgba(234, 210, 245, 0.72)", + "--lime-sidebar-divider": "rgba(134,25,143,0.12)", + "--lime-sidebar-hover": "#f0ddfb", + "--lime-sidebar-active": "#fae8ff", + "--lime-sidebar-active-text": "#86198f", + "--lime-sidebar-search-bg": "#fffaff", + "--lime-sidebar-search-hover": "#f5e7fb", + "--lime-sidebar-search-border-hover": "#dbb5eb", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #fffaff 0%, #f5e7fb 100%)", + "--lime-sidebar-card-border": "rgba(234, 210, 245, 0.7)", + "--lime-home-bg-start": "#fbf0ff", + "--lime-home-bg-mid": "#fffaff", + "--lime-home-bg-end": "#f4fbf7", + "--lime-home-glow-primary": "rgba(192,38,211,0.04)", + "--lime-home-glow-secondary": "rgba(34,197,94,0.05)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #2f1b45 0%, #a21caf 54%, #15803d 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #d946ef, #22c55e)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(192,38,211,0.045), 0 0 14px rgba(34,197,94,0.08)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(192,38,211,0) 0%, rgba(192,38,211,0.034) 32%, rgba(255,255,255,0.26) 50%, rgba(34,197,94,0.045) 68%, rgba(192,38,211,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(250,232,255,0.9) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,255,255,0.98), rgba(236,254,255,0.92))", + "--lime-home-card-border": "rgba(234,210,245,0.86)", + "--lime-home-card-border-muted": "rgba(234,210,245,0.84)", + "--lime-home-card-hover-border": "#dbb5eb", + "--lime-composer-surface": + "linear-gradient(180deg, #ffffff 0%, #fbf0ff 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #ffffff 0%, #fbf0ff 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #ffffff 0%, #fbf0ff 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #ffffff 0%, #f5e7fb 100%)", + "--lime-composer-border": "rgba(219, 181, 235, 0.7)", + "--lime-composer-border-focus": "rgba(192, 38, 211, 0.44)", + "--lime-primary-gradient": + "linear-gradient(135deg,#86198f 0%,#c026d3 54%,#22c55e 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#86198f 0%,#c026d3 100%)", + }), + }, + { + id: "lime-citron", + label: "青柠", + description: "活力清新的黄绿配紫。", + swatches: ["#fbffe8", "#84cc16", "#6d4fb3"], + variables: withPalette({ + "--lime-text": "#25351a", + "--lime-text-muted": "#69735b", + "--lime-surface": "#fffffb", + "--lime-surface-subtle": "#fbffe8", + "--lime-surface-soft": "#f4ffd2", + "--lime-surface-muted": "#ecf7bf", + "--lime-surface-hover": "#e3f2a8", + "--lime-surface-border": "#d9e9a5", + "--lime-surface-border-strong": "#c4d77d", + "--lime-brand-strong": "#4d7c0f", + "--lime-brand": "#84cc16", + "--lime-brand-muted": "#6d4fb3", + "--lime-brand-soft": "#f7fee7", + "--lime-info": "#6d4fb3", + "--lime-info-soft": "#f5f3ff", + "--lime-info-border": "#ddd6fe", + "--lime-focus-ring": "rgba(132, 204, 22, 0.18)", + "--lime-app-bg": "#f5f8e9", + "--lime-shell-surface": + "linear-gradient(180deg, #edf7c7 0%, #fffffb 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #fffffb 0%, #f5f8e9 56%, #fbffe8 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,255,251,0.96) 0%, rgba(245,248,233,0.92) 100%)", + "--lime-stage-surface-top": "#fffffb", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,255,255,0.96) 0%, rgba(247,254,231,0.94) 100%)", + "--lime-card-subtle-border": "rgba(217, 233, 165, 0.76)", + "--lime-divider-subtle": "rgba(77, 124, 15, 0.14)", + "--lime-chrome-rail": "#f4ffd2", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #edf7c7 0%, #f4ffd2 100%)", + "--lime-chrome-surface": "#fbffe8", + "--lime-chrome-active-tab": "#fffffb", + "--lime-chrome-tab-hover": "#ecf7bf", + "--lime-chrome-tab-active-surface": "#fffffb", + "--lime-chrome-border": "rgba(217, 233, 165, 0.76)", + "--lime-chrome-divider": "rgba(217, 233, 165, 0.66)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(132, 204, 22, 0.035), transparent 42%), radial-gradient(circle at 78% 115%, rgba(109, 79, 179, 0.038), transparent 46%), linear-gradient(180deg, #fffffb 0%, #fbffe8 58%, #fffffb 100%)", + "--lime-chrome-stage-seam": "rgba(77, 124, 15, 0.07)", + "--lime-chrome-text": "#25351a", + "--lime-chrome-muted": "#69735b", + "--lime-sidebar-surface": + "linear-gradient(180deg, #edf7c7 0%, #f8fbdc 48%, #fffffb 100%)", + "--lime-sidebar-surface-top": "#edf7c7", + "--lime-sidebar-surface-middle": "#f8fbdc", + "--lime-sidebar-surface-bottom": "#fffffb", + "--lime-sidebar-border": "rgba(217, 233, 165, 0.72)", + "--lime-sidebar-divider": "rgba(77,124,15,0.12)", + "--lime-sidebar-hover": "#ecf7bf", + "--lime-sidebar-active": "#e3f2a8", + "--lime-sidebar-active-text": "#4d7c0f", + "--lime-sidebar-search-bg": "#fffffb", + "--lime-sidebar-search-hover": "#f4ffd2", + "--lime-sidebar-search-border-hover": "#c4d77d", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #fffffb 0%, #ecf7bf 100%)", + "--lime-sidebar-card-border": "rgba(217, 233, 165, 0.7)", + "--lime-home-bg-start": "#f4ffd2", + "--lime-home-bg-mid": "#fffffb", + "--lime-home-bg-end": "#f5f3ff", + "--lime-home-glow-primary": "rgba(132,204,22,0.045)", + "--lime-home-glow-secondary": "rgba(109,79,179,0.045)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #25351a 0%, #4d7c0f 54%, #6d4fb3 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #84cc16, #6d4fb3)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(132,204,22,0.05), 0 0 14px rgba(109,79,179,0.075)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(132,204,22,0) 0%, rgba(132,204,22,0.038) 32%, rgba(255,255,255,0.24) 50%, rgba(109,79,179,0.042) 68%, rgba(132,204,22,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(247,254,231,0.92) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,255,255,0.98), rgba(245,243,255,0.92))", + "--lime-home-card-border": "rgba(217,233,165,0.86)", + "--lime-home-card-border-muted": "rgba(217,233,165,0.84)", + "--lime-home-card-hover-border": "#c4d77d", + "--lime-composer-surface": + "linear-gradient(180deg, #ffffff 0%, #f4ffd2 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #ffffff 0%, #f4ffd2 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #ffffff 0%, #f4ffd2 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #ffffff 0%, #ecf7bf 100%)", + "--lime-composer-border": "rgba(196, 215, 125, 0.7)", + "--lime-composer-border-focus": "rgba(132, 204, 22, 0.46)", + "--lime-primary-gradient": + "linear-gradient(135deg,#4d7c0f 0%,#84cc16 54%,#6d4fb3 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#4d7c0f 0%,#84cc16 100%)", + }), + }, + { + id: "lime-dusk", + label: "黄昏", + description: "柔和温暖的暮色调。", + swatches: ["#fbf4e5", "#7c7f32", "#c1784a"], + variables: withPalette({ + "--lime-text": "#3f3529", + "--lime-text-muted": "#7a6f61", + "--lime-surface": "#fffaf2", + "--lime-surface-subtle": "#fbf4e5", + "--lime-surface-soft": "#f7ead7", + "--lime-surface-muted": "#efdfc8", + "--lime-surface-hover": "#ead4ba", + "--lime-surface-border": "#dcc8aa", + "--lime-surface-border-strong": "#c9ad83", + "--lime-brand-strong": "#65691f", + "--lime-brand": "#7c7f32", + "--lime-brand-muted": "#c1784a", + "--lime-brand-soft": "#f7f3dd", + "--lime-info": "#8a5a44", + "--lime-info-soft": "#fff7ed", + "--lime-info-border": "#fed7aa", + "--lime-warning": "#9a5a1f", + "--lime-warning-soft": "#fff7ed", + "--lime-warning-border": "#fed7aa", + "--lime-focus-ring": "rgba(124, 127, 50, 0.16)", + "--lime-app-bg": "#f3eadc", + "--lime-shell-surface": + "linear-gradient(180deg, #eadfce 0%, #fffaf2 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #fffaf2 0%, #f3eadc 56%, #fbf4e5 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,250,242,0.96) 0%, rgba(243,234,220,0.92) 100%)", + "--lime-stage-surface-top": "#fffaf2", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,250,242,0.96) 0%, rgba(247,234,215,0.94) 100%)", + "--lime-card-subtle-border": "rgba(220, 200, 170, 0.76)", + "--lime-divider-subtle": "rgba(124, 127, 50, 0.14)", + "--lime-chrome-rail": "#f7ead7", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #eadfce 0%, #f7ead7 100%)", + "--lime-chrome-surface": "#fbf4e5", + "--lime-chrome-active-tab": "#fffaf2", + "--lime-chrome-tab-hover": "#efdfc8", + "--lime-chrome-tab-active-surface": "#fffaf2", + "--lime-chrome-border": "rgba(220, 200, 170, 0.76)", + "--lime-chrome-divider": "rgba(220, 200, 170, 0.66)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(124, 127, 50, 0.03), transparent 42%), radial-gradient(circle at 78% 115%, rgba(193, 120, 74, 0.038), transparent 46%), linear-gradient(180deg, #fffaf2 0%, #fbf4e5 58%, #fffaf2 100%)", + "--lime-chrome-stage-seam": "rgba(124, 127, 50, 0.07)", + "--lime-chrome-text": "#3f3529", + "--lime-chrome-muted": "#7a6f61", + "--lime-sidebar-surface": + "linear-gradient(180deg, #eadfce 0%, #f6ecdc 48%, #fffaf2 100%)", + "--lime-sidebar-surface-top": "#eadfce", + "--lime-sidebar-surface-middle": "#f6ecdc", + "--lime-sidebar-surface-bottom": "#fffaf2", + "--lime-sidebar-border": "rgba(220, 200, 170, 0.72)", + "--lime-sidebar-divider": "rgba(124,127,50,0.12)", + "--lime-sidebar-hover": "#efdfc8", + "--lime-sidebar-active": "#f0e8c3", + "--lime-sidebar-active-text": "#65691f", + "--lime-sidebar-search-bg": "#fffaf2", + "--lime-sidebar-search-hover": "#f7ead7", + "--lime-sidebar-search-border-hover": "#c9ad83", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #fffaf2 0%, #efdfc8 100%)", + "--lime-sidebar-card-border": "rgba(220, 200, 170, 0.7)", + "--lime-home-bg-start": "#f7ead7", + "--lime-home-bg-mid": "#fffaf2", + "--lime-home-bg-end": "#f7f3dd", + "--lime-home-glow-primary": "rgba(124,127,50,0.04)", + "--lime-home-glow-secondary": "rgba(193,120,74,0.05)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #3f3529 0%, #65691f 52%, #9a5a1f 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #7c7f32, #c1784a)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(124,127,50,0.045), 0 0 14px rgba(193,120,74,0.08)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(124,127,50,0) 0%, rgba(124,127,50,0.034) 32%, rgba(255,255,255,0.24) 50%, rgba(193,120,74,0.044) 68%, rgba(124,127,50,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,250,242,0.98) 0%, rgba(247,234,215,0.92) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,250,242,0.98), rgba(247,243,221,0.92))", + "--lime-home-card-border": "rgba(220,200,170,0.86)", + "--lime-home-card-border-muted": "rgba(220,200,170,0.84)", + "--lime-home-card-hover-border": "#c9ad83", + "--lime-composer-surface": + "linear-gradient(180deg, #fffaf2 0%, #f7ead7 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #fffaf2 0%, #f7ead7 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #fffaf2 0%, #f7ead7 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #fffaf2 0%, #efdfc8 100%)", + "--lime-composer-border": "rgba(201, 173, 131, 0.72)", + "--lime-composer-border-focus": "rgba(124, 127, 50, 0.44)", + "--lime-primary-gradient": + "linear-gradient(135deg,#65691f 0%,#7c7f32 54%,#c1784a 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#65691f 0%,#7c7f32 100%)", + }), + }, + { + id: "lime-minimal", + label: "极简", + description: "清晰专业的深蓝商务风。", + swatches: ["#f8fafc", "#334155", "#2563eb"], + variables: withPalette({ + "--lime-text": "#1e293b", + "--lime-text-muted": "#64748b", + "--lime-surface": "#ffffff", + "--lime-surface-subtle": "#f8fafc", + "--lime-surface-soft": "#f1f5f9", + "--lime-surface-muted": "#e2e8f0", + "--lime-surface-hover": "#eef2f7", + "--lime-surface-border": "#d8e0ea", + "--lime-surface-border-strong": "#cbd5e1", + "--lime-brand-strong": "#334155", + "--lime-brand": "#2563eb", + "--lime-brand-muted": "#0f766e", + "--lime-brand-soft": "#eff6ff", + "--lime-info": "#0369a1", + "--lime-info-soft": "#f0f9ff", + "--lime-info-border": "#bae6fd", + "--lime-focus-ring": "rgba(37, 99, 235, 0.16)", + "--lime-app-bg": "#f3f6fa", + "--lime-shell-surface": + "linear-gradient(180deg, #eef2f7 0%, #ffffff 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #ffffff 0%, #f3f6fa 56%, #f8fafc 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,255,255,0.96) 0%, rgba(243,246,250,0.92) 100%)", + "--lime-stage-surface-top": "#ffffff", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(248,250,252,0.96) 100%)", + "--lime-card-subtle-border": "rgba(216, 224, 234, 0.8)", + "--lime-divider-subtle": "rgba(51, 65, 85, 0.12)", + "--lime-chrome-rail": "#f1f5f9", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #eef2f7 0%, #f1f5f9 100%)", + "--lime-chrome-surface": "#f8fafc", + "--lime-chrome-active-tab": "#ffffff", + "--lime-chrome-tab-hover": "#e2e8f0", + "--lime-chrome-tab-active-surface": "#ffffff", + "--lime-chrome-border": "rgba(216, 224, 234, 0.78)", + "--lime-chrome-divider": "rgba(216, 224, 234, 0.68)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(37, 99, 235, 0.026), transparent 42%), radial-gradient(circle at 78% 115%, rgba(15, 118, 110, 0.028), transparent 46%), linear-gradient(180deg, #ffffff 0%, #f8fafc 58%, #ffffff 100%)", + "--lime-chrome-stage-seam": "rgba(51, 65, 85, 0.065)", + "--lime-chrome-text": "#1e293b", + "--lime-chrome-muted": "#64748b", + "--lime-sidebar-surface": + "linear-gradient(180deg, #eef2f7 0%, #f8fafc 48%, #ffffff 100%)", + "--lime-sidebar-surface-top": "#eef2f7", + "--lime-sidebar-surface-middle": "#f8fafc", + "--lime-sidebar-surface-bottom": "#ffffff", + "--lime-sidebar-border": "rgba(216, 224, 234, 0.72)", + "--lime-sidebar-divider": "rgba(51,65,85,0.1)", + "--lime-sidebar-hover": "#e2e8f0", + "--lime-sidebar-active": "#eaf2ff", + "--lime-sidebar-active-text": "#1d4ed8", + "--lime-sidebar-search-bg": "#ffffff", + "--lime-sidebar-search-hover": "#f1f5f9", + "--lime-sidebar-search-border-hover": "#cbd5e1", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #ffffff 0%, #eef2f7 100%)", + "--lime-sidebar-card-border": "rgba(216, 224, 234, 0.72)", + "--lime-home-bg-start": "#f1f5f9", + "--lime-home-bg-mid": "#ffffff", + "--lime-home-bg-end": "#eff6ff", + "--lime-home-glow-primary": "rgba(37,99,235,0.032)", + "--lime-home-glow-secondary": "rgba(15,118,110,0.032)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #1e293b 0%, #334155 54%, #2563eb 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #334155, #2563eb)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(37,99,235,0.04), 0 0 14px rgba(51,65,85,0.07)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(37,99,235,0) 0%, rgba(37,99,235,0.032) 32%, rgba(255,255,255,0.24) 50%, rgba(15,118,110,0.032) 68%, rgba(37,99,235,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(248,250,252,0.94) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,255,255,0.98), rgba(239,246,255,0.92))", + "--lime-home-card-border": "rgba(216,224,234,0.88)", + "--lime-home-card-border-muted": "rgba(216,224,234,0.86)", + "--lime-home-card-hover-border": "#cbd5e1", + "--lime-composer-surface": + "linear-gradient(180deg, #ffffff 0%, #f1f5f9 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #ffffff 0%, #f1f5f9 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #ffffff 0%, #f1f5f9 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #ffffff 0%, #eff6ff 100%)", + "--lime-composer-border": "rgba(203, 213, 225, 0.74)", + "--lime-composer-border-focus": "rgba(37, 99, 235, 0.44)", + "--lime-primary-gradient": + "linear-gradient(135deg,#334155 0%,#2563eb 58%,#0f766e 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#334155 0%,#2563eb 100%)", + }), + }, + { + id: "lime-vivid", + label: "活力", + description: "时尚有冲击力的现代科技风。", + swatches: ["#f0fdfa", "#14b8a6", "#f97316"], + variables: withPalette({ + "--lime-text": "#143d3a", + "--lime-text-muted": "#607874", + "--lime-surface": "#ffffff", + "--lime-surface-subtle": "#f0fdfa", + "--lime-surface-soft": "#e8fbf7", + "--lime-surface-muted": "#d7f3ed", + "--lime-surface-hover": "#c8eee7", + "--lime-surface-border": "#bde7df", + "--lime-surface-border-strong": "#98d7cd", + "--lime-brand-strong": "#0f766e", + "--lime-brand": "#14b8a6", + "--lime-brand-muted": "#f97316", + "--lime-brand-soft": "#ccfbf1", + "--lime-info": "#0ea5e9", + "--lime-info-soft": "#f0f9ff", + "--lime-info-border": "#bae6fd", + "--lime-warning": "#c2410c", + "--lime-warning-soft": "#fff7ed", + "--lime-warning-border": "#fed7aa", + "--lime-focus-ring": "rgba(20, 184, 166, 0.18)", + "--lime-app-bg": "#eef9f7", + "--lime-shell-surface": + "linear-gradient(180deg, #dff7f1 0%, #ffffff 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #ffffff 0%, #eef9f7 56%, #f0fdfa 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,255,255,0.96) 0%, rgba(238,249,247,0.92) 100%)", + "--lime-stage-surface-top": "#ffffff", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(240,253,250,0.94) 100%)", + "--lime-card-subtle-border": "rgba(189, 231, 223, 0.78)", + "--lime-divider-subtle": "rgba(15, 118, 110, 0.13)", + "--lime-chrome-rail": "#e8fbf7", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #dff7f1 0%, #e8fbf7 100%)", + "--lime-chrome-surface": "#f0fdfa", + "--lime-chrome-active-tab": "#ffffff", + "--lime-chrome-tab-hover": "#d7f3ed", + "--lime-chrome-tab-active-surface": "#ffffff", + "--lime-chrome-border": "rgba(189, 231, 223, 0.78)", + "--lime-chrome-divider": "rgba(189, 231, 223, 0.66)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(20, 184, 166, 0.034), transparent 42%), radial-gradient(circle at 78% 115%, rgba(249, 115, 22, 0.036), transparent 46%), linear-gradient(180deg, #ffffff 0%, #f0fdfa 58%, #ffffff 100%)", + "--lime-chrome-stage-seam": "rgba(15, 118, 110, 0.07)", + "--lime-chrome-text": "#143d3a", + "--lime-chrome-muted": "#607874", + "--lime-sidebar-surface": + "linear-gradient(180deg, #dff7f1 0%, #f0fdfa 48%, #ffffff 100%)", + "--lime-sidebar-surface-top": "#dff7f1", + "--lime-sidebar-surface-middle": "#f0fdfa", + "--lime-sidebar-surface-bottom": "#ffffff", + "--lime-sidebar-border": "rgba(189, 231, 223, 0.72)", + "--lime-sidebar-divider": "rgba(15,118,110,0.11)", + "--lime-sidebar-hover": "#d7f3ed", + "--lime-sidebar-active": "#ccfbf1", + "--lime-sidebar-active-text": "#0f766e", + "--lime-sidebar-search-bg": "#ffffff", + "--lime-sidebar-search-hover": "#e8fbf7", + "--lime-sidebar-search-border-hover": "#98d7cd", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #ffffff 0%, #d7f3ed 100%)", + "--lime-sidebar-card-border": "rgba(189, 231, 223, 0.7)", + "--lime-home-bg-start": "#e8fbf7", + "--lime-home-bg-mid": "#ffffff", + "--lime-home-bg-end": "#fff7ed", + "--lime-home-glow-primary": "rgba(20,184,166,0.045)", + "--lime-home-glow-secondary": "rgba(249,115,22,0.048)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #143d3a 0%, #0f766e 54%, #c2410c 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #14b8a6, #f97316)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(20,184,166,0.045), 0 0 14px rgba(249,115,22,0.08)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(20,184,166,0) 0%, rgba(20,184,166,0.036) 32%, rgba(255,255,255,0.24) 50%, rgba(249,115,22,0.042) 68%, rgba(20,184,166,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(240,253,250,0.92) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,255,255,0.98), rgba(255,247,237,0.92))", + "--lime-home-card-border": "rgba(189,231,223,0.88)", + "--lime-home-card-border-muted": "rgba(189,231,223,0.84)", + "--lime-home-card-hover-border": "#98d7cd", + "--lime-composer-surface": + "linear-gradient(180deg, #ffffff 0%, #e8fbf7 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #ffffff 0%, #e8fbf7 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #ffffff 0%, #e8fbf7 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #ffffff 0%, #d7f3ed 100%)", + "--lime-composer-border": "rgba(152, 215, 205, 0.72)", + "--lime-composer-border-focus": "rgba(20, 184, 166, 0.46)", + "--lime-primary-gradient": + "linear-gradient(135deg,#0f766e 0%,#14b8a6 54%,#f97316 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#0f766e 0%,#14b8a6 100%)", + }), + }, + { + id: "lime-literary", + label: "文艺", + description: "宁静高雅的灰蓝文艺风。", + swatches: ["#f5f7fb", "#53627a", "#8b7ab8"], + variables: withPalette({ + "--lime-text": "#283244", + "--lime-text-muted": "#6b7280", + "--lime-surface": "#ffffff", + "--lime-surface-subtle": "#f8fafc", + "--lime-surface-soft": "#f2f5f9", + "--lime-surface-muted": "#e8edf4", + "--lime-surface-hover": "#e1e8f1", + "--lime-surface-border": "#d7e0eb", + "--lime-surface-border-strong": "#c3cedc", + "--lime-brand-strong": "#475569", + "--lime-brand": "#64748b", + "--lime-brand-muted": "#8b7ab8", + "--lime-brand-soft": "#f1f5f9", + "--lime-info": "#66738f", + "--lime-info-soft": "#eef2ff", + "--lime-info-border": "#c7d2fe", + "--lime-focus-ring": "rgba(100, 116, 139, 0.16)", + "--lime-app-bg": "#f1f4f8", + "--lime-shell-surface": + "linear-gradient(180deg, #e8edf4 0%, #ffffff 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #ffffff 0%, #f1f4f8 56%, #f8fafc 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,255,255,0.96) 0%, rgba(241,244,248,0.92) 100%)", + "--lime-stage-surface-top": "#ffffff", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(242,245,249,0.94) 100%)", + "--lime-card-subtle-border": "rgba(215, 224, 235, 0.78)", + "--lime-divider-subtle": "rgba(71, 85, 105, 0.12)", + "--lime-chrome-rail": "#f2f5f9", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #e8edf4 0%, #f2f5f9 100%)", + "--lime-chrome-surface": "#f8fafc", + "--lime-chrome-active-tab": "#ffffff", + "--lime-chrome-tab-hover": "#e8edf4", + "--lime-chrome-tab-active-surface": "#ffffff", + "--lime-chrome-border": "rgba(215, 224, 235, 0.78)", + "--lime-chrome-divider": "rgba(215, 224, 235, 0.68)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(100, 116, 139, 0.026), transparent 42%), radial-gradient(circle at 78% 115%, rgba(139, 122, 184, 0.034), transparent 46%), linear-gradient(180deg, #ffffff 0%, #f8fafc 58%, #ffffff 100%)", + "--lime-chrome-stage-seam": "rgba(71, 85, 105, 0.065)", + "--lime-chrome-text": "#283244", + "--lime-chrome-muted": "#6b7280", + "--lime-sidebar-surface": + "linear-gradient(180deg, #e8edf4 0%, #f5f7fb 48%, #ffffff 100%)", + "--lime-sidebar-surface-top": "#e8edf4", + "--lime-sidebar-surface-middle": "#f5f7fb", + "--lime-sidebar-surface-bottom": "#ffffff", + "--lime-sidebar-border": "rgba(215, 224, 235, 0.72)", + "--lime-sidebar-divider": "rgba(71,85,105,0.1)", + "--lime-sidebar-hover": "#e8edf4", + "--lime-sidebar-active": "#eef2ff", + "--lime-sidebar-active-text": "#475569", + "--lime-sidebar-search-bg": "#ffffff", + "--lime-sidebar-search-hover": "#f2f5f9", + "--lime-sidebar-search-border-hover": "#c3cedc", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #ffffff 0%, #e8edf4 100%)", + "--lime-sidebar-card-border": "rgba(215, 224, 235, 0.7)", + "--lime-home-bg-start": "#f2f5f9", + "--lime-home-bg-mid": "#ffffff", + "--lime-home-bg-end": "#eef2ff", + "--lime-home-glow-primary": "rgba(100,116,139,0.032)", + "--lime-home-glow-secondary": "rgba(139,122,184,0.04)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #283244 0%, #53627a 54%, #8b7ab8 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #53627a, #8b7ab8)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(100,116,139,0.04), 0 0 14px rgba(139,122,184,0.075)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(100,116,139,0) 0%, rgba(100,116,139,0.03) 32%, rgba(255,255,255,0.24) 50%, rgba(139,122,184,0.038) 68%, rgba(100,116,139,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,255,255,0.98) 0%, rgba(242,245,249,0.92) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,255,255,0.98), rgba(238,242,255,0.92))", + "--lime-home-card-border": "rgba(215,224,235,0.88)", + "--lime-home-card-border-muted": "rgba(215,224,235,0.84)", + "--lime-home-card-hover-border": "#c3cedc", + "--lime-composer-surface": + "linear-gradient(180deg, #ffffff 0%, #f2f5f9 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #ffffff 0%, #f2f5f9 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #ffffff 0%, #f2f5f9 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #ffffff 0%, #eef2ff 100%)", + "--lime-composer-border": "rgba(195, 206, 220, 0.72)", + "--lime-composer-border-focus": "rgba(100, 116, 139, 0.44)", + "--lime-primary-gradient": + "linear-gradient(135deg,#475569 0%,#64748b 54%,#8b7ab8 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#475569 0%,#64748b 100%)", + }), + }, + { + id: "lime-luxury", + label: "奢华", + description: "尊贵权威的黑金商务风。", + swatches: ["#fbf8ef", "#1f2933", "#c9a23a"], + variables: withPalette({ + "--lime-text": "#2c2a24", + "--lime-text-muted": "#746f62", + "--lime-surface": "#fffdf7", + "--lime-surface-subtle": "#fbf8ef", + "--lime-surface-soft": "#f4efe2", + "--lime-surface-muted": "#ebe2cf", + "--lime-surface-hover": "#e6dac2", + "--lime-surface-border": "#d8cab0", + "--lime-surface-border-strong": "#c9a23a", + "--lime-brand-strong": "#1f2933", + "--lime-brand": "#9d7a22", + "--lime-brand-muted": "#c9a23a", + "--lime-brand-soft": "#f8edd0", + "--lime-info": "#58606a", + "--lime-info-soft": "#f4f6f8", + "--lime-info-border": "#d8dee6", + "--lime-warning": "#8a5a10", + "--lime-warning-soft": "#fff7e6", + "--lime-warning-border": "#f3d28d", + "--lime-focus-ring": "rgba(157, 122, 34, 0.16)", + "--lime-app-bg": "#f1eadc", + "--lime-shell-surface": + "linear-gradient(180deg, #e8dec9 0%, #fffdf7 100%)", + "--lime-stage-surface": + "linear-gradient(180deg, #fffdf7 0%, #f1eadc 56%, #fbf8ef 100%)", + "--lime-stage-surface-soft": + "linear-gradient(180deg, rgba(255,253,247,0.96) 0%, rgba(241,234,220,0.92) 100%)", + "--lime-stage-surface-top": "#fffdf7", + "--lime-card-subtle": + "linear-gradient(180deg, rgba(255,253,247,0.96) 0%, rgba(244,239,226,0.94) 100%)", + "--lime-card-subtle-border": "rgba(216, 202, 176, 0.78)", + "--lime-divider-subtle": "rgba(31, 41, 51, 0.12)", + "--lime-chrome-rail": "#f4efe2", + "--lime-chrome-rail-surface": + "linear-gradient(180deg, #e8dec9 0%, #f4efe2 100%)", + "--lime-chrome-surface": "#fbf8ef", + "--lime-chrome-active-tab": "#fffdf7", + "--lime-chrome-tab-hover": "#ebe2cf", + "--lime-chrome-tab-active-surface": "#fffdf7", + "--lime-chrome-border": "rgba(216, 202, 176, 0.78)", + "--lime-chrome-divider": "rgba(216, 202, 176, 0.68)", + "--lime-chrome-stage-blend": + "radial-gradient(circle at 18% 100%, rgba(31, 41, 51, 0.026), transparent 42%), radial-gradient(circle at 78% 115%, rgba(201, 162, 58, 0.04), transparent 46%), linear-gradient(180deg, #fffdf7 0%, #fbf8ef 58%, #fffdf7 100%)", + "--lime-chrome-stage-seam": "rgba(31, 41, 51, 0.07)", + "--lime-chrome-text": "#2c2a24", + "--lime-chrome-muted": "#746f62", + "--lime-sidebar-surface": + "linear-gradient(180deg, #e8dec9 0%, #f6f0e4 48%, #fffdf7 100%)", + "--lime-sidebar-surface-top": "#e8dec9", + "--lime-sidebar-surface-middle": "#f6f0e4", + "--lime-sidebar-surface-bottom": "#fffdf7", + "--lime-sidebar-border": "rgba(216, 202, 176, 0.72)", + "--lime-sidebar-divider": "rgba(31,41,51,0.1)", + "--lime-sidebar-hover": "#ebe2cf", + "--lime-sidebar-active": "#f8edd0", + "--lime-sidebar-active-text": "#1f2933", + "--lime-sidebar-search-bg": "#fffdf7", + "--lime-sidebar-search-hover": "#f4efe2", + "--lime-sidebar-search-border-hover": "#c9a23a", + "--lime-sidebar-card-surface": + "linear-gradient(180deg, #fffdf7 0%, #ebe2cf 100%)", + "--lime-sidebar-card-border": "rgba(216, 202, 176, 0.7)", + "--lime-home-bg-start": "#f4efe2", + "--lime-home-bg-mid": "#fffdf7", + "--lime-home-bg-end": "#f8edd0", + "--lime-home-glow-primary": "rgba(31,41,51,0.032)", + "--lime-home-glow-secondary": "rgba(201,162,58,0.048)", + "--lime-home-title-gradient": + "linear-gradient(90deg, #1f2933 0%, #2c2a24 52%, #9d7a22 100%)", + "--lime-home-dot-gradient": "linear-gradient(135deg, #1f2933, #c9a23a)", + "--lime-home-dot-shadow": + "0 0 0 8px rgba(31,41,51,0.038), 0 0 14px rgba(201,162,58,0.08)", + "--lime-home-beam-gradient": + "linear-gradient(90deg, rgba(31,41,51,0) 0%, rgba(31,41,51,0.03) 32%, rgba(255,255,255,0.24) 50%, rgba(201,162,58,0.044) 68%, rgba(31,41,51,0) 100%)", + "--lime-home-card-surface": + "linear-gradient(180deg, rgba(255,253,247,0.98) 0%, rgba(244,239,226,0.92) 100%)", + "--lime-home-card-surface-strong": + "linear-gradient(180deg, rgba(255,253,247,0.98), rgba(248,237,208,0.92))", + "--lime-home-card-border": "rgba(216,202,176,0.88)", + "--lime-home-card-border-muted": "rgba(216,202,176,0.84)", + "--lime-home-card-hover-border": "#c9a23a", + "--lime-composer-surface": + "linear-gradient(180deg, #fffdf7 0%, #f4efe2 100%)", + "--lime-composer-shell": + "linear-gradient(180deg, #fffdf7 0%, #f4efe2 100%)", + "--lime-composer-surface-floating": + "linear-gradient(180deg, #fffdf7 0%, #f4efe2 100%)", + "--lime-composer-surface-focus": + "linear-gradient(180deg, #fffdf7 0%, #f8edd0 100%)", + "--lime-composer-border": "rgba(201, 162, 58, 0.48)", + "--lime-composer-border-focus": "rgba(157, 122, 34, 0.44)", + "--lime-primary-gradient": + "linear-gradient(135deg,#1f2933 0%,#2c2a24 54%,#c9a23a 100%)", + "--lime-primary-gradient-simple": + "linear-gradient(135deg,#1f2933 0%,#9d7a22 100%)", + }), + }, ]; const colorSchemeIds = new Set( diff --git a/src/lib/dev-bridge/http-client.test.ts b/src/lib/dev-bridge/http-client.test.ts index 12af11224..236c7a512 100644 --- a/src/lib/dev-bridge/http-client.test.ts +++ b/src/lib/dev-bridge/http-client.test.ts @@ -227,6 +227,36 @@ describe("http-client", () => { }); }); + it("agent 标题生成命令应使用 agent 长超时窗口", async () => { + const fetchMock = vi + .fn() + .mockResolvedValueOnce(new Response(null, { status: 200 })) + .mockImplementationOnce(createAbortablePendingFetch()); + vi.stubGlobal("fetch", fetchMock); + + let settled = false; + const invokePromise = invokeViaHttp("agent_generate_title", { + sessionId: "session-1", + }).then( + () => ({ ok: true as const }), + (error) => ({ ok: false as const, error }), + ); + invokePromise.finally(() => { + settled = true; + }); + + await vi.advanceTimersByTimeAsync(5000); + expect(settled).toBe(false); + + await vi.advanceTimersByTimeAsync(55000); + await expect(invokePromise).resolves.toMatchObject({ + ok: false, + error: expect.objectContaining({ + message: expect.stringContaining("timeout after 60000ms"), + }), + }); + }); + it("bridge 真相命令应使用 5000ms 的请求超时窗口", async () => { const fetchMock = vi .fn() diff --git a/src/lib/dev-bridge/http-client.ts b/src/lib/dev-bridge/http-client.ts index 54d00ea80..66c7a3ade 100644 --- a/src/lib/dev-bridge/http-client.ts +++ b/src/lib/dev-bridge/http-client.ts @@ -29,6 +29,10 @@ const DEV_BRIDGE_PROVIDER_PROBE_COMMANDS = new Set([ "test_api_key_provider_chat", ]); +const DEV_BRIDGE_AGENT_LONG_RUNNING_COMMANDS = new Set([ + "agent_generate_title", +]); + export interface InvokeRequest { cmd: string; args?: unknown; @@ -56,7 +60,10 @@ let bridgeConnectionBackoffUntil = 0; let bridgeHealthProbePromise: Promise | null = null; function resolveBridgeRequestTimeoutMs(cmd: string): number { - if (cmd.startsWith("agent_runtime_")) { + if ( + cmd.startsWith("agent_runtime_") || + DEV_BRIDGE_AGENT_LONG_RUNNING_COMMANDS.has(cmd) + ) { return DEV_BRIDGE_AGENT_RUNTIME_TIMEOUT_MS; } if (DEV_BRIDGE_PROVIDER_PROBE_COMMANDS.has(cmd)) { diff --git a/src/lib/dev-bridge/mockPriorityCommands.test.ts b/src/lib/dev-bridge/mockPriorityCommands.test.ts index 6145a5634..5f58df7cb 100644 --- a/src/lib/dev-bridge/mockPriorityCommands.test.ts +++ b/src/lib/dev-bridge/mockPriorityCommands.test.ts @@ -78,6 +78,19 @@ describe("mockPriorityCommands", () => { expect(shouldDisallowMockFallbackInBrowser("get_provider_ui_state")).toBe( true, ); + expect(shouldDisallowMockFallbackInBrowser("get_api_key_provider")).toBe( + true, + ); + expect( + shouldDisallowMockFallbackInBrowser("add_custom_api_key_provider"), + ).toBe(true); + expect(shouldDisallowMockFallbackInBrowser("update_api_key_provider")).toBe( + true, + ); + expect( + shouldDisallowMockFallbackInBrowser("delete_custom_api_key_provider"), + ).toBe(true); + expect(shouldDisallowMockFallbackInBrowser("add_api_key")).toBe(true); expect(shouldPreferMockInBrowser("session_files_save_file")).toBe(false); expect(shouldDisallowMockFallbackInBrowser("session_files_save_file")).toBe( true, diff --git a/src/lib/dev-bridge/mockPriorityCommands.ts b/src/lib/dev-bridge/mockPriorityCommands.ts index ae37dd2c1..34d95375f 100644 --- a/src/lib/dev-bridge/mockPriorityCommands.ts +++ b/src/lib/dev-bridge/mockPriorityCommands.ts @@ -74,6 +74,12 @@ const mockPriorityCommands = new Set([ "launch_browser_session", "launch_browser_profile_runtime_assist_cmd", "get_browser_action_audit_logs", + "voice_models_list_catalog", + "voice_models_get_install_state", + "voice_models_download", + "voice_models_delete", + "voice_models_set_default", + "voice_models_test_transcribe_file", ]); /** @@ -127,6 +133,11 @@ const bridgeTruthCommands = new Set([ "get_default_provider", "get_provider_ui_state", "get_api_key_providers", + "get_api_key_provider", + "add_custom_api_key_provider", + "update_api_key_provider", + "delete_custom_api_key_provider", + "add_api_key", "get_model_registry", "get_model_registry_provider_ids", "get_models_for_provider", @@ -137,6 +148,7 @@ const bridgeTruthCommands = new Set([ "fetch_provider_models_auto", "create_image_generation_task_artifact", "create_audio_generation_task_artifact", + "complete_audio_generation_task_artifact", "get_media_task_artifact", "list_media_task_artifacts", "cancel_media_task_artifact", diff --git a/src/lib/governance/agentCommandCatalog.json b/src/lib/governance/agentCommandCatalog.json index 4c067e9f8..d3155a7e1 100644 --- a/src/lib/governance/agentCommandCatalog.json +++ b/src/lib/governance/agentCommandCatalog.json @@ -62,6 +62,7 @@ "site_save_adapter_result", "create_image_generation_task_artifact", "create_audio_generation_task_artifact", + "complete_audio_generation_task_artifact", "get_media_task_artifact", "list_media_task_artifacts", "cancel_media_task_artifact" diff --git a/src/lib/governance/modalityArtifactGraph.json b/src/lib/governance/modalityArtifactGraph.json new file mode 100644 index 000000000..17255bfb1 --- /dev/null +++ b/src/lib/governance/modalityArtifactGraph.json @@ -0,0 +1,300 @@ +{ + "version": 1, + "status": "current", + "owner": "docs/roadmap/warp/artifact-graph.md", + "artifact_kinds": [ + { + "kind": "image_task", + "lifecycle": "current", + "modality": "image", + "implementation_status": "current", + "truth_sources": ["image_task_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["image_workbench"], + "evidence_events": [ + "runtime_identity_bound", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "created_at", + "updated_at" + ], + "current_contracts": ["image_generation"], + "notes": "图片生成任务的主事实源，承载 routing、payload、进度和输出关联。" + }, + { + "kind": "image_output", + "lifecycle": "current", + "modality": "image", + "implementation_status": "current", + "truth_sources": ["image_task_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["image_workbench"], + "evidence_events": ["executor_invoked", "artifact_written"], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "created_at", + "updated_at" + ], + "current_contracts": ["image_generation"], + "notes": "图片生成结果，不应自动降级成普通文件卡。" + }, + { + "kind": "audio_task", + "lifecycle": "current", + "modality": "audio", + "implementation_status": "current", + "truth_sources": ["audio_task_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["audio_player"], + "evidence_events": [ + "runtime_identity_bound", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "audio_output_status", + "created_at", + "updated_at" + ], + "current_contracts": ["voice_generation"], + "notes": "标准 audio_generate task file，voice_generation 的 primary source。" + }, + { + "kind": "audio_output", + "lifecycle": "current", + "modality": "audio", + "implementation_status": "current", + "truth_sources": ["audio_task_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["audio_player"], + "evidence_events": ["executor_invoked", "artifact_written"], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "audio_output_status", + "audio_output_path", + "created_at", + "updated_at" + ], + "current_contracts": ["voice_generation"], + "notes": "音频完成/失败输出，必须保留 mime、duration、provider、error code 等事实。" + }, + { + "kind": "transcript", + "lifecycle": "current", + "modality": "audio", + "implementation_status": "partial", + "truth_sources": ["transcript_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["transcript_viewer", "document_viewer"], + "evidence_events": [ + "runtime_identity_bound", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "transcript_status", + "transcript_path", + "transcript_source_path", + "transcript_source_url", + "transcript_language", + "transcript_output_format", + "transcript_error_code", + "created_at", + "updated_at" + ], + "current_contracts": ["audio_transcription"], + "notes": "audio_transcription 的 transcript 领域产物；当前已由 transcription_generate task 写入 transcript.pending，并由 lime-transcription-worker 回写 transcript.completed/failed，再通过媒体任务索引恢复到聊天任务卡和 .lime/runtime/transcription-generate 运行时文档；运行时文档会读取 transcript_path 文本内容供复制校对，并从 JSON/SRT/VTT 解析时间轴与说话人段落表；保存校对稿时只写回同一 ArtifactDocument 新版本与 transcriptCorrection metadata，不改写原始 ASR 输出，并在 viewer 中展示校对稿状态与 transcriptCorrectionDiffSummary；Evidence transcriptIndex 与 Replay 检查继续消费同一事实源，不能由 generic_file 冒充。" + }, + { + "kind": "browser_session", + "lifecycle": "current", + "modality": "browser", + "implementation_status": "partial", + "truth_sources": ["browser_action_trace", "runtime_timeline_event"], + "viewer_surfaces": ["browser_replay_viewer"], + "evidence_events": [ + "browser_action_requested", + "browser_observation_recorded", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "browser_session_id", + "action_count", + "last_url", + "created_at", + "updated_at" + ], + "current_contracts": ["browser_control"], + "notes": "浏览器会话/动作 trace 已进入 evidence/replay，并由 snapshotIndex.browserActionIndex 提供 session/action/URL 可查询摘要；权限 profile 与完整 viewer 仍需继续补齐。" + }, + { + "kind": "browser_snapshot", + "lifecycle": "current", + "modality": "browser", + "implementation_status": "partial", + "truth_sources": ["browser_action_trace", "runtime_timeline_event"], + "viewer_surfaces": ["browser_replay_viewer"], + "evidence_events": [ + "browser_action_requested", + "browser_observation_recorded", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "browser_session_id", + "observation_count", + "screenshot_count", + "last_url", + "created_at", + "updated_at" + ], + "current_contracts": ["browser_control"], + "notes": "截图/DOM/URL/network observation 的领域产物，不等同 image_output；当前 evidence 索引已按 browser_snapshot 记录 observation/screenshot 摘要。" + }, + { + "kind": "pdf_extract", + "lifecycle": "current", + "modality": "document", + "implementation_status": "partial", + "truth_sources": ["pdf_extract_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["document_viewer"], + "evidence_events": [ + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "source_path", + "created_at", + "updated_at" + ], + "current_contracts": ["pdf_extract"], + "notes": "PDF 抽取结果需要页码、引用和来源恢复；当前 evidence 已接入，专属 artifact/viewer 仍未完全闭环。" + }, + { + "kind": "report_document", + "lifecycle": "current", + "modality": "document", + "implementation_status": "partial", + "truth_sources": [ + "report_document_artifact", + "research_timeline_event", + "runtime_timeline_event" + ], + "viewer_surfaces": ["document_viewer", "report_viewer"], + "evidence_events": [ + "web_search_performed", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "source_count", + "created_at", + "updated_at" + ], + "current_contracts": ["pdf_extract", "web_research", "text_transform"], + "notes": "报告/分析/总结类文档产物，不能只留聊天总结。" + }, + { + "kind": "presentation_document", + "lifecycle": "current", + "modality": "document", + "implementation_status": "planned", + "truth_sources": ["presentation_artifact", "runtime_timeline_event"], + "viewer_surfaces": ["presentation_viewer", "document_viewer"], + "evidence_events": [ + "runtime_identity_bound", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "created_at", + "updated_at" + ], + "current_contracts": [], + "notes": "为 presentation_generate contract 预留；当前不应被 report_document 或 generic_file 偷换。" + }, + { + "kind": "webpage_artifact", + "lifecycle": "current", + "modality": "document", + "implementation_status": "planned", + "truth_sources": [ + "webpage_artifact", + "research_timeline_event", + "runtime_timeline_event" + ], + "viewer_surfaces": ["webpage_viewer", "document_viewer"], + "evidence_events": [ + "web_search_performed", + "executor_invoked", + "artifact_written" + ], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "url", + "created_at", + "updated_at" + ], + "current_contracts": ["web_research"], + "notes": "网页生成或网页读取产物，后续需要从 report_document 分离 viewer mapping。" + }, + { + "kind": "generic_file", + "lifecycle": "compat", + "modality": "mixed", + "implementation_status": "current", + "truth_sources": ["runtime_timeline_event", "report_document_artifact"], + "viewer_surfaces": ["generic_file_viewer", "document_viewer"], + "evidence_events": ["executor_invoked", "artifact_written"], + "task_index_fields": [ + "task_id", + "contract_key", + "artifact_kind", + "status", + "path", + "created_at", + "updated_at" + ], + "current_contracts": ["text_transform"], + "notes": "只允许作为兜底；新增多模态主结果不得默认落到 generic_file。" + } + ] +} diff --git a/src/lib/governance/modalityExecutionProfiles.json b/src/lib/governance/modalityExecutionProfiles.json new file mode 100644 index 000000000..a54b1a5e4 --- /dev/null +++ b/src/lib/governance/modalityExecutionProfiles.json @@ -0,0 +1,516 @@ +{ + "version": 1, + "status": "current", + "owner": "docs/roadmap/warp/execution-profile.md", + "profiles": [ + { + "profile_key": "image_generation_profile", + "lifecycle": "current", + "supported_contracts": ["image_generation"], + "model_role_slots": ["image_generation_model"], + "permission_profile_keys": [ + "write_artifacts", + "media_upload", + "ask_user_question" + ], + "executor_adapter_keys": ["skill:image_generate"], + "artifact_policy": { + "write_mode": "domain_task_artifact", + "artifact_kinds": ["image_task", "image_output"], + "viewer_surfaces": ["image_workbench"] + }, + "limecore_policy_refs": [ + "model_catalog", + "provider_offer", + "tenant_feature_flags" + ], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "ask_user_on_permission_denied", + "surface_capability_gap", + "do_not_fallback_to_legacy_cli" + ], + "evidence_events": [ + "runtime_identity_bound", + "routing_decision_made", + "executor_invoked", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "task_id", + "contract_key", + "routing_slot", + "executor_kind", + "artifact_kind" + ], + "notes": "图片生成 profile 只允许写 image_task/image_output，入口不得直接写 artifact 或退回 legacy CLI。" + }, + { + "profile_key": "browser_control_profile", + "lifecycle": "current", + "supported_contracts": ["browser_control"], + "model_role_slots": ["browser_reasoning_model"], + "permission_profile_keys": [ + "browser_control", + "web_search", + "ask_user_question" + ], + "executor_adapter_keys": ["browser:browser_assist"], + "artifact_policy": { + "write_mode": "runtime_observation_trace", + "artifact_kinds": ["browser_session", "browser_snapshot"], + "viewer_surfaces": ["browser_replay_viewer"] + }, + "limecore_policy_refs": ["tenant_feature_flags", "gateway_policy"], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "ask_user_on_browser_permission_denied", + "surface_observation_unavailable", + "do_not_fallback_to_web_search" + ], + "evidence_events": [ + "runtime_identity_bound", + "execution_profile_decision", + "browser_action_requested", + "browser_observation_recorded", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "contract_key", + "browser_session_id", + "action_count", + "last_url", + "observation_count" + ], + "notes": "浏览器 profile 必须保留 typed action 与 observation trace，不能把浏览器需求改写成普通搜索。" + }, + { + "profile_key": "pdf_extract_profile", + "lifecycle": "current", + "supported_contracts": ["pdf_extract"], + "model_role_slots": ["base_model"], + "permission_profile_keys": [ + "read_files", + "write_artifacts", + "ask_user_question" + ], + "executor_adapter_keys": ["skill:pdf_read"], + "artifact_policy": { + "write_mode": "domain_document_artifact", + "artifact_kinds": ["pdf_extract", "report_document"], + "viewer_surfaces": ["document_viewer"] + }, + "limecore_policy_refs": ["tenant_feature_flags"], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "ask_user_on_file_permission_denied", + "surface_file_unavailable", + "do_not_fallback_to_frontend_pdf_parse" + ], + "evidence_events": [ + "runtime_identity_bound", + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "task_id", + "contract_key", + "source_path", + "artifact_kind" + ], + "notes": "PDF profile 固定走 Skill(pdf_read) 与文件读取证据，页码/引用 viewer 仍在后续补齐。" + }, + { + "profile_key": "voice_generation_profile", + "lifecycle": "current", + "supported_contracts": ["voice_generation"], + "model_role_slots": ["voice_generation_model"], + "permission_profile_keys": [ + "write_artifacts", + "media_upload", + "service_api_call", + "ask_user_question" + ], + "executor_adapter_keys": ["service_skill:voice_runtime"], + "artifact_policy": { + "write_mode": "domain_task_artifact", + "artifact_kinds": ["audio_task", "audio_output"], + "viewer_surfaces": ["audio_player"] + }, + "limecore_policy_refs": [ + "client_scenes", + "tenant_feature_flags", + "provider_offer" + ], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "surface_provider_unconfigured", + "surface_capability_gap", + "do_not_fallback_to_legacy_tts_test_command" + ], + "evidence_events": [ + "runtime_identity_bound", + "execution_profile_decision", + "executor_invoked", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "task_id", + "contract_key", + "provider_id", + "model", + "audio_output_status" + ], + "notes": "配音 profile 使用本地 ServiceSkill 与 audio_task/audio_output，不把 LimeCore 当默认执行器。" + }, + { + "profile_key": "audio_transcription_profile", + "lifecycle": "current", + "supported_contracts": ["audio_transcription"], + "model_role_slots": ["audio_transcription_model"], + "permission_profile_keys": [ + "read_files", + "write_artifacts", + "media_upload", + "service_api_call", + "ask_user_question" + ], + "executor_adapter_keys": ["skill:transcription_generate"], + "artifact_policy": { + "write_mode": "domain_task_artifact", + "artifact_kinds": ["transcript"], + "viewer_surfaces": ["transcript_viewer", "document_viewer"] + }, + "limecore_policy_refs": [ + "model_catalog", + "provider_offer", + "tenant_feature_flags" + ], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "surface_provider_unconfigured", + "surface_file_unavailable", + "do_not_fallback_to_frontend_asr" + ], + "evidence_events": [ + "runtime_identity_bound", + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "task_id", + "contract_key", + "transcript_status", + "transcript_path", + "provider_id", + "model" + ], + "notes": "转写 profile 固定消费 transcription_generate task 与 transcriptIndex，不允许 frontend ASR 或 generic_file transcript 旁路。" + }, + { + "profile_key": "web_research_profile", + "lifecycle": "current", + "supported_contracts": ["web_research"], + "model_role_slots": ["report_generation_model"], + "permission_profile_keys": [ + "web_search", + "write_artifacts", + "ask_user_question" + ], + "executor_adapter_keys": ["skill:research"], + "artifact_policy": { + "write_mode": "timeline_backed_document_artifact", + "artifact_kinds": ["report_document", "webpage_artifact"], + "viewer_surfaces": ["document_viewer", "report_viewer", "webpage_viewer"] + }, + "limecore_policy_refs": [ + "gateway_policy", + "tenant_feature_flags", + "model_catalog" + ], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "surface_web_search_unavailable", + "surface_source_unavailable", + "do_not_fallback_to_model_memory_only" + ], + "evidence_events": [ + "runtime_identity_bound", + "web_search_performed", + "executor_invoked", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "contract_key", + "entry_source", + "source_count", + "artifact_kind" + ], + "notes": "联网研究 profile 覆盖搜索、深搜、站点搜索与报告型子入口，来源索引和 report viewer 仍需继续补齐。" + }, + { + "profile_key": "text_transform_profile", + "lifecycle": "current", + "supported_contracts": ["text_transform"], + "model_role_slots": ["base_model"], + "permission_profile_keys": [ + "read_files", + "write_artifacts", + "ask_user_question" + ], + "executor_adapter_keys": ["skill:text_transform"], + "artifact_policy": { + "write_mode": "document_or_compat_file_artifact", + "artifact_kinds": ["report_document", "generic_file"], + "viewer_surfaces": ["document_viewer", "generic_file_viewer"] + }, + "limecore_policy_refs": ["tenant_feature_flags", "model_catalog"], + "user_lock_policy": "honor_explicit_model_lock_with_capability_check", + "fallback_behavior": [ + "surface_file_unavailable", + "surface_capability_gap", + "keep_generic_file_as_compat_only" + ], + "evidence_events": [ + "runtime_identity_bound", + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "audit_fields": [ + "session_id", + "thread_id", + "turn_id", + "contract_key", + "entry_source", + "artifact_kind" + ], + "notes": "文本转换 profile 暂允许 generic_file 作为 compat fallback，但新增文档主结果必须继续向 report_document/document_viewer 收敛。" + } + ], + "executor_adapters": [ + { + "adapter_key": "skill:image_generate", + "lifecycle": "current", + "executor_kind": "skill", + "binding_key": "image_generate", + "supported_contracts": ["image_generation"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": false, + "supports_artifact": true, + "artifact_output_kinds": ["image_task", "image_output"], + "permission_requirements": [ + "write_artifacts", + "media_upload", + "ask_user_question" + ], + "credential_requirements": ["provider_offer", "model_catalog"], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "executor_error" + ], + "evidence_events": ["executor_invoked", "artifact_written"], + "notes": "Image Skill 负责写标准 image task/output artifact，不支持 resume。" + }, + { + "adapter_key": "browser:browser_assist", + "lifecycle": "current", + "executor_kind": "browser", + "binding_key": "browser_assist", + "supported_contracts": ["browser_control"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": true, + "supports_artifact": true, + "artifact_output_kinds": ["browser_session", "browser_snapshot"], + "permission_requirements": [ + "browser_control", + "web_search", + "ask_user_question" + ], + "credential_requirements": [], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "executor_error", + "observation_unavailable" + ], + "evidence_events": [ + "browser_action_requested", + "browser_observation_recorded", + "artifact_written" + ], + "notes": "Browser Assist 是 typed action executor，必须产出 observation，不允许降级成 WebSearch。" + }, + { + "adapter_key": "skill:pdf_read", + "lifecycle": "current", + "executor_kind": "skill", + "binding_key": "pdf_read", + "supported_contracts": ["pdf_extract"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": false, + "supports_artifact": true, + "artifact_output_kinds": ["pdf_extract", "report_document"], + "permission_requirements": [ + "read_files", + "write_artifacts", + "ask_user_question" + ], + "credential_requirements": [], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "file_unavailable", + "executor_error" + ], + "evidence_events": [ + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "notes": "PDF Skill 保留真实文件读取证据，页码/引用 viewer 后续继续补。" + }, + { + "adapter_key": "service_skill:voice_runtime", + "lifecycle": "current", + "executor_kind": "service_skill", + "binding_key": "voice_runtime", + "supported_contracts": ["voice_generation"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": false, + "supports_artifact": true, + "artifact_output_kinds": ["audio_task", "audio_output"], + "permission_requirements": [ + "write_artifacts", + "media_upload", + "service_api_call", + "ask_user_question" + ], + "credential_requirements": ["provider_offer"], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "executor_error" + ], + "evidence_events": ["executor_invoked", "artifact_written"], + "notes": "Voice runtime 通过本地 ServiceSkill/worker 写 audio_task/audio_output，不默认云执行。" + }, + { + "adapter_key": "skill:transcription_generate", + "lifecycle": "current", + "executor_kind": "skill", + "binding_key": "transcription_generate", + "supported_contracts": ["audio_transcription"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": false, + "supports_artifact": true, + "artifact_output_kinds": ["transcript"], + "permission_requirements": [ + "read_files", + "write_artifacts", + "media_upload", + "service_api_call", + "ask_user_question" + ], + "credential_requirements": ["provider_offer", "model_catalog"], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "file_unavailable", + "executor_error" + ], + "evidence_events": [ + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "notes": "Transcription Skill 写 transcription_generate task 与 transcript，不允许 frontend ASR 旁路。" + }, + { + "adapter_key": "skill:research", + "lifecycle": "current", + "executor_kind": "skill", + "binding_key": "research", + "supported_contracts": ["web_research"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": true, + "supports_artifact": true, + "artifact_output_kinds": ["report_document", "webpage_artifact"], + "permission_requirements": [ + "web_search", + "write_artifacts", + "ask_user_question" + ], + "credential_requirements": ["gateway_policy", "model_catalog"], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "executor_error", + "source_unavailable" + ], + "evidence_events": [ + "web_search_performed", + "executor_invoked", + "artifact_written" + ], + "notes": "Research adapter 覆盖 research/site_search/report_generate trace，后续继续补来源索引。" + }, + { + "adapter_key": "skill:text_transform", + "lifecycle": "current", + "executor_kind": "skill", + "binding_key": "text_transform", + "supported_contracts": ["text_transform"], + "supports_progress": true, + "supports_cancel": true, + "supports_resume": false, + "supports_artifact": true, + "artifact_output_kinds": ["report_document", "generic_file"], + "permission_requirements": [ + "read_files", + "write_artifacts", + "ask_user_question" + ], + "credential_requirements": ["model_catalog"], + "failure_mapping": [ + "permission_denied", + "capability_gap", + "file_unavailable", + "executor_error" + ], + "evidence_events": [ + "file_read_authorized", + "executor_invoked", + "artifact_written" + ], + "notes": "Text transform adapter 覆盖 summary/translation/analysis，generic_file 只能作为 compat fallback。" + } + ] +} diff --git a/src/lib/governance/modalityExecutionProfiles.test.ts b/src/lib/governance/modalityExecutionProfiles.test.ts new file mode 100644 index 000000000..e53c13eea --- /dev/null +++ b/src/lib/governance/modalityExecutionProfiles.test.ts @@ -0,0 +1,119 @@ +import { describe, expect, it } from "vitest"; +import { + resolveBrowserControlRuntimeContractBinding, + resolveImageGenerationRuntimeContractBinding, + resolveTextTransformRuntimeContractBinding, +} from "./modalityRuntimeContracts"; +import { + resolveExecutorAdapterKey, + resolveModalityExecutionProfileBinding, +} from "./modalityExecutionProfiles"; + +describe("modalityExecutionProfiles", () => { + it("应从 executor binding 解析标准 adapter key", () => { + expect( + resolveExecutorAdapterKey({ + executor_kind: "skill", + binding_key: "image_generate", + }), + ).toBe("skill:image_generate"); + expect(resolveExecutorAdapterKey({ executor_kind: "skill" })).toBeNull(); + }); + + it("image_generation 应解析到 profile 与 executor adapter", () => { + const binding = resolveImageGenerationRuntimeContractBinding(); + + expect(binding).toMatchObject({ + executionProfileKey: "image_generation_profile", + executorAdapterKey: "skill:image_generate", + runtimeContract: { + execution_profile: expect.objectContaining({ + profile_key: "image_generation_profile", + model_role_slots: ["image_generation_model"], + artifact_policy: expect.objectContaining({ + artifact_kinds: ["image_task", "image_output"], + viewer_surfaces: ["image_workbench"], + }), + }), + executor_adapter: expect.objectContaining({ + adapter_key: "skill:image_generate", + supports_progress: true, + supports_cancel: true, + supports_resume: false, + artifact_output_kinds: ["image_task", "image_output"], + }), + }, + }); + }); + + it("browser_control profile 应保留浏览器权限与 resume 能力", () => { + const binding = resolveBrowserControlRuntimeContractBinding(); + + expect(binding.executionProfile).toEqual( + expect.objectContaining({ + profile_key: "browser_control_profile", + permission_profile_keys: expect.arrayContaining([ + "browser_control", + "web_search", + ]), + fallback_behavior: expect.arrayContaining([ + "do_not_fallback_to_web_search", + ]), + }), + ); + expect(binding.executorAdapter).toEqual( + expect.objectContaining({ + adapter_key: "browser:browser_assist", + supports_resume: true, + artifact_output_kinds: expect.arrayContaining([ + "browser_session", + "browser_snapshot", + ]), + }), + ); + }); + + it("text_transform 只能把 generic_file 作为 compat fallback", () => { + const binding = resolveTextTransformRuntimeContractBinding(); + + expect(binding.executionProfile).toEqual( + expect.objectContaining({ + profile_key: "text_transform_profile", + artifact_policy: expect.objectContaining({ + artifact_kinds: ["report_document", "generic_file"], + }), + fallback_behavior: expect.arrayContaining([ + "keep_generic_file_as_compat_only", + ]), + }), + ); + }); + + it("直接 resolver 应按 contract key 返回同一份 profile 绑定", () => { + const binding = resolveModalityExecutionProfileBinding({ + contractKey: "audio_transcription", + executorBinding: { + executor_kind: "skill", + binding_key: "transcription_generate", + }, + }); + + expect(binding).toEqual( + expect.objectContaining({ + profileKey: "audio_transcription_profile", + executorAdapterKey: "skill:transcription_generate", + executionProfile: expect.objectContaining({ + model_role_slots: ["audio_transcription_model"], + permission_profile_keys: expect.arrayContaining([ + "read_files", + "write_artifacts", + "service_api_call", + ]), + }), + executorAdapter: expect.objectContaining({ + artifact_output_kinds: ["transcript"], + }), + }), + ); + }); +}); diff --git a/src/lib/governance/modalityExecutionProfiles.ts b/src/lib/governance/modalityExecutionProfiles.ts new file mode 100644 index 000000000..0b1dfecf4 --- /dev/null +++ b/src/lib/governance/modalityExecutionProfiles.ts @@ -0,0 +1,227 @@ +import profileRegistry from "./modalityExecutionProfiles.json"; + +interface ModalityExecutionProfileRecord { + profile_key?: string; + lifecycle?: string; + supported_contracts?: unknown; + model_role_slots?: unknown; + permission_profile_keys?: unknown; + executor_adapter_keys?: unknown; + artifact_policy?: unknown; + limecore_policy_refs?: unknown; + user_lock_policy?: string; + fallback_behavior?: unknown; + audit_fields?: unknown; +} + +interface ModalityExecutorAdapterRecord { + adapter_key?: string; + lifecycle?: string; + executor_kind?: string; + binding_key?: string; + supported_contracts?: unknown; + supports_progress?: unknown; + supports_cancel?: unknown; + supports_resume?: unknown; + supports_artifact?: unknown; + artifact_output_kinds?: unknown; + permission_requirements?: unknown; + credential_requirements?: unknown; + failure_mapping?: unknown; +} + +interface ModalityExecutorBindingRecord { + executor_kind?: unknown; + binding_key?: unknown; +} + +export interface ModalityArtifactPolicySnapshot { + write_mode?: string; + artifact_kinds: string[]; + viewer_surfaces: string[]; +} + +export interface ModalityExecutionProfileSnapshot { + profile_key: string; + supported_contracts: string[]; + model_role_slots: string[]; + permission_profile_keys: string[]; + executor_adapter_keys: string[]; + artifact_policy?: ModalityArtifactPolicySnapshot; + limecore_policy_refs: string[]; + user_lock_policy?: string; + fallback_behavior: string[]; + audit_fields: string[]; +} + +export interface ModalityExecutorAdapterSnapshot { + adapter_key: string; + executor_kind: string; + binding_key: string; + supported_contracts: string[]; + supports_progress: boolean; + supports_cancel: boolean; + supports_resume: boolean; + supports_artifact: boolean; + artifact_output_kinds: string[]; + permission_requirements: string[]; + credential_requirements: string[]; + failure_mapping: string[]; +} + +export interface ModalityExecutionProfileBinding { + profileKey: string; + executorAdapterKey: string | null; + executionProfile: ModalityExecutionProfileSnapshot; + executorAdapter?: ModalityExecutorAdapterSnapshot; +} + +function asRecord(value: unknown): T | null { + return value && typeof value === "object" && !Array.isArray(value) + ? (value as T) + : null; +} + +function readTrimmedString(value: unknown): string | null { + return typeof value === "string" && value.trim() ? value.trim() : null; +} + +function readStringArray(value: unknown): string[] { + return Array.isArray(value) + ? value + .map((item) => readTrimmedString(item)) + .filter((item): item is string => Boolean(item)) + : []; +} + +function readBoolean(value: unknown): boolean { + return value === true; +} + +function getProfiles(): ModalityExecutionProfileRecord[] { + return Array.isArray(profileRegistry.profiles) + ? profileRegistry.profiles + .map((profile) => asRecord(profile)) + .filter((profile): profile is ModalityExecutionProfileRecord => + Boolean(profile), + ) + : []; +} + +function getAdapters(): ModalityExecutorAdapterRecord[] { + return Array.isArray(profileRegistry.executor_adapters) + ? profileRegistry.executor_adapters + .map((adapter) => asRecord(adapter)) + .filter((adapter): adapter is ModalityExecutorAdapterRecord => + Boolean(adapter), + ) + : []; +} + +function toArtifactPolicySnapshot( + value: unknown, +): ModalityArtifactPolicySnapshot | undefined { + const policy = asRecord<{ + write_mode?: unknown; + artifact_kinds?: unknown; + viewer_surfaces?: unknown; + }>(value); + if (!policy) { + return undefined; + } + return { + write_mode: readTrimmedString(policy.write_mode) ?? undefined, + artifact_kinds: readStringArray(policy.artifact_kinds), + viewer_surfaces: readStringArray(policy.viewer_surfaces), + }; +} + +function toProfileSnapshot( + profile: ModalityExecutionProfileRecord, +): ModalityExecutionProfileSnapshot | null { + const profileKey = readTrimmedString(profile.profile_key); + if (!profileKey) { + return null; + } + return { + profile_key: profileKey, + supported_contracts: readStringArray(profile.supported_contracts), + model_role_slots: readStringArray(profile.model_role_slots), + permission_profile_keys: readStringArray(profile.permission_profile_keys), + executor_adapter_keys: readStringArray(profile.executor_adapter_keys), + artifact_policy: toArtifactPolicySnapshot(profile.artifact_policy), + limecore_policy_refs: readStringArray(profile.limecore_policy_refs), + user_lock_policy: readTrimmedString(profile.user_lock_policy) ?? undefined, + fallback_behavior: readStringArray(profile.fallback_behavior), + audit_fields: readStringArray(profile.audit_fields), + }; +} + +function toAdapterSnapshot( + adapter: ModalityExecutorAdapterRecord, +): ModalityExecutorAdapterSnapshot | null { + const adapterKey = readTrimmedString(adapter.adapter_key); + const executorKind = readTrimmedString(adapter.executor_kind); + const bindingKey = readTrimmedString(adapter.binding_key); + if (!adapterKey || !executorKind || !bindingKey) { + return null; + } + return { + adapter_key: adapterKey, + executor_kind: executorKind, + binding_key: bindingKey, + supported_contracts: readStringArray(adapter.supported_contracts), + supports_progress: readBoolean(adapter.supports_progress), + supports_cancel: readBoolean(adapter.supports_cancel), + supports_resume: readBoolean(adapter.supports_resume), + supports_artifact: readBoolean(adapter.supports_artifact), + artifact_output_kinds: readStringArray(adapter.artifact_output_kinds), + permission_requirements: readStringArray(adapter.permission_requirements), + credential_requirements: readStringArray(adapter.credential_requirements), + failure_mapping: readStringArray(adapter.failure_mapping), + }; +} + +export function resolveExecutorAdapterKey( + executorBinding: unknown, +): string | null { + const executor = asRecord(executorBinding); + const executorKind = readTrimmedString(executor?.executor_kind); + const bindingKey = readTrimmedString(executor?.binding_key); + return executorKind && bindingKey ? `${executorKind}:${bindingKey}` : null; +} + +export function resolveModalityExecutionProfileBinding(params: { + contractKey: string; + executorBinding?: unknown; +}): ModalityExecutionProfileBinding | null { + const contractKey = params.contractKey.trim(); + if (!contractKey) { + return null; + } + + const profile = getProfiles().find((candidate) => + readStringArray(candidate.supported_contracts).includes(contractKey), + ); + const executionProfile = profile ? toProfileSnapshot(profile) : null; + if (!executionProfile) { + return null; + } + + const adapterKey = + resolveExecutorAdapterKey(params.executorBinding) ?? + executionProfile.executor_adapter_keys[0] ?? + null; + const executorAdapter = adapterKey + ? getAdapters() + .map(toAdapterSnapshot) + .find((adapter) => adapter?.adapter_key === adapterKey) + : undefined; + + return { + profileKey: executionProfile.profile_key, + executorAdapterKey: adapterKey, + executionProfile, + executorAdapter: executorAdapter ?? undefined, + }; +} diff --git a/src/lib/governance/modalityRuntimeContracts.json b/src/lib/governance/modalityRuntimeContracts.json index 2d0f8e471..b8f6642de 100644 --- a/src/lib/governance/modalityRuntimeContracts.json +++ b/src/lib/governance/modalityRuntimeContracts.json @@ -253,7 +253,83 @@ ] }, "owner_surface": "browser_runtime", - "bound_entries": [] + "bound_entries": [ + { + "entry_key": "at_browser_command", + "entry_kind": "command", + "display_name": "@浏览器 / @browser / @browse", + "launch_metadata_path": "harness.browser_assist", + "entry_source": "at_browser_command", + "default_input_mapping": ["user_text", "url", "browser_state"], + "entry_visibility_policy": [ + "browser_runtime_available", + "profile_allows_browser_control" + ] + }, + { + "entry_key": "at_browser_agent_command", + "entry_kind": "command", + "display_name": "@Browser Agent", + "launch_metadata_path": "harness.browser_assist", + "entry_source": "at_browser_agent_command", + "default_input_mapping": ["user_text", "url", "browser_state"], + "entry_visibility_policy": [ + "browser_runtime_available", + "profile_allows_browser_control" + ] + }, + { + "entry_key": "at_mini_tester_command", + "entry_kind": "command", + "display_name": "@Mini Tester", + "launch_metadata_path": "harness.browser_assist", + "entry_source": "at_mini_tester_command", + "default_input_mapping": [ + "user_text", + "url", + "browser_state", + "test_goal" + ], + "entry_visibility_policy": [ + "browser_runtime_available", + "profile_allows_browser_control" + ] + }, + { + "entry_key": "at_web_scheduler_command", + "entry_kind": "command", + "display_name": "@Web Scheduler", + "launch_metadata_path": "harness.browser_assist", + "entry_source": "at_web_scheduler_command", + "default_input_mapping": [ + "user_text", + "url", + "browser_state", + "schedule_goal" + ], + "entry_visibility_policy": [ + "browser_runtime_available", + "profile_allows_browser_control" + ] + }, + { + "entry_key": "at_web_manage_command", + "entry_kind": "command", + "display_name": "@Web Manage", + "launch_metadata_path": "harness.browser_assist", + "entry_source": "at_web_manage_command", + "default_input_mapping": [ + "user_text", + "url", + "browser_state", + "operation_goal" + ], + "entry_visibility_policy": [ + "browser_runtime_available", + "profile_allows_browser_control" + ] + } + ] }, { "contract_key": "pdf_extract", @@ -422,6 +498,100 @@ } ] }, + { + "contract_key": "audio_transcription", + "lifecycle": "current", + "modality": "audio", + "runtime_identity": [ + "session_id", + "thread_id", + "turn_id", + "task_id", + "content_id", + "artifact_id" + ], + "input_context_kinds": [ + "audio_reference", + "video_reference", + "text_prompt", + "workspace_context" + ], + "required_capabilities": [ + "text_generation", + "audio_transcription" + ], + "permission_profile_keys": [ + "read_files", + "write_artifacts", + "media_upload", + "service_api_call", + "ask_user_question" + ], + "routing_slot": "audio_transcription_model", + "executor_binding": { + "executor_kind": "skill", + "binding_key": "transcription_generate", + "current_path": "Runtime contract -> Skill(transcription_generate) -> transcription task file / transcript artifact", + "supports_progress": true, + "supports_cancel": true, + "supports_resume": false, + "supports_artifact": true, + "failure_mapping": [ + "permission_denied", + "capability_gap", + "file_unavailable", + "executor_error" + ] + }, + "truth_source": ["transcript_artifact", "runtime_timeline_event"], + "artifact_kinds": ["transcript"], + "viewer_surface": ["transcript_viewer", "document_viewer"], + "evidence_events": [ + "runtime_identity_bound", + "model_routing_decision", + "executor_invoked", + "artifact_written" + ], + "limecore_policy_refs": [ + "model_catalog", + "provider_offer", + "tenant_feature_flags" + ], + "fallback_policy": [ + "ask_on_missing_audio_source", + "block_on_no_audio_transcription_model" + ], + "detour_policy": { + "allowed": ["ask_user_question"], + "denied": [ + "frontend_direct_asr", + "generic_file_transcript", + "tool_search_before_transcription_skill", + "web_search_before_transcription_skill" + ] + }, + "owner_surface": "agent_runtime", + "bound_entries": [ + { + "entry_key": "at_transcription_command", + "entry_kind": "command", + "display_name": "@转写 / @transcribe / @Audio Extractor", + "launch_metadata_path": "harness.transcription_skill_launch.transcription_task", + "entry_source": "at_transcription_command", + "default_input_mapping": [ + "user_text", + "audio_or_video_reference", + "transcript_options", + "workspace_context" + ], + "entry_visibility_policy": [ + "skill_catalog_visible", + "profile_allows_audio_transcription", + "requires_audio_or_video_source" + ] + } + ] + }, { "contract_key": "web_research", "lifecycle": "current", diff --git a/src/lib/governance/modalityRuntimeContracts.test.ts b/src/lib/governance/modalityRuntimeContracts.test.ts index 44549fd3d..a49b0f3e2 100644 --- a/src/lib/governance/modalityRuntimeContracts.test.ts +++ b/src/lib/governance/modalityRuntimeContracts.test.ts @@ -1,7 +1,9 @@ import { describe, expect, it } from "vitest"; import { resolveBrowserControlRuntimeContractBinding, + resolveBrowserControlEntrySource, isImageGenerationBoundEntrySource, + resolveAudioTranscriptionRuntimeContractBinding, resolveImageGenerationRuntimeContractBinding, resolvePdfExtractRuntimeContractBinding, resolveTextTransformRuntimeContractBinding, @@ -63,7 +65,28 @@ describe("modalityRuntimeContracts", () => { "browser_control_planning", ]), ); - expect(contract.boundEntrySources).toEqual([]); + expect(contract.boundEntrySources).toEqual( + expect.arrayContaining([ + "at_browser_command", + "at_browser_agent_command", + "at_mini_tester_command", + ]), + ); + }); + + it("browser_control entry source 应从 contract registry 派生", () => { + expect(resolveBrowserControlEntrySource("@浏览器")).toBe( + "at_browser_command", + ); + expect(resolveBrowserControlEntrySource("@Browser Agent")).toBe( + "at_browser_agent_command", + ); + expect(resolveBrowserControlEntrySource("@Mini Tester")).toBe( + "at_mini_tester_command", + ); + expect(resolveBrowserControlEntrySource("@unknown")).toBe( + "at_browser_command", + ); }); it("pdf_extract contract 应提供 Skill(pdf_read) 底层运行字段与上层入口绑定", () => { @@ -118,6 +141,30 @@ describe("modalityRuntimeContracts", () => { ); }); + it("audio_transcription contract 应提供 Skill(transcription_generate) 底层运行字段与上层入口绑定", () => { + const contract = resolveAudioTranscriptionRuntimeContractBinding(); + + expect(contract).toMatchObject({ + contractKey: "audio_transcription", + modality: "audio", + routingSlot: "audio_transcription_model", + runtimeContract: expect.objectContaining({ + contract_key: "audio_transcription", + routing_slot: "audio_transcription_model", + executor_binding: expect.objectContaining({ + executor_kind: "skill", + binding_key: "transcription_generate", + }), + }), + }); + expect(contract.requiredCapabilities).toEqual( + expect.arrayContaining(["text_generation", "audio_transcription"]), + ); + expect(contract.boundEntrySources).toEqual( + expect.arrayContaining(["at_transcription_command"]), + ); + }); + it("web_research contract 应提供 Skill(research) 底层运行字段与上层入口绑定", () => { const contract = resolveWebResearchRuntimeContractBinding(); diff --git a/src/lib/governance/modalityRuntimeContracts.ts b/src/lib/governance/modalityRuntimeContracts.ts index 25a71b843..56053f545 100644 --- a/src/lib/governance/modalityRuntimeContracts.ts +++ b/src/lib/governance/modalityRuntimeContracts.ts @@ -1,4 +1,9 @@ import registry from "./modalityRuntimeContracts.json"; +import { + resolveModalityExecutionProfileBinding, + type ModalityExecutionProfileSnapshot, + type ModalityExecutorAdapterSnapshot, +} from "./modalityExecutionProfiles"; export const IMAGE_GENERATION_CONTRACT_KEY = "image_generation"; export const IMAGE_GENERATION_DEFAULT_ENTRY_SOURCE = "at_image_command"; @@ -8,6 +13,9 @@ export const PDF_EXTRACT_CONTRACT_KEY = "pdf_extract"; export const PDF_EXTRACT_DEFAULT_ENTRY_SOURCE = "at_pdf_read_command"; export const VOICE_GENERATION_CONTRACT_KEY = "voice_generation"; export const VOICE_GENERATION_DEFAULT_ENTRY_SOURCE = "at_voice_command"; +export const AUDIO_TRANSCRIPTION_CONTRACT_KEY = "audio_transcription"; +export const AUDIO_TRANSCRIPTION_DEFAULT_ENTRY_SOURCE = + "at_transcription_command"; export const WEB_RESEARCH_CONTRACT_KEY = "web_research"; export const WEB_RESEARCH_DEFAULT_ENTRY_SOURCE = "at_search_command"; export const TEXT_TRANSFORM_CONTRACT_KEY = "text_transform"; @@ -40,6 +48,12 @@ const FALLBACK_VOICE_GENERATION_REQUIRED_CAPABILITIES = [ ] as const; const FALLBACK_VOICE_GENERATION_ROUTING_SLOT = "voice_generation_model"; const FALLBACK_VOICE_GENERATION_MODALITY = "audio"; +const FALLBACK_AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES = [ + "text_generation", + "audio_transcription", +] as const; +const FALLBACK_AUDIO_TRANSCRIPTION_ROUTING_SLOT = "audio_transcription_model"; +const FALLBACK_AUDIO_TRANSCRIPTION_MODALITY = "audio"; const FALLBACK_WEB_RESEARCH_REQUIRED_CAPABILITIES = [ "text_generation", "web_search", @@ -55,6 +69,15 @@ const FALLBACK_TEXT_TRANSFORM_REQUIRED_CAPABILITIES = [ ] as const; const FALLBACK_TEXT_TRANSFORM_ROUTING_SLOT = "base_model"; const FALLBACK_TEXT_TRANSFORM_MODALITY = "document"; +const BROWSER_CONTROL_ENTRY_SOURCE_BY_TRIGGER: Record = { + "@浏览器": BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE, + "@browser": BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE, + "@browse": BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE, + "@browser agent": "at_browser_agent_command", + "@mini tester": "at_mini_tester_command", + "@web scheduler": "at_web_scheduler_command", + "@web manage": "at_web_manage_command", +}; interface ModalityRuntimeContractEntryBinding { entry_key?: string; @@ -80,12 +103,18 @@ export interface ModalityRuntimeContractBinding { modality: string; requiredCapabilities: string[]; routingSlot: string; + executionProfileKey?: string; + executorAdapterKey?: string; + executionProfile?: ModalityExecutionProfileSnapshot; + executorAdapter?: ModalityExecutorAdapterSnapshot; runtimeContract: { contract_key: string; modality: string; routing_slot: string; required_capabilities: string[]; executor_binding?: unknown; + execution_profile?: ModalityExecutionProfileSnapshot; + executor_adapter?: ModalityExecutorAdapterSnapshot; }; boundEntrySources: string[]; } @@ -97,6 +126,8 @@ export type BrowserControlRuntimeContractBinding = export type PdfExtractRuntimeContractBinding = ModalityRuntimeContractBinding; export type VoiceGenerationRuntimeContractBinding = ModalityRuntimeContractBinding; +export type AudioTranscriptionRuntimeContractBinding = + ModalityRuntimeContractBinding; export type WebResearchRuntimeContractBinding = ModalityRuntimeContractBinding; export type TextTransformRuntimeContractBinding = ModalityRuntimeContractBinding; @@ -152,6 +183,10 @@ export function resolveModalityRuntimeContractBinding(params: { : [...params.fallbackRequiredCapabilities]; const routingSlot = readTrimmedString(contract?.routing_slot) ?? params.fallbackRoutingSlot; + const profileBinding = resolveModalityExecutionProfileBinding({ + contractKey, + executorBinding: contract?.executor_binding, + }); const boundEntrySources = Array.from( new Set( (contract?.bound_entries ?? []) @@ -165,12 +200,18 @@ export function resolveModalityRuntimeContractBinding(params: { modality, requiredCapabilities, routingSlot, + executionProfileKey: profileBinding?.profileKey, + executorAdapterKey: profileBinding?.executorAdapterKey ?? undefined, + executionProfile: profileBinding?.executionProfile, + executorAdapter: profileBinding?.executorAdapter, runtimeContract: { contract_key: contractKey, modality, routing_slot: routingSlot, required_capabilities: requiredCapabilities, executor_binding: contract?.executor_binding, + execution_profile: profileBinding?.executionProfile, + executor_adapter: profileBinding?.executorAdapter, }, boundEntrySources, }; @@ -196,6 +237,20 @@ export function resolveBrowserControlRuntimeContractBinding(): BrowserControlRun }); } +export function resolveBrowserControlEntrySource( + trigger: string | null | undefined, +): string { + const normalizedTrigger = trigger?.trim().toLowerCase() || ""; + const candidate = + BROWSER_CONTROL_ENTRY_SOURCE_BY_TRIGGER[normalizedTrigger] || + BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE; + const boundEntrySources = + resolveBrowserControlRuntimeContractBinding().boundEntrySources; + return boundEntrySources.includes(candidate) + ? candidate + : BROWSER_CONTROL_DEFAULT_ENTRY_SOURCE; +} + export function resolvePdfExtractRuntimeContractBinding(): PdfExtractRuntimeContractBinding { return resolveModalityRuntimeContractBinding({ contractKey: PDF_EXTRACT_CONTRACT_KEY, @@ -215,6 +270,16 @@ export function resolveVoiceGenerationRuntimeContractBinding(): VoiceGenerationR }); } +export function resolveAudioTranscriptionRuntimeContractBinding(): AudioTranscriptionRuntimeContractBinding { + return resolveModalityRuntimeContractBinding({ + contractKey: AUDIO_TRANSCRIPTION_CONTRACT_KEY, + fallbackModality: FALLBACK_AUDIO_TRANSCRIPTION_MODALITY, + fallbackRequiredCapabilities: + FALLBACK_AUDIO_TRANSCRIPTION_REQUIRED_CAPABILITIES, + fallbackRoutingSlot: FALLBACK_AUDIO_TRANSCRIPTION_ROUTING_SLOT, + }); +} + export function resolveWebResearchRuntimeContractBinding(): WebResearchRuntimeContractBinding { return resolveModalityRuntimeContractBinding({ contractKey: WEB_RESEARCH_CONTRACT_KEY, diff --git a/src/lib/model/limeHubLocalDevModels.ts b/src/lib/model/limeHubLocalDevModels.ts index 44cd5ad48..78800c6a2 100644 --- a/src/lib/model/limeHubLocalDevModels.ts +++ b/src/lib/model/limeHubLocalDevModels.ts @@ -10,18 +10,7 @@ import { } from "@/lib/model/inferModelCapabilities"; import type { EnhancedModelMetadata } from "@/lib/types/modelRegistry"; -export const LIME_HUB_LOCAL_DEV_MODEL_IDS = [ - "gpt-5.5", - "gpt-5.4", - "claude-sonnet-4-5", - "gemini-2.5-pro", - "kimi-coding-plan", - "glm-coding-plan", - "minimax-coding-plan", - "mimo-coding-plan", - "deepseek-coding-plan", - "deepseek-v4-flash", -]; +export const LIME_HUB_LOCAL_DEV_MODEL_IDS = ["gpt-5.5", "gpt-5.4"]; interface LimeHubModelProviderInput { key: string; diff --git a/src/lib/tauri-mock/core.test.ts b/src/lib/tauri-mock/core.test.ts index 16b0d0e22..625aa0718 100644 --- a/src/lib/tauri-mock/core.test.ts +++ b/src/lib/tauri-mock/core.test.ts @@ -700,16 +700,53 @@ describe("tauri-mock/core invoke", () => { total: 1, modality_runtime_contracts: expect.objectContaining({ contract_keys: ["voice_generation"], + audio_output_count: 1, + audio_output_statuses: [{ status: "pending", count: 1 }], snapshots: expect.arrayContaining([ expect.objectContaining({ task_type: "audio_generate", contract_key: "voice_generation", routing_event: "executor_invoked", + audio_output_status: "pending", }), ]), }), }), ); + + await expect( + invoke("complete_audio_generation_task_artifact", { + request: { + projectRootPath: "/mock/workspace", + taskRef: "task-audio-mock-1", + audioPath: ".lime/runtime/audio/task-audio-mock-1.mp3", + mimeType: "audio/mpeg", + durationMs: 1800, + providerId: "limecore", + model: "voice-pro", + }, + }), + ).resolves.toEqual( + expect.objectContaining({ + task_type: "audio_generate", + task_family: "audio", + normalized_status: "succeeded", + record: expect.objectContaining({ + payload: expect.objectContaining({ + audio_path: ".lime/runtime/audio/task-audio-mock-1.mp3", + audio_output: expect.objectContaining({ + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-mock-1.mp3", + duration_ms: 1800, + }), + }), + result: expect.objectContaining({ + status: "completed", + audio_path: ".lime/runtime/audio/task-audio-mock-1.mp3", + }), + }), + }), + ); }); it("OpenClaw 环境状态命令在 bridge 失败时回退默认 mock", async () => { diff --git a/src/lib/tauri-mock/core.ts b/src/lib/tauri-mock/core.ts index f24ef4176..5f114b05e 100644 --- a/src/lib/tauri-mock/core.ts +++ b/src/lib/tauri-mock/core.ts @@ -125,6 +125,10 @@ function resolveMockMediaTaskProtocol( const requestedModality = request.modality; const requestedTaskRef = typeof request.taskRef === "string" ? request.taskRef.toLowerCase() : ""; + const isTranscription = + requestedTaskType === "transcription_generate" || + requestedContractKey === "audio_transcription" || + requestedTaskRef.includes("transcription_generate"); const isAudio = requestedTaskType === "audio_generate" || requestedTaskFamily === "audio" || @@ -132,6 +136,28 @@ function resolveMockMediaTaskProtocol( requestedModality === "audio" || requestedTaskRef.includes("audio_generate"); + if (isTranscription) { + return { + taskType: "transcription_generate", + taskFamily: "audio", + defaultTaskId: "task-transcription-mock-1", + contractKey: "audio_transcription", + modality: "audio", + requiredCapabilities: ["text_generation", "audio_transcription"], + routingSlot: "audio_transcription_model", + runtimeContract: { + contract_key: "audio_transcription", + modality: "audio", + required_capabilities: ["text_generation", "audio_transcription"], + routing_slot: "audio_transcription_model", + executor_binding: { + executor_kind: "agent_tool", + binding_key: "transcription_generate", + }, + }, + }; + } + if (isAudio) { return { taskType: "audio_generate", @@ -228,19 +254,23 @@ function buildMockMediaTaskOutput( request.runtime_contract ?? protocol.runtimeContract; const payload = - protocol.taskFamily === "audio" + protocol.taskType === "transcription_generate" ? { prompt, - source_text: prompt, raw_text: request.rawText ?? request.raw_text ?? null, - voice: request.voice ?? null, - voice_style: request.voiceStyle ?? request.voice_style ?? null, - target_language: - request.targetLanguage ?? request.target_language ?? null, + source_url: request.sourceUrl ?? request.source_url ?? null, + source_path: request.sourcePath ?? request.source_path ?? null, + language: request.language ?? null, + output_format: request.outputFormat ?? request.output_format ?? "txt", + speaker_labels: + request.speakerLabels ?? request.speaker_labels ?? null, + timestamps: request.timestamps ?? null, provider_id: request.providerId ?? request.provider_id ?? null, model: request.model ?? null, entry_source: - request.entrySource ?? request.entry_source ?? "at_voice_command", + request.entrySource ?? + request.entry_source ?? + "at_transcription_command", modality_contract_key: request.modalityContractKey ?? request.modality_contract_key ?? @@ -253,40 +283,88 @@ function buildMockMediaTaskOutput( routing_slot: request.routingSlot ?? request.routing_slot ?? protocol.routingSlot, runtime_contract: runtimeContract, - audio_output: { - kind: "audio_output", - status: "pending", - audio_path: request.audioPath ?? request.audio_path ?? null, - mime_type: request.mimeType ?? request.mime_type ?? "audio/mpeg", - duration_ms: request.durationMs ?? request.duration_ms ?? null, - source_text: prompt, - voice: request.voice ?? null, + requested_target: + request.requestedTarget ?? request.requested_target ?? "transcript", + transcript: { + kind: "transcript", + status: request.transcriptStatus ?? "pending", + transcript_path: + request.transcriptPath ?? request.transcript_path ?? null, + source_url: request.sourceUrl ?? request.source_url ?? null, + source_path: request.sourcePath ?? request.source_path ?? null, + language: request.language ?? null, + output_format: + request.outputFormat ?? request.output_format ?? "txt", + provider_id: request.providerId ?? request.provider_id ?? null, + model: request.model ?? null, + modality_contract_key: protocol.contractKey, + modality: protocol.modality, + routing_slot: protocol.routingSlot, }, } - : { - prompt, - mode: request.mode ?? "generate", - size: request.size ?? "1024x1024", - count: request.count ?? 1, - provider_id: request.providerId ?? request.provider_id ?? null, - model: request.model ?? null, - modality_contract_key: - request.modalityContractKey ?? - request.modality_contract_key ?? - protocol.contractKey, - modality: request.modality ?? protocol.modality, - required_capabilities: - request.requiredCapabilities ?? - request.required_capabilities ?? - protocol.requiredCapabilities, - routing_slot: - request.routingSlot ?? request.routing_slot ?? protocol.routingSlot, - runtime_contract: runtimeContract, - model_capability_assessment: - request.modelCapabilityAssessment ?? - request.model_capability_assessment ?? - null, - }; + : protocol.taskFamily === "audio" + ? { + prompt, + source_text: prompt, + raw_text: request.rawText ?? request.raw_text ?? null, + voice: request.voice ?? null, + voice_style: request.voiceStyle ?? request.voice_style ?? null, + target_language: + request.targetLanguage ?? request.target_language ?? null, + provider_id: request.providerId ?? request.provider_id ?? null, + model: request.model ?? null, + entry_source: + request.entrySource ?? request.entry_source ?? "at_voice_command", + modality_contract_key: + request.modalityContractKey ?? + request.modality_contract_key ?? + protocol.contractKey, + modality: request.modality ?? protocol.modality, + required_capabilities: + request.requiredCapabilities ?? + request.required_capabilities ?? + protocol.requiredCapabilities, + routing_slot: + request.routingSlot ?? + request.routing_slot ?? + protocol.routingSlot, + runtime_contract: runtimeContract, + audio_output: { + kind: "audio_output", + status: "pending", + audio_path: request.audioPath ?? request.audio_path ?? null, + mime_type: request.mimeType ?? request.mime_type ?? "audio/mpeg", + duration_ms: request.durationMs ?? request.duration_ms ?? null, + source_text: prompt, + voice: request.voice ?? null, + }, + } + : { + prompt, + mode: request.mode ?? "generate", + size: request.size ?? "1024x1024", + count: request.count ?? 1, + provider_id: request.providerId ?? request.provider_id ?? null, + model: request.model ?? null, + modality_contract_key: + request.modalityContractKey ?? + request.modality_contract_key ?? + protocol.contractKey, + modality: request.modality ?? protocol.modality, + required_capabilities: + request.requiredCapabilities ?? + request.required_capabilities ?? + protocol.requiredCapabilities, + routing_slot: + request.routingSlot ?? + request.routing_slot ?? + protocol.routingSlot, + runtime_contract: runtimeContract, + model_capability_assessment: + request.modelCapabilityAssessment ?? + request.model_capability_assessment ?? + null, + }; const record = { task_id: taskId, task_type: protocol.taskType, @@ -329,6 +407,72 @@ function buildMockMediaTaskOutput( }; } +function buildMockCompletedAudioTaskOutput(args: any) { + const request = args?.request ?? args ?? {}; + const output = buildMockMediaTaskOutput(args, { + task_type: "audio_generate", + status: "succeeded", + normalized_status: "succeeded", + }); + const record = output.record as Record; + const payload = record.payload as Record; + const audioPath = + request.audioPath ?? + request.audio_path ?? + request.audioUrl ?? + request.audio_url ?? + payload.audio_path ?? + "/mock/workspace/.lime/runtime/audio/task-audio-mock-1.mp3"; + const mimeType = + request.mimeType ?? request.mime_type ?? payload.mime_type ?? "audio/mpeg"; + const durationMs = + request.durationMs ?? + request.duration_ms ?? + payload.duration_ms ?? + payload.audio_output?.duration_ms ?? + 1200; + const providerId = + request.providerId ?? request.provider_id ?? payload.provider_id ?? null; + const model = request.model ?? payload.model ?? null; + const audioOutput = { + ...(payload.audio_output ?? {}), + kind: "audio_output", + status: "completed", + audio_path: audioPath, + mime_type: mimeType, + duration_ms: durationMs, + provider_id: providerId, + model, + modality_contract_key: "voice_generation", + modality: "audio", + routing_slot: "voice_generation_model", + }; + + payload.audio_path = audioPath; + payload.mime_type = mimeType; + payload.duration_ms = durationMs; + payload.provider_id = providerId; + payload.model = model; + payload.audio_output = audioOutput; + record.result = { + kind: "audio_generation_result", + status: "completed", + audio_output: audioOutput, + outputs: [audioOutput], + audio_path: audioPath, + mime_type: mimeType, + duration_ms: durationMs, + }; + record.progress = { + phase: "succeeded", + percent: 100, + message: "音频任务已完成，audio_output 已回写。", + preview_slots: [], + }; + + return output; +} + type MockBrowserProfileRecord = { id: string; profile_key: string; @@ -5808,6 +5952,8 @@ const defaultMocks: Record = { buildMockMediaTaskOutput(args, { task_type: "audio_generate", }), + complete_audio_generation_task_artifact: (args: any) => + buildMockCompletedAudioTaskOutput(args), get_media_task_artifact: (args: any) => buildMockMediaTaskOutput(args), list_media_task_artifacts: (args: any) => { const request = args?.request ?? args ?? {}; @@ -5844,6 +5990,14 @@ const defaultMocks: Record = { | Record | null | undefined; + const audioOutput = payload?.audio_output as + | Record + | null + | undefined; + const transcript = payload?.transcript as + | Record + | null + | undefined; return { task_id: item.task_id, task_type: item.task_type, @@ -5853,7 +6007,8 @@ const defaultMocks: Record = { provider_id: payload?.provider_id ?? null, model: payload?.model ?? null, routing_event: - payload?.modality_contract_key === "voice_generation" + payload?.modality_contract_key === "voice_generation" || + payload?.modality_contract_key === "audio_transcription" ? "executor_invoked" : "model_routing_decision", routing_outcome: @@ -5864,8 +6019,56 @@ const defaultMocks: Record = { payload?.modality_contract_key === "image_generation" ? (assessment?.supports_image_generation ?? null) : null, + audio_output_status: audioOutput?.status ?? null, + audio_output_path: audioOutput?.audio_path ?? null, + audio_output_mime_type: audioOutput?.mime_type ?? null, + audio_output_duration_ms: audioOutput?.duration_ms ?? null, + audio_output_error_code: audioOutput?.error_code ?? null, + audio_output_retryable: audioOutput?.retryable ?? null, + transcript_status: transcript?.status ?? null, + transcript_path: transcript?.transcript_path ?? null, + transcript_source_url: transcript?.source_url ?? null, + transcript_source_path: transcript?.source_path ?? null, + transcript_language: transcript?.language ?? null, + transcript_output_format: transcript?.output_format ?? null, + transcript_error_code: transcript?.error_code ?? null, + transcript_retryable: transcript?.retryable ?? null, }; }); + const audioOutputStatuses = snapshots.reduce( + (items, snapshot) => { + if (!snapshot.audio_output_status) { + return items; + } + const existing = items.find( + (item) => item.status === snapshot.audio_output_status, + ); + if (existing) { + existing.count += 1; + } else { + items.push({ status: snapshot.audio_output_status, count: 1 }); + } + return items; + }, + [] as Array<{ status: string; count: number }>, + ); + const transcriptStatuses = snapshots.reduce( + (items, snapshot) => { + if (!snapshot.transcript_status) { + return items; + } + const existing = items.find( + (item) => item.status === snapshot.transcript_status, + ); + if (existing) { + existing.count += 1; + } else { + items.push({ status: snapshot.transcript_status, count: 1 }); + } + return items; + }, + [] as Array<{ status: string; count: number }>, + ); return { success: true, workspace_root: request.projectRootPath ?? "/mock/workspace", @@ -5896,6 +6099,24 @@ const defaultMocks: Record = { (item) => item.model_capability_assessment_source === "model_registry", ).length, + audio_output_count: snapshots.filter((item) => item.audio_output_status) + .length, + audio_output_statuses: audioOutputStatuses, + audio_output_error_codes: [ + ...new Set( + snapshots + .map((item) => item.audio_output_error_code) + .filter(Boolean), + ), + ], + transcript_count: snapshots.filter((item) => item.transcript_status) + .length, + transcript_statuses: transcriptStatuses, + transcript_error_codes: [ + ...new Set( + snapshots.map((item) => item.transcript_error_code).filter(Boolean), + ), + ], snapshots, }, tasks, @@ -6465,6 +6686,91 @@ const defaultMocks: Record = { language: "zh-CN", }, ], + voice_models_list_catalog: () => [ + { + id: "sensevoice-small-int8-2024-07-17", + name: "SenseVoice Small INT8", + provider: "FunAudioLLM / sherpa-onnx", + description: + "本地离线 ASR，支持中文、英文、日文、韩文和粤语；模型按需下载到用户数据目录。", + version: "2024-07-17", + languages: ["zh", "en", "ja", "ko", "yue"], + size_bytes: 262144000, + download_url: + "https://models.example.com/voice/sensevoice-small-int8-2024-07-17/sherpa-onnx-sense-voice-zh-en-ja-ko-yue-int8-2024-07-17.tar.bz2", + vad_model_id: "silero-vad-onnx", + vad_download_url: + "https://models.example.com/voice/silero-vad-onnx/silero_vad.onnx", + runtime: "sherpa-onnx", + bundled: false, + checksum_sha256: null, + }, + ], + voice_models_get_install_state: () => ({ + model_id: "sensevoice-small-int8-2024-07-17", + installed: false, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: null, + tokens_file: null, + vad_file: null, + installed_bytes: 0, + last_verified_at: Math.floor(Date.now() / 1000), + missing_files: ["model.int8.onnx", "tokens.txt", "silero_vad.onnx"], + default_credential_id: null, + }), + voice_models_download: () => ({ + state: { + model_id: "sensevoice-small-int8-2024-07-17", + installed: true, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: + "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17/model.int8.onnx", + tokens_file: + "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17/tokens.txt", + vad_file: + "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17/silero_vad.onnx", + installed_bytes: 262144000, + last_verified_at: Math.floor(Date.now() / 1000), + missing_files: [], + default_credential_id: null, + }, + }), + voice_models_delete: () => ({ + model_id: "sensevoice-small-int8-2024-07-17", + installed: false, + installing: false, + install_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + model_file: null, + tokens_file: null, + vad_file: null, + installed_bytes: 0, + last_verified_at: Math.floor(Date.now() / 1000), + missing_files: ["model.int8.onnx", "tokens.txt", "silero_vad.onnx"], + default_credential_id: null, + }), + voice_models_set_default: () => ({ + id: "sensevoice-local-sensevoice-small-int8-2024-07-17", + provider: "sensevoice_local", + name: "SenseVoice Small 本地", + is_default: true, + disabled: false, + language: "auto", + sensevoice_config: { + model_id: "sensevoice-small-int8-2024-07-17", + model_dir: "/mock/lime/models/voice/sensevoice-small-int8-2024-07-17", + use_itn: true, + num_threads: 4, + vad_model_id: "silero-vad-onnx", + }, + }), + voice_models_test_transcribe_file: () => ({ + text: "这是一段 SenseVoice Small 本地测试转写结果。", + duration_secs: 3.2, + sample_rate: 16000, + language: "auto", + }), get_voice_instructions: () => [ { id: "default", @@ -6490,6 +6796,10 @@ const defaultMocks: Record = { registered_shortcut: null, translate_shortcut_registered: false, registered_translate_shortcut: null, + fn_supported: false, + fn_registered: false, + fn_fallback_shortcut: "CommandOrControl+Shift+V", + fn_note: "Fn 按住录音当前仅支持 macOS；已使用普通语音快捷键回退。", }), save_voice_input_config: () => ({}), save_voice_instruction: () => ({}), @@ -6515,7 +6825,7 @@ const defaultMocks: Record = { // Update 相关 check_update: () => ({ - current_version: "1.24.0", + current_version: "1.25.0", latest_version: null, has_update: false, download_url: "https://github.com/limecloud/lime/releases", @@ -6526,7 +6836,7 @@ const defaultMocks: Record = { error: null, }), check_for_updates: () => ({ - current: "1.24.0", + current: "1.25.0", latest: null, hasUpdate: false, downloadUrl: "https://github.com/limecloud/lime/releases", diff --git a/src/lib/windowDrag.test.ts b/src/lib/windowDrag.test.ts new file mode 100644 index 000000000..f73583949 --- /dev/null +++ b/src/lib/windowDrag.test.ts @@ -0,0 +1,111 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { + isWindowDragInteractiveTarget, + shouldStartWindowDragFromMouseEvent, + startWindowDragFromMouseEvent, +} from "./windowDrag"; + +const { mockHasTauriInvokeCapability, mockStartDragging } = vi.hoisted(() => ({ + mockHasTauriInvokeCapability: vi.fn(() => true), + mockStartDragging: vi.fn(), +})); + +vi.mock("@/lib/tauri-runtime", () => ({ + hasTauriInvokeCapability: mockHasTauriInvokeCapability, +})); + +vi.mock("@tauri-apps/api/window", () => ({ + getCurrentWindow: () => ({ + startDragging: mockStartDragging, + }), +})); + +function buildMouseEventLike(params: { + button?: number; + currentTarget?: EventTarget | null; + defaultPrevented?: boolean; + target?: EventTarget | null; +}) { + const target = params.target ?? document.createElement("div"); + return { + button: params.button ?? 0, + currentTarget: params.currentTarget ?? target, + defaultPrevented: params.defaultPrevented ?? false, + target, + preventDefault: vi.fn(), + }; +} + +describe("windowDrag", () => { + beforeEach(() => { + mockHasTauriInvokeCapability.mockReturnValue(true); + mockStartDragging.mockReset(); + mockStartDragging.mockResolvedValue(undefined); + }); + + it("桌面环境左键点击非交互区域时应启动窗口拖拽", async () => { + const event = buildMouseEventLike({}); + + await expect( + startWindowDragFromMouseEvent(event, { source: "test" }), + ).resolves.toBe(true); + + expect(event.preventDefault).toHaveBeenCalledTimes(1); + expect(mockStartDragging).toHaveBeenCalledTimes(1); + }); + + it("非 Tauri 环境不应启动窗口拖拽", async () => { + mockHasTauriInvokeCapability.mockReturnValue(false); + const event = buildMouseEventLike({}); + + await expect(startWindowDragFromMouseEvent(event)).resolves.toBe(false); + + expect(event.preventDefault).not.toHaveBeenCalled(); + expect(mockStartDragging).not.toHaveBeenCalled(); + }); + + it("非左键点击不应启动窗口拖拽", () => { + const event = buildMouseEventLike({ button: 2 }); + + expect(shouldStartWindowDragFromMouseEvent(event)).toBe(false); + expect(mockStartDragging).not.toHaveBeenCalled(); + }); + + it("交互控件及其子元素不应启动窗口拖拽", () => { + const button = document.createElement("button"); + const label = document.createElement("span"); + button.appendChild(label); + + expect(isWindowDragInteractiveTarget(label)).toBe(true); + expect( + shouldStartWindowDragFromMouseEvent( + buildMouseEventLike({ currentTarget: document.createElement("div"), target: label }), + ), + ).toBe(false); + }); + + it("要求只允许自身命中时，子元素不应启动窗口拖拽", () => { + const container = document.createElement("div"); + const child = document.createElement("div"); + container.appendChild(child); + + expect( + shouldStartWindowDragFromMouseEvent( + buildMouseEventLike({ currentTarget: container, target: child }), + { allowDescendantTargets: false }, + ), + ).toBe(false); + }); + + it("声明 no-drag 的区域不应启动窗口拖拽", () => { + const noDragRegion = document.createElement("div"); + noDragRegion.dataset.limeNoWindowDrag = "true"; + + expect(isWindowDragInteractiveTarget(noDragRegion)).toBe(true); + expect( + shouldStartWindowDragFromMouseEvent( + buildMouseEventLike({ target: noDragRegion }), + ), + ).toBe(false); + }); +}); diff --git a/src/lib/windowDrag.ts b/src/lib/windowDrag.ts new file mode 100644 index 000000000..4f7f56a2b --- /dev/null +++ b/src/lib/windowDrag.ts @@ -0,0 +1,98 @@ +import { getCurrentWindow } from "@tauri-apps/api/window"; +import { hasTauriInvokeCapability } from "@/lib/tauri-runtime"; + +const WINDOW_DRAG_INTERACTIVE_SELECTOR = [ + "a", + "button", + "input", + "select", + "textarea", + "summary", + "[contenteditable='true']", + "[role='button']", + "[role='checkbox']", + "[role='combobox']", + "[role='link']", + "[role='menuitem']", + "[role='option']", + "[role='radio']", + "[role='slider']", + "[role='switch']", + "[tabindex]:not([tabindex='-1'])", + "[data-tauri-no-drag]", + "[data-lime-no-window-drag]", +].join(","); + +interface WindowDragMouseEventLike { + button: number; + currentTarget: EventTarget | null; + defaultPrevented?: boolean; + target: EventTarget | null; + preventDefault: () => void; +} + +interface StartWindowDragOptions { + allowDescendantTargets?: boolean; + source?: string; +} + +function resolveTargetElement(target: EventTarget | null): Element | null { + if (target instanceof Element) { + return target; + } + + if (target instanceof Node) { + return target.parentElement; + } + + return null; +} + +export function isWindowDragInteractiveTarget( + target: EventTarget | null, +): boolean { + return Boolean( + resolveTargetElement(target)?.closest(WINDOW_DRAG_INTERACTIVE_SELECTOR), + ); +} + +export function shouldStartWindowDragFromMouseEvent( + event: WindowDragMouseEventLike, + options: StartWindowDragOptions = {}, +): boolean { + if (!hasTauriInvokeCapability() || event.defaultPrevented) { + return false; + } + + if (event.button !== 0) { + return false; + } + + if ( + options.allowDescendantTargets === false && + event.target !== event.currentTarget + ) { + return false; + } + + return !isWindowDragInteractiveTarget(event.target); +} + +export async function startWindowDragFromMouseEvent( + event: WindowDragMouseEventLike, + options: StartWindowDragOptions = {}, +): Promise { + if (!shouldStartWindowDragFromMouseEvent(event, options)) { + return false; + } + + event.preventDefault(); + + try { + await getCurrentWindow().startDragging(); + return true; + } catch (error) { + console.warn(`[窗口] 启动主窗口拖拽失败: ${options.source ?? "unknown"}`, error); + return false; + } +}