From 581ecb311a460305fcf60cb899498bc3398a5027 Mon Sep 17 00:00:00 2001 From: Ma Date: Sun, 7 Jun 2026 10:17:08 +0800 Subject: [PATCH] chore(test): add studio e2e benchmark checklist --- package.json | 2 + scripts/audit-semantic-patterns.mjs | 139 ++++++++++++ scripts/studio-e2e-benchmark-cases.json | 284 ++++++++++++++++++++++++ scripts/studio-e2e-benchmark.mjs | 102 +++++++++ 4 files changed, 527 insertions(+) create mode 100644 scripts/audit-semantic-patterns.mjs create mode 100644 scripts/studio-e2e-benchmark-cases.json create mode 100644 scripts/studio-e2e-benchmark.mjs diff --git a/package.json b/package.json index e6083739..ff2d752c 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,9 @@ "test": "pnpm -r test", "lint": "pnpm -r lint", "typecheck": "pnpm -r typecheck", + "audit:semantic-patterns": "node scripts/audit-semantic-patterns.mjs", "verify:publish-manifests": "node scripts/verify-no-workspace-protocol.mjs packages/core packages/cli packages/studio", + "benchmark:studio-e2e": "node scripts/studio-e2e-benchmark.mjs", "release": "pnpm build && pnpm test" }, "engines": { diff --git a/scripts/audit-semantic-patterns.mjs b/scripts/audit-semantic-patterns.mjs new file mode 100644 index 00000000..fd51774b --- /dev/null +++ b/scripts/audit-semantic-patterns.mjs @@ -0,0 +1,139 @@ +#!/usr/bin/env node +import { readdir, readFile } from "node:fs/promises"; +import { join, relative } from "node:path"; + +const ROOT = process.cwd(); +const SCAN_DIRS = [ + "packages/core/src", + "packages/studio/src", + "packages/cli/src", +]; + +const SOURCE_EXTENSIONS = new Set([".ts", ".tsx"]); + +const ACTION_SURFACE_PATHS = [ + "packages/core/src/agent/", + "packages/core/src/interaction/", + "packages/studio/src/api/server.ts", + "packages/studio/src/pages/BookCreate.tsx", + "packages/cli/src/commands/agent.ts", + "packages/cli/src/tui/", +]; + +const SEMANTIC_HINTS = [ + "instruction", + "intent", + "requestedIntent", + "actionSource", + "free-text", + "continue", + "write next", + "create book", + "edit", + "rewrite", + "revise", + "play", + "short", + "fanfic", + "spinoff", + "imitation", + "续写", + "继续", + "建书", + "短篇", + "互动", + "番外", + "仿写", + "同人", +]; + +const PATTERN_TOKENS = [ + ".match(", + ".test(", + ".includes(", + ".startsWith(", + ".endsWith(", + "new RegExp(", +]; + +function hasAny(text, words) { + const lower = text.toLowerCase(); + return words.some((word) => lower.includes(word.toLowerCase())); +} + +function isActionSurface(path) { + const rel = path.replace(`${ROOT}/`, ""); + return ACTION_SURFACE_PATHS.some((prefix) => rel.startsWith(prefix)); +} + +function isLikelySemanticDecision(path, line, windowText) { + if (!isActionSurface(path)) return false; + if (!PATTERN_TOKENS.some((token) => line.includes(token))) return false; + if (!hasAny(`${line}\n${windowText}`, SEMANTIC_HINTS)) return false; + if (line.includes("CHAT_EDIT_TEXT_EXTENSIONS")) return false; + if (line.includes("SAFE_ROLE_TRUTH_FILE_RE")) return false; + if (line.includes("CODE_FENCE_RE") || line.includes("DIRECTIVE_CLOSE_RE")) return false; + if (line.includes("safeSessionId")) return false; + if (line.includes("genreId")) return false; + if (line.includes("relPath") || line.includes("targetPath") || line.includes("requestedPath")) return false; + if (line.includes("entry.name") || line.includes("base.includes") || line.includes("escapeRegExp")) return false; + if (line.includes("raw.includes(\"/\")") || line.includes("raw.includes(\"\\\\\")")) return false; + if (line.includes("Manual text edit requires review")) return false; + if (path.endsWith("project-tools.ts") && line.includes("includes(")) return false; + if (path.endsWith("i18n.ts")) return false; + if (line.includes("rel || rel.startsWith")) return false; + if (line.includes("trimmed.match(/^([A-Za-z_]")) return false; + if (line.includes("file.includes(\"/\")")) return false; + if (line.includes("startsWith(\"[Tool results]\")")) return false; + if (line.includes("already processing") || line.includes("prompt.*queue")) return false; + if (line.includes("trimmed.startsWith(\"#\")")) return false; + if (line.includes("actionSource") && line.includes("startsWith(\"/\")")) return false; + if (line.includes("startsWith(\"/\")")) return false; + if (line.includes("endsWith(") && !hasAny(windowText, ["instruction", "intent"])) return false; + return true; +} + +async function walk(dir) { + const out = []; + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + if (entry.isDirectory()) { + if (entry.name === "node_modules" || entry.name === "dist" || entry.name === "__tests__") continue; + out.push(...await walk(path)); + } else if (SOURCE_EXTENSIONS.has(path.slice(path.lastIndexOf(".")))) { + out.push(path); + } + } + return out; +} + +const files = []; +for (const dir of SCAN_DIRS) { + files.push(...await walk(join(ROOT, dir))); +} + +const findings = []; +for (const file of files) { + const content = await readFile(file, "utf-8"); + const lines = content.split(/\r?\n/); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + const windowText = lines.slice(Math.max(0, index - 4), Math.min(lines.length, index + 5)).join("\n"); + if (isLikelySemanticDecision(file, line, windowText)) { + findings.push({ + file: relative(ROOT, file), + line: index + 1, + text: line.trim(), + }); + } + } +} + +if (process.argv.includes("--json")) { + console.log(JSON.stringify({ findings }, null, 2)); +} else { + console.log(`Semantic/template-pattern audit candidates: ${findings.length}`); + for (const finding of findings) { + console.log(`${finding.file}:${finding.line} ${finding.text}`); + } +} diff --git a/scripts/studio-e2e-benchmark-cases.json b/scripts/studio-e2e-benchmark-cases.json new file mode 100644 index 00000000..0cda6456 --- /dev/null +++ b/scripts/studio-e2e-benchmark-cases.json @@ -0,0 +1,284 @@ +{ + "version": 1, + "name": "InkOS Studio Real-Model E2E Benchmark", + "purpose": "A stable manual/automated run sheet for true Studio UI + real LLM regression runs. Cases intentionally use user-like prompts rather than exact prompt engineering.", + "defaultModelHint": { + "text": "kkaiapi / deepseek-v4-flash or deepseek-v4-pro", + "cover": "configured Studio cover provider" + }, + "categories": [ + { + "id": "long_book", + "label": "长篇", + "entry": "Studio 创作 -> 新建书籍 -> 写第一章 -> 对话编辑角色卡", + "cases": [ + { + "id": "long-01", + "prompt": "写一本县城悬疑长篇，主角是被停职的女法医，第一人称，重点是亲情债和旧案反转。", + "acceptance": [ + "创建前出现确认卡，不直接建书", + "确认后生成 book 并可进入书籍页", + "第一章落盘且人称基本保持第一人称", + "能通过对话要求修改某张角色卡，角色卡文件发生变化" + ] + }, + { + "id": "long-02", + "prompt": "开一部长篇商战文，主角是破产厂二代，工业调查占七成，感情线三成，第一章从假账仓库开始。", + "acceptance": [ + "确认卡保留题材比例", + "第一章不脱离工厂/假账/调查主线", + "写完后审稿或状态不应把比例要求丢掉" + ] + }, + { + "id": "long-03", + "prompt": "写一本英文 progression fantasy，主角从失败药剂学徒开始，风格要干净，不要中文网文口吻。", + "acceptance": [ + "语言进入英文链路", + "章节长度按英文词数目标，不按中文字数硬套", + "生成物不混入中文流程标签" + ] + } + ] + }, + { + "id": "short", + "label": "短篇", + "entry": "Studio 创作 -> InkOS Short", + "cases": [ + { + "id": "short-01", + "prompt": "写一篇女频短篇，离婚协议、三年隐忍、财产证据反杀。", + "acceptance": [ + "确认后一次生成完整短篇", + "产出 final markdown、sales package、cover prompt", + "封面失败时正文/简介/卖点仍保留" + ] + }, + { + "id": "short-02", + "prompt": "来一个亲情吸血题材，主角发现母亲的账本不是偏心证据，而是在救她。", + "acceptance": [ + "题材不被路由成长篇", + "标题、简介和正文服务同一个核心冲突", + "章节不出现大面积空章或剧情梗概化" + ] + }, + { + "id": "short-03", + "prompt": "做一个规则怪谈短篇，场景是凌晨便利店，主角靠收银小票发现规则漏洞。", + "acceptance": [ + "规则和证据链清楚", + "正文不是纯设定说明", + "封面提示词体现手机端短篇封面风格" + ] + } + ] + }, + { + "id": "play_guided", + "label": "分支互动", + "entry": "Studio 创作 -> InkOS Play -> guided", + "cases": [ + { + "id": "guided-01", + "prompt": "分支互动，旧戏院夜巡，我是临时维修员，系统每轮给我几个可选动作。", + "acceptance": [ + "启动前出现确认卡", + "世界启动后每轮有 2-4 个可点击动作", + "HUD 不遮挡发送按钮" + ] + }, + { + "id": "guided-02", + "prompt": "做一个民国当铺互动，我是新来的账房，账本里有一页不能翻。", + "acceptance": [ + "点击建议动作能推进回合", + "关键物品进入状态或图谱", + "不会把用户自由讨论误当动作" + ] + }, + { + "id": "guided-03", + "prompt": "分支互动，太空救援船，我是通信员，船上有人伪造求救信号。", + "acceptance": [ + "题材状态词不泄漏英文模板", + "实体/关系边持续增长", + "每回合场景回应和图谱状态基本一致" + ] + } + ] + }, + { + "id": "play_open", + "label": "开放世界", + "entry": "Studio 创作 -> InkOS Play -> open", + "cases": [ + { + "id": "open-01", + "prompt": "开放世界，我在暴雨里的县医院档案室醒来，手机只有一条未发送短信。", + "acceptance": [ + "启动后允许自由输入动作", + "自由动作不会被误路由到建书/短篇", + "renderer 新出现的关键实体会被对账进状态" + ] + }, + { + "id": "open-02", + "prompt": "开放互动，古堡晚宴，我是临时钢琴师，主人要求我不要弹第三首曲子。", + "acceptance": [ + "世界规则在后续回合保持", + "玩家身份稳定", + "状态/HUD 不靠英文模板词展示" + ] + }, + { + "id": "open-03", + "prompt": "自由行动，赛博城地下诊所，我欠黑市医生一笔账，门外有人找我。", + "acceptance": [ + "资源/压力能进入状态", + "关系和证据能被图谱保存", + "长会话后不会明显丢失当前目标" + ] + } + ] + }, + { + "id": "fanfic", + "label": "同人", + "entry": "Studio 工具 -> 导入 -> 同人创作", + "cases": [ + { + "id": "fanfic-01", + "prompt": "根据一段正典材料做同人新书，写原作空白处，不要改原角色底色。", + "acceptance": [ + "入口等待后台创建，不短轮询误报失败", + "正典约束写入 fanfic_canon 或 parent_canon", + "第一章不明显 OOC" + ] + }, + { + "id": "fanfic-02", + "prompt": "做一个 AU 同人，保留人物关系张力，但换成县城医院背景。", + "acceptance": [ + "AU 与正典约束分开", + "不会把同人请求路由成普通长篇", + "创建完成后跳转到新书" + ] + }, + { + "id": "fanfic-03", + "prompt": "做 CP 向同人，重点是误解解除，不写露骨内容。", + "acceptance": [ + "禁忌要求进入书籍规则或作者意图", + "角色关系在角色卡中可见", + "后续写章能读取这些约束" + ] + } + ] + }, + { + "id": "imitation", + "label": "仿写", + "entry": "Studio 工具 -> 导入 -> 仿写", + "cases": [ + { + "id": "imitation-01", + "prompt": "参考一段县城生活质感，仿写一个新故事，不要照搬人物和情节。", + "acceptance": [ + "入口等待后台创建完成", + "风格约束写入 style_guide", + "新书不直接复制原文人物名" + ] + }, + { + "id": "imitation-02", + "prompt": "仿写冷峻克制的叙述感，做一个短章开头，不要大段心理独白。", + "acceptance": [ + "风格要求可追溯", + "生成章节能体现叙述节制", + "不混入模板化分析文字" + ] + }, + { + "id": "imitation-03", + "prompt": "仿写轻喜剧对话密度，做成都市家庭题材。", + "acceptance": [ + "对话风格进入写作约束", + "书籍创建后可继续写章", + "角色卡与风格指南不相互覆盖" + ] + } + ] + }, + { + "id": "spinoff", + "label": "番外", + "entry": "Studio 工具 -> 导入 -> 番外", + "cases": [ + { + "id": "spinoff-01", + "prompt": "给已有书做一个番外，讲配角收到一封迟到十年的信。", + "acceptance": [ + "创建时保留父书上下文", + "不污染父书主线章节", + "后台创建完成后自动跳转" + ] + }, + { + "id": "spinoff-02", + "prompt": "番外写结局后的小饭馆重逢，重点是余味，不改主线事实。", + "acceptance": [ + "既成事实不被覆盖", + "番外书有独立 bookId", + "角色卡能继承并局部调整" + ] + }, + { + "id": "spinoff-03", + "prompt": "写一个反派视角番外，让他有动机但不要洗白。", + "acceptance": [ + "视角切换进入规则", + "动机不破坏主线因果", + "后续章节能维持番外定位" + ] + } + ] + }, + { + "id": "continuation", + "label": "续写", + "entry": "Studio 工具 -> 导入 -> 章节导入/续写", + "cases": [ + { + "id": "continuation-01", + "prompt": "导入两章已有正文后继续写第三章，要求保持第一人称和旧角色关系。", + "acceptance": [ + "导入章节正文被原样保留", + "重新导入同章节不会留下旧标题残留文件", + "续写读取 parent/已有章节上下文" + ] + }, + { + "id": "continuation-02", + "prompt": "导入一个英文开头，继续写下一章，保持英文。", + "acceptance": [ + "语言不被切回中文", + "导入章节 wordCount 非 0", + "下一章不丢原主角" + ] + }, + { + "id": "continuation-03", + "prompt": "导入一段短剧剧本，改成小说续写。", + "acceptance": [ + "系统能区分导入材料和新章节", + "续写不是复述剧本", + "状态/角色卡从导入内容提取" + ] + } + ] + } + ] +} diff --git a/scripts/studio-e2e-benchmark.mjs b/scripts/studio-e2e-benchmark.mjs new file mode 100644 index 00000000..bbecf705 --- /dev/null +++ b/scripts/studio-e2e-benchmark.mjs @@ -0,0 +1,102 @@ +#!/usr/bin/env node +import { readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const CASES_PATH = resolve(__dirname, "studio-e2e-benchmark-cases.json"); + +function usage() { + return [ + "Usage:", + " node scripts/studio-e2e-benchmark.mjs list", + " node scripts/studio-e2e-benchmark.mjs markdown [output.md]", + "", + "This script does not call LLMs. It materializes the fixed Studio real-model", + "E2E benchmark run sheet so a human or browser automation can run it with", + "the configured Studio model service.", + ].join("\n"); +} + +async function loadCases() { + return JSON.parse(await readFile(CASES_PATH, "utf-8")); +} + +function flattenCases(spec) { + return spec.categories.flatMap((category) => + category.cases.map((testCase) => ({ + category, + testCase, + })), + ); +} + +function renderList(spec) { + const lines = [ + `${spec.name} v${spec.version}`, + `Default text model: ${spec.defaultModelHint?.text ?? "(project config)"}`, + `Default cover model: ${spec.defaultModelHint?.cover ?? "(project config)"}`, + "", + ]; + for (const { category, testCase } of flattenCases(spec)) { + lines.push(`${testCase.id} [${category.label}] ${testCase.prompt}`); + } + return lines.join("\n"); +} + +function renderMarkdown(spec) { + const lines = [ + `# ${spec.name}`, + "", + `Version: ${spec.version}`, + "", + spec.purpose, + "", + `- Text model hint: ${spec.defaultModelHint?.text ?? "(project config)"}`, + `- Cover model hint: ${spec.defaultModelHint?.cover ?? "(project config)"}`, + "", + ]; + + for (const category of spec.categories) { + lines.push(`## ${category.label}`); + lines.push(""); + lines.push(`Entry: ${category.entry}`); + lines.push(""); + for (const testCase of category.cases) { + lines.push(`### ${testCase.id}`); + lines.push(""); + lines.push(`Prompt: ${testCase.prompt}`); + lines.push(""); + lines.push("Acceptance:"); + for (const item of testCase.acceptance) { + lines.push(`- [ ] ${item}`); + } + lines.push(""); + lines.push("Result:"); + lines.push("- Status: TODO"); + lines.push("- Artifact: "); + lines.push("- Notes: "); + lines.push(""); + } + } + return lines.join("\n"); +} + +const command = process.argv[2] ?? "list"; +const output = process.argv[3]; + +const spec = await loadCases(); + +if (command === "list") { + console.log(renderList(spec)); +} else if (command === "markdown") { + const markdown = renderMarkdown(spec); + if (output) { + await writeFile(resolve(process.cwd(), output), markdown, "utf-8"); + } else { + console.log(markdown); + } +} else { + console.error(usage()); + process.exitCode = 1; +}