chore(test): add studio e2e benchmark checklist

package.json

@@ -26,7 +26,9 @@
     "test": "pnpm -r test",
     "lint": "pnpm -r lint",
     "typecheck": "pnpm -r typecheck",
+    "audit:semantic-patterns": "node scripts/audit-semantic-patterns.mjs",
     "verify:publish-manifests": "node scripts/verify-no-workspace-protocol.mjs packages/core packages/cli packages/studio",
+    "benchmark:studio-e2e": "node scripts/studio-e2e-benchmark.mjs",
     "release": "pnpm build && pnpm test"
   },
   "engines": {

scripts/audit-semantic-patterns.mjs

@@ -0,0 +1,139 @@
+#!/usr/bin/env node
+import { readdir, readFile } from "node:fs/promises";
+import { join, relative } from "node:path";
+
+const ROOT = process.cwd();
+const SCAN_DIRS = [
+  "packages/core/src",
+  "packages/studio/src",
+  "packages/cli/src",
+];
+
+const SOURCE_EXTENSIONS = new Set([".ts", ".tsx"]);
+
+const ACTION_SURFACE_PATHS = [
+  "packages/core/src/agent/",
+  "packages/core/src/interaction/",
+  "packages/studio/src/api/server.ts",
+  "packages/studio/src/pages/BookCreate.tsx",
+  "packages/cli/src/commands/agent.ts",
+  "packages/cli/src/tui/",
+];
+
+const SEMANTIC_HINTS = [
+  "instruction",
+  "intent",
+  "requestedIntent",
+  "actionSource",
+  "free-text",
+  "continue",
+  "write next",
+  "create book",
+  "edit",
+  "rewrite",
+  "revise",
+  "play",
+  "short",
+  "fanfic",
+  "spinoff",
+  "imitation",
+  "续写",
+  "继续",
+  "建书",
+  "短篇",
+  "互动",
+  "番外",
+  "仿写",
+  "同人",
+];
+
+const PATTERN_TOKENS = [
+  ".match(",
+  ".test(",
+  ".includes(",
+  ".startsWith(",
+  ".endsWith(",
+  "new RegExp(",
+];
+
+function hasAny(text, words) {
+  const lower = text.toLowerCase();
+  return words.some((word) => lower.includes(word.toLowerCase()));
+}
+
+function isActionSurface(path) {
+  const rel = path.replace(`${ROOT}/`, "");
+  return ACTION_SURFACE_PATHS.some((prefix) => rel.startsWith(prefix));
+}
+
+function isLikelySemanticDecision(path, line, windowText) {
+  if (!isActionSurface(path)) return false;
+  if (!PATTERN_TOKENS.some((token) => line.includes(token))) return false;
+  if (!hasAny(`${line}\n${windowText}`, SEMANTIC_HINTS)) return false;
+  if (line.includes("CHAT_EDIT_TEXT_EXTENSIONS")) return false;
+  if (line.includes("SAFE_ROLE_TRUTH_FILE_RE")) return false;
+  if (line.includes("CODE_FENCE_RE") || line.includes("DIRECTIVE_CLOSE_RE")) return false;
+  if (line.includes("safeSessionId")) return false;
+  if (line.includes("genreId")) return false;
+  if (line.includes("relPath") || line.includes("targetPath") || line.includes("requestedPath")) return false;
+  if (line.includes("entry.name") || line.includes("base.includes") || line.includes("escapeRegExp")) return false;
+  if (line.includes("raw.includes(\"/\")") || line.includes("raw.includes(\"\\\\\")")) return false;
+  if (line.includes("Manual text edit requires review")) return false;
+  if (path.endsWith("project-tools.ts") && line.includes("includes(")) return false;
+  if (path.endsWith("i18n.ts")) return false;
+  if (line.includes("rel || rel.startsWith")) return false;
+  if (line.includes("trimmed.match(/^([A-Za-z_]")) return false;
+  if (line.includes("file.includes(\"/\")")) return false;
+  if (line.includes("startsWith(\"[Tool results]\")")) return false;
+  if (line.includes("already processing") || line.includes("prompt.*queue")) return false;
+  if (line.includes("trimmed.startsWith(\"#\")")) return false;
+  if (line.includes("actionSource") && line.includes("startsWith(\"/\")")) return false;
+  if (line.includes("startsWith(\"/\")")) return false;
+  if (line.includes("endsWith(") && !hasAny(windowText, ["instruction", "intent"])) return false;
+  return true;
+}
+
+async function walk(dir) {
+  const out = [];
+  for (const entry of await readdir(dir, { withFileTypes: true })) {
+    const path = join(dir, entry.name);
+    if (entry.isDirectory()) {
+      if (entry.name === "node_modules" || entry.name === "dist" || entry.name === "__tests__") continue;
+      out.push(...await walk(path));
+    } else if (SOURCE_EXTENSIONS.has(path.slice(path.lastIndexOf(".")))) {
+      out.push(path);
+    }
+  }
+  return out;
+}
+
+const files = [];
+for (const dir of SCAN_DIRS) {
+  files.push(...await walk(join(ROOT, dir)));
+}
+
+const findings = [];
+for (const file of files) {
+  const content = await readFile(file, "utf-8");
+  const lines = content.split(/\r?\n/);
+  for (let index = 0; index < lines.length; index += 1) {
+    const line = lines[index];
+    const windowText = lines.slice(Math.max(0, index - 4), Math.min(lines.length, index + 5)).join("\n");
+    if (isLikelySemanticDecision(file, line, windowText)) {
+      findings.push({
+        file: relative(ROOT, file),
+        line: index + 1,
+        text: line.trim(),
+      });
+    }
+  }
+}
+
+if (process.argv.includes("--json")) {
+  console.log(JSON.stringify({ findings }, null, 2));
+} else {
+  console.log(`Semantic/template-pattern audit candidates: ${findings.length}`);
+  for (const finding of findings) {
+    console.log(`${finding.file}:${finding.line}  ${finding.text}`);
+  }
+}

scripts/studio-e2e-benchmark-cases.json

@@ -0,0 +1,284 @@
+{
+  "version": 1,
+  "name": "InkOS Studio Real-Model E2E Benchmark",
+  "purpose": "A stable manual/automated run sheet for true Studio UI + real LLM regression runs. Cases intentionally use user-like prompts rather than exact prompt engineering.",
+  "defaultModelHint": {
+    "text": "kkaiapi / deepseek-v4-flash or deepseek-v4-pro",
+    "cover": "configured Studio cover provider"
+  },
+  "categories": [
+    {
+      "id": "long_book",
+      "label": "长篇",
+      "entry": "Studio 创作 -> 新建书籍 -> 写第一章 -> 对话编辑角色卡",
+      "cases": [
+        {
+          "id": "long-01",
+          "prompt": "写一本县城悬疑长篇，主角是被停职的女法医，第一人称，重点是亲情债和旧案反转。",
+          "acceptance": [
+            "创建前出现确认卡，不直接建书",
+            "确认后生成 book 并可进入书籍页",
+            "第一章落盘且人称基本保持第一人称",
+            "能通过对话要求修改某张角色卡，角色卡文件发生变化"
+          ]
+        },
+        {
+          "id": "long-02",
+          "prompt": "开一部长篇商战文，主角是破产厂二代，工业调查占七成，感情线三成，第一章从假账仓库开始。",
+          "acceptance": [
+            "确认卡保留题材比例",
+            "第一章不脱离工厂/假账/调查主线",
+            "写完后审稿或状态不应把比例要求丢掉"
+          ]
+        },
+        {
+          "id": "long-03",
+          "prompt": "写一本英文 progression fantasy，主角从失败药剂学徒开始，风格要干净，不要中文网文口吻。",
+          "acceptance": [
+            "语言进入英文链路",
+            "章节长度按英文词数目标，不按中文字数硬套",
+            "生成物不混入中文流程标签"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "short",
+      "label": "短篇",
+      "entry": "Studio 创作 -> InkOS Short",
+      "cases": [
+        {
+          "id": "short-01",
+          "prompt": "写一篇女频短篇，离婚协议、三年隐忍、财产证据反杀。",
+          "acceptance": [
+            "确认后一次生成完整短篇",
+            "产出 final markdown、sales package、cover prompt",
+            "封面失败时正文/简介/卖点仍保留"
+          ]
+        },
+        {
+          "id": "short-02",
+          "prompt": "来一个亲情吸血题材，主角发现母亲的账本不是偏心证据，而是在救她。",
+          "acceptance": [
+            "题材不被路由成长篇",
+            "标题、简介和正文服务同一个核心冲突",
+            "章节不出现大面积空章或剧情梗概化"
+          ]
+        },
+        {
+          "id": "short-03",
+          "prompt": "做一个规则怪谈短篇，场景是凌晨便利店，主角靠收银小票发现规则漏洞。",
+          "acceptance": [
+            "规则和证据链清楚",
+            "正文不是纯设定说明",
+            "封面提示词体现手机端短篇封面风格"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "play_guided",
+      "label": "分支互动",
+      "entry": "Studio 创作 -> InkOS Play -> guided",
+      "cases": [
+        {
+          "id": "guided-01",
+          "prompt": "分支互动，旧戏院夜巡，我是临时维修员，系统每轮给我几个可选动作。",
+          "acceptance": [
+            "启动前出现确认卡",
+            "世界启动后每轮有 2-4 个可点击动作",
+            "HUD 不遮挡发送按钮"
+          ]
+        },
+        {
+          "id": "guided-02",
+          "prompt": "做一个民国当铺互动，我是新来的账房，账本里有一页不能翻。",
+          "acceptance": [
+            "点击建议动作能推进回合",
+            "关键物品进入状态或图谱",
+            "不会把用户自由讨论误当动作"
+          ]
+        },
+        {
+          "id": "guided-03",
+          "prompt": "分支互动，太空救援船，我是通信员，船上有人伪造求救信号。",
+          "acceptance": [
+            "题材状态词不泄漏英文模板",
+            "实体/关系边持续增长",
+            "每回合场景回应和图谱状态基本一致"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "play_open",
+      "label": "开放世界",
+      "entry": "Studio 创作 -> InkOS Play -> open",
+      "cases": [
+        {
+          "id": "open-01",
+          "prompt": "开放世界，我在暴雨里的县医院档案室醒来，手机只有一条未发送短信。",
+          "acceptance": [
+            "启动后允许自由输入动作",
+            "自由动作不会被误路由到建书/短篇",
+            "renderer 新出现的关键实体会被对账进状态"
+          ]
+        },
+        {
+          "id": "open-02",
+          "prompt": "开放互动，古堡晚宴，我是临时钢琴师，主人要求我不要弹第三首曲子。",
+          "acceptance": [
+            "世界规则在后续回合保持",
+            "玩家身份稳定",
+            "状态/HUD 不靠英文模板词展示"
+          ]
+        },
+        {
+          "id": "open-03",
+          "prompt": "自由行动，赛博城地下诊所，我欠黑市医生一笔账，门外有人找我。",
+          "acceptance": [
+            "资源/压力能进入状态",
+            "关系和证据能被图谱保存",
+            "长会话后不会明显丢失当前目标"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "fanfic",
+      "label": "同人",
+      "entry": "Studio 工具 -> 导入 -> 同人创作",
+      "cases": [
+        {
+          "id": "fanfic-01",
+          "prompt": "根据一段正典材料做同人新书，写原作空白处，不要改原角色底色。",
+          "acceptance": [
+            "入口等待后台创建，不短轮询误报失败",
+            "正典约束写入 fanfic_canon 或 parent_canon",
+            "第一章不明显 OOC"
+          ]
+        },
+        {
+          "id": "fanfic-02",
+          "prompt": "做一个 AU 同人，保留人物关系张力，但换成县城医院背景。",
+          "acceptance": [
+            "AU 与正典约束分开",
+            "不会把同人请求路由成普通长篇",
+            "创建完成后跳转到新书"
+          ]
+        },
+        {
+          "id": "fanfic-03",
+          "prompt": "做 CP 向同人，重点是误解解除，不写露骨内容。",
+          "acceptance": [
+            "禁忌要求进入书籍规则或作者意图",
+            "角色关系在角色卡中可见",
+            "后续写章能读取这些约束"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "imitation",
+      "label": "仿写",
+      "entry": "Studio 工具 -> 导入 -> 仿写",
+      "cases": [
+        {
+          "id": "imitation-01",
+          "prompt": "参考一段县城生活质感，仿写一个新故事，不要照搬人物和情节。",
+          "acceptance": [
+            "入口等待后台创建完成",
+            "风格约束写入 style_guide",
+            "新书不直接复制原文人物名"
+          ]
+        },
+        {
+          "id": "imitation-02",
+          "prompt": "仿写冷峻克制的叙述感，做一个短章开头，不要大段心理独白。",
+          "acceptance": [
+            "风格要求可追溯",
+            "生成章节能体现叙述节制",
+            "不混入模板化分析文字"
+          ]
+        },
+        {
+          "id": "imitation-03",
+          "prompt": "仿写轻喜剧对话密度，做成都市家庭题材。",
+          "acceptance": [
+            "对话风格进入写作约束",
+            "书籍创建后可继续写章",
+            "角色卡与风格指南不相互覆盖"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "spinoff",
+      "label": "番外",
+      "entry": "Studio 工具 -> 导入 -> 番外",
+      "cases": [
+        {
+          "id": "spinoff-01",
+          "prompt": "给已有书做一个番外，讲配角收到一封迟到十年的信。",
+          "acceptance": [
+            "创建时保留父书上下文",
+            "不污染父书主线章节",
+            "后台创建完成后自动跳转"
+          ]
+        },
+        {
+          "id": "spinoff-02",
+          "prompt": "番外写结局后的小饭馆重逢，重点是余味，不改主线事实。",
+          "acceptance": [
+            "既成事实不被覆盖",
+            "番外书有独立 bookId",
+            "角色卡能继承并局部调整"
+          ]
+        },
+        {
+          "id": "spinoff-03",
+          "prompt": "写一个反派视角番外，让他有动机但不要洗白。",
+          "acceptance": [
+            "视角切换进入规则",
+            "动机不破坏主线因果",
+            "后续章节能维持番外定位"
+          ]
+        }
+      ]
+    },
+    {
+      "id": "continuation",
+      "label": "续写",
+      "entry": "Studio 工具 -> 导入 -> 章节导入/续写",
+      "cases": [
+        {
+          "id": "continuation-01",
+          "prompt": "导入两章已有正文后继续写第三章，要求保持第一人称和旧角色关系。",
+          "acceptance": [
+            "导入章节正文被原样保留",
+            "重新导入同章节不会留下旧标题残留文件",
+            "续写读取 parent/已有章节上下文"
+          ]
+        },
+        {
+          "id": "continuation-02",
+          "prompt": "导入一个英文开头，继续写下一章，保持英文。",
+          "acceptance": [
+            "语言不被切回中文",
+            "导入章节 wordCount 非 0",
+            "下一章不丢原主角"
+          ]
+        },
+        {
+          "id": "continuation-03",
+          "prompt": "导入一段短剧剧本，改成小说续写。",
+          "acceptance": [
+            "系统能区分导入材料和新章节",
+            "续写不是复述剧本",
+            "状态/角色卡从导入内容提取"
+          ]
+        }
+      ]
+    }
+  ]
+}

scripts/studio-e2e-benchmark.mjs

@@ -0,0 +1,102 @@
+#!/usr/bin/env node
+import { readFile, writeFile } from "node:fs/promises";
+import { dirname, resolve } from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CASES_PATH = resolve(__dirname, "studio-e2e-benchmark-cases.json");
+
+function usage() {
+  return [
+    "Usage:",
+    "  node scripts/studio-e2e-benchmark.mjs list",
+    "  node scripts/studio-e2e-benchmark.mjs markdown [output.md]",
+    "",
+    "This script does not call LLMs. It materializes the fixed Studio real-model",
+    "E2E benchmark run sheet so a human or browser automation can run it with",
+    "the configured Studio model service.",
+  ].join("\n");
+}
+
+async function loadCases() {
+  return JSON.parse(await readFile(CASES_PATH, "utf-8"));
+}
+
+function flattenCases(spec) {
+  return spec.categories.flatMap((category) =>
+    category.cases.map((testCase) => ({
+      category,
+      testCase,
+    })),
+  );
+}
+
+function renderList(spec) {
+  const lines = [
+    `${spec.name} v${spec.version}`,
+    `Default text model: ${spec.defaultModelHint?.text ?? "(project config)"}`,
+    `Default cover model: ${spec.defaultModelHint?.cover ?? "(project config)"}`,
+    "",
+  ];
+  for (const { category, testCase } of flattenCases(spec)) {
+    lines.push(`${testCase.id} [${category.label}] ${testCase.prompt}`);
+  }
+  return lines.join("\n");
+}
+
+function renderMarkdown(spec) {
+  const lines = [
+    `# ${spec.name}`,
+    "",
+    `Version: ${spec.version}`,
+    "",
+    spec.purpose,
+    "",
+    `- Text model hint: ${spec.defaultModelHint?.text ?? "(project config)"}`,
+    `- Cover model hint: ${spec.defaultModelHint?.cover ?? "(project config)"}`,
+    "",
+  ];
+
+  for (const category of spec.categories) {
+    lines.push(`## ${category.label}`);
+    lines.push("");
+    lines.push(`Entry: ${category.entry}`);
+    lines.push("");
+    for (const testCase of category.cases) {
+      lines.push(`### ${testCase.id}`);
+      lines.push("");
+      lines.push(`Prompt: ${testCase.prompt}`);
+      lines.push("");
+      lines.push("Acceptance:");
+      for (const item of testCase.acceptance) {
+        lines.push(`- [ ] ${item}`);
+      }
+      lines.push("");
+      lines.push("Result:");
+      lines.push("- Status: TODO");
+      lines.push("- Artifact: ");
+      lines.push("- Notes: ");
+      lines.push("");
+    }
+  }
+  return lines.join("\n");
+}
+
+const command = process.argv[2] ?? "list";
+const output = process.argv[3];
+
+const spec = await loadCases();
+
+if (command === "list") {
+  console.log(renderList(spec));
+} else if (command === "markdown") {
+  const markdown = renderMarkdown(spec);
+  if (output) {
+    await writeFile(resolve(process.cwd(), output), markdown, "utf-8");
+  } else {
+    console.log(markdown);
+  }
+} else {
+  console.error(usage());
+  process.exitCode = 1;
+}