chore: release v1.33.0

This commit is contained in:
coso
2026-05-11 01:36:44 +08:00
parent 27707daf23
commit d47f84cdd2
320 changed files with 49585 additions and 7201 deletions
+316
View File
@@ -0,0 +1,316 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import { createAgentQcGuiFlowReport } from "./lib/agent-qc-gui-flow-core.mjs";
import { buildQCLoopJobPayload, validateQCLoopJobPayload } from "./lib/agent-qc-qcloop-job-core.mjs";
import { createAgentQcReport } from "./lib/agent-qc-report-core.mjs";
import {
buildAgentQcCompletionAudit,
renderAgentQcCompletionAuditMarkdown,
} from "./lib/agent-qc-completion-audit-core.mjs";
function parseArgs(argv) {
const result = {
check: false,
format: "markdown",
help: false,
outputPath: "",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC Completion Audit
用法:
npm run agent-qc:audit
npm run agent-qc:audit -- --format json
node scripts/agent-qc-completion-audit.mjs --check
选项:
--format FMT markdown | json
--output PATH 写入文件;默认 stdout
--check 整体目标未完成时非 0 退出
-h, --help 显示帮助
`);
}
function exists(filePath) {
return fs.existsSync(path.resolve(process.cwd(), filePath));
}
function readText(filePath) {
return fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8");
}
function readJson(filePath) {
return JSON.parse(readText(filePath));
}
function readOptionalEvidencePack(filePath) {
if (!exists(filePath)) {
return { exists: false, status: "", scenarioCount: 0, scenarioIds: [] };
}
try {
const pack = readJson(filePath);
return summarizeEvidencePack(pack, { exists: true });
} catch {
return { exists: true, status: "invalid-json", scenarioCount: 0, scenarioIds: [] };
}
}
function readOptionalJson(filePath, fallback) {
if (!exists(filePath)) {
return fallback;
}
try {
return readJson(filePath);
} catch {
return { ...fallback, status: "invalid-json" };
}
}
function summarizeEvidencePack(pack, overrides = {}) {
return {
...overrides,
status: pack?.verdict?.status || "",
scenarioCount: Array.isArray(pack?.scenarioResults)
? pack.scenarioResults.length
: 0,
scenarioIds: Array.isArray(pack?.scenarioResults)
? pack.scenarioResults
.map((result) => String(result?.scenarioId || "").trim())
.filter(Boolean)
: [],
};
}
function readEvidenceSidecars(dirPath) {
const resolvedDirPath = path.resolve(process.cwd(), dirPath);
if (!fs.existsSync(resolvedDirPath)) {
return [];
}
return fs
.readdirSync(resolvedDirPath)
.filter((fileName) => /^agent-qc-evidence\..+\.json$/.test(fileName))
.sort()
.map((fileName) => {
const relativePath = path.posix.join(dirPath, fileName);
try {
const pack = readJson(relativePath);
return summarizeEvidencePack(pack, { path: relativePath });
} catch {
return { path: relativePath, status: "invalid-json", scenarioCount: 0, scenarioIds: [] };
}
});
}
function readQcloopStatusSidecars(dirPath) {
const resolvedDirPath = path.resolve(process.cwd(), dirPath);
if (!fs.existsSync(resolvedDirPath)) {
return [];
}
return fs
.readdirSync(resolvedDirPath)
.filter((fileName) => /^qcloop-status\..+\.json$/.test(fileName))
.sort()
.map((fileName) => {
const relativePath = path.posix.join(dirPath, fileName);
try {
const status = readJson(relativePath);
return {
path: relativePath,
verdictStatus: status?.verdict?.status || "",
verdictSummary: status?.verdict?.summary || "",
counts: status?.counts || null,
};
} catch {
return { path: relativePath, verdictStatus: "invalid-json", verdictSummary: "", counts: null };
}
});
}
function loadFacts() {
const scenarioManifest = readJson("docs/test/agent-qc-scenarios.manifest.json");
const guiFlowManifest = readJson("docs/test/agent-qc-gui-flows.manifest.json");
const evidenceSchema = readJson("docs/test/agent-qc-evidence.schema.json");
const packageJson = readJson("package.json");
const scenarioReport = createAgentQcReport({ manifest: scenarioManifest, packageJson, evidenceSchema });
scenarioReport.p0ScenarioIds = Array.isArray(scenarioManifest?.scenarios)
? scenarioManifest.scenarios
.filter((scenario) => String(scenario?.risk || "").toUpperCase() === "P0")
.map((scenario) => String(scenario?.id || "").trim())
.filter(Boolean)
: [];
const guiFlowReport = createAgentQcGuiFlowReport({ flowManifest: guiFlowManifest, scenarioManifest });
const qcloopPayload = buildQCLoopJobPayload(scenarioManifest, { risks: ["P0"], generatedAt: "2026-05-10T00:00:00.000Z" });
const qcloopPayloadValidation = validateQCLoopJobPayload(qcloopPayload);
const evidenceCore = exists("scripts/lib/agent-qc-evidence-core.mjs")
? readText("scripts/lib/agent-qc-evidence-core.mjs")
: "";
const releaseSummaryCore = exists("scripts/lib/agent-qc-release-summary-core.mjs")
? readText("scripts/lib/agent-qc-release-summary-core.mjs")
: "";
const guiOwnerCore = exists("scripts/lib/agent-qc-gui-owner-core.mjs")
? readText("scripts/lib/agent-qc-gui-owner-core.mjs")
: "";
const staleOwnerInterventionDoc = exists("docs/tests/lime-agent-qc-stale-owner-intervention.md")
? readText("docs/tests/lime-agent-qc-stale-owner-intervention.md")
: "";
const nightly = exists(".github/workflows/harness-nightly.yml") ? readText(".github/workflows/harness-nightly.yml") : "";
const release = exists(".github/workflows/release.yml") ? readText(".github/workflows/release.yml") : "";
return {
files: {
agentOpsQc: exists("docs/tests/agent-ops-qc.md"),
p0Scenarios: exists("docs/tests/agent-qc-p0-scenarios.md"),
limeRolloutPlan: exists("docs/tests/lime-agent-qc-rollout-plan.md"),
testsReadme: exists("docs/tests/README.md"),
evidenceSchema: exists("docs/test/agent-qc-evidence.schema.json"),
qcloopJobScript: exists("scripts/agent-qc-qcloop-job.mjs"),
guiOwnerCheckScript: exists("scripts/agent-qc-gui-owner-check.mjs"),
qcloopStatusScript: exists("scripts/agent-qc-qcloop-status.mjs"),
qcloopPreflightScript: exists("scripts/agent-qc-qcloop-preflight.mjs"),
qcloopOperationsDoc: exists("docs/tests/lime-agent-qc-qcloop-operations.md"),
evidenceContractDoc: exists("docs/tests/lime-agent-qc-evidence-contract.md"),
staleOwnerInterventionDoc: exists("docs/tests/lime-agent-qc-stale-owner-intervention.md"),
exportEvidenceScript: exists("scripts/agent-qc-export-evidence.mjs"),
releaseSummaryScript: exists("scripts/agent-qc-release-summary.mjs"),
realGuiEvidence: exists(".lime/qc/gui-evidence"),
},
realEvidencePack: readOptionalEvidencePack(".lime/qc/agent-qc-evidence.json"),
localVerify: readOptionalJson(".lime/qc/verify-local-current.json", {
exists: false,
status: "",
failedStage: "",
error: "",
}),
guiSmoke: readOptionalJson(".lime/qc/verify-gui-smoke-current.json", {
exists: false,
status: "",
failedStage: "",
error: "",
}),
realEvidenceSidecars: readEvidenceSidecars(".lime/qc"),
qcloopStatusSidecars: readQcloopStatusSidecars(".lime/qc"),
scenarioReport,
guiFlowReport,
qcloopPayload: {
valid: qcloopPayloadValidation.valid,
itemCount: qcloopPayload.items.length,
verifierHasWorkerOutput:
qcloopPayload.verifier_prompt_template.includes("{{stdout}}") ||
qcloopPayload.verifier_prompt_template.includes("{{output}}"),
verifierHasAttemptStatus: qcloopPayload.verifier_prompt_template.includes(
"{{attempt_status}}",
),
verifierHasExitCode: qcloopPayload.verifier_prompt_template.includes("{{exit_code}}"),
workerPromptHasPreflight:
qcloopPayload.prompt_template.includes("agent-qc:qcloop-preflight") &&
qcloopPayload.prompt_template.includes("--require-devbridge"),
workerPromptHasStructuredEvidence:
qcloopPayload.prompt_template.includes("QCLOOP_WORKER_RESULT=PASS|FAIL|BLOCKED") &&
qcloopPayload.prompt_template.includes("QCLOOP_EVIDENCE_SUMMARY_JSON=<json>"),
verifierRequiresStructuredEvidence:
qcloopPayload.verifier_prompt_template.includes("QCLOOP_EVIDENCE_SUMMARY_JSON"),
verifierRequiresStrictJson:
qcloopPayload.verifier_prompt_template.includes('{"pass": true|false') &&
qcloopPayload.verifier_prompt_template.includes("不要 Markdown"),
},
structuredEvidence: {
exporterParsesSummary:
evidenceCore.includes("collectQCLoopEvidenceSummaries") &&
evidenceCore.includes("qcloop:evidence_summary_missing") &&
evidenceCore.includes("qcloop:evidence_summary_invalid_json"),
releaseSummaryRejectsWeakRefs:
releaseSummaryCore.includes("weakEvidenceScenarioIds") &&
releaseSummaryCore.includes("hasStructuredEvidenceRef") &&
releaseSummaryCore.includes("缺少结构化 evidenceRefs"),
},
staleOwnerIntervention: {
guiOwnerReportHasDecisionPacket:
guiOwnerCore.includes("ownerIntervention") &&
guiOwnerCore.includes("requiredConfirmationText") &&
guiOwnerCore.includes("prohibitedUntilConfirmed"),
guiOwnerReportHasWatchHistory:
guiOwnerCore.includes("createAgentQcGuiOwnerWatchEntry") &&
exists("scripts/agent-qc-gui-owner-check.mjs") &&
readText("scripts/agent-qc-gui-owner-check.mjs").includes("--watch-history-output"),
docMentionsDecisionPacket:
staleOwnerInterventionDoc.includes("ownerIntervention") &&
staleOwnerInterventionDoc.includes("stale-owner-intervention-request.json"),
docMentionsWatchHistory:
staleOwnerInterventionDoc.includes("watch-history-output") ||
staleOwnerInterventionDoc.includes("stale-owner-watch-history.jsonl"),
},
nightly: {
hasAgentQcReport: nightly.includes("agent-qc-report"),
hasGuiFlowReport: nightly.includes("agent-qc-gui-flow-report"),
hasReleasePreview: nightly.includes("release-agent-qc-preview"),
},
release: {
hasHardGate:
release.includes("agent-qc-release-summary") &&
release.includes("--check") &&
!release.includes("--allow-missing-evidence"),
requiresP0ScenarioCoverage:
release.includes("--require-scenario-manifest") &&
release.includes("docs/test/agent-qc-scenarios.manifest.json") &&
release.includes("--require-risk") &&
release.includes("P0"),
},
};
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const audit = buildAgentQcCompletionAudit(loadFacts());
const content =
options.format === "json"
? `${JSON.stringify(audit, null, 2)}\n`
: renderAgentQcCompletionAuditMarkdown(audit);
writeOutput(options.outputPath, content);
if (options.check && audit.status !== "complete") {
process.exit(1);
}
}
main();
+215
View File
@@ -0,0 +1,215 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
buildAgentQcEvidencePack,
validateEvidencePackShape,
} from "./lib/agent-qc-evidence-core.mjs";
function parseArgs(argv) {
const result = {
baseUrl: "http://127.0.0.1:8080",
changedFiles: [],
check: false,
diffBase: "",
format: "json",
help: false,
itemsJson: "",
jobId: "",
jobJson: "",
outputPath: "",
ref: "",
repo: "lime",
riskTags: [],
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--base-url" && argv[index + 1]) {
result.baseUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--job-id" && argv[index + 1]) {
result.jobId = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--job-json" && argv[index + 1]) {
result.jobJson = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--items-json" && argv[index + 1]) {
result.itemsJson = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--repo" && argv[index + 1]) {
result.repo = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--ref" && argv[index + 1]) {
result.ref = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--diff-base" && argv[index + 1]) {
result.diffBase = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--changed-file" && argv[index + 1]) {
result.changedFiles.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--risk-tag" && argv[index + 1]) {
result.riskTags.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC Evidence Export
用法:
npm run agent-qc:export-evidence -- --job-id "qcloop-job-id" --output ./.lime/qc/evidence.json
node scripts/agent-qc-export-evidence.mjs --job-json ./job.json --items-json ./items.json --check
选项:
--job-id ID 从 qcloop HTTP API 读取 /api/jobs/:id 和 /api/items?job_id=:id
--base-url URL qcloop API 地址,默认 http://127.0.0.1:8080
--job-json PATH 离线 job JSON 文件
--items-json PATH 离线 items JSON 文件
--output PATH 写入 Evidence Pack;默认 stdout
--repo NAME subject.repo,默认 lime
--ref REF subject.ref
--diff-base REF subject.diffBase
--changed-file PATH 追加 changedFiles,可重复
--risk-tag TAG 追加 riskTags,可重复
--check 只校验输出形状,非法时非 0 退出
--format FMT 输出格式:json | summary
-h, --help 显示帮助
`);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
}
async function readJsonUrl(url) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`请求失败 ${response.status}: ${url}`);
}
return response.json();
}
async function loadQCLoopData(options) {
if (options.jobJson || options.itemsJson) {
if (!options.jobJson || !options.itemsJson) {
throw new Error("离线模式必须同时提供 --job-json 与 --items-json。");
}
return {
job: readJsonFile(options.jobJson),
items: readJsonFile(options.itemsJson),
};
}
if (!options.jobId) {
throw new Error("必须提供 --job-id,或同时提供 --job-json / --items-json。");
}
const baseUrl = options.baseUrl.replace(/\/$/, "");
return {
job: await readJsonUrl(`${baseUrl}/api/jobs/${encodeURIComponent(options.jobId)}`),
items: await readJsonUrl(`${baseUrl}/api/items/?job_id=${encodeURIComponent(options.jobId)}`),
};
}
function renderSummary(pack, validation) {
const lines = [
`runId=${pack.runId}`,
`status=${pack.verdict.status}`,
`scenarios=${pack.scenarioResults.length}`,
`valid=${validation.valid}`,
`summary=${pack.verdict.summary}`,
];
if (validation.issues.length > 0) {
lines.push(`issues=${validation.issues.join("; ")}`);
}
return `${lines.join("\n")}\n`;
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
async function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const { job, items } = await loadQCLoopData(options);
const pack = buildAgentQcEvidencePack({
job,
items,
options: {
repo: options.repo,
ref: options.ref,
diffBase: options.diffBase,
changedFiles: options.changedFiles,
riskTags: options.riskTags,
},
});
const validation = validateEvidencePackShape(pack);
const content = options.format === "summary" ? renderSummary(pack, validation) : `${JSON.stringify(pack, null, 2)}\n`;
writeOutput(options.outputPath, content);
if (options.check && !validation.valid) {
for (const issue of validation.issues) {
console.error(`[agent-qc-evidence] ${issue}`);
}
process.exit(1);
}
}
main().catch((error) => {
console.error(`[agent-qc-evidence] ${error.message}`);
process.exit(1);
});
+114
View File
@@ -0,0 +1,114 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
createAgentQcGuiFlowReport,
renderAgentQcGuiFlowMarkdown,
} from "./lib/agent-qc-gui-flow-core.mjs";
const DEFAULT_FLOW_MANIFEST_PATH = "docs/test/agent-qc-gui-flows.manifest.json";
const DEFAULT_SCENARIO_MANIFEST_PATH = "docs/test/agent-qc-scenarios.manifest.json";
function parseArgs(argv) {
const result = {
check: false,
flowManifestPath: DEFAULT_FLOW_MANIFEST_PATH,
format: "markdown",
help: false,
outputPath: "",
scenarioManifestPath: DEFAULT_SCENARIO_MANIFEST_PATH,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--flow-manifest" && argv[index + 1]) {
result.flowManifestPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--scenario-manifest" && argv[index + 1]) {
result.scenarioManifestPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC GUI Flow Report
用法:
npm run agent-qc:gui-flow:report
npm run agent-qc:gui-flow:check
选项:
--flow-manifest PATH GUI flow manifest,默认 docs/test/agent-qc-gui-flows.manifest.json
--scenario-manifest PATH Agent QC scenario manifest,默认 docs/test/agent-qc-scenarios.manifest.json
--format FMT markdown | json
--output PATH 写入文件;默认 stdout
--check 非法时非 0 退出
-h, --help 显示帮助
`);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const report = createAgentQcGuiFlowReport({
flowManifest: readJsonFile(options.flowManifestPath),
scenarioManifest: readJsonFile(options.scenarioManifestPath),
});
const content =
options.format === "json"
? `${JSON.stringify(report, null, 2)}\n`
: renderAgentQcGuiFlowMarkdown(report);
writeOutput(options.outputPath, content);
if (options.check && !report.valid) {
process.exit(1);
}
}
main();
+155
View File
@@ -0,0 +1,155 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
createAgentQcGuiOwnerWatchEntry,
createAgentQcGuiOwnerReport,
renderAgentQcGuiOwnerSummary,
} from "./lib/agent-qc-gui-owner-core.mjs";
function parseArgs(argv) {
const result = {
check: false,
format: "summary",
help: false,
manifestPath: "docs/test/agent-qc-scenarios.manifest.json",
maxActiveOwners: 0,
outputPath: "",
statusDir: ".lime/qc",
watchHistoryOutputPath: "",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--manifest" && argv[index + 1]) {
result.manifestPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--status-dir" && argv[index + 1]) {
result.statusDir = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--max-active-owners" && argv[index + 1]) {
result.maxActiveOwners = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--watch-history-output" && argv[index + 1]) {
result.watchHistoryOutputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC GUI Owner Check
用法:
npm run agent-qc:gui-owner-check
npm run agent-qc:gui-owner-check -- --check
node scripts/agent-qc-gui-owner-check.mjs --format json --output ./.lime/qc/gui-owner-current.json
选项:
--manifest PATH scenario manifest,默认 docs/test/agent-qc-scenarios.manifest.json
--status-dir PATH qcloop status sidecar 目录,默认 .lime/qc
--max-active-owners N 允许的 active GUI owner 数,默认 0
--format FMT summary | json
--output PATH 写入文件;默认 stdout
--watch-history-output PATH 追加 JSONL 观察记录
--check active owner 超过上限时非 0 退出
-h, --help 显示帮助
`);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
}
function readQcloopStatusSidecars(statusDir) {
const resolvedDir = path.resolve(process.cwd(), statusDir);
if (!fs.existsSync(resolvedDir)) {
return [];
}
return fs
.readdirSync(resolvedDir)
.filter((fileName) => /^qcloop-status\..+\.json$/.test(fileName))
.sort()
.map((fileName) => {
const relativePath = path.posix.join(statusDir, fileName);
try {
return { path: relativePath, status: readJsonFile(relativePath) };
} catch {
return { path: relativePath, status: { job: { id: relativePath, status: "invalid-json" }, items: [] } };
}
});
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function appendJsonLine(outputPath, entry) {
if (!outputPath) {
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.appendFileSync(resolvedOutputPath, `${JSON.stringify(entry)}\n`, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const report = createAgentQcGuiOwnerReport({
manifest: readJsonFile(options.manifestPath),
statusSidecars: readQcloopStatusSidecars(options.statusDir),
maxActiveOwners: options.maxActiveOwners,
});
const content =
options.format === "json"
? `${JSON.stringify(report, null, 2)}\n`
: renderAgentQcGuiOwnerSummary(report);
writeOutput(options.outputPath, content);
appendJsonLine(options.watchHistoryOutputPath, createAgentQcGuiOwnerWatchEntry(report));
if (options.check && report.verdict.status !== "pass") {
process.exit(1);
}
}
main();
+318
View File
@@ -0,0 +1,318 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
const GUI_OWNER_PATTERNS = [
"verify:gui-smoke",
"smoke:workspace-ready",
"smoke:browser-runtime",
"smoke:site-adapters",
"smoke:agent-service-skill-entry",
"smoke:agent-runtime-tool-surface",
"smoke:knowledge-gui",
"smoke:design-canvas",
"claw-chat-ready-streaming",
"browser-runtime-site-adapter",
"workspace-ready-session-restore",
"release-package-startup-smoke",
];
const CARGO_OWNER_PATTERNS = [
"cargo ",
"cargo-fmt",
"rustc ",
"clippy-driver",
"tauri dev",
];
const QCLOOP_OWNER_PATTERNS = [
"qcloop --db",
"./qcloop --db",
"qcloop serve",
"qcloop-worker",
"qcloop_worker_result",
"agent qc p0",
"只读执行 lime agent qc p0",
];
function parseArgs(argv) {
const result = {
check: false,
format: "summary",
help: false,
markdownOutputPath: "",
maxActiveGuiSmoke: 0,
maxCargoOrRust: 0,
maxQcloopRelated: 0,
outputPath: "",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--markdown-output" && argv[index + 1]) {
result.markdownOutputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--max-active-gui-smoke" && argv[index + 1]) {
result.maxActiveGuiSmoke = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--max-cargo-or-rust" && argv[index + 1]) {
result.maxCargoOrRust = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--max-qcloop-related" && argv[index + 1]) {
result.maxQcloopRelated = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC Process Owner Check
用法:
npm run agent-qc:process-owner-check
node scripts/agent-qc-process-owner-check.mjs --format json --output ./.lime/qc/gui-process-owner-current.json --markdown-output ./.lime/qc/gui-process-owner-current.md
选项:
--format FMT summary | json,默认 summary
--output PATH 写入 JSON 或 summary;默认 stdout
--markdown-output PATH 额外写入 Markdown 摘要
--max-active-gui-smoke N 允许的 GUI smoke / deep flow owner 数,默认 0
--max-cargo-or-rust N 允许的 Cargo / Rust 构建进程数,默认 0
--max-qcloop-related N 允许的 qcloop 相关进程数,默认 0
--check owner 超过上限时非 0 退出
-h, --help 显示帮助
`);
}
function parseUnixPsLine(line) {
const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
if (!match) {
return null;
}
return {
pid: Number(match[1]),
ppid: Number(match[2]),
pgid: Number(match[3]),
stat: match[4],
etime: match[5],
command: match[6].trim(),
};
}
function collectUnixProcesses() {
const output = execFileSync("ps", ["-axo", "pid,ppid,pgid,stat,etime,command"], {
encoding: "utf8",
});
return output
.split(/\r?\n/)
.slice(1)
.map(parseUnixPsLine)
.filter(Boolean)
.filter((entry) => entry.pid !== process.pid && entry.command.length > 0);
}
function collectWindowsProcesses() {
const output = execFileSync(
"powershell.exe",
[
"-NoProfile",
"-Command",
"Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CommandLine | ConvertTo-Json -Compress",
],
{ encoding: "utf8" },
).trim();
const parsed = output ? JSON.parse(output) : [];
return (Array.isArray(parsed) ? parsed : [parsed])
.map((entry) => ({
pid: Number(entry?.ProcessId || 0),
ppid: Number(entry?.ParentProcessId || 0),
pgid: null,
stat: "unknown",
etime: "unknown",
command: String(entry?.CommandLine || "").trim(),
}))
.filter((entry) => entry.pid > 0 && entry.pid !== process.pid && entry.command.length > 0);
}
function collectProcesses() {
return process.platform === "win32" ? collectWindowsProcesses() : collectUnixProcesses();
}
function commandHasAny(command, patterns) {
const normalized = String(command || "").toLowerCase();
return patterns.some((pattern) => normalized.includes(pattern.toLowerCase()));
}
function uniqueByPid(entries) {
const seen = new Set();
const result = [];
for (const entry of entries) {
if (seen.has(entry.pid)) {
continue;
}
seen.add(entry.pid);
result.push(entry);
}
return result.sort((left, right) => left.pid - right.pid);
}
function createReport({
generatedAt = new Date().toISOString(),
maxActiveGuiSmoke = 0,
maxCargoOrRust = 0,
maxQcloopRelated = 0,
} = {}) {
const processes = collectProcesses();
const activeGuiSmokeProcesses = uniqueByPid(
processes.filter((entry) => commandHasAny(entry.command, GUI_OWNER_PATTERNS)),
);
const qcloopProcesses = uniqueByPid(
processes.filter((entry) => commandHasAny(entry.command, QCLOOP_OWNER_PATTERNS)),
);
const cargoProcesses = uniqueByPid(
processes.filter((entry) => commandHasAny(entry.command, CARGO_OWNER_PATTERNS)),
);
const counts = {
activeGuiSmoke: activeGuiSmokeProcesses.length,
cargoOrRust: cargoProcesses.length,
qcloopRelated: qcloopProcesses.length,
};
const busy =
counts.activeGuiSmoke > maxActiveGuiSmoke ||
counts.cargoOrRust > maxCargoOrRust ||
counts.qcloopRelated > maxQcloopRelated;
return {
schemaVersion: "v1",
generatedAt,
platform: process.platform,
maxActiveGuiSmoke,
maxCargoOrRust,
maxQcloopRelated,
verdict: {
status: busy ? "busy" : "pass",
summary: `activeGuiSmoke=${counts.activeGuiSmoke}, cargoOrRust=${counts.cargoOrRust}, qcloopRelated=${counts.qcloopRelated}`,
nextAction: busy
? "Do not start full verify:local or another GUI P0 batch while these owners are active; continue read-only observation or wait for natural completion."
: "No active raw GUI smoke, Cargo/Rust, or qcloop owner was observed; heavy gates may run if qcloop GUI owner and release evidence gates are also clear.",
},
activeGuiSmokeProcesses,
qcloopProcesses,
cargoProcesses,
guardrails: [
"best-effort process snapshot only",
"do not kill or restart listed processes from this sidecar",
"use this sidecar to decide whether heavy GUI or verify gates should wait",
],
};
}
function renderSummary(report) {
const lines = [
`status=${report.verdict.status}`,
`summary=${report.verdict.summary}`,
`generatedAt=${report.generatedAt}`,
`platform=${report.platform}`,
];
return `${lines.join("\n")}\n`;
}
function renderMarkdown(report) {
const lines = [
"# Agent QC raw process owner snapshot",
"",
`- Generated at: ${report.generatedAt}`,
`- Status: ${report.verdict.status}`,
`- Summary: ${report.verdict.summary}`,
`- Next action: ${report.verdict.nextAction}`,
"",
"## Active GUI smoke / deep flow processes",
"",
];
appendProcessList(lines, report.activeGuiSmokeProcesses);
lines.push("", "## qcloop related processes", "");
appendProcessList(lines, report.qcloopProcesses);
lines.push("", "## Cargo / Rust processes", "");
appendProcessList(lines, report.cargoProcesses);
lines.push("", "## Guardrails", "");
for (const guardrail of report.guardrails) {
lines.push(`- ${guardrail}`);
}
return `${lines.join("\n")}\n`;
}
function appendProcessList(lines, entries) {
if (!entries?.length) {
lines.push("- none");
return;
}
for (const entry of entries) {
lines.push(
`- pid=${entry.pid} ppid=${entry.ppid} pgid=${entry.pgid} stat=${entry.stat} etime=${entry.etime} command=${entry.command}`,
);
}
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const report = createReport(options);
const content =
options.format === "json"
? `${JSON.stringify(report, null, 2)}\n`
: renderSummary(report);
writeOutput(options.outputPath, content);
if (options.markdownOutputPath) {
writeOutput(options.markdownOutputPath, renderMarkdown(report));
}
if (options.check && report.verdict.status !== "pass") {
process.exit(1);
}
}
main();
+258
View File
@@ -0,0 +1,258 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
function parseArgs(argv) {
const result = {
checkTerminal: false,
dbPath: ".lime/qc/qcloop-isolated-worker-preflight.db",
format: "summary",
help: false,
jobId: "",
markdownOutputPath: "",
outputPath: "",
processOwnerPath: ".lime/qc/gui-process-owner-current.json",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--db" && argv[index + 1]) {
result.dbPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--job-id" && argv[index + 1]) {
result.jobId = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--process-owner" && argv[index + 1]) {
result.processOwnerPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--markdown-output" && argv[index + 1]) {
result.markdownOutputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check-terminal") {
result.checkTerminal = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC qcloop DB Lease Snapshot
用法:
npm run agent-qc:qcloop-db-lease -- --job-id <qcloop-job-id>
node scripts/agent-qc-qcloop-db-lease.mjs --db ./.lime/qc/qcloop-isolated-worker-preflight.db --job-id <id> --format json --output ./.lime/qc/qcloop-db-lease-current.json
选项:
--db PATH qcloop SQLite DB,默认 .lime/qc/qcloop-isolated-worker-preflight.db
--job-id ID qcloop job id,必填
--process-owner PATH raw process owner sidecar,默认 .lime/qc/gui-process-owner-current.json
--format FMT summary | json,默认 summary
--output PATH 写入文件;默认 stdout
--markdown-output PATH 额外写入 Markdown 摘要
--check-terminal job 未终态或仍有 running item 时非 0 退出
-h, --help 显示帮助
`);
}
function sqlQuote(value) {
return `'${String(value).replaceAll("'", "''")}'`;
}
function sqliteJson(dbPath, sql) {
const output = execFileSync("sqlite3", ["-readonly", "-json", dbPath, sql], {
encoding: "utf8",
}).trim();
return output ? JSON.parse(output) : [];
}
function readJsonIfExists(filePath) {
const resolved = path.resolve(process.cwd(), filePath);
if (!fs.existsSync(resolved)) {
return null;
}
return JSON.parse(fs.readFileSync(resolved, "utf8"));
}
function extractScenarioId(itemPreview) {
return String(itemPreview || "").match(/"scenario_id"\s*:\s*"([^"]+)"/)?.[1] || null;
}
function createDbLeaseReport({
dbPath,
generatedAt = new Date().toISOString(),
jobId,
processOwnerPath,
}) {
if (!jobId) {
throw new Error("--job-id is required");
}
const quotedJobId = sqlQuote(jobId);
const jobs = sqliteJson(
dbPath,
`select id, name, status, created_at, finished_at from batch_jobs where id=${quotedJobId}`,
);
const items = sqliteJson(
dbPath,
`select id, batch_job_id, status, current_attempt_no, current_qc_no, tokens_used, lock_owner, lock_expires_at, queued_at, last_error, created_at, finished_at, substr(item_value,1,700) as item_preview from batch_items where batch_job_id=${quotedJobId} order by created_at, id`,
);
const attempts = sqliteJson(
dbPath,
`select attempts.id, attempts.batch_item_id, attempts.attempt_no, attempts.run_no, attempts.attempt_type, attempts.status, attempts.exit_code, attempts.tokens_used, attempts.started_at, attempts.finished_at, length(attempts.stdout) as stdout_len, length(attempts.stderr) as stderr_len, substr(attempts.stdout,1,700) as stdout_preview, substr(attempts.stderr,1,700) as stderr_preview from attempts join batch_items on batch_items.id=attempts.batch_item_id where batch_items.batch_job_id=${quotedJobId} order by attempts.started_at, attempts.id`,
);
const processOwner = readJsonIfExists(processOwnerPath) || { qcloopProcesses: [] };
const processSnapshot = (processOwner.qcloopProcesses || []).filter((entry) => {
const command = String(entry?.command || "");
return command.includes(jobId) || command.includes("18080") || command.includes("browser-runtime-site-adapter");
});
const activeItem = items.find((item) => item.status === "running") || null;
const activeAttempt = activeItem
? attempts.find((attempt) => attempt.batch_item_id === activeItem.id && attempt.status === "running") || null
: null;
const observedWorker = processSnapshot.find((entry) =>
String(entry?.command || "").includes("browser-runtime-site-adapter"),
) || null;
const observedQcloop = processSnapshot.find((entry) =>
String(entry?.command || "").includes("serve --addr 127.0.0.1:18080"),
) || null;
const terminalStatuses = new Set(["completed", "failed", "cancelled", "canceled"]);
const jobStatus = jobs[0]?.status || "unknown";
const terminal = terminalStatuses.has(String(jobStatus).toLowerCase()) && !activeItem;
return {
schemaVersion: "v1",
generatedAt,
jobId,
dbPath,
processOwnerPath,
summary: {
status: jobStatus,
terminal,
activeItemId: activeItem?.id || null,
activeScenario: extractScenarioId(activeItem?.item_preview),
lockOwner: activeItem?.lock_owner || null,
lockExpiresAt: activeItem?.lock_expires_at || null,
activeAttemptId: activeAttempt?.id || null,
activeAttemptStatus: activeAttempt?.status || null,
stdoutLength: activeAttempt?.stdout_len ?? null,
stderrLength: activeAttempt?.stderr_len ?? null,
observedWorkerPid: observedWorker?.pid || null,
observedQcloopPid: observedQcloop?.pid || null,
finding: activeItem
? "qcloop DB lease, active attempt, and process snapshot still show a running no-output stale GUI owner; no intervention was performed."
: "No running item observed in this DB snapshot; verify qcloop status sidecar before changing gates.",
},
jobs,
items,
attempts,
processSnapshot,
guardrails: [
"read-only DB observation only",
"do not kill / pause / interrupt stale worker without owner confirmation",
"do not modify qcloop SQLite DB",
"do not start another full GUI P0 batch while GUI owner is blocked",
"do not overwrite .lime/qc/agent-qc-evidence.json before a real 8/8 P0 pass",
],
};
}
function renderSummary(report) {
const summary = report.summary;
return [
`status=${summary.status}`,
`terminal=${summary.terminal}`,
`activeItem=${summary.activeItemId || "none"}`,
`activeScenario=${summary.activeScenario || "none"}`,
`lockOwner=${summary.lockOwner || "none"}`,
`lockExpiresAt=${summary.lockExpiresAt || "none"}`,
`activeAttempt=${summary.activeAttemptId || "none"}:${summary.activeAttemptStatus || "none"}`,
`stdoutStderr=${summary.stdoutLength ?? "null"}/${summary.stderrLength ?? "null"}`,
`observedWorkerPid=${summary.observedWorkerPid || "none"}`,
`observedQcloopPid=${summary.observedQcloopPid || "none"}`,
`finding=${summary.finding}`,
].join("\n") + "\n";
}
function renderMarkdown(report) {
const summary = report.summary;
return [
"# qcloop DB lease snapshot",
"",
`- Generated at: ${report.generatedAt}`,
`- Job: ${report.jobId}`,
`- DB: ${report.dbPath}`,
`- Status: ${summary.status}`,
`- Terminal: ${summary.terminal}`,
`- Active item: ${summary.activeItemId || "none"}`,
`- Active scenario: ${summary.activeScenario || "none"}`,
`- Lock owner: ${summary.lockOwner || "none"}`,
`- Lock expires at: ${summary.lockExpiresAt || "none"}`,
`- Active attempt: ${summary.activeAttemptId || "none"} / ${summary.activeAttemptStatus || "none"}`,
`- stdout/stderr length: ${summary.stdoutLength ?? "null"} / ${summary.stderrLength ?? "null"}`,
`- Observed worker PID: ${summary.observedWorkerPid || "none"}`,
`- Observed qcloop PID: ${summary.observedQcloopPid || "none"}`,
"",
"## Finding",
"",
summary.finding,
"",
"## Guardrails",
"",
...report.guardrails.map((guardrail) => `- ${guardrail}`),
"",
].join("\n");
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const report = createDbLeaseReport(options);
const content = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderSummary(report);
writeOutput(options.outputPath, content);
if (options.markdownOutputPath) {
writeOutput(options.markdownOutputPath, renderMarkdown(report));
}
if (options.checkTerminal && !report.summary.terminal) {
process.exit(1);
}
}
main();
+195
View File
@@ -0,0 +1,195 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
buildQCLoopJobPayload,
renderQCLoopCurl,
validateQCLoopJobPayload,
} from "./lib/agent-qc-qcloop-job-core.mjs";
const DEFAULT_MANIFEST_PATH = "docs/test/agent-qc-scenarios.manifest.json";
function parseArgs(argv) {
const result = {
baseUrl: "http://127.0.0.1:8080",
check: false,
cwd: process.cwd(),
executorProvider: "codex",
executionMode: "standard",
format: "json",
help: false,
includeAll: false,
manifestPath: DEFAULT_MANIFEST_PATH,
maxQcRounds: 0,
maxExecutorRetries: undefined,
name: "",
outputPath: "",
risks: [],
scenarioIds: [],
tokenBudgetPerItem: 0,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--manifest" && argv[index + 1]) {
result.manifestPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--risk" && argv[index + 1]) {
result.risks.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--scenario" && argv[index + 1]) {
result.scenarioIds.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--all") {
result.includeAll = true;
continue;
}
if (arg === "--name" && argv[index + 1]) {
result.name = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--cwd" && argv[index + 1]) {
result.cwd = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--executor-provider" && argv[index + 1]) {
result.executorProvider = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--execution-mode" && argv[index + 1]) {
result.executionMode = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--max-qc-rounds" && argv[index + 1]) {
result.maxQcRounds = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--max-executor-retries" && argv[index + 1]) {
result.maxExecutorRetries = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--token-budget-per-item" && argv[index + 1]) {
result.tokenBudgetPerItem = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--base-url" && argv[index + 1]) {
result.baseUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC qcloop Job Payload
用法:
npm run agent-qc:qcloop-job -- --risk P0 --output ./tmp/qcloop-p0-job.json
node scripts/agent-qc-qcloop-job.mjs --scenario command-bridge-contract --format curl
选项:
--risk RISK 选择风险等级,可重复;默认 P0
--scenario ID 选择指定 scenario,可重复;优先级高于 --risk
--all 选择全部 scenario
--name NAME qcloop job 名称
--cwd PATH worker 目标仓库目录,默认当前目录
--executor-provider NAME qcloop executor_provider,默认 codex
--execution-mode MODE standard | goal_assisted,默认 standard
--max-qc-rounds N 覆盖 max_qc_rounds
--max-executor-retries N 覆盖 max_executor_retries,0-5
--token-budget-per-item N 覆盖 token_budget_per_item
--format FMT json | curl
--base-url URL curl 模式 qcloop API 地址,默认 http://127.0.0.1:8080
--output PATH 写入文件;默认 stdout
--check payload 非法时非 0 退出
-h, --help 显示帮助
`);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const manifest = readJsonFile(options.manifestPath);
const payload = buildQCLoopJobPayload(manifest, {
includeAll: options.includeAll,
risks: options.risks.length > 0 ? options.risks : ["P0"],
scenarioIds: options.scenarioIds,
name: options.name,
cwd: options.cwd,
executorProvider: options.executorProvider,
executionMode: options.executionMode,
maxQcRounds: options.maxQcRounds,
maxExecutorRetries: options.maxExecutorRetries,
tokenBudgetPerItem: options.tokenBudgetPerItem,
});
const validation = validateQCLoopJobPayload(payload);
const content =
options.format === "curl"
? renderQCLoopCurl(payload, { baseUrl: options.baseUrl })
: `${JSON.stringify({ ...payload, _validation: validation }, null, 2)}\n`;
writeOutput(options.outputPath, content);
if (options.check && !validation.valid) {
for (const issue of validation.issues) {
console.error(`[agent-qc-qcloop-job] ${issue}`);
}
process.exit(1);
}
}
main();
+149
View File
@@ -0,0 +1,149 @@
#!/usr/bin/env node
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { buildQCLoopPreflightReport } from "./lib/agent-qc-qcloop-preflight-core.mjs";
function parseArgs(argv) {
const result = {
check: false,
expectedCwd: "",
format: "summary",
help: false,
healthUrl: "http://127.0.0.1:3030/health",
requireDevBridge: false,
timeoutMs: 15000,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--expected-cwd" && argv[index + 1]) {
result.expectedCwd = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--health-url" && argv[index + 1]) {
result.healthUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--timeout-ms" && argv[index + 1]) {
result.timeoutMs = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--require-devbridge") {
result.requireDevBridge = true;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC qcloop Worker Preflight
用法:
npm run agent-qc:qcloop-preflight -- --check
npm run agent-qc:qcloop-preflight -- --require-devbridge --timeout-ms 15000 --check
选项:
--expected-cwd PATH 预期仓库 cwd;不传则只检查 cwd 存在
--require-devbridge 要求 http://127.0.0.1:3030/health 可访问
--health-url URL DevBridge health 地址,默认 http://127.0.0.1:3030/health
--timeout-ms N DevBridge health 超时,默认 15000
--format FMT summary | json
--check preflight blocked 时非 0 退出
-h, --help 显示帮助
`);
}
function checkTmpWritable() {
const filePath = path.join(os.tmpdir(), `lime-agent-qc-preflight-${process.pid}-${Date.now()}.tmp`);
try {
fs.writeFileSync(filePath, "ok", "utf8");
fs.unlinkSync(filePath);
return true;
} catch {
return false;
}
}
async function checkDevBridgeHealth({ url, timeoutMs }) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), Number.isFinite(timeoutMs) ? timeoutMs : 15000);
try {
const response = await fetch(url, { signal: controller.signal });
const body = await response.text();
let status = response.ok ? "ok" : `http-${response.status}`;
try {
const parsed = JSON.parse(body);
status = parsed?.status || status;
} catch {
// health body 不是 JSON 时保留 HTTP 状态。
}
return { ok: response.ok, status, url };
} catch (error) {
return { ok: false, error: `${error.name || "Error"}: ${error.message}`, url };
} finally {
clearTimeout(timeout);
}
}
function renderSummary(report) {
const lines = [
`status=${report.status}`,
`summary=${report.summary}`,
`QCLOOP_PREFLIGHT_RESULT=${report.status === "pass" ? "PASS" : "BLOCKED"}`,
];
for (const check of report.checks) {
lines.push(`- ${check.id} ${check.passed ? "PASS" : "BLOCKED"}: ${check.detail}`);
}
return `${lines.join("\n")}\n`;
}
async function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const devBridge = options.requireDevBridge
? await checkDevBridgeHealth({ url: options.healthUrl, timeoutMs: options.timeoutMs })
: null;
const report = buildQCLoopPreflightReport({
cwd: process.cwd(),
expectedCwd: options.expectedCwd,
tmpWritable: checkTmpWritable(),
devBridge,
});
const content = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderSummary(report);
process.stdout.write(content);
if (options.check && report.status !== "pass") {
process.exit(1);
}
}
main().catch((error) => {
console.error(`[agent-qc-qcloop-preflight] ${error.message}`);
process.exit(1);
});
+237
View File
@@ -0,0 +1,237 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
buildQCLoopStatusReport,
validateQCLoopStatusReport,
} from "./lib/agent-qc-qcloop-status-core.mjs";
function parseArgs(argv) {
const result = {
baseUrl: "http://127.0.0.1:8080",
checkTerminal: false,
failOnStale: false,
format: "summary",
help: false,
itemsJson: "",
jobId: "",
jobJson: "",
outputPath: "",
staleMinutes: 30,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--base-url" && argv[index + 1]) {
result.baseUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--job-id" && argv[index + 1]) {
result.jobId = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--job-json" && argv[index + 1]) {
result.jobJson = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--items-json" && argv[index + 1]) {
result.itemsJson = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--stale-minutes" && argv[index + 1]) {
result.staleMinutes = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check-terminal") {
result.checkTerminal = true;
continue;
}
if (arg === "--fail-on-stale") {
result.failOnStale = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC qcloop Status
用法:
npm run agent-qc:qcloop-status -- --job-id "qcloop-job-id"
npm run agent-qc:qcloop-status -- --job-id "qcloop-job-id" --format json --output ./.lime/qc/qcloop-status.json
node scripts/agent-qc-qcloop-status.mjs --job-json ./job.json --items-json ./items.json
选项:
--job-id ID 从 qcloop HTTP API 读取 /api/jobs/:id 和 /api/items?job_id=:id
--base-url URL qcloop API 地址,默认 http://127.0.0.1:8080
--job-json PATH 离线 job JSON 文件
--items-json PATH 离线 items JSON 文件
--output PATH 写入报告;默认 stdout
--stale-minutes N running item 超过 N 分钟且无 stdout/stderr 时标记 stale,默认 30
--format FMT 输出格式:summary | json
--check-terminal job 未 complete 或存在失败 / 卡住 item 时非 0 退出
--fail-on-stale 只要存在 stale item 就非 0 退出
-h, --help 显示帮助
`);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
}
async function readJsonUrl(url) {
const response = await fetch(url);
if (!response.ok) {
throw new Error(`请求失败 ${response.status}: ${url}`);
}
return response.json();
}
async function loadQCLoopData(options) {
if (options.jobJson || options.itemsJson) {
if (!options.jobJson || !options.itemsJson) {
throw new Error("离线模式必须同时提供 --job-json 与 --items-json。");
}
return {
job: readJsonFile(options.jobJson),
items: readJsonFile(options.itemsJson),
};
}
if (!options.jobId) {
throw new Error("必须提供 --job-id,或同时提供 --job-json / --items-json。");
}
const baseUrl = options.baseUrl.replace(/\/$/, "");
return {
job: await readJsonUrl(`${baseUrl}/api/jobs/${encodeURIComponent(options.jobId)}`),
items: await readJsonUrl(`${baseUrl}/api/items/?job_id=${encodeURIComponent(options.jobId)}`),
};
}
function renderItemLine(item) {
const parts = [
`- ${item.scenarioId}`,
`status=${item.qcloopStatus}`,
`worker=${item.worker.status}`,
`attempt=${item.currentAttemptNo}`,
`qc=${item.currentQcNo}`,
];
if (item.worker.durationMinutes !== null) {
parts.push(`duration=${item.worker.durationMinutes}m`);
}
parts.push(`stdout=${item.worker.stdoutLength}`);
parts.push(`stderr=${item.worker.stderrLength}`);
if (item.stale) {
parts.push(`stale=${item.staleReasons.join("; ")}`);
}
if (item.qc.feedback) {
parts.push(`feedback=${item.qc.feedback.replace(/\s+/g, " ").slice(0, 180)}`);
}
return parts.join(" ");
}
function renderSummary(report, validation) {
const lines = [
`job=${report.job.id}`,
`name=${report.job.name}`,
`status=${report.job.status}`,
`verdict=${report.verdict.status}`,
`items=${report.counts.total} terminal=${report.counts.terminal} nonTerminal=${report.counts.nonTerminal}`,
`success=${report.counts.success} failed=${report.counts.failed} exhausted=${report.counts.exhausted} running=${report.counts.running} pending=${report.counts.pending} stale=${report.counts.stale}`,
`summary=${report.verdict.summary}`,
`nextAction=${report.verdict.nextAction}`,
`valid=${validation.valid}`,
];
if (validation.issues.length > 0) {
lines.push(`issues=${validation.issues.join("; ")}`);
}
const activeItems = report.items.filter(
(item) =>
!item.terminal ||
item.stale ||
item.qcloopStatus === "failed" ||
item.qcloopStatus === "exhausted",
);
if (activeItems.length > 0) {
lines.push("items:");
for (const item of activeItems) {
lines.push(renderItemLine(item));
}
}
return `${lines.join("\n")}\n`;
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
async function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const { job, items } = await loadQCLoopData(options);
const report = buildQCLoopStatusReport({
job,
items,
options: {
staleMinutes: options.staleMinutes,
},
});
const validation = validateQCLoopStatusReport(report);
const content = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderSummary(report, validation);
writeOutput(options.outputPath, content);
if (!validation.valid) {
for (const issue of validation.issues) {
console.error(`[agent-qc-qcloop-status] ${issue}`);
}
process.exit(1);
}
if (options.failOnStale && report.counts.stale > 0) {
process.exit(2);
}
if (options.checkTerminal && report.verdict.status !== "complete") {
process.exit(3);
}
}
main().catch((error) => {
console.error(`[agent-qc-qcloop-status] ${error.message}`);
process.exit(1);
});
+184
View File
@@ -0,0 +1,184 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
buildAgentQcReleaseSummary,
renderAgentQcReleaseMarkdown,
validateReleaseSummary,
} from "./lib/agent-qc-release-summary-core.mjs";
function parseArgs(argv) {
const result = {
check: false,
evidencePaths: [],
format: "markdown",
harnessSummaryPath: "",
harnessTrendPath: "",
help: false,
outputPath: "",
requireEvidence: true,
requiredRisks: [],
requiredScenarioManifestPath: "",
tag: "",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--evidence" && argv[index + 1]) {
result.evidencePaths.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--harness-summary" && argv[index + 1]) {
result.harnessSummaryPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--harness-trend" && argv[index + 1]) {
result.harnessTrendPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--tag" && argv[index + 1]) {
result.tag = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--require-scenario-manifest" && argv[index + 1]) {
result.requiredScenarioManifestPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--require-risk" && argv[index + 1]) {
result.requiredRisks.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--allow-missing-evidence") {
result.requireEvidence = false;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC Release Summary
用法:
npm run agent-qc:release-summary -- --evidence ./.lime/qc/agent-qc-evidence.json --require-scenario-manifest docs/test/agent-qc-scenarios.manifest.json --require-risk P0 --tag v1.2.3
node scripts/agent-qc-release-summary.mjs --evidence evidence.json --harness-summary summary.json --harness-trend trend.json --output release-qc.md --check
选项:
--evidence PATH Agent QC Evidence Pack,可重复
--harness-summary PATH harness-eval-summary.json
--harness-trend PATH harness-eval-trend.json
--tag TAG release tag
--require-scenario-manifest PATH 要求 Evidence Pack 覆盖该 manifest 中的场景
--require-risk RISK 搭配 --require-scenario-manifest 使用,可重复;默认 P0
--output PATH 写入文件;默认 stdout
--format FMT markdown | json
--check 非 pass 或缺证据时非 0 退出
--allow-missing-evidence 允许无 Evidence Pack,适合本地预览,不适合发布门禁
-h, --help 显示帮助
`);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
}
function readOptionalJson(filePath) {
if (!filePath) {
return null;
}
return readJsonFile(filePath);
}
function loadRequiredScenarioIds(manifestPath, risks) {
if (!manifestPath) {
return [];
}
const manifest = readJsonFile(manifestPath);
const requiredRisks = new Set((risks.length > 0 ? risks : ["P0"]).map((risk) => risk.toUpperCase()));
return Array.isArray(manifest?.scenarios)
? manifest.scenarios
.filter((scenario) => requiredRisks.has(String(scenario?.risk || "").toUpperCase()))
.map((scenario) => String(scenario?.id || "").trim())
.filter(Boolean)
: [];
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const evidencePacks = options.evidencePaths.map((sourcePath) => ({
sourcePath,
pack: readJsonFile(sourcePath),
}));
const requiredScenarioIds = loadRequiredScenarioIds(
options.requiredScenarioManifestPath,
options.requiredRisks,
);
const summary = buildAgentQcReleaseSummary({
evidencePacks,
harnessSummary: readOptionalJson(options.harnessSummaryPath),
harnessTrend: readOptionalJson(options.harnessTrendPath),
requiredScenarioIds,
tag: options.tag,
});
const validation = validateReleaseSummary(summary, {
requireEvidence: options.requireEvidence,
});
const content =
options.format === "json"
? `${JSON.stringify({ ...summary, validation }, null, 2)}\n`
: renderAgentQcReleaseMarkdown(summary);
writeOutput(options.outputPath, content);
if (options.check && !validation.valid) {
for (const issue of validation.issues) {
console.error(`[agent-qc-release] ${issue}`);
}
process.exit(1);
}
}
main();
+119
View File
@@ -0,0 +1,119 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
createAgentQcReport,
readJsonFile,
renderAgentQcMarkdownReport,
} from "./lib/agent-qc-report-core.mjs";
const DEFAULT_MANIFEST_PATH = "docs/test/agent-qc-scenarios.manifest.json";
const DEFAULT_SCHEMA_PATH = "docs/test/agent-qc-evidence.schema.json";
function parseArgs(argv) {
const result = {
check: false,
format: "markdown",
help: false,
manifestPath: DEFAULT_MANIFEST_PATH,
outputPath: "",
schemaPath: DEFAULT_SCHEMA_PATH,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--manifest" && argv[index + 1]) {
result.manifestPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--schema" && argv[index + 1]) {
result.schemaPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Agent QC 场景报告
用法:
npm run agent-qc:report
npm run agent-qc:report:json
npm run agent-qc:check
node scripts/agent-qc-report.mjs --manifest docs/test/agent-qc-scenarios.manifest.json
选项:
--check 如果 manifest 或 evidence schema 不合法,以非 0 退出
--format FMT 输出格式:markdown | json
--manifest PATH Agent QC manifest 路径
--schema PATH Evidence schema 路径
--output PATH 写入报告文件;默认输出到 stdout
-h, --help 显示帮助
`);
}
function resolvePath(targetPath) {
return path.resolve(process.cwd(), targetPath);
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = resolvePath(outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const manifest = readJsonFile(resolvePath(options.manifestPath));
const schema = readJsonFile(resolvePath(options.schemaPath));
const packageJson = readJsonFile(resolvePath("package.json"));
const report = createAgentQcReport({ manifest, packageJson, evidenceSchema: schema });
const content =
options.format === "json"
? `${JSON.stringify(report, null, 2)}\n`
: renderAgentQcMarkdownReport(report);
writeOutput(options.outputPath, content);
if (options.check && !report.valid) {
process.exit(1);
}
}
main();
@@ -0,0 +1,652 @@
#!/usr/bin/env node
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
import {
buildApprovalSandboxSmokeEvidence,
renderApprovalSandboxTranscriptLines,
} from "./lib/agent-runtime-approval-sandbox-smoke-core.mjs";
import { runVitestSmoke } from "./lib/vitest-smoke-runner.mjs";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, "..");
const DEFAULT_OUTPUT = path.join(
rootDir,
".lime/qc/runtime-approval-sandbox-smoke.json",
);
const DEFAULT_HEALTH_URL = "http://127.0.0.1:3030/health";
const DEFAULT_INVOKE_URL = "http://127.0.0.1:3030/invoke";
const DEFAULT_TIMEOUT_MS = 60_000;
const DEFAULT_INTERVAL_MS = 1_000;
const DEFAULT_PROVIDER_PREFERENCE =
process.env.LIME_AGENT_QC_PROVIDER ||
process.env.LIME_E2E_PROVIDER ||
process.env.LIME_DEFAULT_PROVIDER ||
"";
const DEFAULT_MODEL_PREFERENCE =
process.env.LIME_AGENT_QC_MODEL ||
process.env.LIME_E2E_MODEL ||
process.env.LIME_DEFAULT_MODEL ||
"";
const APPROVAL_POLICY = "on-request";
const SANDBOX_POLICY = "workspace-write";
const PROVIDER_PICK_ORDER = ["deepseek", "doubao", "lime-hub"];
function printHelp() {
console.log(`
Lime Agent Runtime Approval / Sandbox Smoke
用途:
生成 tool / approval / sandbox 边界的确定性 smoke 证据,补齐 qcloop P0 场景的可审查摘要。
用法:
node scripts/agent-runtime-approval-sandbox-smoke.mjs [选项]
选项:
--output <path> 写入 evidence JSON,默认 ./.lime/qc/runtime-approval-sandbox-smoke.json
--health-url <url> DevBridge health 地址,默认 ${DEFAULT_HEALTH_URL}
--invoke-url <url> DevBridge invoke 地址,默认 ${DEFAULT_INVOKE_URL}
--timeout-ms <ms> live runtime 等待超时,默认 ${DEFAULT_TIMEOUT_MS}
--interval-ms <ms> live runtime 轮询间隔,默认 ${DEFAULT_INTERVAL_MS}
--provider-preference <id> live runtime 使用的 provider 偏好;默认读取 LIME_AGENT_QC_PROVIDER / LIME_E2E_PROVIDER,未设置时自动选本地启用 provider
--model-preference <name> live runtime 使用的 model 偏好;默认读取 LIME_AGENT_QC_MODEL / LIME_E2E_MODEL,未设置时自动选 provider 的首个自定义模型
--skip-live-runtime 跳过 DevBridge live runtime transcript,仅生成确定性投影摘要
--no-write 只运行校验并打印摘要,不写 evidence JSON
-h, --help 显示帮助
`);
}
function parseArgs(argv) {
const options = {
output: DEFAULT_OUTPUT,
healthUrl: DEFAULT_HEALTH_URL,
invokeUrl: DEFAULT_INVOKE_URL,
timeoutMs: DEFAULT_TIMEOUT_MS,
intervalMs: DEFAULT_INTERVAL_MS,
providerPreference: DEFAULT_PROVIDER_PREFERENCE,
modelPreference: DEFAULT_MODEL_PREFERENCE,
liveRuntime: true,
write: true,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--output" && argv[index + 1]) {
options.output = path.resolve(rootDir, String(argv[index + 1]));
index += 1;
continue;
}
if (arg === "--no-write") {
options.write = false;
continue;
}
if (arg === "--health-url" && argv[index + 1]) {
options.healthUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--invoke-url" && argv[index + 1]) {
options.invokeUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--timeout-ms" && argv[index + 1]) {
options.timeoutMs = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--interval-ms" && argv[index + 1]) {
options.intervalMs = Number(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--provider-preference" && argv[index + 1]) {
options.providerPreference = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--model-preference" && argv[index + 1]) {
options.modelPreference = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--skip-live-runtime" || arg === "--no-live-runtime") {
options.liveRuntime = false;
continue;
}
if (arg === "--help" || arg === "-h") {
printHelp();
process.exit(0);
}
}
if (!Number.isFinite(options.timeoutMs) || options.timeoutMs < 10_000) {
throw new Error("--timeout-ms 必须是 >= 10000 的数字");
}
if (!Number.isFinite(options.intervalMs) || options.intervalMs < 100) {
throw new Error("--interval-ms 必须是 >= 100 的数字");
}
return options;
}
function runVitest(label, args) {
return runVitestSmoke({
rootDir,
label,
args,
logPrefix: "smoke:agent-runtime-approval-sandbox",
});
}
function writeEvidence(outputPath, evidence) {
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
fs.writeFileSync(outputPath, `${JSON.stringify(evidence, null, 2)}\n`);
return outputPath;
}
function writeEvidenceWithFallback(outputPath, evidence) {
try {
return writeEvidence(outputPath, evidence);
} catch (error) {
if (outputPath !== DEFAULT_OUTPUT) {
throw error;
}
const fallbackPath = path.join(os.tmpdir(), "lime-runtime-approval-sandbox-smoke.json");
const writtenPath = writeEvidence(fallbackPath, evidence);
console.warn(
`[smoke:agent-runtime-approval-sandbox] 默认 evidence 写入失败,已回退到 ${writtenPath}: ${error.message}`,
);
return writtenPath;
}
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function assert(condition, message) {
if (!condition) {
throw new Error(message);
}
}
async function waitForHealth(options) {
const startedAt = Date.now();
let lastError = null;
while (Date.now() - startedAt < options.timeoutMs) {
try {
const response = await fetch(options.healthUrl, {
method: "GET",
signal: AbortSignal.timeout(Math.min(options.intervalMs, 5_000)),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
const payload = text ? JSON.parse(text) : {};
console.log(
`[smoke:agent-runtime-approval-sandbox] DevBridge 已就绪 (${Date.now() - startedAt}ms)${
payload?.status ? ` status=${payload.status}` : ""
}`,
);
return payload;
} catch (error) {
lastError = error;
await sleep(options.intervalMs);
}
}
const detail =
lastError instanceof Error
? lastError.message
: String(lastError || "unknown error");
throw new Error(
`[smoke:agent-runtime-approval-sandbox] DevBridge 未就绪,无法采集 live runtime transcript: ${detail}`,
);
}
async function invoke(options, cmd, args, timeoutMs = options.timeoutMs) {
const response = await fetch(options.invokeUrl, {
method: "POST",
headers: {
"content-type": "application/json",
},
body: JSON.stringify({ cmd, args }),
signal: AbortSignal.timeout(timeoutMs),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`${cmd} HTTP ${response.status}: ${text}`);
}
const payload = text ? JSON.parse(text) : null;
if (payload?.error) {
throw new Error(`${cmd} error: ${payload.error}`);
}
return payload?.result;
}
function normalizeProviderId(provider) {
return String(provider?.id || provider?.provider_id || provider?.providerId || "")
.trim();
}
function providerEnabled(provider) {
return provider?.enabled !== false;
}
function pickModelPreference(provider) {
const candidates = [
...(Array.isArray(provider?.custom_models) ? provider.custom_models : []),
...(Array.isArray(provider?.customModels) ? provider.customModels : []),
...(Array.isArray(provider?.models) ? provider.models : []),
]
.map((value) =>
typeof value === "string"
? value
: String(value?.name || value?.id || value?.model || "").trim(),
)
.filter(Boolean);
return (
candidates.find((value) => /flash|mini|lite/i.test(value)) ||
candidates[0] ||
""
);
}
function pickProvider(providers, preferredProviderId) {
const enabled = providers.filter((provider) => providerEnabled(provider));
if (preferredProviderId) {
return (
enabled.find((provider) => normalizeProviderId(provider) === preferredProviderId) ||
providers.find((provider) => normalizeProviderId(provider) === preferredProviderId) ||
null
);
}
for (const providerId of PROVIDER_PICK_ORDER) {
const match = enabled.find((provider) => normalizeProviderId(provider) === providerId);
if (match) {
return match;
}
}
return enabled[0] || null;
}
async function resolveProviderPreference(options) {
const explicitProvider = String(options.providerPreference || "").trim();
const explicitModel = String(options.modelPreference || "").trim();
if (explicitProvider && explicitModel) {
return {
providerPreference: explicitProvider,
modelPreference: explicitModel,
source: "explicit",
};
}
const providers = await invoke(options, "get_api_key_providers", {}, 30_000);
const selected = pickProvider(Array.isArray(providers) ? providers : [], explicitProvider);
const providerId = normalizeProviderId(selected);
if (!providerId) {
throw new Error(
"[smoke:agent-runtime-approval-sandbox] 未找到可用 provider;请传 --provider-preference / --model-preference 或先配置本地 provider",
);
}
let providerDetail = selected;
try {
providerDetail =
(await invoke(options, "get_api_key_provider", { id: providerId }, 30_000)) ||
selected;
} catch (error) {
console.warn(
`[smoke:agent-runtime-approval-sandbox] 读取 provider 详情失败,使用列表摘要继续: ${error.message}`,
);
}
const modelPreference = explicitModel || pickModelPreference(providerDetail);
if (!modelPreference) {
throw new Error(
`[smoke:agent-runtime-approval-sandbox] provider ${providerId} 缺少可用模型;请传 --model-preference`,
);
}
return {
providerPreference: providerId,
modelPreference,
source: explicitProvider || explicitModel ? "partial-explicit" : "auto-enabled-provider",
};
}
function pickPermissionState(threadRead) {
return threadRead?.permission_state || threadRead?.permissionState || {};
}
function pendingRequests(threadRead) {
return threadRead?.pending_requests || threadRead?.pendingRequests || [];
}
function latestTurnStatus(threadRead) {
return (
threadRead?.diagnostics?.latest_turn_status ||
threadRead?.diagnostics?.latestTurnStatus ||
threadRead?.runtime_summary?.latestTurnStatus ||
threadRead?.runtimeSummary?.latestTurnStatus ||
threadRead?.status ||
null
);
}
function permissionField(permissionState, snakeKey, camelKey) {
return permissionState?.[snakeKey] ?? permissionState?.[camelKey] ?? null;
}
function summarizeThreadRead(threadRead) {
const permissionState = pickPermissionState(threadRead);
const requests = pendingRequests(threadRead);
return {
threadStatus: threadRead?.status || null,
latestTurnStatus: latestTurnStatus(threadRead),
primaryBlockingKind:
threadRead?.diagnostics?.primary_blocking_kind ||
threadRead?.diagnostics?.primaryBlockingKind ||
null,
primaryBlockingSummary:
threadRead?.diagnostics?.primary_blocking_summary ||
threadRead?.diagnostics?.primaryBlockingSummary ||
null,
pendingRequestCount: requests.length,
permissionStatus: permissionField(permissionState, "status", "status"),
confirmationStatus: permissionField(
permissionState,
"confirmation_status",
"confirmationStatus",
),
confirmationRequestId: permissionField(
permissionState,
"confirmation_request_id",
"confirmationRequestId",
),
confirmationSource: permissionField(
permissionState,
"confirmation_source",
"confirmationSource",
),
askProfileKeys:
permissionField(permissionState, "ask_profile_keys", "askProfileKeys") || [],
requiredProfileKeys:
permissionField(
permissionState,
"required_profile_keys",
"requiredProfileKeys",
) || [],
};
}
async function waitForThreadRead(options, sessionId, predicate, label) {
const startedAt = Date.now();
let lastSummary = null;
while (Date.now() - startedAt < options.timeoutMs) {
const threadRead = await invoke(options, "agent_runtime_get_thread_read", {
sessionId,
});
lastSummary = summarizeThreadRead(threadRead);
if (predicate(threadRead, lastSummary)) {
return { threadRead, summary: lastSummary };
}
await sleep(options.intervalMs);
}
throw new Error(
`[smoke:agent-runtime-approval-sandbox] ${label} 超时,最后线程摘要: ${JSON.stringify(lastSummary)}`,
);
}
function buildRuntimeContractMetadata() {
return {
harness: {
browser_assist: {
runtime_contract: {
contract_key: "browser_control",
routing_slot: "browser_reasoning_model",
execution_profile: {
profile_key: "browser_control_profile",
},
executor_adapter: {
adapter_key: "browser:browser_assist",
},
executor_binding: {
executor_kind: "browser",
binding_key: "browser_assist",
},
},
},
},
};
}
async function runPermissionDecisionFlow(options, workspaceId, providerPreference, decision) {
const confirmed = decision === "resolved";
const responseLabel = confirmed ? "允许本次执行" : "拒绝";
const sessionId = await invoke(options, "agent_runtime_create_session", {
workspaceId,
name: `Agent QC approval ${decision} ${Date.now()}`,
runStartHooks: false,
});
const turnId = `qc-approval-${decision}-${Date.now()}-${process.pid}`;
const eventName = `aster_stream_${sessionId}_${turnId}`;
const submittedPolicies = {
approvalPolicy: APPROVAL_POLICY,
sandboxPolicy: SANDBOX_POLICY,
};
await invoke(options, "agent_runtime_submit_turn", {
request: {
message:
"Agent QC runtime permission confirmation smoke:应在模型执行前创建真实权限确认请求。",
session_id: sessionId,
workspace_id: workspaceId,
event_name: eventName,
turn_id: turnId,
turn_config: {
provider_preference: providerPreference.providerPreference,
model_preference: providerPreference.modelPreference,
approval_policy: APPROVAL_POLICY,
sandbox_policy: SANDBOX_POLICY,
metadata: buildRuntimeContractMetadata(),
},
skip_pre_submit_resume: true,
},
});
const requested = await waitForThreadRead(
options,
sessionId,
(_threadRead, summary) =>
summary.confirmationStatus === "requested" &&
summary.pendingRequestCount > 0 &&
String(summary.confirmationRequestId || "").includes(turnId),
`等待 ${decision} flow 进入权限确认 requested`,
);
const requestId = requested.summary.confirmationRequestId;
assert(requestId, `${decision} flow 缺少 confirmationRequestId`);
await invoke(options, "agent_runtime_respond_action", {
request: {
session_id: sessionId,
request_id: requestId,
action_type: "elicitation",
confirmed,
response: JSON.stringify({ answer: responseLabel }),
user_data: { answer: responseLabel },
event_name: eventName,
action_scope: {
session_id: sessionId,
thread_id: sessionId,
turn_id: turnId,
},
},
});
const completed = await waitForThreadRead(
options,
sessionId,
(_threadRead, summary) =>
summary.pendingRequestCount === 0 &&
summary.confirmationStatus === decision,
`等待 ${decision} flow 写回确认结果`,
);
return {
decision: confirmed ? "resolved" : "denied",
sessionId,
turnId,
requestId,
providerPreference,
submittedPolicies,
before: requested.summary,
respond: {
confirmed,
responseLabel,
},
after: completed.summary,
};
}
async function collectLiveRuntimeTranscript(options) {
const health = await waitForHealth(options);
const providerPreference = await resolveProviderPreference(options);
console.log(
`[smoke:agent-runtime-approval-sandbox] live runtime provider: provider=${providerPreference.providerPreference} model=${providerPreference.modelPreference} source=${providerPreference.source}`,
);
const workspace = await invoke(options, "get_or_create_default_project");
const workspaceId = workspace?.id;
assert(workspaceId, "get_or_create_default_project 缺少 workspace id");
const deniedFlow = await runPermissionDecisionFlow(
options,
workspaceId,
providerPreference,
"denied",
);
const resolvedFlow = await runPermissionDecisionFlow(
options,
workspaceId,
providerPreference,
"resolved",
);
const flows = [deniedFlow, resolvedFlow];
return {
kind: "devbridge-runtime-permission-confirmation",
health,
workspaceId,
providerPreference,
flows,
assertions: {
devBridgeHealthy: health?.status === "ok" || Boolean(health),
permissionRequestCreatedBeforeModel: flows.every(
(flow) =>
flow.before.permissionStatus === "requires_confirmation" &&
flow.before.confirmationStatus === "requested" &&
flow.before.pendingRequestCount > 0 &&
flow.before.latestTurnStatus === "failed" &&
String(flow.before.confirmationRequestId || "").includes(flow.turnId),
),
deniedDecisionClearsPendingRequest:
deniedFlow.after.confirmationStatus === "denied" &&
deniedFlow.after.pendingRequestCount === 0,
resolvedDecisionClearsPendingRequest:
resolvedFlow.after.confirmationStatus === "resolved" &&
resolvedFlow.after.pendingRequestCount === 0,
approvalPolicySubmitted: flows.every(
(flow) => flow.submittedPolicies.approvalPolicy === APPROVAL_POLICY,
),
sandboxPolicySubmitted: flows.every(
(flow) => flow.submittedPolicies.sandboxPolicy === SANDBOX_POLICY,
),
},
};
}
async function main() {
const options = parseArgs(process.argv.slice(2));
const commandResults = [];
commandResults.push(
runVitest("submit preferences 应透传 approval / sandbox policy", [
"src/components/agent/chat/utils/buildUserInputSubmitOp.test.ts",
"src/components/agent/chat/hooks/agentStreamUserInputSubmission.test.ts",
"src/components/agent/chat/hooks/agentStreamSubmitExecution.test.ts",
"src/components/agent/chat/hooks/agentStreamSubmitOpController.test.ts",
]),
);
commandResults.push(
runVitest("runtime projection 应保留 permission / tool lifecycle", [
"src/components/agent/chat/projection/agentUiEventProjection.test.ts",
"-t",
"runtime permission metadata|工具输入、成功输出和失败输出|plan approval",
]),
);
commandResults.push(
runVitest("消息与时间线不应把权限确认误渲染为失败", [
"src/components/agent/chat/components/AgentThreadTimeline.test.tsx",
"src/components/agent/chat/components/MessageList.test.tsx",
"-t",
"运行时权限确认",
]),
);
commandResults.push(
runVitest("Harness 面板应展示工具权限、待审批与 sandbox 来源", [
"src/components/agent/chat/components/HarnessStatusPanel.test.tsx",
"-t",
"存在工具库存时应展示工具与权限区块及来源统计|待审批区块应通过 artifact protocol",
]),
);
const liveRuntimeTranscript = options.liveRuntime
? await collectLiveRuntimeTranscript(options)
: null;
const evidence = buildApprovalSandboxSmokeEvidence({
commandResults,
generatedAt: new Date().toISOString(),
liveRuntimeTranscript,
});
if (options.liveRuntime && !evidence.coverage.liveRuntimeTranscript) {
throw new Error(
"[smoke:agent-runtime-approval-sandbox] live runtime transcript 未满足发布门禁断言",
);
}
for (const line of renderApprovalSandboxTranscriptLines(evidence)) {
console.log(line);
}
if (options.write) {
const evidencePath = writeEvidenceWithFallback(options.output, evidence);
console.log(`\n[smoke:agent-runtime-approval-sandbox] evidence: ${evidencePath}`);
}
console.log("\n[smoke:agent-runtime-approval-sandbox] 通过");
}
main().catch((error) => {
console.error(
error instanceof Error ? error.message : String(error || "unknown error"),
);
process.exit(1);
});
+10 -22
View File
@@ -1,36 +1,21 @@
#!/usr/bin/env node
import { spawnSync } from "node:child_process";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
import { runVitestSmoke } from "./lib/vitest-smoke-runner.mjs";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, "..");
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
function runVitest(label, args) {
console.log(`\n[smoke:agent-runtime-tool-surface] > ${label}`);
const result = spawnSync(
npmCommand,
["exec", "--", "vitest", "run", ...args],
{
cwd: rootDir,
stdio: "inherit",
env: process.env,
},
);
if (result.error) {
throw result.error;
}
if (typeof result.status === "number" && result.status !== 0) {
const error = new Error(`[smoke:agent-runtime-tool-surface] ${label} 失败`);
error.exitCode = result.status;
throw error;
}
return runVitestSmoke({
rootDir,
label,
args,
logPrefix: "smoke:agent-runtime-tool-surface",
});
}
function main() {
@@ -50,6 +35,9 @@ function main() {
"src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts",
]);
console.log(
'[smoke:agent-runtime-tool-surface] surface.summary: runtime inventory/strip/harness inventory 已一致透传;unsafeToolExposed=false; sources=runtime_tools|persisted_tools|default_policy',
);
console.log("\n[smoke:agent-runtime-tool-surface] 通过");
}
@@ -35,7 +35,55 @@ function runVitest(label, args) {
}
}
function runCargoTest(label, args) {
console.log(`\n[smoke:agent-service-skill-entry] > ${label}`);
const result = spawnSync("cargo", ["test", ...args], {
cwd: path.join(rootDir, "src-tauri"),
stdio: "inherit",
env: process.env,
});
if (result.error) {
throw result.error;
}
if (typeof result.status === "number" && result.status !== 0) {
const error = new Error(
`[smoke:agent-service-skill-entry] ${label} 失败`,
);
error.exitCode = result.status;
throw error;
}
}
function main() {
runVitest("Skill Forge 前端 metadata 与工作台显式启用链路", [
"src/lib/api/capabilityDrafts.test.ts",
"src/lib/api/agentRuntime/inventoryClient.test.ts",
"src/components/agent/chat/utils/workspaceSkillBindingsMetadata.test.ts",
"src/components/agent/chat/utils/harnessRequestMetadata.test.ts",
"src/features/capability-drafts/workspaceSkillAgentAutomationDraft.test.ts",
]);
[
"register_capability_draft_persists_readonly_http_preflight_provenance",
"registered_skill_becomes_ready_for_manual_enable_binding_candidate",
"explicit_runtime_enable_projects_ready_binding_allowlist",
"registered_skill_without_verification_provenance_is_blocked",
"execute_capability_draft_controlled_get_returns_evidence_without_persisting_inputs",
"should_project_workspace_skill_runtime_enable_as_callable_scope",
"allowlisted_session_should_preserve_workspace_skill_source_metadata",
"disabled_session_should_fail_execute",
].forEach((testName) => {
runCargoTest(`Skill Forge Rust 定向测试: ${testName}`, [
"--manifest-path",
"Cargo.toml",
testName,
"--",
"--exact",
]);
});
runVitest("服务技能入口路由与挂起参数", [
"src/components/skills/SkillsWorkspacePage.test.tsx",
"src/components/agent/chat/workspace/useWorkspaceServiceSkillEntryActions.test.tsx",
+52 -3
View File
@@ -25,7 +25,9 @@ const INVOKE_RETRY_DELAY_MS = 1_000;
const POST_HEALTH_SETTLE_MS = 3_000;
const POST_LAUNCH_SETTLE_MS = 1_500;
const READ_PAGE_TIMEOUT_MS = 45_000;
const SMOKE_PROFILE_KEY = "smoke-browser-runtime";
const DEFAULT_SMOKE_PROFILE_KEY =
process.env.LIME_BROWSER_RUNTIME_SMOKE_PROFILE_KEY ||
`smoke-browser-runtime-${process.pid}`;
function printHelp() {
console.log(`
@@ -43,6 +45,7 @@ Lime Browser Runtime Smoke
--timeout-ms <ms> 等待健康检查超时,默认 90000
--interval-ms <ms> 健康检查轮询间隔,默认 1000
--launch-url <url> 启动浏览器会话的 URL,默认使用内置 data: 测试页
--profile-key <key> smoke 专用浏览器 profile key,默认按进程隔离
--open-window 显式打开浏览器窗口
--headless 以无界面浏览器会话执行 smoke,避免弹出空白 Chrome
--stream-mode <mode> events | frames | both,默认 both
@@ -85,6 +88,11 @@ function parseArgs(argv) {
index += 1;
continue;
}
if (arg === "--profile-key" && argv[index + 1]) {
options.profileKey = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--open-window") {
options.openWindow = true;
continue;
@@ -111,6 +119,9 @@ function parseArgs(argv) {
if (!options.launchUrl) {
throw new Error("--launch-url 不能为空");
}
if (!options.profileKey) {
options.profileKey = DEFAULT_SMOKE_PROFILE_KEY;
}
return options;
}
@@ -239,8 +250,9 @@ async function main() {
await waitForHealth(options);
await sleep(POST_HEALTH_SETTLE_MS);
const profileKey = SMOKE_PROFILE_KEY;
const profileKey = options.profileKey;
let sessionId = null;
let cleanupStatus = "skipped";
try {
await closeSmokeProfileSession(
@@ -310,6 +322,38 @@ async function main() {
"browser_execute_action 未返回对应的 target_id",
);
const consoleResult = await invoke(options, "browser_execute_action", {
request: {
profile_key: profileKey,
action: "read_console_messages",
args: { since: 0 },
timeout_ms: Math.min(options.timeoutMs, READ_PAGE_TIMEOUT_MS),
},
});
assert(
consoleResult?.success === true,
"browser_execute_action(read_console_messages) 未成功",
);
const consoleCount = Array.isArray(consoleResult?.data?.messages)
? consoleResult.data.messages.length
: 0;
const networkResult = await invoke(options, "browser_execute_action", {
request: {
profile_key: profileKey,
action: "read_network_requests",
args: { since: 0 },
timeout_ms: Math.min(options.timeoutMs, READ_PAGE_TIMEOUT_MS),
},
});
assert(
networkResult?.success === true,
"browser_execute_action(read_network_requests) 未成功",
);
const networkCount = Array.isArray(networkResult?.data?.events)
? networkResult.data.events.length
: 0;
const auditLogs = await invoke(options, "get_browser_action_audit_logs", {
limit: 10,
});
@@ -344,7 +388,7 @@ async function main() {
);
console.log(
`[smoke:browser-runtime] 通过 session=${sessionId} target=${sessionState.target_id} profile=${profileKey}`,
`[smoke:browser-runtime] 通过 session=${sessionId} target=${sessionState.target_id} profile=${profileKey} consoleEvents=${consoleCount} networkEvents=${networkCount}`,
);
} finally {
if (sessionId) {
@@ -354,7 +398,9 @@ async function main() {
session_id: sessionId,
},
});
cleanupStatus = "pass";
} catch (error) {
cleanupStatus = "failed";
console.warn(
`[smoke:browser-runtime] 清理会话失败: ${
error instanceof Error ? error.message : String(error)
@@ -368,6 +414,9 @@ async function main() {
profileKey,
"关闭 smoke profile 失败",
);
console.log(
`[smoke:browser-runtime] cleanup=${cleanupStatus} session=${sessionId ?? "none"} profile=${profileKey}`,
);
}
}
File diff suppressed because it is too large Load Diff
+109
View File
@@ -0,0 +1,109 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import {
analyzeTranslations,
applyTranslationFixes,
formatTranslationReport,
hasTranslationIssues,
} from "./detect-missing-translations";
const tempDirs: string[] = [];
function createTempResourcesDir() {
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lime-i18n-check-"));
tempDirs.push(dir);
return dir;
}
function writeResource(
resourcesDir: string,
locale: string,
namespace: string,
resource: Record<string, unknown>,
) {
const filePath = path.join(resourcesDir, locale, `${namespace}.json`);
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, `${JSON.stringify(resource, null, 2)}\n`);
}
afterEach(() => {
for (const dir of tempDirs.splice(0)) {
fs.rmSync(dir, { force: true, recursive: true });
}
});
describe("detect-missing-translations", () => {
it("应在所有 locale 与 source key 一致时通过", () => {
const resourcesDir = createTempResourcesDir();
writeResource(resourcesDir, "zh-CN", "common", { "common.save": "保存" });
writeResource(resourcesDir, "en-US", "common", { "common.save": "Save" });
const result = analyzeTranslations({ resourcesDir });
expect(hasTranslationIssues(result)).toBe(false);
expect(formatTranslationReport(result)).toContain("通过");
});
it("应发现缺失 namespace、缺失 key 与多余 key", () => {
const resourcesDir = createTempResourcesDir();
writeResource(resourcesDir, "zh-CN", "common", {
"common.cancel": "取消",
"common.save": "保存",
});
writeResource(resourcesDir, "zh-CN", "settings", {
"settings.title": "设置",
});
writeResource(resourcesDir, "en-US", "common", {
"common.extra": "Extra",
"common.save": "Save",
});
const result = analyzeTranslations({ resourcesDir });
expect(hasTranslationIssues(result)).toBe(true);
expect(result.issues).toEqual([
{
locale: "en-US",
missingNamespaces: ["settings"],
extraNamespaces: [],
namespaces: [
{
namespace: "common",
missingKeys: ["common.cancel"],
extraKeys: ["common.extra"],
},
],
},
]);
});
it("--fix 语义应补齐缺失 namespace 与缺失 key,但保留人工已有翻译", () => {
const resourcesDir = createTempResourcesDir();
writeResource(resourcesDir, "zh-CN", "common", {
"common.cancel": "取消",
"common.save": "保存",
});
writeResource(resourcesDir, "zh-CN", "settings", {
"settings.title": "设置",
});
writeResource(resourcesDir, "en-US", "common", {
"common.save": "Save",
});
applyTranslationFixes({ resourcesDir });
const result = analyzeTranslations({ resourcesDir });
const common = JSON.parse(
fs.readFileSync(path.join(resourcesDir, "en-US", "common.json"), "utf8"),
) as Record<string, string>;
expect(hasTranslationIssues(result)).toBe(false);
expect(common["common.save"]).toBe("Save");
expect(common["common.cancel"]).toBe("取消");
expect(
fs.existsSync(path.join(resourcesDir, "en-US", "settings.json")),
).toBe(true);
});
});
+356
View File
@@ -0,0 +1,356 @@
#!/usr/bin/env tsx
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath, pathToFileURL } from "node:url";
export interface TranslationCheckOptions {
resourcesDir: string;
sourceLocale?: string;
}
export interface NamespaceIssue {
namespace: string;
missingKeys: string[];
extraKeys: string[];
}
export interface LocaleIssue {
locale: string;
missingNamespaces: string[];
extraNamespaces: string[];
namespaces: NamespaceIssue[];
}
export interface TranslationCheckResult {
resourcesDir: string;
sourceLocale: string;
locales: string[];
namespaces: string[];
sourceKeyCount: number;
issues: LocaleIssue[];
}
interface CliOptions extends TranslationCheckOptions {
fix: boolean;
verbose: boolean;
}
const DEFAULT_SOURCE_LOCALE = "zh-CN";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const REPO_ROOT = path.resolve(__dirname, "..");
const DEFAULT_RESOURCES_DIR = path.join(REPO_ROOT, "src", "i18n", "resources");
function isRecord(value: unknown): value is Record<string, unknown> {
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
}
function readJsonObject(filePath: string): Record<string, unknown> {
const raw = fs.readFileSync(filePath, "utf8");
const parsed = JSON.parse(raw) as unknown;
if (!isRecord(parsed)) {
throw new Error(`Translation resource must be a JSON object: ${filePath}`);
}
return parsed;
}
function listJsonNamespaces(localeDir: string): string[] {
if (!fs.existsSync(localeDir)) {
return [];
}
return fs
.readdirSync(localeDir, { withFileTypes: true })
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
.map((entry) => path.basename(entry.name, ".json"))
.sort((left, right) => left.localeCompare(right));
}
function listLocaleDirs(resourcesDir: string): string[] {
if (!fs.existsSync(resourcesDir)) {
throw new Error(`Translation resources directory does not exist: ${resourcesDir}`);
}
return fs
.readdirSync(resourcesDir, { withFileTypes: true })
.filter((entry) => entry.isDirectory())
.map((entry) => entry.name)
.sort((left, right) => left.localeCompare(right));
}
export function flattenResource(
resource: Record<string, unknown>,
prefix = "",
): Record<string, unknown> {
const flattened: Record<string, unknown> = {};
for (const [key, value] of Object.entries(resource)) {
const nextKey = prefix ? `${prefix}.${key}` : key;
if (isRecord(value)) {
Object.assign(flattened, flattenResource(value, nextKey));
} else {
flattened[nextKey] = value;
}
}
return flattened;
}
function sortedDifference(left: Iterable<string>, right: Set<string>): string[] {
return [...left]
.filter((item) => !right.has(item))
.sort((a, b) => a.localeCompare(b));
}
function readNamespaceKeys(localeDir: string, namespace: string): Set<string> {
const filePath = path.join(localeDir, `${namespace}.json`);
if (!fs.existsSync(filePath)) {
return new Set();
}
return new Set(Object.keys(flattenResource(readJsonObject(filePath))));
}
function countSourceKeys(sourceLocaleDir: string, namespaces: string[]): number {
return namespaces.reduce(
(total, namespace) => total + readNamespaceKeys(sourceLocaleDir, namespace).size,
0,
);
}
export function hasTranslationIssues(result: TranslationCheckResult): boolean {
return result.issues.some(
(issue) =>
issue.missingNamespaces.length > 0 ||
issue.extraNamespaces.length > 0 ||
issue.namespaces.some(
(namespaceIssue) =>
namespaceIssue.missingKeys.length > 0 || namespaceIssue.extraKeys.length > 0,
),
);
}
export function analyzeTranslations(
options: TranslationCheckOptions,
): TranslationCheckResult {
const resourcesDir = path.resolve(options.resourcesDir);
const sourceLocale = options.sourceLocale || DEFAULT_SOURCE_LOCALE;
const locales = listLocaleDirs(resourcesDir);
const sourceLocaleDir = path.join(resourcesDir, sourceLocale);
if (!locales.includes(sourceLocale)) {
throw new Error(`Source locale is missing from resources: ${sourceLocale}`);
}
const namespaces = listJsonNamespaces(sourceLocaleDir);
const sourceNamespaceSet = new Set(namespaces);
const issues: LocaleIssue[] = [];
for (const locale of locales) {
if (locale === sourceLocale) {
continue;
}
const localeDir = path.join(resourcesDir, locale);
const localeNamespaces = listJsonNamespaces(localeDir);
const localeNamespaceSet = new Set(localeNamespaces);
const missingNamespaces = sortedDifference(namespaces, localeNamespaceSet);
const extraNamespaces = sortedDifference(localeNamespaces, sourceNamespaceSet);
const namespaceIssues: NamespaceIssue[] = [];
for (const namespace of namespaces) {
if (!localeNamespaceSet.has(namespace)) {
continue;
}
const sourceKeys = readNamespaceKeys(sourceLocaleDir, namespace);
const targetKeys = readNamespaceKeys(localeDir, namespace);
const missingKeys = sortedDifference(sourceKeys, targetKeys);
const extraKeys = sortedDifference(targetKeys, sourceKeys);
if (missingKeys.length > 0 || extraKeys.length > 0) {
namespaceIssues.push({ namespace, missingKeys, extraKeys });
}
}
if (
missingNamespaces.length > 0 ||
extraNamespaces.length > 0 ||
namespaceIssues.length > 0
) {
issues.push({
locale,
missingNamespaces,
extraNamespaces,
namespaces: namespaceIssues,
});
}
}
return {
resourcesDir,
sourceLocale,
locales,
namespaces,
sourceKeyCount: countSourceKeys(sourceLocaleDir, namespaces),
issues,
};
}
function sortObjectByKey(resource: Record<string, unknown>): Record<string, unknown> {
return Object.fromEntries(
Object.entries(resource).sort(([left], [right]) => left.localeCompare(right)),
);
}
function writeJsonObject(filePath: string, resource: Record<string, unknown>): void {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
fs.writeFileSync(filePath, `${JSON.stringify(sortObjectByKey(resource), null, 2)}${os.EOL}`);
}
export function applyTranslationFixes(options: TranslationCheckOptions): void {
const resourcesDir = path.resolve(options.resourcesDir);
const sourceLocale = options.sourceLocale || DEFAULT_SOURCE_LOCALE;
const sourceLocaleDir = path.join(resourcesDir, sourceLocale);
const sourceNamespaces = listJsonNamespaces(sourceLocaleDir);
const locales = listLocaleDirs(resourcesDir).filter((locale) => locale !== sourceLocale);
for (const locale of locales) {
const localeDir = path.join(resourcesDir, locale);
for (const namespace of sourceNamespaces) {
const sourcePath = path.join(sourceLocaleDir, `${namespace}.json`);
const targetPath = path.join(localeDir, `${namespace}.json`);
const sourceResource = flattenResource(readJsonObject(sourcePath));
const targetResource = fs.existsSync(targetPath)
? flattenResource(readJsonObject(targetPath))
: {};
let changed = false;
for (const [key, value] of Object.entries(sourceResource)) {
if (!(key in targetResource)) {
targetResource[key] = value;
changed = true;
}
}
if (changed || !fs.existsSync(targetPath)) {
writeJsonObject(targetPath, targetResource);
}
}
}
}
function formatList(items: string[]): string {
return items.length === 0 ? "-" : items.join(", ");
}
export function formatTranslationReport(
result: TranslationCheckResult,
options: { verbose?: boolean } = {},
): string {
const lines: string[] = [];
const issueCount = result.issues.length;
lines.push(
`[i18n:check] resources=${path.relative(REPO_ROOT, result.resourcesDir) || result.resourcesDir} source=${result.sourceLocale} locales=${result.locales.length} namespaces=${result.namespaces.length} sourceKeys=${result.sourceKeyCount}`,
);
if (!hasTranslationIssues(result)) {
lines.push("[i18n:check] 通过:所有 locale 与 source namespace/key 保持一致。");
if (options.verbose) {
lines.push(`[i18n:check] locale 列表: ${result.locales.join(", ")}`);
lines.push(`[i18n:check] namespace 列表: ${result.namespaces.join(", ")}`);
}
return lines.join(os.EOL);
}
lines.push(`[i18n:check] 发现 ${issueCount} 个 locale 存在翻译结构差异。`);
for (const localeIssue of result.issues) {
lines.push(`[i18n:check] locale=${localeIssue.locale}`);
if (localeIssue.missingNamespaces.length > 0) {
lines.push(
` missing namespaces: ${formatList(localeIssue.missingNamespaces)}`,
);
}
if (localeIssue.extraNamespaces.length > 0) {
lines.push(` extra namespaces: ${formatList(localeIssue.extraNamespaces)}`);
}
for (const namespaceIssue of localeIssue.namespaces) {
lines.push(` namespace=${namespaceIssue.namespace}`);
if (namespaceIssue.missingKeys.length > 0) {
lines.push(` missing keys: ${formatList(namespaceIssue.missingKeys)}`);
}
if (namespaceIssue.extraKeys.length > 0) {
lines.push(` extra keys: ${formatList(namespaceIssue.extraKeys)}`);
}
}
}
return lines.join(os.EOL);
}
function parseCliArgs(argv: string[]): CliOptions {
const options: CliOptions = {
fix: false,
resourcesDir: DEFAULT_RESOURCES_DIR,
sourceLocale: DEFAULT_SOURCE_LOCALE,
verbose: false,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
const next = argv[index + 1];
if (arg === "--fix") {
options.fix = true;
continue;
}
if (arg === "--verbose") {
options.verbose = true;
continue;
}
if (arg === "--resources-dir" && next) {
options.resourcesDir = next;
index += 1;
continue;
}
if (arg === "--source-locale" && next) {
options.sourceLocale = next;
index += 1;
continue;
}
if (arg === "-h" || arg === "--help") {
console.log(`Usage: npm run detect-translations -- [--fix] [--verbose] [--resources-dir <dir>] [--source-locale <locale>]`);
process.exit(0);
}
throw new Error(`Unknown argument: ${arg}`);
}
return options;
}
export function runCli(argv = process.argv.slice(2)): number {
const options = parseCliArgs(argv);
if (options.fix) {
applyTranslationFixes(options);
}
const result = analyzeTranslations(options);
console.log(formatTranslationReport(result, { verbose: options.verbose }));
return hasTranslationIssues(result) ? 1 : 0;
}
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
try {
process.exitCode = runCli();
} catch (error) {
console.error(
`[i18n:check] 失败:${error instanceof Error ? error.message : String(error)}`,
);
process.exitCode = 1;
}
}
+160
View File
@@ -0,0 +1,160 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
import {
createI18nPatchMetricsReport,
renderI18nPatchMetricsTextReport,
} from "./lib/i18n-patch-metrics-report-core.mjs";
const DEFAULT_INPUT_PATH = ".lime/i18n/patch-metrics.json";
function parseNumberArg(value) {
if (value === undefined) {
return undefined;
}
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : undefined;
}
function parseArgs(argv) {
const result = {
check: false,
format: "text",
help: false,
inputPath: DEFAULT_INPUT_PATH,
maxMatchedSegments: undefined,
maxReplacedNodes: undefined,
maxRuns: undefined,
outputPath: "",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--check") {
result.check = true;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--input" && argv[index + 1]) {
result.inputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--max-matched-segments" && argv[index + 1]) {
result.maxMatchedSegments = parseNumberArg(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--max-replaced-nodes" && argv[index + 1]) {
result.maxReplacedNodes = parseNumberArg(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--max-runs" && argv[index + 1]) {
result.maxRuns = parseNumberArg(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime i18n Patch Metrics Report
用法:
npm run i18n:patch-report
npm run i18n:patch-report:json
node scripts/i18n-patch-metrics-report.mjs --input .lime/i18n/patch-metrics.json --format json
node scripts/i18n-patch-metrics-report.mjs --check --max-matched-segments 0 --max-replaced-nodes 0
输入:
默认读取 ${DEFAULT_INPUT_PATH}。该文件应来自 GUI / Playwright 导出的 window.__I18N_METRICS__ 或 getI18nPatchMetricsReport() JSON。
选项:
--input PATH Patch metrics JSON 路径
--output PATH 写入报告文件;默认输出到 stdout
--format FMT 输出格式:text | json
--check 如果门限问题存在,以非 0 退出
--max-matched-segments NUM 允许的最大命中文本段数
--max-replaced-nodes NUM 允许的最大替换节点数
--max-runs NUM 允许的最大 Patch 运行次数
-h, --help 显示帮助
`);
}
function resolvePath(targetPath) {
return path.resolve(process.cwd(), targetPath);
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
}
function writeOutput(outputPath, content) {
if (!outputPath) {
process.stdout.write(content);
return;
}
const resolvedOutputPath = resolvePath(outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(resolvedOutputPath, content, "utf8");
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const resolvedInputPath = resolvePath(options.inputPath);
const metrics = readJsonFile(resolvedInputPath);
const report = createI18nPatchMetricsReport({
metrics,
sourcePath: path.relative(process.cwd(), resolvedInputPath),
thresholds: {
maxMatchedSegments: options.maxMatchedSegments,
maxReplacedNodes: options.maxReplacedNodes,
maxRuns: options.maxRuns,
},
});
const content =
options.format === "json"
? `${JSON.stringify(report, null, 2)}\n`
: renderI18nPatchMetricsTextReport(report);
writeOutput(options.outputPath, content);
if (options.check && report.thresholdIssues.length > 0) {
process.exit(1);
}
}
main();
+43
View File
@@ -12,6 +12,7 @@ const DEFAULTS = {
invokeUrl: "http://127.0.0.1:3030/invoke",
timeoutMs: 180_000,
intervalMs: 1_000,
i18nPatchMetricsOutput: "",
};
const INVOKE_TIMEOUT_CEILING_MS = 180_000;
@@ -135,6 +136,8 @@ Lime Knowledge GUI Smoke
--invoke-url <url> DevBridge invoke 地址,默认 http://127.0.0.1:3030/invoke
--timeout-ms <ms> 总超时,默认 180000
--interval-ms <ms> 轮询间隔,默认 1000
--i18n-patch-metrics-output <path>
导出 window.__I18N_METRICS__ JSON,供 i18n:patch-report 消费
-h, --help 显示帮助
`);
}
@@ -175,6 +178,12 @@ function parseArgs(argv) {
continue;
}
if (arg === "--i18n-patch-metrics-output" && argv[index + 1]) {
options.i18nPatchMetricsOutput = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--help" || arg === "-h") {
printHelp();
process.exit(0);
@@ -202,6 +211,39 @@ function logStage(label) {
console.log(`[smoke:knowledge-gui] stage=${label}`);
}
async function exportI18nPatchMetrics(page, outputPath) {
if (!outputPath) {
return;
}
try {
const metrics = await page.evaluate(() => {
const globalMetrics = window.__I18N_METRICS__;
if (!globalMetrics) {
return null;
}
return JSON.parse(JSON.stringify(globalMetrics));
});
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
fs.writeFileSync(
resolvedOutputPath,
JSON.stringify(metrics ?? {}, null, 2),
"utf8",
);
console.log(
`[smoke:knowledge-gui] i18n Patch metrics: ${resolvedOutputPath}`,
);
} catch (error) {
console.warn(
`[smoke:knowledge-gui] 导出 i18n Patch metrics 失败: ${
error instanceof Error ? error.message : String(error)
}`,
);
}
}
function isTransientInvokeError(error) {
return (
error?.name === "TimeoutError" ||
@@ -1096,6 +1138,7 @@ async function runPlaywrightGuiFlow(options) {
);
}
} finally {
await exportI18nPatchMetrics(page, options.i18nPatchMetricsOutput);
await context.close().catch(() => undefined);
fs.rmSync(userDataDir, { recursive: true, force: true });
}
+16 -1
View File
@@ -302,8 +302,23 @@ async function openKnowledgeOverview(page, options) {
text.includes("存到哪里?") ||
text.includes("整理新资料")
) {
if (text.includes("项目资料状态说明")) {
const backButton = page.getByRole("button", {
name: "回到项目资料",
exact: true,
});
if (await backButton.isVisible().catch(() => false)) {
await backButton.click({ timeout: options.timeoutMs });
await waitText(page, options, "状态说明返回项目资料首页", [
"让 Lime 记住这个项目",
"项目资料清单",
]);
return;
}
}
await clickSideNav(page, options, "灵感");
await waitText(page, options, "灵感页", ["灵感"]);
await waitText(page, options, "灵感页", ["收藏过的想法"]);
}
await clickSideNav(page, options, "项目资料");
@@ -0,0 +1,292 @@
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function normalizeIdList(value) {
return asArray(value)
.map((item) => String(item || "").trim())
.filter(Boolean);
}
function createItem(id, title, passed, evidence, gap = "") {
return { id, title, passed: Boolean(passed), evidence: evidence || "", gap: passed ? "" : gap };
}
function summarizeQcloopStatusSidecar(entry) {
const counts = entry.counts || {};
return [
`${entry.path} verdict=${entry.verdictStatus || "unknown"}`,
`success=${counts.success ?? 0}`,
`running=${counts.running ?? 0}`,
`pending=${counts.pending ?? 0}`,
`stale=${counts.stale ?? 0}`,
].join(" ");
}
function buildAgentQcCompletionAudit(facts) {
const requiredQcloopScenarioIds = normalizeIdList(facts.scenarioReport?.p0ScenarioIds);
const requiredQcloopScenarioCount =
requiredQcloopScenarioIds.length || facts.scenarioReport?.p0ScenarioCount || 1;
const realEvidenceScenarioIds = new Set(normalizeIdList(facts.realEvidencePack?.scenarioIds));
const missingQcloopScenarioIds = requiredQcloopScenarioIds.filter(
(scenarioId) => !realEvidenceScenarioIds.has(scenarioId),
);
const hasRequiredQcloopCoverage =
requiredQcloopScenarioIds.length > 0
? missingQcloopScenarioIds.length === 0
: facts.realEvidencePack?.scenarioCount >= requiredQcloopScenarioCount;
const sidecarEvidenceSummary = asArray(facts.realEvidenceSidecars)
.map((entry) => `${entry.path} status=${entry.status || "unknown"} scenarios=${entry.scenarioCount || 0}`)
.join("; ");
const qcloopStatusSidecarSummary = asArray(facts.qcloopStatusSidecars)
.map((entry) => {
const counts = entry.counts
? ` success=${entry.counts.success ?? 0} running=${entry.counts.running ?? 0} pending=${entry.counts.pending ?? 0} stale=${entry.counts.stale ?? 0}`
: "";
return `${entry.path} verdict=${entry.verdictStatus || "unknown"}${counts}`;
})
.join("; ");
const staleQcloopStatusSidecars = asArray(facts.qcloopStatusSidecars).filter(
(entry) => entry.verdictStatus === "stale" || Number(entry.counts?.stale || 0) > 0,
);
const activeQcloopStatusSidecars = asArray(facts.qcloopStatusSidecars).filter(
(entry) =>
entry.verdictStatus === "running" ||
entry.verdictStatus === "stale" ||
Number(entry.counts?.running || 0) > 0 ||
Number(entry.counts?.pending || 0) > 0,
);
const qcloopEvidenceText = facts.realEvidencePack?.exists
? `.lime/qc/agent-qc-evidence.json status=${facts.realEvidencePack.status || "unknown"} scenarios=${facts.realEvidencePack.scenarioCount || 0}/${requiredQcloopScenarioCount}${missingQcloopScenarioIds.length > 0 ? ` missing=${missingQcloopScenarioIds.join(",")}` : ""}`
: sidecarEvidenceSummary
? `未发现 .lime/qc/agent-qc-evidence.json;sidecars: ${sidecarEvidenceSummary}`
: "未发现 .lime/qc/agent-qc-evidence.json";
const qcloopEvidenceWithStatusText = qcloopStatusSidecarSummary
? `${qcloopEvidenceText}; qcloopStatus: ${qcloopStatusSidecarSummary}`
: qcloopEvidenceText;
const qcloopGapFragments = [];
if (facts.realEvidencePack?.exists && facts.realEvidencePack.status !== "pass") {
qcloopGapFragments.push(
`官方 Evidence Pack 当前 status=${facts.realEvidencePack.status || "unknown"},不能发布`,
);
}
if (!facts.realEvidencePack?.exists && sidecarEvidenceSummary) {
qcloopGapFragments.push("已有 sidecar Evidence Pack,但尚未生成 pass 的官方 .lime/qc/agent-qc-evidence.json");
}
if (missingQcloopScenarioIds.length > 0) {
qcloopGapFragments.push(`尚未覆盖 P0 场景:${missingQcloopScenarioIds.join(",")}`);
}
if (activeQcloopStatusSidecars.length > 0) {
qcloopGapFragments.push(
`仍有 qcloop status sidecar 未终态:${activeQcloopStatusSidecars
.map(summarizeQcloopStatusSidecar)
.join("; ")}`,
);
}
if (staleQcloopStatusSidecars.length > 0) {
qcloopGapFragments.push(
`其中 stale sidecar:${staleQcloopStatusSidecars.map((entry) => entry.path).join(", ")}`,
);
}
const qcloopGapText =
qcloopGapFragments.length > 0
? `${qcloopGapFragments.join(";")}。`
: "尚未运行真实 qcloop 批次并导出 pass Evidence Pack。";
const localVerifyEvidenceText = facts.localVerify?.status
? `status=${facts.localVerify.status}${facts.localVerify.failedStage ? ` failedStage=${facts.localVerify.failedStage}` : ""}`
: "未发现 .lime/qc/verify-local-current.json";
const guiSmokeEvidenceText = facts.guiSmoke?.status
? `; latestGuiSmoke status=${facts.guiSmoke.status}${facts.guiSmoke.failedStage ? ` failedStage=${facts.guiSmoke.failedStage}` : ""}`
: "";
const localVerifyGapFragments = [];
if (facts.localVerify?.status) {
localVerifyGapFragments.push(
`verify:local 当前 status=${facts.localVerify.status}${facts.localVerify.error ? `;${facts.localVerify.error}` : ""}`,
);
} else {
localVerifyGapFragments.push("缺少 verify:local 当前结果 sidecar,无法证明仓库统一本地校验通过");
}
if (facts.guiSmoke?.status && facts.guiSmoke.status !== "pass") {
localVerifyGapFragments.push(
`最近一次 verify:gui-smoke status=${facts.guiSmoke.status}${facts.guiSmoke.error ? `;${facts.guiSmoke.error}` : ""}`,
);
}
const items = [
createItem(
"docs-tests-standard",
"docs/tests 下存在 Agent QC 人读测试文档",
facts.files?.agentOpsQc &&
facts.files?.p0Scenarios &&
facts.files?.limeRolloutPlan &&
facts.files?.testsReadme,
"docs/tests/agent-ops-qc.md, docs/tests/agent-qc-p0-scenarios.md, docs/tests/lime-agent-qc-rollout-plan.md, docs/tests/README.md",
"缺少 docs/tests 测试体系文档。",
),
createItem(
"scenario-manifest",
"Agent QC scenario manifest 有效",
facts.scenarioReport?.valid === true && facts.scenarioReport?.scenarioCount > 0,
`scenarioCount=${facts.scenarioReport?.scenarioCount ?? 0}`,
"scenario manifest 未通过校验或没有场景。",
),
createItem(
"gui-flow-manifest",
"GUI / Playwright MCP flow manifest 有效",
facts.guiFlowReport?.valid === true && facts.guiFlowReport?.flowCount > 0,
`flowCount=${facts.guiFlowReport?.flowCount ?? 0}`,
"GUI flow manifest 未通过校验或没有 flow。",
),
createItem(
"evidence-schema",
"Agent QC Evidence Pack schema 存在",
facts.files?.evidenceSchema,
"docs/test/agent-qc-evidence.schema.json",
"缺少 Evidence Pack schema。",
),
createItem(
"qcloop-payload-generator",
"可从 manifest 生成 qcloop job payload",
facts.files?.qcloopJobScript && facts.qcloopPayload?.valid === true && facts.qcloopPayload?.itemCount > 0,
`itemCount=${facts.qcloopPayload?.itemCount ?? 0}`,
"qcloop payload 生成器不可用或没有 item。",
),
createItem(
"qcloop-verifier-evidence-placeholders",
"qcloop verifier prompt 带 worker evidence 占位符",
facts.qcloopPayload?.verifierHasWorkerOutput === true &&
facts.qcloopPayload?.verifierHasAttemptStatus === true &&
facts.qcloopPayload?.verifierHasExitCode === true,
`stdout=${Boolean(facts.qcloopPayload?.verifierHasWorkerOutput)} attempt_status=${Boolean(facts.qcloopPayload?.verifierHasAttemptStatus)} exit_code=${Boolean(facts.qcloopPayload?.verifierHasExitCode)}`,
"qcloop verifier prompt 缺少 {{stdout}} / {{attempt_status}} / {{exit_code}},verifier 无法审查 worker 证据。",
),
createItem(
"structured-evidence-contract",
"qcloop worker / verifier / exporter 强制结构化 evidence summary",
facts.files?.evidenceContractDoc &&
facts.qcloopPayload?.workerPromptHasStructuredEvidence === true &&
facts.qcloopPayload?.verifierRequiresStructuredEvidence === true &&
facts.qcloopPayload?.verifierRequiresStrictJson === true &&
facts.structuredEvidence?.exporterParsesSummary === true &&
facts.structuredEvidence?.releaseSummaryRejectsWeakRefs === true,
`doc=${Boolean(facts.files?.evidenceContractDoc)} worker=${Boolean(facts.qcloopPayload?.workerPromptHasStructuredEvidence)} verifier=${Boolean(facts.qcloopPayload?.verifierRequiresStructuredEvidence)} strictJson=${Boolean(facts.qcloopPayload?.verifierRequiresStrictJson)} exporter=${Boolean(facts.structuredEvidence?.exporterParsesSummary)} release=${Boolean(facts.structuredEvidence?.releaseSummaryRejectsWeakRefs)}`,
"结构化 evidence summary 契约未被文档、payload worker prompt、verifier prompt、exporter 与 release summary gate 同时强制,可能再次接受浅层 qcloop success。",
),
createItem(
"qcloop-evidence-exporter",
"可把 qcloop job/items 导出为 Evidence Pack",
facts.files?.exportEvidenceScript,
"scripts/agent-qc-export-evidence.mjs",
"缺少 qcloop Evidence Pack 导出入口。",
),
createItem(
"qcloop-status-monitor",
"可只读监控 qcloop 运行批次和 stale item",
facts.files?.qcloopStatusScript && facts.files?.qcloopOperationsDoc,
"scripts/agent-qc-qcloop-status.mjs, docs/tests/lime-agent-qc-qcloop-operations.md",
"缺少 qcloop 只读状态监控脚本或运维手册。",
),
createItem(
"gui-owner-check",
"可在启动 GUI P0 前阻断并发 qcloop GUI owner",
facts.files?.guiOwnerCheckScript && facts.files?.qcloopOperationsDoc,
"scripts/agent-qc-gui-owner-check.mjs, docs/tests/lime-agent-qc-qcloop-operations.md",
"缺少 GUI owner 并发检查脚本或运维手册。",
),
createItem(
"stale-owner-intervention-protocol",
"stale GUI qcloop owner 有只读取证和 owner 确认协议",
facts.files?.staleOwnerInterventionDoc &&
facts.files?.qcloopOperationsDoc &&
facts.staleOwnerIntervention?.guiOwnerReportHasDecisionPacket === true &&
facts.staleOwnerIntervention?.docMentionsDecisionPacket === true &&
facts.staleOwnerIntervention?.guiOwnerReportHasWatchHistory === true &&
facts.staleOwnerIntervention?.docMentionsWatchHistory === true,
`doc=${Boolean(facts.files?.staleOwnerInterventionDoc)} operations=${Boolean(facts.files?.qcloopOperationsDoc)} ownerIntervention=${Boolean(facts.staleOwnerIntervention?.guiOwnerReportHasDecisionPacket)} docDecisionPacket=${Boolean(facts.staleOwnerIntervention?.docMentionsDecisionPacket)} watchHistory=${Boolean(facts.staleOwnerIntervention?.guiOwnerReportHasWatchHistory)} docWatchHistory=${Boolean(facts.staleOwnerIntervention?.docMentionsWatchHistory)}`,
"缺少 stale GUI owner 处置协议、机器可读 ownerIntervention 或 watch history 输出;容易在未确认时误杀 worker、启动并发 GUI P0,或丢失长期观察证据。",
),
createItem(
"qcloop-worker-preflight",
"qcloop worker 执行前有 cwd / tmp / DevBridge preflight",
facts.files?.qcloopPreflightScript &&
facts.qcloopPayload?.workerPromptHasPreflight === true,
`script=${Boolean(facts.files?.qcloopPreflightScript)} prompt=${Boolean(facts.qcloopPayload?.workerPromptHasPreflight)}`,
"缺少 qcloop worker preflight 脚本,或 payload prompt 没有要求执行前置环境检查。",
),
createItem(
"release-summary",
"可把 Evidence Pack 汇总为 release note 质量证据",
facts.files?.releaseSummaryScript,
"scripts/agent-qc-release-summary.mjs",
"缺少 release summary 入口。",
),
createItem(
"nightly-artifacts",
"nightly 上传 Agent QC 标准资产报告",
facts.nightly?.hasAgentQcReport && facts.nightly?.hasGuiFlowReport && facts.nightly?.hasReleasePreview,
".github/workflows/harness-nightly.yml artifacts/agent-qc/*",
"nightly 未完整上传 Agent QC report / GUI flow report / release preview。",
),
createItem(
"real-qcloop-evidence",
"存在真实 qcloop Agent QC Evidence Pack",
facts.realEvidencePack?.status === "pass" &&
hasRequiredQcloopCoverage,
qcloopEvidenceWithStatusText,
qcloopGapText,
),
createItem(
"real-gui-evidence",
"存在真实 GUI / Playwright MCP evidence",
facts.files?.realGuiEvidence,
facts.files?.realGuiEvidence ? ".lime/qc/gui-evidence" : "未发现 .lime/qc/gui-evidence",
"尚未执行真实 GUI / Playwright MCP flow 并保存证据。",
),
createItem(
"release-hard-gate",
"release workflow 强制 Evidence Pack pass 且覆盖 P0 场景",
facts.release?.hasHardGate === true && facts.release?.requiresP0ScenarioCoverage === true,
facts.release?.hasHardGate && facts.release?.requiresP0ScenarioCoverage
? ".github/workflows/release.yml contains agent-qc release gate and P0 scenario coverage"
: "release.yml 未完整强制 agent-qc release gate / P0 scenario coverage",
"release workflow 尚未强制真实 Evidence Pack pass 或 P0 场景覆盖。",
),
createItem(
"local-verify-gate",
"仓库统一本地校验 verify:local 通过",
facts.localVerify?.status === "pass",
`${localVerifyEvidenceText}${guiSmokeEvidenceText}`,
`${localVerifyGapFragments.join(";")}。`,
),
];
const passedCount = items.filter((item) => item.passed).length;
const failed = items.filter((item) => !item.passed);
return {
status: failed.length === 0 ? "complete" : "incomplete",
passedCount,
failedCount: failed.length,
totalCount: items.length,
completionRatio: items.length === 0 ? 0 : passedCount / items.length,
items,
gaps: failed.map((item) => ({ id: item.id, title: item.title, gap: item.gap })),
};
}
function renderAgentQcCompletionAuditMarkdown(audit) {
const lines = audit.items.map((item) => {
const marker = item.passed ? "PASS" : "MISS";
return `- ${marker} ${item.id}: ${item.title};证据:${item.evidence}${item.gap ? `;缺口:${item.gap}` : ""}`;
});
return `# Agent QC Completion Audit
- Status: ${audit.status}
- Passed: ${audit.passedCount}/${audit.totalCount}
- Completion: ${Math.round(audit.completionRatio * 100)}%
## Checklist
${lines.join("\n")}
`;
}
export { buildAgentQcCompletionAudit, renderAgentQcCompletionAuditMarkdown };
@@ -0,0 +1,299 @@
import { describe, expect, it } from "vitest";
import {
buildAgentQcCompletionAudit,
renderAgentQcCompletionAuditMarkdown,
} from "./agent-qc-completion-audit-core.mjs";
function completeFacts() {
return {
files: {
agentOpsQc: true,
p0Scenarios: true,
limeRolloutPlan: true,
testsReadme: true,
evidenceSchema: true,
qcloopJobScript: true,
guiOwnerCheckScript: true,
qcloopStatusScript: true,
qcloopPreflightScript: true,
qcloopOperationsDoc: true,
evidenceContractDoc: true,
staleOwnerInterventionDoc: true,
exportEvidenceScript: true,
releaseSummaryScript: true,
realGuiEvidence: true,
},
realEvidencePack: {
exists: true,
status: "pass",
scenarioCount: 1,
scenarioIds: ["command-bridge-contract"],
},
localVerify: {
status: "pass",
failedStage: "",
error: "",
},
realEvidenceSidecars: [],
qcloopStatusSidecars: [],
scenarioReport: {
valid: true,
scenarioCount: 12,
p0ScenarioCount: 1,
p0ScenarioIds: ["command-bridge-contract"],
},
guiFlowReport: { valid: true, flowCount: 4 },
qcloopPayload: {
valid: true,
itemCount: 8,
verifierHasWorkerOutput: true,
verifierHasAttemptStatus: true,
verifierHasExitCode: true,
workerPromptHasPreflight: true,
workerPromptHasStructuredEvidence: true,
verifierRequiresStructuredEvidence: true,
verifierRequiresStrictJson: true,
},
structuredEvidence: {
exporterParsesSummary: true,
releaseSummaryRejectsWeakRefs: true,
},
staleOwnerIntervention: {
guiOwnerReportHasDecisionPacket: true,
docMentionsDecisionPacket: true,
guiOwnerReportHasWatchHistory: true,
docMentionsWatchHistory: true,
},
nightly: { hasAgentQcReport: true, hasGuiFlowReport: true, hasReleasePreview: true },
release: { hasHardGate: true, requiresP0ScenarioCoverage: true },
};
}
describe("agent-qc-completion-audit-core", () => {
it("所有事实齐全时应标记 complete", () => {
const audit = buildAgentQcCompletionAudit(completeFacts());
expect(audit.status).toBe("complete");
expect(audit.failedCount).toBe(0);
});
it("缺真实 evidence 时应保持 incomplete", () => {
const facts = completeFacts();
facts.realEvidencePack = { exists: false, status: "", scenarioCount: 0, scenarioIds: [] };
facts.files.realGuiEvidence = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("real-qcloop-evidence");
expect(audit.gaps.map((gap) => gap.id)).toContain("real-gui-evidence");
});
it("缺正式 evidence 但存在 sidecar evidence 时应在审计证据中显示 sidecar 状态", () => {
const facts = completeFacts();
facts.realEvidencePack = { exists: false, status: "", scenarioCount: 0, scenarioIds: [] };
facts.realEvidenceSidecars = [
{
path: ".lime/qc/agent-qc-evidence.p0-v2.json",
status: "fail",
scenarioCount: 8,
scenarioIds: ["command-bridge-contract"],
},
];
const audit = buildAgentQcCompletionAudit(facts);
const item = audit.items.find((entry) => entry.id === "real-qcloop-evidence");
expect(audit.status).toBe("incomplete");
expect(item?.evidence).toContain("sidecars");
expect(item?.evidence).toContain("status=fail");
});
it("真实 qcloop 批次 stale 时应在缺口中显示 stale sidecar", () => {
const facts = completeFacts();
facts.realEvidencePack = {
exists: true,
status: "fail",
scenarioCount: 8,
scenarioIds: ["command-bridge-contract"],
};
facts.qcloopStatusSidecars = [
{
path: ".lime/qc/qcloop-status.isolated-p0-full-v1-stale-check.json",
verdictStatus: "stale",
counts: { success: 4, running: 1, pending: 3, stale: 1 },
},
];
const audit = buildAgentQcCompletionAudit(facts);
const item = audit.items.find((entry) => entry.id === "real-qcloop-evidence");
expect(audit.status).toBe("incomplete");
expect(item?.evidence).toContain("qcloopStatus");
expect(item?.gap).toContain("stale");
expect(item?.gap).toContain("qcloop-status.isolated-p0-full-v1-stale-check.json");
});
it("真实 qcloop 批次仍在运行时应在缺口中显示未终态 sidecar 摘要", () => {
const facts = completeFacts();
facts.realEvidencePack = {
exists: true,
status: "fail",
scenarioCount: 8,
scenarioIds: ["command-bridge-contract"],
};
facts.qcloopStatusSidecars = [
{
path: ".lime/qc/qcloop-status.fastmini-readonly-p0-v1-current.json",
verdictStatus: "running",
counts: { success: 0, running: 1, pending: 2, stale: 0 },
},
];
const audit = buildAgentQcCompletionAudit(facts);
const item = audit.items.find((entry) => entry.id === "real-qcloop-evidence");
expect(audit.status).toBe("incomplete");
expect(item?.gap).toContain("未终态");
expect(item?.gap).toContain("running=1");
expect(item?.gap).toContain("qcloop-status.fastmini-readonly-p0-v1-current.json");
});
it("qcloop verifier prompt 缺少 worker evidence 占位符时应保持 incomplete", () => {
const facts = completeFacts();
facts.qcloopPayload.verifierHasWorkerOutput = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain(
"qcloop-verifier-evidence-placeholders",
);
});
it("结构化 evidence summary 契约缺失时应保持 incomplete", () => {
const facts = completeFacts();
facts.qcloopPayload.workerPromptHasStructuredEvidence = false;
facts.structuredEvidence.exporterParsesSummary = false;
facts.structuredEvidence.releaseSummaryRejectsWeakRefs = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("structured-evidence-contract");
});
it("缺少 qcloop 状态监控入口时应保持 incomplete", () => {
const facts = completeFacts();
facts.files.qcloopStatusScript = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("qcloop-status-monitor");
});
it("缺少 qcloop worker preflight 时应保持 incomplete", () => {
const facts = completeFacts();
facts.qcloopPayload.workerPromptHasPreflight = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("qcloop-worker-preflight");
});
it("缺少 stale owner 机器可读决策输出时应保持 incomplete", () => {
const facts = completeFacts();
facts.staleOwnerIntervention.guiOwnerReportHasDecisionPacket = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("stale-owner-intervention-protocol");
expect(audit.items.find((item) => item.id === "stale-owner-intervention-protocol")?.evidence).toContain(
"ownerIntervention=false",
);
});
it("缺少 stale owner watch history 输出时应保持 incomplete", () => {
const facts = completeFacts();
facts.staleOwnerIntervention.guiOwnerReportHasWatchHistory = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("stale-owner-intervention-protocol");
expect(audit.items.find((item) => item.id === "stale-owner-intervention-protocol")?.evidence).toContain(
"watchHistory=false",
);
});
it("release hard gate 缺失时应给出缺口", () => {
const facts = completeFacts();
facts.release.hasHardGate = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("release-hard-gate");
});
it("release 未强制 P0 scenario 覆盖时应给出缺口", () => {
const facts = completeFacts();
facts.release.requiresP0ScenarioCoverage = false;
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("release-hard-gate");
});
it("verify:local 失败时应给出缺口", () => {
const facts = completeFacts();
facts.localVerify = {
status: "fail",
failedStage: "typecheck",
error: "example typecheck failure",
};
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.gaps.map((gap) => gap.id)).toContain("local-verify-gate");
expect(audit.items.find((item) => item.id === "local-verify-gate")?.evidence).toContain(
"failedStage=typecheck",
);
});
it("真实 evidence 数量足够但缺 P0 scenario id 时仍应保持 incomplete", () => {
const facts = completeFacts();
facts.scenarioReport.p0ScenarioCount = 2;
facts.scenarioReport.p0ScenarioIds = [
"command-bridge-contract",
"tool-approval-sandbox-boundary",
];
facts.realEvidencePack = {
exists: true,
status: "pass",
scenarioCount: 2,
scenarioIds: ["command-bridge-contract", "non-p0-placeholder"],
};
const audit = buildAgentQcCompletionAudit(facts);
expect(audit.status).toBe("incomplete");
expect(audit.items.find((item) => item.id === "real-qcloop-evidence")?.evidence).toContain(
"missing=tool-approval-sandbox-boundary",
);
});
it("应渲染 Markdown 审计报告", () => {
const audit = buildAgentQcCompletionAudit(completeFacts());
const markdown = renderAgentQcCompletionAuditMarkdown(audit);
expect(markdown).toContain("Agent QC Completion Audit");
expect(markdown).toContain("Status: complete");
});
});
+497
View File
@@ -0,0 +1,497 @@
const QCLoopTerminalStatus = new Set(["success", "failed", "exhausted"]);
const QCLoopEvidenceSummaryMarker = "QCLOOP_EVIDENCE_SUMMARY_JSON=";
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeStatus(value) {
return isNonEmptyString(value) ? value.trim().toLowerCase() : "unknown";
}
function collectAttemptStdout(item) {
return asArray(item?.attempts)
.map((attempt) => (typeof attempt?.stdout === "string" ? attempt.stdout : ""))
.join("\n");
}
function collectAttemptStderr(item) {
return asArray(item?.attempts)
.map((attempt) => (typeof attempt?.stderr === "string" ? attempt.stderr : ""))
.join("\n");
}
function collectAttemptOutput(item) {
return [collectAttemptStdout(item), collectAttemptStderr(item)].join("\n");
}
function isQCLoopWorkerBlocked(item) {
return /QCLOOP_WORKER_RESULT\s*[:=]\s*BLOCKED/i.test(collectAttemptStdout(item));
}
function isQCLoopWorkerEnvironmentBlocked(item) {
return [
/QCLOOP_CODEX_BIN\s+不可用/i,
/QCLOOP_CODEX_EXTRA_ARGS\s+解析失败/i,
/QCLOOP_CODEX_SANDBOX=.*无效/i,
/fork\/exec .* no such file or directory/i,
/incorrect api key provided/i,
/failed to connect to websocket: HTTP error: 401 Unauthorized/i,
].some((pattern) => pattern.test(collectAttemptOutput(item)));
}
function isQCLoopWorkerSelfReportedPass(item) {
return /QCLOOP_WORKER_RESULT\s*[:=]\s*PASS/i.test(collectAttemptStdout(item));
}
function parseQCLoopEvidenceSummaryLine(line) {
if (!isNonEmptyString(line)) {
return { summary: null, error: "" };
}
const markerIndex = line.indexOf(QCLoopEvidenceSummaryMarker);
if (markerIndex < 0) {
return { summary: null, error: "" };
}
const rawJson = line.slice(markerIndex + QCLoopEvidenceSummaryMarker.length).trim();
if (!rawJson) {
return { summary: null, error: "empty" };
}
try {
const parsed = JSON.parse(rawJson);
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
return { summary: null, error: "not_object" };
}
return { summary: parsed, error: "" };
} catch {
return { summary: null, error: "invalid_json" };
}
}
function collectQCLoopEvidenceSummaries(item) {
const summaries = [];
const parseErrors = [];
const stdout = collectAttemptStdout(item);
for (const line of stdout.split(/\r?\n/)) {
if (!line.includes(QCLoopEvidenceSummaryMarker)) {
continue;
}
const parsed = parseQCLoopEvidenceSummaryLine(line);
if (parsed.summary) {
summaries.push(parsed.summary);
} else {
parseErrors.push(parsed.error || "unknown");
}
}
return { summaries, parseErrors };
}
function latestQCLoopEvidenceSummary(item) {
const { summaries } = collectQCLoopEvidenceSummaries(item);
return summaries.at(-1) || null;
}
function normalizeEvidenceResult(value) {
const normalized = normalizeStatus(value);
if (["pass", "passed", "success"].includes(normalized)) {
return "pass";
}
if (["blocked", "block"].includes(normalized)) {
return "blocked";
}
if (["fail", "failed", "failure", "error"].includes(normalized)) {
return "fail";
}
return "unknown";
}
function hasFailedQCLoopExecution(item) {
const status = normalizeStatus(item?.status);
if (status === "failed" || status === "exhausted") {
return true;
}
if (asArray(item?.attempts).some((attempt) => normalizeStatus(attempt?.status) === "failed")) {
return true;
}
return asArray(item?.qc_rounds).some((round) => normalizeStatus(round?.status) === "fail");
}
function mapQCLoopItemStatus(status, item = null) {
const normalized = normalizeStatus(status);
const evidenceSummary = item ? latestQCLoopEvidenceSummary(item) : null;
const evidenceResult = evidenceSummary ? normalizeEvidenceResult(evidenceSummary.result) : "unknown";
if (evidenceResult === "blocked") {
return "blocked";
}
if (evidenceResult === "fail") {
return "fail";
}
if (normalized === "success") {
return item ? (evidenceSummary ? "pass" : "fail") : "pass";
}
if (isQCLoopWorkerBlocked(item) || isQCLoopWorkerEnvironmentBlocked(item)) {
return "blocked";
}
if (normalized === "failed" || normalized === "exhausted") {
return "fail";
}
if (normalized === "running" || normalized === "pending") {
return "blocked";
}
return "needs-human-review";
}
function mapQCLoopJobStatus(status, itemStatuses) {
const normalized = normalizeStatus(status);
if (itemStatuses.some((itemStatus) => itemStatus === "fail")) {
return "fail";
}
if (itemStatuses.some((itemStatus) => itemStatus === "blocked")) {
return "blocked";
}
if (itemStatuses.some((itemStatus) => itemStatus === "needs-human-review")) {
return "needs-human-review";
}
if (normalized === "completed") {
return "pass";
}
if (normalized === "failed") {
return "fail";
}
if (normalized === "paused" || normalized === "running" || normalized === "pending") {
return "blocked";
}
return "needs-human-review";
}
function parseScenarioId(itemValue) {
if (!isNonEmptyString(itemValue)) {
return "unknown-scenario";
}
const trimmed = itemValue.trim();
if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
try {
const parsed = JSON.parse(trimmed);
for (const key of ["scenarioId", "scenario_id", "id", "name", "entry"]) {
if (isNonEmptyString(parsed?.[key])) {
return parsed[key].trim();
}
}
} catch {
// 非 JSON item 按普通字符串处理。
}
}
return trimmed.split(/[\s,|]+/)[0] || "unknown-scenario";
}
function collectItemFailureModes(item) {
const modes = [];
const status = normalizeStatus(item?.status);
const { summaries, parseErrors } = collectQCLoopEvidenceSummaries(item);
if (parseErrors.length > 0) {
modes.push("qcloop:evidence_summary_invalid_json");
}
if (status === "success" && summaries.length === 0 && parseErrors.length === 0) {
modes.push("qcloop:evidence_summary_missing");
}
if (status === "failed") {
modes.push("qcloop:item_failed");
}
if (status === "exhausted") {
modes.push("qcloop:max_qc_rounds_exhausted");
}
if (status === "running" || status === "pending") {
modes.push("qcloop:item_not_terminal");
}
if (isNonEmptyString(item?.last_error)) {
modes.push("qcloop:last_error_present");
}
if (isQCLoopWorkerBlocked(item)) {
modes.push("qcloop:worker_blocked");
}
if (isQCLoopWorkerEnvironmentBlocked(item)) {
modes.push("qcloop:worker_environment_blocked");
}
if (isQCLoopWorkerSelfReportedPass(item) && hasFailedQCLoopExecution(item)) {
modes.push("qcloop:worker_self_report_pass_not_verified");
}
for (const attempt of asArray(item?.attempts)) {
if (normalizeStatus(attempt.status) === "failed") {
modes.push(`qcloop:attempt_${attempt.attempt_no ?? "unknown"}_failed`);
}
}
for (const round of asArray(item?.qc_rounds)) {
if (normalizeStatus(round.status) === "fail") {
modes.push(`qcloop:qc_${round.qc_no ?? "unknown"}_failed`);
}
}
return Array.from(new Set(modes));
}
function collectEvidenceSummaryArtifactRefs(summary) {
const refs = [];
for (const command of asArray(summary?.commands)) {
for (const key of ["stdout_artifact", "stderr_artifact", "logRef", "log_ref"]) {
if (isNonEmptyString(command?.[key])) {
refs.push(command[key].trim());
}
}
}
for (const evidence of asArray(summary?.evidence_required)) {
if (isNonEmptyString(evidence?.artifact_path)) {
refs.push(evidence.artifact_path.trim());
}
}
for (const artifact of asArray(summary?.artifacts)) {
if (typeof artifact === "string" && artifact.trim()) {
refs.push(artifact.trim());
continue;
}
if (isNonEmptyString(artifact?.path)) {
refs.push(artifact.path.trim());
}
}
if (isNonEmptyString(summary?.gui_session_owner)) {
refs.push(`gui-session-owner:${summary.gui_session_owner.trim()}`);
}
if (isNonEmptyString(summary?.release_scope)) {
refs.push(`release-scope:${summary.release_scope.trim()}`);
}
return refs;
}
function collectEvidenceRefs(item) {
const refs = [];
if (isNonEmptyString(item?.id)) {
refs.push(`qcloop:item:${item.id}`);
}
for (const attempt of asArray(item?.attempts)) {
if (isNonEmptyString(attempt?.id)) {
refs.push(`qcloop:attempt:${attempt.id}`);
}
}
for (const round of asArray(item?.qc_rounds)) {
if (isNonEmptyString(round?.id)) {
refs.push(`qcloop:qc:${round.id}`);
}
}
for (const summary of collectQCLoopEvidenceSummaries(item).summaries) {
refs.push(...collectEvidenceSummaryArtifactRefs(summary));
}
return Array.from(new Set(refs));
}
function findArtifactRefByKind(summary, kind) {
for (const artifact of asArray(summary?.artifacts)) {
if (typeof artifact === "string") {
if (artifact.includes(kind)) {
return artifact;
}
continue;
}
if (artifact?.kind === kind && isNonEmptyString(artifact?.path)) {
return artifact.path.trim();
}
}
return "";
}
function summarizeQCLoopItem(item) {
const attempts = asArray(item?.attempts);
const qcRounds = asArray(item?.qc_rounds);
const failedQcRound = qcRounds.find((round) => normalizeStatus(round.status) === "fail");
const latestQcRound = qcRounds.at(-1);
const status = mapQCLoopItemStatus(item?.status, item);
const evidenceSummary = latestQCLoopEvidenceSummary(item);
return {
scenarioId: parseScenarioId(item?.item_value),
status,
executor: "qcloop",
attempts: attempts.length,
evidenceRefs: collectEvidenceRefs(item),
failureModes: collectItemFailureModes(item),
humanReview: {
required: status === "needs-human-review",
reason:
status === "needs-human-review"
? "qcloop item 状态无法自动归类,需要人工审核。"
: "无需人工审核。",
decision: status === "needs-human-review" ? "not-reviewed" : "approved",
},
runtimeTranscriptRef:
findArtifactRefByKind(evidenceSummary, "runtime-transcript") ||
(attempts.length > 0 ? `qcloop:item:${item.id}:attempts` : ""),
guiTraceRef: findArtifactRefByKind(evidenceSummary, "gui-trace"),
consoleErrorCount: 0,
networkErrorCount: 0,
_summary: {
itemId: item?.id ?? "",
itemValue: item?.item_value ?? "",
qcloopStatus: item?.status ?? "unknown",
currentAttemptNo: item?.current_attempt_no ?? 0,
currentQcNo: item?.current_qc_no ?? 0,
latestFeedback: failedQcRound?.feedback || latestQcRound?.feedback || item?.last_error || "",
evidenceSummaryPresent: Boolean(evidenceSummary),
},
};
}
function stripPrivateSummary(scenarioResult) {
const { _summary, ...publicResult } = scenarioResult;
return publicResult;
}
function collectChangedFiles(options) {
return asArray(options.changedFiles).filter(isNonEmptyString);
}
function buildAgentQcEvidencePack({ job, items, options = {} }) {
const generatedAt = options.generatedAt || new Date().toISOString();
const scenarioSummaries = asArray(items).map(summarizeQCLoopItem);
const scenarioResults = scenarioSummaries.map(stripPrivateSummary);
const itemStatuses = scenarioResults.map((result) => result.status);
const verdictStatus = mapQCLoopJobStatus(job?.status, itemStatuses);
const failedItems = scenarioSummaries.filter((result) => result.status === "fail");
const blockedItems = scenarioSummaries.filter((result) => result.status === "blocked");
const reviewItems = scenarioSummaries.filter((result) => result.status === "needs-human-review");
const commandStatus = verdictStatus === "pass" ? "pass" : verdictStatus;
return {
schemaVersion: "v1",
runId: options.runId || `qcloop-${job?.id || "unknown"}-${Date.parse(generatedAt) || Date.now()}`,
generatedAt,
subject: {
repo: options.repo || "lime",
ref: options.ref || "unknown",
diffBase: options.diffBase || "",
changedFiles: collectChangedFiles(options),
riskTags: ["agent-qc", "qcloop", ...(options.riskTags || [])],
},
laneResults: [
{
laneId: "L4-behavior-eval",
status: commandStatus,
commands: [
{
command: `qcloop job ${job?.id || "unknown"}`,
status: commandStatus,
artifactRefs: [
`qcloop:job:${job?.id || "unknown"}`,
`qcloop:items:${job?.id || "unknown"}`,
],
},
],
notes: [
`qcloop job status: ${job?.status || "unknown"}`,
`items: ${scenarioResults.length}`,
`failed: ${failedItems.length}`,
`blocked: ${blockedItems.length}`,
`needs review: ${reviewItems.length}`,
],
},
],
scenarioResults,
verdict: {
status: verdictStatus,
summary: renderVerdictSummary({ job, scenarioResults, failedItems, blockedItems, reviewItems }),
blockers: renderBlockers({ failedItems, blockedItems, reviewItems }),
waivers: [],
nextAction: renderNextAction(verdictStatus),
},
};
}
function renderVerdictSummary({ job, scenarioResults, failedItems, blockedItems, reviewItems }) {
return `qcloop job ${job?.id || "unknown"} (${job?.status || "unknown"}) 导出 ${scenarioResults.length} 个场景结果:失败 ${failedItems.length},阻断 ${blockedItems.length},需人审 ${reviewItems.length}。`;
}
function renderBlockers({ failedItems, blockedItems, reviewItems }) {
const blockers = [];
for (const result of failedItems) {
let reason = result._summary.latestFeedback || "qcloop item failed";
if (result.failureModes.includes("qcloop:evidence_summary_missing")) {
reason = "qcloop item 已 success,但 stdout 缺少 QCLOOP_EVIDENCE_SUMMARY_JSON;不能作为 release pass。";
} else if (result.failureModes.includes("qcloop:evidence_summary_invalid_json")) {
reason = "qcloop stdout 包含 QCLOOP_EVIDENCE_SUMMARY_JSON,但 JSON 无法解析。";
} else if (result.failureModes.includes("qcloop:worker_self_report_pass_not_verified")) {
reason = `worker stdout 自报 QCLOOP_WORKER_RESULT=PASS,但 qcloop / verifier 未通过;latest verifier feedback: ${result._summary.latestFeedback || "无"}`;
}
blockers.push(`${result.scenarioId}: ${reason}`);
}
for (const result of blockedItems) {
const reason = result.failureModes.includes("qcloop:worker_environment_blocked")
? "qcloop worker 环境阻断;请检查内层 CLI 二进制、认证、sandbox 或 extra args。"
: result.failureModes.includes("qcloop:worker_blocked")
? `qcloop worker 明确输出 QCLOOP_WORKER_RESULT=BLOCKED;latest verifier feedback: ${result._summary.latestFeedback || "无"}`
: `qcloop item 未进入终态 (${result._summary.qcloopStatus})`;
blockers.push(`${result.scenarioId}: ${reason}`);
}
for (const result of reviewItems) {
blockers.push(`${result.scenarioId}: 状态无法自动归类,需要人工审核`);
}
return blockers;
}
function renderNextAction(status) {
if (status === "pass") {
return "可把 Evidence Pack 挂到 PR / release 证据摘要。";
}
if (status === "fail") {
return "优先修复 failed/exhausted item,并把高价值失败沉淀为 replay 或稳定回归。";
}
if (status === "blocked") {
return "先恢复 qcloop job 到终态,或记录环境/权限阻断原因。";
}
return "安排人工审核 verifier feedback,再决定修复、waive 或补测试。";
}
function validateEvidencePackShape(pack) {
const issues = [];
for (const field of ["schemaVersion", "runId", "generatedAt", "subject", "laneResults", "scenarioResults", "verdict"]) {
if (!(field in pack)) {
issues.push(`缺少字段 ${field}`);
}
}
if (pack.schemaVersion !== "v1") {
issues.push("schemaVersion 必须是 v1");
}
if (!Array.isArray(pack.laneResults)) {
issues.push("laneResults 必须是数组");
}
if (!Array.isArray(pack.scenarioResults)) {
issues.push("scenarioResults 必须是数组");
}
if (!pack.verdict || !["pass", "fail", "blocked", "needs-human-review", "waived"].includes(pack.verdict.status)) {
issues.push("verdict.status 不合法");
}
return {
valid: issues.length === 0,
issues,
};
}
export {
buildAgentQcEvidencePack,
collectEvidenceRefs,
collectItemFailureModes,
collectQCLoopEvidenceSummaries,
isQCLoopWorkerEnvironmentBlocked,
isQCLoopWorkerBlocked,
isQCLoopWorkerSelfReportedPass,
mapQCLoopItemStatus,
mapQCLoopJobStatus,
parseScenarioId,
parseQCLoopEvidenceSummaryLine,
summarizeQCLoopItem,
validateEvidencePackShape,
};
+339
View File
@@ -0,0 +1,339 @@
import { describe, expect, it } from "vitest";
import {
buildAgentQcEvidencePack,
collectQCLoopEvidenceSummaries,
isQCLoopWorkerEnvironmentBlocked,
isQCLoopWorkerBlocked,
isQCLoopWorkerSelfReportedPass,
mapQCLoopItemStatus,
parseQCLoopEvidenceSummaryLine,
parseScenarioId,
validateEvidencePackShape,
} from "./agent-qc-evidence-core.mjs";
function evidenceSummary(overrides = {}) {
return `QCLOOP_EVIDENCE_SUMMARY_JSON=${JSON.stringify({
scenario_id: "command-bridge-contract",
result: "pass",
commands: [
{
command: "npm run test:contracts",
exit_code: 0,
stdout_artifact: ".lime/qc/test-contracts.stdout.txt",
},
],
evidence_required: [
{
name: "contract command log",
status: "pass",
evidence: "test:contracts passed",
artifact_path: ".lime/qc/test-contracts.stdout.txt",
},
],
evidence_layers_covered: ["deterministic-smoke"],
failure_modes: [{ name: "mock fallback drift", status: "excluded", evidence: "contract check passed" }],
artifacts: [{ path: ".lime/qc/contract-summary.json", kind: "deterministic-smoke", redacted: true }],
blockers: [],
gui_session_owner: "",
release_scope: "source-tree-startup-smoke",
...overrides,
})}`;
}
describe("agent-qc-evidence-core", () => {
it("应把 qcloop item 状态映射为 Evidence Pack 状态", () => {
expect(mapQCLoopItemStatus("success")).toBe("pass");
expect(mapQCLoopItemStatus("failed")).toBe("fail");
expect(mapQCLoopItemStatus("exhausted")).toBe("fail");
expect(mapQCLoopItemStatus("running")).toBe("blocked");
});
it("应解析 qcloop 结构化 evidence summary marker", () => {
const parsed = parseQCLoopEvidenceSummaryLine(evidenceSummary());
expect(parsed.error).toBe("");
expect(parsed.summary?.scenario_id).toBe("command-bridge-contract");
const item = { attempts: [{ stdout: `log\n${evidenceSummary()}\n` }] };
const summaries = collectQCLoopEvidenceSummaries(item);
expect(summaries.summaries).toHaveLength(1);
expect(summaries.parseErrors).toEqual([]);
});
it("应把 worker 明确报告 BLOCKED 的 exhausted item 映射为 blocked", () => {
const item = {
status: "exhausted",
attempts: [
{
stdout: [
"QCLOOP_WORKER_RESULT=BLOCKED",
"失败 check: devbridge-health",
"错误: TypeError: fetch failed",
].join("\n"),
},
],
};
expect(isQCLoopWorkerBlocked(item)).toBe(true);
expect(mapQCLoopItemStatus("exhausted", item)).toBe("blocked");
});
it("应把内层 CLI 二进制或认证配置错误映射为环境 blocked", () => {
const item = {
status: "failed",
attempts: [
{
status: "failed",
stderr:
"QCLOOP_CODEX_BIN 不可用: /opt/homebrew/bin/codex: fork/exec /opt/homebrew/bin/codex: no such file or directory",
},
],
qc_rounds: [{ status: "fail", feedback: "verifier 输出格式错误" }],
};
expect(isQCLoopWorkerEnvironmentBlocked(item)).toBe(true);
expect(mapQCLoopItemStatus("failed", item)).toBe("blocked");
});
it("应标记 worker 自报 PASS 但 qcloop/verifier 未通过的冲突", () => {
const conflictItem = {
id: "item-pass-conflict",
item_value: "tool-approval-sandbox-boundary",
status: "failed",
current_attempt_no: 2,
current_qc_no: 1,
attempts: [
{
id: "attempt-pass-conflict",
status: "failed",
attempt_no: 2,
stdout: "QCLOOP_WORKER_RESULT=PASS\nsmoke pass, but no live transcript",
},
],
qc_rounds: [
{
id: "qc-pass-conflict",
status: "fail",
qc_no: 1,
feedback: "缺少 live runtime transcript。",
},
],
};
const pack = buildAgentQcEvidencePack({
job: { id: "job-pass-conflict", status: "running" },
items: [conflictItem],
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
});
expect(isQCLoopWorkerSelfReportedPass(conflictItem)).toBe(true);
expect(pack.verdict.status).toBe("fail");
expect(pack.scenarioResults[0].failureModes).toContain(
"qcloop:worker_self_report_pass_not_verified",
);
expect(pack.verdict.blockers.join("\n")).toContain(
"worker stdout 自报 QCLOOP_WORKER_RESULT=PASS",
);
});
it("应从 item_value 中提取 scenario id", () => {
expect(parseScenarioId("claw-chat-ready-streaming 基础聊天")).toBe(
"claw-chat-ready-streaming",
);
expect(parseScenarioId('{"scenario_id":"command-bridge-contract"}')).toBe(
"command-bridge-contract",
);
expect(parseScenarioId('{"name":"agent-ui-performance-summary-ttft-fields"}')).toBe(
"agent-ui-performance-summary-ttft-fields",
);
expect(parseScenarioId('{"entry":"lime-agent-runtime-client-check"}')).toBe(
"lime-agent-runtime-client-check",
);
});
it("应把成功 qcloop job 导出为 pass evidence pack", () => {
const pack = buildAgentQcEvidencePack({
job: { id: "job-1", status: "completed" },
items: [
{
id: "item-1",
item_value: "command-bridge-contract",
status: "success",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [
{
id: "attempt-1",
status: "success",
attempt_no: 1,
stdout: evidenceSummary(),
},
],
qc_rounds: [{ id: "qc-1", status: "pass", qc_no: 1 }],
},
],
options: {
generatedAt: "2026-05-10T00:00:00.000Z",
ref: "test-ref",
changedFiles: ["docs/tests/agent-ops-qc.md"],
},
});
expect(pack.verdict.status).toBe("pass");
expect(pack.scenarioResults[0].scenarioId).toBe("command-bridge-contract");
expect(pack.scenarioResults[0].evidenceRefs).toContain("qcloop:attempt:attempt-1");
expect(pack.scenarioResults[0].evidenceRefs).toContain(".lime/qc/contract-summary.json");
expect(pack.scenarioResults[0].evidenceRefs).toContain("release-scope:source-tree-startup-smoke");
expect(validateEvidencePackShape(pack).valid).toBe(true);
});
it("应拒绝缺少 QCLOOP_EVIDENCE_SUMMARY_JSON 的 success item,避免旧浅层 pass 进入 release gate", () => {
const pack = buildAgentQcEvidencePack({
job: { id: "job-missing-summary", status: "completed" },
items: [
{
id: "item-missing-summary",
item_value: "command-bridge-contract",
status: "success",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [{ id: "attempt-missing-summary", status: "success", attempt_no: 1 }],
qc_rounds: [{ id: "qc-missing-summary", status: "pass", qc_no: 1 }],
},
],
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
});
expect(pack.verdict.status).toBe("fail");
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:evidence_summary_missing");
expect(pack.verdict.blockers.join("\n")).toContain("缺少 QCLOOP_EVIDENCE_SUMMARY_JSON");
});
it("应拒绝无法解析的 QCLOOP_EVIDENCE_SUMMARY_JSON", () => {
const pack = buildAgentQcEvidencePack({
job: { id: "job-invalid-summary", status: "completed" },
items: [
{
id: "item-invalid-summary",
item_value: "command-bridge-contract",
status: "success",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [
{
id: "attempt-invalid-summary",
status: "success",
attempt_no: 1,
stdout: "QCLOOP_EVIDENCE_SUMMARY_JSON={not-json}",
},
],
qc_rounds: [{ id: "qc-invalid-summary", status: "pass", qc_no: 1 }],
},
],
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
});
expect(pack.verdict.status).toBe("fail");
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:evidence_summary_invalid_json");
expect(pack.verdict.blockers.join("\n")).toContain("JSON 无法解析");
});
it("应把耗尽 qcloop item 导出为 fail evidence pack", () => {
const pack = buildAgentQcEvidencePack({
job: { id: "job-2", status: "failed" },
items: [
{
id: "item-2",
item_value: "tool-approval-sandbox-boundary",
status: "exhausted",
current_attempt_no: 3,
current_qc_no: 3,
attempts: [{ id: "attempt-2", status: "failed", attempt_no: 3 }],
qc_rounds: [
{
id: "qc-2",
status: "fail",
qc_no: 3,
feedback: "缺少 approval 证据。",
},
],
},
],
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
});
expect(pack.verdict.status).toBe("fail");
expect(pack.verdict.blockers.join("\n")).toContain("缺少 approval 证据");
expect(pack.scenarioResults[0].failureModes).toContain(
"qcloop:max_qc_rounds_exhausted",
);
});
it("应把 preflight blocked 的耗尽 item 导出为 blocked evidence pack", () => {
const pack = buildAgentQcEvidencePack({
job: { id: "job-3", status: "completed" },
items: [
{
id: "item-3",
item_value: "claw-chat-ready-streaming",
status: "exhausted",
current_attempt_no: 3,
current_qc_no: 3,
attempts: [
{
id: "attempt-3",
status: "success",
attempt_no: 3,
stdout: "QCLOOP_WORKER_RESULT=BLOCKED\nDevBridge preflight: BLOCKED",
},
],
qc_rounds: [
{
id: "qc-3",
status: "fail",
qc_no: 3,
feedback: "DevBridge preflight blocked。",
},
],
},
],
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
});
expect(pack.verdict.status).toBe("blocked");
expect(pack.verdict.blockers.join("\n")).toContain("DevBridge preflight blocked");
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:worker_blocked");
});
it("应把 qcloop worker 环境阻断导出为 blocked evidence pack,且不泄露原始 stderr", () => {
const pack = buildAgentQcEvidencePack({
job: { id: "job-env", status: "failed" },
items: [
{
id: "item-env",
item_value: "command-bridge-contract",
status: "failed",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [
{
id: "attempt-env",
status: "failed",
attempt_no: 1,
stdout: "",
stderr:
"QCLOOP_CODEX_BIN 不可用: /opt/homebrew/bin/codex: fork/exec /opt/homebrew/bin/codex: no such file or directory",
},
],
qc_rounds: [{ id: "qc-env", status: "fail", qc_no: 1, feedback: "verifier 输出格式错误" }],
},
],
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
});
expect(pack.verdict.status).toBe("blocked");
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:worker_environment_blocked");
expect(pack.verdict.blockers.join("\n")).toContain("qcloop worker 环境阻断");
expect(pack.verdict.blockers.join("\n")).not.toContain("/opt/homebrew/bin/codex");
});
});
+163
View File
@@ -0,0 +1,163 @@
const REQUIRED_FLOW_FIELDS = [
"id",
"scenarioId",
"risk",
"surface",
"preflight",
"steps",
"assertions",
"evidenceRequired",
];
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function addIssue(issues, severity, path, message) {
issues.push({ severity, path, message });
}
function validateStringArray(value, path, issues) {
if (!Array.isArray(value) || value.length === 0) {
addIssue(issues, "error", path, "必须是非空数组。");
return;
}
value.forEach((entry, index) => {
if (!isNonEmptyString(entry)) {
addIssue(issues, "error", `${path}[${index}]`, "必须是非空字符串。");
}
});
}
function validateAgentQcGuiFlowManifest(flowManifest, scenarioManifest) {
const issues = [];
if (!flowManifest || typeof flowManifest !== "object") {
return {
valid: false,
issues: [{ severity: "error", path: "manifest", message: "GUI flow manifest 必须是 JSON object。" }],
};
}
if (flowManifest.manifestVersion !== "v1") {
addIssue(issues, "error", "manifestVersion", "当前只接受 manifestVersion=v1。");
}
const scenarios = new Set(asArray(scenarioManifest?.scenarios).map((scenario) => scenario.id).filter(Boolean));
const flows = asArray(flowManifest.flows);
const flowIds = new Set();
if (flows.length === 0) {
addIssue(issues, "error", "flows", "至少需要一个 GUI flow。");
}
flows.forEach((flow, index) => {
const prefix = `flows[${index}]`;
for (const field of REQUIRED_FLOW_FIELDS) {
if (!(field in flow)) {
addIssue(issues, "error", `${prefix}.${field}`, `缺少 ${field}。`);
}
}
if (!isNonEmptyString(flow.id)) {
addIssue(issues, "error", `${prefix}.id`, "flow id 必须是非空字符串。");
} else if (flowIds.has(flow.id)) {
addIssue(issues, "error", `${prefix}.id`, `重复 flow id: ${flow.id}`);
} else {
flowIds.add(flow.id);
}
if (!isNonEmptyString(flow.scenarioId)) {
addIssue(issues, "error", `${prefix}.scenarioId`, "scenarioId 必须是非空字符串。");
} else if (!scenarios.has(flow.scenarioId)) {
addIssue(issues, "error", `${prefix}.scenarioId`, `引用了不存在的 Agent QC scenario: ${flow.scenarioId}`);
}
for (const field of ["preflight", "steps", "assertions", "evidenceRequired"]) {
validateStringArray(flow[field], `${prefix}.${field}`, issues);
}
});
const errors = issues.filter((issue) => issue.severity === "error");
return {
valid: errors.length === 0,
issues,
};
}
function summarizeAgentQcGuiFlowManifest(flowManifest, validation) {
const flows = asArray(flowManifest?.flows);
const riskCounts = new Map();
for (const flow of flows) {
const risk = flow.risk || "unknown";
riskCounts.set(risk, (riskCounts.get(risk) ?? 0) + 1);
}
return {
title: flowManifest?.title || "Lime Agent QC GUI Flow Manifest",
manifestVersion: flowManifest?.manifestVersion || "unknown",
valid: validation.valid,
issueCount: validation.issues.length,
errorCount: validation.issues.filter((issue) => issue.severity === "error").length,
warnCount: validation.issues.filter((issue) => issue.severity === "warn").length,
flowCount: flows.length,
risks: Object.fromEntries(Array.from(riskCounts.entries()).sort(([left], [right]) => left.localeCompare(right))),
flows: flows.map((flow) => ({
id: flow.id,
scenarioId: flow.scenarioId,
risk: flow.risk,
surface: flow.surface,
stepCount: asArray(flow.steps).length,
assertionCount: asArray(flow.assertions).length,
evidenceCount: asArray(flow.evidenceRequired).length,
})),
issues: validation.issues,
};
}
function renderAgentQcGuiFlowMarkdown(summary) {
const flowLines = summary.flows.length
? summary.flows
.map(
(flow) =>
`- ${flow.id}: ${flow.scenarioId};${flow.risk};${flow.surface};steps ${flow.stepCount} / assertions ${flow.assertionCount} / evidence ${flow.evidenceCount}`,
)
.join("\n")
: "- 无";
const issueLines = summary.issues.length
? summary.issues.map((issue) => `- ${issue.severity.toUpperCase()} ${issue.path}: ${issue.message}`).join("\n")
: "- 无";
return `# ${summary.title}
## 结论
- Manifest: ${summary.manifestVersion}
- 状态: ${summary.valid ? "PASS" : "FAIL"}
- Flow 数: ${summary.flowCount}
- Issue: ${summary.issueCount}(error ${summary.errorCount} / warn ${summary.warnCount})
## Flow 清单
${flowLines}
## 校验问题
${issueLines}
`;
}
function createAgentQcGuiFlowReport({ flowManifest, scenarioManifest }) {
const validation = validateAgentQcGuiFlowManifest(flowManifest, scenarioManifest);
return summarizeAgentQcGuiFlowManifest(flowManifest, validation);
}
export {
createAgentQcGuiFlowReport,
renderAgentQcGuiFlowMarkdown,
summarizeAgentQcGuiFlowManifest,
validateAgentQcGuiFlowManifest,
};
@@ -0,0 +1,69 @@
import { describe, expect, it } from "vitest";
import {
createAgentQcGuiFlowReport,
renderAgentQcGuiFlowMarkdown,
validateAgentQcGuiFlowManifest,
} from "./agent-qc-gui-flow-core.mjs";
const scenarioManifest = {
scenarios: [
{ id: "claw-chat-ready-streaming" },
{ id: "workspace-ready-session-restore" },
],
};
const flowManifest = {
manifestVersion: "v1",
title: "GUI Flow Manifest",
flows: [
{
id: "gui-claw-chat-ready-streaming",
scenarioId: "claw-chat-ready-streaming",
risk: "P0",
surface: "desktop_gui",
preflight: ["check bridge"],
steps: ["open", "send"],
assertions: ["ready"],
evidenceRequired: ["trace"],
},
],
};
describe("agent-qc-gui-flow-core", () => {
it("应接受引用现有 scenario 的 GUI flow manifest", () => {
const result = validateAgentQcGuiFlowManifest(flowManifest, scenarioManifest);
expect(result.valid).toBe(true);
expect(result.issues).toEqual([]);
});
it("应拒绝引用不存在 scenario 的 flow", () => {
const invalid = structuredClone(flowManifest);
invalid.flows[0].scenarioId = "missing-scenario";
const result = validateAgentQcGuiFlowManifest(invalid, scenarioManifest);
expect(result.valid).toBe(false);
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
"missing-scenario",
);
});
it("应汇总 flow 数、步骤数和证据数", () => {
const report = createAgentQcGuiFlowReport({ flowManifest, scenarioManifest });
expect(report.valid).toBe(true);
expect(report.flowCount).toBe(1);
expect(report.flows[0].stepCount).toBe(2);
expect(report.flows[0].evidenceCount).toBe(1);
});
it("应渲染 Markdown 报告", () => {
const report = createAgentQcGuiFlowReport({ flowManifest, scenarioManifest });
const markdown = renderAgentQcGuiFlowMarkdown(report);
expect(markdown).toContain("GUI Flow Manifest");
expect(markdown).toContain("gui-claw-chat-ready-streaming");
});
});
+239
View File
@@ -0,0 +1,239 @@
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeScenarioId(value) {
return String(value || "").trim();
}
function collectGuiScenarioIds(manifest) {
return asArray(manifest?.scenarios)
.filter((scenario) =>
asArray(scenario?.evidenceRequired).includes("GUI session owner / isolation statement"),
)
.map((scenario) => normalizeScenarioId(scenario?.id))
.filter(Boolean);
}
function summarizeActiveGuiItems(status, guiScenarioIds) {
const guiScenarioIdSet = new Set(guiScenarioIds);
return asArray(status?.items)
.filter((item) => guiScenarioIdSet.has(normalizeScenarioId(item?.scenarioId)))
.filter((item) => item?.terminal !== true)
.map((item) => ({
scenarioId: normalizeScenarioId(item?.scenarioId),
qcloopStatus: item?.qcloopStatus || "unknown",
evidenceStatus: item?.evidenceStatus || "unknown",
stale: Boolean(item?.stale),
staleSeconds: item?.staleSeconds ?? null,
workerStatus: item?.worker?.status || "unknown",
workerDurationSeconds: item?.worker?.durationSeconds ?? null,
}));
}
function createAgentQcGuiOwnerReport({
manifest,
statusSidecars = [],
generatedAt = new Date().toISOString(),
maxActiveOwners = 0,
} = {}) {
const guiScenarioIds = collectGuiScenarioIds(manifest);
const sidecarsByJobId = new Map();
for (const sidecar of asArray(statusSidecars)) {
const status = sidecar?.status || sidecar;
const jobId = status?.job?.id || sidecar?.path || "unknown";
const entries = sidecarsByJobId.get(jobId) || [];
entries.push(sidecar);
sidecarsByJobId.set(jobId, entries);
}
const activeOwnerByJobId = new Map();
for (const sidecars of sidecarsByJobId.values()) {
const sidecar = selectRepresentativeSidecar(sidecars);
const status = sidecar?.status || sidecar;
const activeItems = summarizeActiveGuiItems(status, guiScenarioIds);
if (activeItems.length === 0) {
continue;
}
const jobId = status?.job?.id || sidecar?.path || "unknown";
const existing = activeOwnerByJobId.get(jobId);
const owner = {
path: sidecar?.path || "",
jobId,
jobName: status?.job?.name || "",
jobStatus: status?.job?.status || "unknown",
verdictStatus: status?.verdict?.status || "unknown",
counts: status?.counts || null,
activeItems,
};
if (!existing) {
activeOwnerByJobId.set(jobId, owner);
continue;
}
const existingStaleCount = existing.activeItems.filter((item) => item.stale).length;
const ownerStaleCount = owner.activeItems.filter((item) => item.stale).length;
if (ownerStaleCount > existingStaleCount) {
activeOwnerByJobId.set(jobId, owner);
}
}
const activeOwners = Array.from(activeOwnerByJobId.values()).sort((left, right) =>
left.jobId.localeCompare(right.jobId),
);
const ownerCount = activeOwners.length;
const staleOwners = activeOwners.filter((owner) =>
owner.activeItems.some((item) => item.stale),
);
const staleOwnerCount = staleOwners.length;
const oldestStaleSeconds = Math.max(
0,
...activeOwners.flatMap((owner) =>
owner.activeItems
.filter((item) => item.stale)
.map((item) => Number(item.staleSeconds || 0)),
),
);
const pass = ownerCount <= maxActiveOwners;
const ownerIntervention =
staleOwnerCount > 0 ? createOwnerIntervention(activeOwners) : null;
return {
schemaVersion: "v1",
generatedAt,
maxActiveOwners,
guiScenarioIds,
ownerCount,
staleOwnerCount,
oldestStaleSeconds,
activeOwners,
ownerIntervention,
verdict: {
status: pass ? "pass" : "blocked",
summary: pass
? `active GUI qcloop owner=${ownerCount},未超过上限 ${maxActiveOwners}。`
: `active GUI qcloop owner=${ownerCount},超过上限 ${maxActiveOwners}${staleOwnerCount > 0 ? `;其中 stale owner=${staleOwnerCount},最长 ${oldestStaleSeconds}s` : ""}。`,
nextAction: pass
? "可以在其他 release gate 满足后启动新的 GUI P0 批次。"
: staleOwnerCount > 0
? "不要启动新的 GUI P0 批次;当前存在 stale GUI owner,只能继续只读观察,或由该 owner 明确确认后处理。"
: "不要启动新的 GUI P0 批次;等待现有 running 批次自然结束或由 owner 明确处理。",
},
};
}
function createOwnerIntervention(activeOwners) {
const staleOwners = activeOwners.filter((owner) =>
owner.activeItems.some((item) => item.stale),
);
const primaryOwner = staleOwners[0];
const primaryJobId = primaryOwner?.jobId || "<job-id>";
return {
status: "requires_owner_confirmation",
jobIds: staleOwners.map((owner) => owner.jobId),
requiredConfirmationText: `确认处理 stale GUI owner ${primaryJobId},可以终止 PID <pid> 并记录 sidecar。`,
prohibitedUntilConfirmed: [
"kill / pause / interrupt stale worker",
"modify qcloop SQLite DB",
"start another full GUI P0 batch",
"overwrite .lime/qc/agent-qc-evidence.json",
"git commit / push / tag / release",
],
evidenceRefs: staleOwners.map((owner) => owner.path).filter(Boolean),
nextAction:
"先刷新 qcloop status、GUI owner、DB lease 和进程证据;只有 owner 明确确认后才能处理 stale worker。",
};
}
function isTerminalJobStatus(status) {
const jobStatus = String(status?.job?.status || "").toLowerCase();
return (
status?.job?.terminal === true ||
["completed", "failed", "cancelled", "canceled"].includes(jobStatus)
);
}
function selectRepresentativeSidecar(sidecars) {
const terminal = asArray(sidecars).find((sidecar) => isTerminalJobStatus(sidecar?.status || sidecar));
if (terminal) {
return terminal;
}
return asArray(sidecars).reduce((selected, candidate) => {
if (!selected) {
return candidate;
}
const selectedStatus = selected?.status || selected;
const candidateStatus = candidate?.status || candidate;
const selectedStale = Number(selectedStatus?.counts?.stale || 0);
const candidateStale = Number(candidateStatus?.counts?.stale || 0);
if (candidateStale > selectedStale) {
return candidate;
}
const selectedRunning = Number(selectedStatus?.counts?.running || 0);
const candidateRunning = Number(candidateStatus?.counts?.running || 0);
return candidateRunning > selectedRunning ? candidate : selected;
}, null);
}
function renderAgentQcGuiOwnerSummary(report) {
const lines = [
`status=${report.verdict.status}`,
`activeOwners=${report.ownerCount}`,
`staleOwners=${report.staleOwnerCount ?? 0}`,
`oldestStaleSeconds=${report.oldestStaleSeconds ?? 0}`,
`maxActiveOwners=${report.maxActiveOwners}`,
`guiScenarios=${report.guiScenarioIds.join(",")}`,
`summary=${report.verdict.summary}`,
];
for (const owner of report.activeOwners) {
const items = owner.activeItems
.map((item) =>
`${item.scenarioId}:${item.qcloopStatus}${item.stale ? `:stale=${item.staleSeconds ?? 0}s` : ""}`,
)
.join(",");
lines.push(`owner=${owner.jobId} jobStatus=${owner.jobStatus} verdict=${owner.verdictStatus} path=${owner.path} items=${items}`);
}
return `${lines.join("\n")}\n`;
}
function createAgentQcGuiOwnerWatchEntry(report) {
return {
schemaVersion: "v1",
observedAt: report?.generatedAt || new Date().toISOString(),
verdictStatus: report?.verdict?.status || "unknown",
ownerCount: report?.ownerCount ?? 0,
staleOwnerCount: report?.staleOwnerCount ?? 0,
oldestStaleSeconds: report?.oldestStaleSeconds ?? 0,
activeOwners: asArray(report?.activeOwners).map((owner) => ({
jobId: owner.jobId,
jobStatus: owner.jobStatus,
verdictStatus: owner.verdictStatus,
path: owner.path,
activeItems: asArray(owner.activeItems).map((item) => ({
scenarioId: item.scenarioId,
qcloopStatus: item.qcloopStatus,
evidenceStatus: item.evidenceStatus,
stale: Boolean(item.stale),
staleSeconds: item.staleSeconds ?? null,
workerStatus: item.workerStatus,
workerDurationSeconds: item.workerDurationSeconds ?? null,
})),
})),
};
}
export {
collectGuiScenarioIds,
createAgentQcGuiOwnerWatchEntry,
createAgentQcGuiOwnerReport,
createOwnerIntervention,
renderAgentQcGuiOwnerSummary,
};
+198
View File
@@ -0,0 +1,198 @@
import { describe, expect, it } from "vitest";
import {
collectGuiScenarioIds,
createAgentQcGuiOwnerWatchEntry,
createAgentQcGuiOwnerReport,
renderAgentQcGuiOwnerSummary,
} from "./agent-qc-gui-owner-core.mjs";
const manifest = {
scenarios: [
{
id: "claw-chat-ready-streaming",
evidenceRequired: ["GUI session owner / isolation statement"],
},
{
id: "command-bridge-contract",
evidenceRequired: ["contract command log"],
},
],
};
describe("agent-qc-gui-owner-core", () => {
it("应从 manifest 中识别 GUI owner 场景", () => {
expect(collectGuiScenarioIds(manifest)).toEqual(["claw-chat-ready-streaming"]);
});
it("没有 active GUI sidecar 时应 pass", () => {
const report = createAgentQcGuiOwnerReport({
manifest,
statusSidecars: [
{
path: ".lime/qc/qcloop-status.contract.json",
status: {
job: { id: "job-1", status: "completed" },
verdict: { status: "complete" },
items: [
{
scenarioId: "command-bridge-contract",
qcloopStatus: "success",
terminal: true,
},
],
},
},
],
});
expect(report.verdict.status).toBe("pass");
expect(report.ownerCount).toBe(0);
});
it("active GUI sidecar 超过上限时应 blocked", () => {
const report = createAgentQcGuiOwnerReport({
manifest,
statusSidecars: [
{
path: ".lime/qc/qcloop-status.gui.json",
status: {
job: { id: "job-gui", status: "running" },
verdict: { status: "stale" },
counts: { running: 1, stale: 1 },
items: [
{
scenarioId: "claw-chat-ready-streaming",
qcloopStatus: "running",
evidenceStatus: "blocked",
terminal: false,
stale: true,
staleSeconds: 600,
worker: { status: "running", durationSeconds: 600 },
},
],
},
},
],
maxActiveOwners: 0,
});
expect(report.verdict.status).toBe("blocked");
expect(report.ownerCount).toBe(1);
expect(report.staleOwnerCount).toBe(1);
expect(report.oldestStaleSeconds).toBe(600);
expect(report.verdict.nextAction).toContain("只读观察");
expect(report.ownerIntervention?.status).toBe("requires_owner_confirmation");
expect(report.ownerIntervention?.requiredConfirmationText).toContain("job-gui");
expect(report.ownerIntervention?.prohibitedUntilConfirmed).toContain(
"start another full GUI P0 batch",
);
expect(report.activeOwners[0].activeItems[0].scenarioId).toBe(
"claw-chat-ready-streaming",
);
});
it("同一 job 的多个 sidecar 应去重", () => {
const sidecar = {
job: { id: "job-gui", status: "running" },
verdict: { status: "running" },
items: [
{
scenarioId: "claw-chat-ready-streaming",
qcloopStatus: "running",
terminal: false,
},
],
};
const report = createAgentQcGuiOwnerReport({
manifest,
statusSidecars: [
{ path: ".lime/qc/qcloop-status.gui-current.json", status: sidecar },
{ path: ".lime/qc/qcloop-status.gui-stale.json", status: sidecar },
],
});
expect(report.ownerCount).toBe(1);
});
it("同一 job 存在终态 sidecar 时应忽略旧 running sidecar", () => {
const runningSidecar = {
job: { id: "job-gui", status: "running", terminal: false },
verdict: { status: "running" },
items: [
{
scenarioId: "claw-chat-ready-streaming",
qcloopStatus: "running",
terminal: false,
},
],
};
const terminalSidecar = {
job: { id: "job-gui", status: "failed", terminal: true },
verdict: { status: "fail" },
items: [
{
scenarioId: "claw-chat-ready-streaming",
qcloopStatus: "exhausted",
terminal: true,
},
],
};
const report = createAgentQcGuiOwnerReport({
manifest,
statusSidecars: [
{ path: ".lime/qc/qcloop-status.gui-current.json", status: runningSidecar },
{ path: ".lime/qc/qcloop-status.gui-completed.json", status: terminalSidecar },
],
});
expect(report.ownerCount).toBe(0);
expect(report.ownerIntervention).toBeNull();
expect(report.verdict.status).toBe("pass");
});
it("应渲染 summary", () => {
const report = createAgentQcGuiOwnerReport({ manifest, statusSidecars: [] });
const summary = renderAgentQcGuiOwnerSummary(report);
expect(summary).toContain("status=pass");
expect(summary).toContain("staleOwners=0");
expect(summary).toContain("guiScenarios=claw-chat-ready-streaming");
});
it("应生成 watch history JSONL entry", () => {
const report = createAgentQcGuiOwnerReport({
manifest,
generatedAt: "2026-05-10T00:00:00.000Z",
statusSidecars: [
{
path: ".lime/qc/qcloop-status.gui.json",
status: {
job: { id: "job-gui", status: "running" },
verdict: { status: "stale" },
items: [
{
scenarioId: "claw-chat-ready-streaming",
qcloopStatus: "running",
evidenceStatus: "blocked",
terminal: false,
stale: true,
staleSeconds: 900,
worker: { status: "running", durationSeconds: 900 },
},
],
},
},
],
});
const entry = createAgentQcGuiOwnerWatchEntry(report);
expect(entry.observedAt).toBe("2026-05-10T00:00:00.000Z");
expect(entry.verdictStatus).toBe("blocked");
expect(entry.activeOwners[0].jobId).toBe("job-gui");
expect(entry.activeOwners[0].activeItems[0].staleSeconds).toBe(900);
});
});
+273
View File
@@ -0,0 +1,273 @@
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeRiskList(risks) {
return asArray(risks)
.map((risk) => String(risk).trim().toUpperCase())
.filter(Boolean);
}
function selectScenarios(manifest, { risks = ["P0"], scenarioIds = [], includeAll = false } = {}) {
const normalizedRisks = new Set(normalizeRiskList(risks));
const normalizedScenarioIds = new Set(asArray(scenarioIds).filter(isNonEmptyString));
const scenarios = asArray(manifest?.scenarios);
return scenarios.filter((scenario) => {
if (includeAll) {
return true;
}
if (normalizedScenarioIds.size > 0) {
return normalizedScenarioIds.has(scenario.id);
}
return normalizedRisks.has(String(scenario.risk || "").toUpperCase());
});
}
const WORKER_RESULT_MARKER = "QCLOOP_WORKER_RESULT=";
const WORKER_EVIDENCE_SUMMARY_MARKER = "QCLOOP_EVIDENCE_SUMMARY_JSON=";
const WORKER_EVIDENCE_CONTRACT_BLOCK = `
---
Agent QC 结构化证据输出契约:
- 最终 stdout 必须包含单行 \`QCLOOP_WORKER_RESULT=PASS|FAIL|BLOCKED\`。
- 最终 stdout 必须包含单行 \`QCLOOP_EVIDENCE_SUMMARY_JSON=<json>\`;\`<json>\` 必须是单个 JSON object,不要 Markdown / code fence。
- \`QCLOOP_EVIDENCE_SUMMARY_JSON\` 至少包含:scenario_id、result、commands[]、evidence_required[]、evidence_layers_covered[]、failure_modes[]、artifacts[]、blockers[]、gui_session_owner、release_scope。
- evidence_required[] 每项包含 name、status(pass|fail|blocked|missing)、evidence、artifact_path;failure_modes[] 每项包含 name、status(covered|excluded|hit|unknown)、evidence。
- 不得输出密钥、token、用户私密内容或未经脱敏的请求 / 响应正文。`;
const VERIFIER_JSON_CONTRACT_BLOCK = `
---
qcloop verifier 输出格式硬约束:
- 只输出一个合法 JSON object,不要 Markdown、不要代码块、不要前后缀文本。
- JSON 必须至少包含 {"pass": true|false, "feedback": "..."}。
- 可选字段 missingEvidence、nextAction、evidenceStatus、scenarioId 必须仍在同一个 JSON object 内。
- 如果 worker stdout 缺少 QCLOOP_EVIDENCE_SUMMARY_JSON=<json>,或该 JSON 无法逐项证明 evidenceRequired / failureModes,必须输出 {"pass": false, "feedback": "..."}。`;
function ensureWorkerEvidenceContract(template) {
const normalized = isNonEmptyString(template)
? template
: "在 Lime 仓库中执行 Agent QC 场景 {{item}},收集证据并给出结论。";
if (
normalized.includes(WORKER_RESULT_MARKER) &&
normalized.includes(WORKER_EVIDENCE_SUMMARY_MARKER)
) {
return normalized;
}
return `${normalized}${WORKER_EVIDENCE_CONTRACT_BLOCK}`;
}
const VERIFIER_EVIDENCE_BLOCK = `
---
qcloop worker 执行证据(自动占位符替换):
- attempt_status: {{attempt_status}}
- attempt_type: {{attempt_type}}
- exit_code: {{exit_code}}
worker stdout:
{{stdout}}
qcloop issue ledger:
{{issue_ledger}}
判定要求:如果 stdout 缺少场景 verifier / evidence_required / failure_modes 所需证据,必须 pass=false;不能因为命令名看似正确就通过。`;
function ensureVerifierMetadataPlaceholders(template) {
const missingLines = [];
if (!template.includes("{{attempt_status}}")) {
missingLines.push("- attempt_status: {{attempt_status}}");
}
if (!template.includes("{{attempt_type}}")) {
missingLines.push("- attempt_type: {{attempt_type}}");
}
if (!template.includes("{{exit_code}}")) {
missingLines.push("- exit_code: {{exit_code}}");
}
if (!template.includes("{{qc_history}}") && !template.includes("{{issue_ledger}}")) {
missingLines.push("- issue_ledger: {{issue_ledger}}");
}
if (missingLines.length === 0) {
return template;
}
return `${template}
---
qcloop worker 执行元数据(自动占位符替换):
${missingLines.join("\n")}`;
}
function ensureVerifierEvidencePlaceholders(template) {
let normalized = isNonEmptyString(template)
? template
: "审查 Agent QC 场景 {{item}} 的输出是否满足证据要求,只输出 JSON verdict。";
if (!normalized.includes("{{stdout}}") && !normalized.includes("{{output}}")) {
normalized = `${normalized}${VERIFIER_EVIDENCE_BLOCK}`;
}
normalized = ensureVerifierMetadataPlaceholders(normalized);
if (!normalized.includes("QCLOOP_EVIDENCE_SUMMARY_JSON")) {
normalized = `${normalized}${VERIFIER_JSON_CONTRACT_BLOCK}`;
} else if (!normalized.includes('{"pass": true|false')) {
normalized = `${normalized}${VERIFIER_JSON_CONTRACT_BLOCK}`;
}
return normalized;
}
function buildQCLoopItemValue(scenario, { cwd = "" } = {}) {
return JSON.stringify({
scenario_id: scenario.id,
title: scenario.title,
risk: scenario.risk,
cwd,
executor: scenario.executor,
lanes: scenario.lanes,
commands: scenario.commands || [],
goal: scenario.goal,
verifier: scenario.verifier,
evidence_required: scenario.evidenceRequired || [],
evidence_layers: scenario.evidenceLayers || [],
failure_modes: scenario.failureModes || [],
});
}
function buildQCLoopJobPayload(manifest, options = {}) {
const selectedScenarios = selectScenarios(manifest, options);
const qcloop = manifest?.qcloop || {};
const riskLabel = options.includeAll
? "all"
: asArray(options.scenarioIds).length > 0
? "selected"
: normalizeRiskList(options.risks || ["P0"]).join("+") || "P0";
const name =
options.name ||
`lime-agent-qc-${riskLabel.toLowerCase()}-${new Date(options.generatedAt || Date.now()).toISOString().slice(0, 10)}`;
const cwdLine = options.cwd
? `目标仓库 cwd: ${options.cwd}。执行任何命令前必须先切换到该目录,并在输出中记录 pwd。`
: "";
const workerPromptTemplate = ensureWorkerEvidenceContract(
options.promptTemplate ||
qcloop.workerPromptTemplate ||
"在 Lime 仓库中执行 Agent QC 场景 {{item}},收集证据并给出结论。",
);
return {
name,
prompt_template: [cwdLine, workerPromptTemplate].filter(Boolean).join("\n\n"),
verifier_prompt_template: ensureVerifierEvidencePlaceholders(
options.verifierPromptTemplate || qcloop.verifierPromptTemplate,
),
max_qc_rounds: Number(options.maxQcRounds || qcloop.recommendedMaxQcRounds || 3),
max_executor_retries: Number(
options.maxExecutorRetries ?? qcloop.recommendedMaxExecutorRetries ?? 1,
),
token_budget_per_item: Number(options.tokenBudgetPerItem || 0),
execution_mode: options.executionMode || "standard",
executor_provider: options.executorProvider || "codex",
items: selectedScenarios.map((scenario) =>
buildQCLoopItemValue(scenario, { cwd: options.cwd || "" }),
),
};
}
function validateVerifierPromptTemplate(template, issues) {
if (!isNonEmptyString(template)) {
issues.push("缺少 verifier_prompt_template。");
return;
}
if (!template.includes("{{stdout}}") && !template.includes("{{output}}")) {
issues.push("verifier_prompt_template 必须包含 {{stdout}} 或 {{output}},否则 verifier 看不到 worker 输出。");
}
for (const placeholder of ["{{attempt_status}}", "{{exit_code}}"]) {
if (!template.includes(placeholder)) {
issues.push(`verifier_prompt_template 必须包含 ${placeholder}。`);
}
}
if (!template.includes("{{qc_history}}") && !template.includes("{{issue_ledger}}")) {
issues.push("verifier_prompt_template 必须包含 {{qc_history}} 或 {{issue_ledger}}。");
}
if (!template.includes(WORKER_EVIDENCE_SUMMARY_MARKER)) {
issues.push(`verifier_prompt_template 必须要求审查 ${WORKER_EVIDENCE_SUMMARY_MARKER}。`);
}
if (!template.includes('{"pass": true|false')) {
issues.push('verifier_prompt_template 必须声明只输出 {"pass": true|false, "feedback": "..."} JSON。');
}
}
function validateQCLoopJobPayload(payload) {
const issues = [];
if (!isNonEmptyString(payload?.name)) {
issues.push("缺少 name。 ");
}
if (!isNonEmptyString(payload?.prompt_template)) {
issues.push("缺少 prompt_template。 ");
} else {
if (!payload.prompt_template.includes("{{item}}")) {
issues.push("prompt_template 必须包含 {{item}}。 ");
}
if (!payload.prompt_template.includes(WORKER_RESULT_MARKER)) {
issues.push(`prompt_template 必须要求输出 ${WORKER_RESULT_MARKER}。 `);
}
if (!payload.prompt_template.includes(WORKER_EVIDENCE_SUMMARY_MARKER)) {
issues.push(`prompt_template 必须要求输出 ${WORKER_EVIDENCE_SUMMARY_MARKER}。 `);
}
}
validateVerifierPromptTemplate(payload?.verifier_prompt_template, issues);
if (!Number.isInteger(payload?.max_qc_rounds) || payload.max_qc_rounds <= 0) {
issues.push("max_qc_rounds 必须是正整数。 ");
}
if (
!Number.isInteger(payload?.max_executor_retries) ||
payload.max_executor_retries < 0 ||
payload.max_executor_retries > 5
) {
issues.push("max_executor_retries 必须是 0 到 5 的整数。 ");
}
if (!Array.isArray(payload?.items) || payload.items.length === 0) {
issues.push("items 不能为空。 ");
}
for (const [index, item] of asArray(payload?.items).entries()) {
if (!isNonEmptyString(item)) {
issues.push(`items[${index}] 不能为空。`);
continue;
}
try {
const parsed = JSON.parse(item);
if (!isNonEmptyString(parsed.scenario_id)) {
issues.push(`items[${index}] 缺少 scenario_id。`);
}
} catch {
issues.push(`items[${index}] 必须是 JSON 字符串。`);
}
}
return {
valid: issues.length === 0,
issues,
};
}
function renderQCLoopCurl(payload, { baseUrl = "http://127.0.0.1:8080" } = {}) {
const body = JSON.stringify(payload, null, 2);
return `curl -sS -X POST "${baseUrl.replace(/\/$/, "")}/api/jobs" \\
-H "Content-Type: application/json" \\
--data-binary @- <<'JSON'\n${body}\nJSON\n`;
}
export {
buildQCLoopItemValue,
buildQCLoopJobPayload,
renderQCLoopCurl,
selectScenarios,
validateQCLoopJobPayload,
};
@@ -0,0 +1,159 @@
import { describe, expect, it } from "vitest";
import {
buildQCLoopJobPayload,
renderQCLoopCurl,
selectScenarios,
validateQCLoopJobPayload,
} from "./agent-qc-qcloop-job-core.mjs";
const manifest = {
qcloop: {
recommendedMaxQcRounds: 3,
workerPromptTemplate: "worker {{item}}",
verifierPromptTemplate: "verifier {{item}}",
},
scenarios: [
{
id: "command-bridge-contract",
title: "命令桥接四侧一致",
risk: "P0",
executor: "npm_command",
lanes: ["L1-contract-bridge"],
commands: ["npm run test:contracts"],
goal: "goal",
verifier: "verifier",
evidenceRequired: ["evidence"],
evidenceLayers: ["deterministic-smoke"],
failureModes: ["failure"],
},
{
id: "knowledge-ingest-retrieve-summarize",
title: "Knowledge",
risk: "P1",
executor: "mixed",
lanes: ["L3-product-surface"],
goal: "goal",
verifier: "verifier",
evidenceRequired: ["evidence"],
failureModes: ["failure"],
},
],
};
describe("agent-qc-qcloop-job-core", () => {
it("默认应选择 P0 场景", () => {
const scenarios = selectScenarios(manifest);
expect(scenarios.map((scenario) => scenario.id)).toEqual(["command-bridge-contract"]);
});
it("应按风险等级生成 qcloop job payload", () => {
const payload = buildQCLoopJobPayload(manifest, {
risks: ["P1"],
cwd: "/workspace/lime",
generatedAt: "2026-05-10T00:00:00.000Z",
maxExecutorRetries: 0,
});
expect(payload.name).toBe("lime-agent-qc-p1-2026-05-10");
expect(payload.prompt_template).toContain("目标仓库 cwd: /workspace/lime");
expect(payload.prompt_template).toContain("QCLOOP_WORKER_RESULT=PASS|FAIL|BLOCKED");
expect(payload.prompt_template).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON=");
expect(payload.items).toHaveLength(1);
expect(payload.max_executor_retries).toBe(0);
expect(payload.verifier_prompt_template).toContain("{{stdout}}");
expect(payload.verifier_prompt_template).toContain("{{exit_code}}");
expect(payload.verifier_prompt_template).toContain("{{issue_ledger}}");
expect(payload.verifier_prompt_template).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON");
expect(payload.verifier_prompt_template).toContain('{"pass": true|false');
expect(JSON.parse(payload.items[0]).scenario_id).toBe(
"knowledge-ingest-retrieve-summarize",
);
expect(JSON.parse(payload.items[0]).cwd).toBe("/workspace/lime");
expect(validateQCLoopJobPayload(payload).valid).toBe(true);
});
it("应把 evidenceLayers 带入 qcloop item,方便 worker 声明证据深度", () => {
const payload = buildQCLoopJobPayload(manifest, {
scenarioIds: ["command-bridge-contract"],
});
expect(JSON.parse(payload.items[0]).evidence_layers).toEqual([
"deterministic-smoke",
]);
});
it("应支持按 scenario id 精确选择", () => {
const payload = buildQCLoopJobPayload(manifest, {
scenarioIds: ["command-bridge-contract"],
name: "selected-job",
});
expect(payload.name).toBe("selected-job");
expect(payload.items).toHaveLength(1);
expect(JSON.parse(payload.items[0]).scenario_id).toBe("command-bridge-contract");
});
it("validateQCLoopJobPayload 应阻止缺少 verifier stdout 占位符的 payload", () => {
const payload = buildQCLoopJobPayload(manifest);
payload.verifier_prompt_template = "verifier {{item}}";
const validation = validateQCLoopJobPayload(payload);
expect(validation.valid).toBe(false);
expect(validation.issues.join("\n")).toContain("{{stdout}}");
});
it("validateQCLoopJobPayload 应阻止缺少结构化证据契约的 payload", () => {
const payload = buildQCLoopJobPayload(manifest);
payload.prompt_template = "worker {{item}}";
payload.verifier_prompt_template =
"verifier {{item}} status={{attempt_status}} code={{exit_code}} stdout={{stdout}} ledger={{issue_ledger}}";
const validation = validateQCLoopJobPayload(payload);
expect(validation.valid).toBe(false);
expect(validation.issues.join("\n")).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON=");
});
it("validateQCLoopJobPayload 应阻止非法 max_executor_retries", () => {
const payload = buildQCLoopJobPayload(manifest);
payload.max_executor_retries = 6;
const validation = validateQCLoopJobPayload(payload);
expect(validation.valid).toBe(false);
expect(validation.issues.join("\n")).toContain("max_executor_retries");
});
it("verifier_prompt_template 不应重复追加已有 stdout 占位符的模板", () => {
const payload = buildQCLoopJobPayload(
{
...manifest,
qcloop: {
...manifest.qcloop,
verifierPromptTemplate: "verifier {{item}} stdout={{stdout}}",
},
},
{ scenarioIds: ["command-bridge-contract"] },
);
expect(payload.verifier_prompt_template.match(/{{stdout}}/g)).toHaveLength(1);
});
it("应生成可复制的 curl 命令", () => {
const payload = buildQCLoopJobPayload(manifest);
const curl = renderQCLoopCurl(payload, { baseUrl: "http://localhost:8080" });
expect(curl).toContain("POST \"http://localhost:8080/api/jobs\"");
expect(curl).toContain("command-bridge-contract");
});
it("默认 curl 应使用 IPv4 loopback,避免 localhost 代理或 IPv6 解析干扰", () => {
const payload = buildQCLoopJobPayload(manifest);
const curl = renderQCLoopCurl(payload);
expect(curl).toContain("POST \"http://127.0.0.1:8080/api/jobs\"");
});
});
@@ -0,0 +1,69 @@
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizePathText(value) {
return isNonEmptyString(value) ? value.trim().replace(/\\/g, "/") : "";
}
function createCheck(id, title, passed, detail = "") {
return {
id,
title,
passed: Boolean(passed),
detail,
};
}
function buildQCLoopPreflightReport({ cwd, expectedCwd = "", tmpWritable = false, devBridge = null } = {}) {
const checks = [];
const normalizedCwd = normalizePathText(cwd);
const normalizedExpectedCwd = normalizePathText(expectedCwd);
checks.push(createCheck("cwd-present", "当前工作目录可读", Boolean(normalizedCwd), normalizedCwd));
if (normalizedExpectedCwd) {
checks.push(
createCheck(
"cwd-expected",
"当前工作目录匹配预期仓库",
normalizedCwd === normalizedExpectedCwd,
`cwd=${normalizedCwd || "unknown"} expected=${normalizedExpectedCwd}`,
),
);
}
checks.push(
createCheck(
"tmp-writable",
"临时目录可写",
tmpWritable === true,
tmpWritable ? "tmp write ok" : "tmp write failed",
),
);
if (devBridge) {
checks.push(
createCheck(
"devbridge-health",
"DevBridge health 可访问",
devBridge.ok === true,
devBridge.ok
? `status=${devBridge.status || "ok"}`
: `url=${devBridge.url || "unknown"} error=${devBridge.error || "unknown"}`,
),
);
}
const failed = checks.filter((check) => !check.passed);
return {
schemaVersion: "v1",
status: failed.length === 0 ? "pass" : "blocked",
checks,
failedChecks: failed.map((check) => check.id),
summary:
failed.length === 0
? "qcloop worker preflight 通过。"
: `qcloop worker preflight 阻断:${failed.map((check) => check.id).join(", ")}。`,
};
}
export { buildQCLoopPreflightReport, createCheck };
@@ -0,0 +1,45 @@
import { describe, expect, it } from "vitest";
import { buildQCLoopPreflightReport } from "./agent-qc-qcloop-preflight-core.mjs";
describe("agent-qc-qcloop-preflight-core", () => {
it("基础环境通过时应返回 pass", () => {
const report = buildQCLoopPreflightReport({
cwd: "/repo/lime",
expectedCwd: "/repo/lime",
tmpWritable: true,
});
expect(report.status).toBe("pass");
expect(report.failedChecks).toEqual([]);
});
it("cwd 不匹配时应返回 blocked", () => {
const report = buildQCLoopPreflightReport({
cwd: "/repo/qcloop",
expectedCwd: "/repo/lime",
tmpWritable: true,
});
expect(report.status).toBe("blocked");
expect(report.failedChecks).toContain("cwd-expected");
});
it("DevBridge 不可访问时应返回 blocked", () => {
const report = buildQCLoopPreflightReport({
cwd: "/repo/lime",
tmpWritable: true,
devBridge: {
ok: false,
url: "http://127.0.0.1:3030/health",
error: "TypeError: fetch failed",
},
});
expect(report.status).toBe("blocked");
expect(report.failedChecks).toContain("devbridge-health");
expect(report.checks.find((check) => check.id === "devbridge-health")?.detail).toContain(
"fetch failed",
);
});
});
+289
View File
@@ -0,0 +1,289 @@
import { mapQCLoopItemStatus, parseScenarioId } from "./agent-qc-evidence-core.mjs";
const TERMINAL_ITEM_STATUSES = new Set(["success", "failed", "exhausted"]);
const TERMINAL_JOB_STATUSES = new Set(["completed", "failed", "canceled", "cancelled"]);
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeStatus(value) {
return isNonEmptyString(value) ? value.trim().toLowerCase() : "unknown";
}
function parseTimestampMs(value) {
if (!isNonEmptyString(value)) {
return null;
}
const timestamp = Date.parse(value);
return Number.isFinite(timestamp) ? timestamp : null;
}
function stringLength(value) {
return typeof value === "string" ? value.length : 0;
}
function durationMinutes(startedAt, finishedAt, nowMs) {
const startedMs = parseTimestampMs(startedAt);
if (startedMs === null) {
return null;
}
const finishedMs = parseTimestampMs(finishedAt) ?? nowMs;
return Math.max(0, Math.round((finishedMs - startedMs) / 60000));
}
function durationSeconds(startedAt, finishedAt, nowMs) {
const startedMs = parseTimestampMs(startedAt);
if (startedMs === null) {
return null;
}
const finishedMs = parseTimestampMs(finishedAt) ?? nowMs;
return Math.max(0, Math.round((finishedMs - startedMs) / 1000));
}
function latestEntry(entries) {
const normalizedEntries = asArray(entries).filter(Boolean);
return normalizedEntries.length > 0 ? normalizedEntries[normalizedEntries.length - 1] : null;
}
function summarizeLatestWorker(item, nowMs) {
const latestAttempt = latestEntry(item?.attempts);
if (!latestAttempt) {
return {
status: "unknown",
exitCode: null,
startedAt: null,
finishedAt: null,
durationMinutes: null,
durationSeconds: null,
stdoutLength: 0,
stderrLength: 0,
outputLength: 0,
};
}
const stdoutLength = stringLength(latestAttempt.stdout);
const stderrLength = stringLength(latestAttempt.stderr);
return {
status: normalizeStatus(latestAttempt.status),
exitCode: latestAttempt.exit_code ?? null,
startedAt: latestAttempt.started_at ?? null,
finishedAt: latestAttempt.finished_at ?? null,
durationMinutes: durationMinutes(latestAttempt.started_at, latestAttempt.finished_at, nowMs),
durationSeconds: durationSeconds(latestAttempt.started_at, latestAttempt.finished_at, nowMs),
stdoutLength,
stderrLength,
outputLength: stdoutLength + stderrLength,
};
}
function summarizeLatestQc(item) {
const latestQc = latestEntry(item?.qc_rounds);
if (!latestQc) {
return {
status: "unknown",
feedback: "",
};
}
return {
status: normalizeStatus(latestQc.status),
feedback: latestQc.feedback || "",
};
}
function classifyItemStaleness(item, worker, { staleMinutes = 30 } = {}) {
const itemStatus = normalizeStatus(item?.status);
const workerStatus = worker.status;
const isRunning = itemStatus === "running" || workerStatus === "running";
const isTerminal = TERMINAL_ITEM_STATUSES.has(itemStatus);
const reasons = [];
if (!isRunning || isTerminal || staleMinutes <= 0) {
return { stale: false, reasons };
}
if (worker.startedAt === null) {
reasons.push("running item 缺少 worker started_at,无法判断进度");
}
if (worker.durationMinutes !== null && worker.durationMinutes >= staleMinutes && worker.outputLength === 0) {
reasons.push(`worker 运行 ${worker.durationMinutes} 分钟且 stdout/stderr 为空`);
}
return {
stale: reasons.length > 0,
reasons,
};
}
function summarizeItem(item, options = {}) {
const nowMs = options.nowMs ?? Date.now();
const worker = summarizeLatestWorker(item, nowMs);
const qc = summarizeLatestQc(item);
const itemStatus = normalizeStatus(item?.status);
const terminal = TERMINAL_ITEM_STATUSES.has(itemStatus);
const staleness = classifyItemStaleness(item, worker, options);
return {
scenarioId: parseScenarioId(item?.item_value),
itemId: item?.id || "",
qcloopStatus: itemStatus,
evidenceStatus: mapQCLoopItemStatus(item?.status, item),
terminal,
stale: staleness.stale,
staleReasons: staleness.reasons,
staleSeconds: staleness.stale ? worker.durationSeconds : null,
currentAttemptNo: item?.current_attempt_no ?? 0,
currentQcNo: item?.current_qc_no ?? 0,
worker,
qc,
};
}
function countByStatus(items) {
const counts = {
total: items.length,
success: 0,
failed: 0,
exhausted: 0,
running: 0,
pending: 0,
unknown: 0,
terminal: 0,
nonTerminal: 0,
stale: 0,
};
for (const item of items) {
const status = item.qcloopStatus;
if (status in counts) {
counts[status] += 1;
} else {
counts.unknown += 1;
}
if (item.terminal) {
counts.terminal += 1;
} else {
counts.nonTerminal += 1;
}
if (item.stale) {
counts.stale += 1;
}
}
return counts;
}
function buildVerdict(job, items, counts) {
const jobStatus = normalizeStatus(job?.status);
const terminalProblemItems = items.filter((item) => item.qcloopStatus === "failed" || item.qcloopStatus === "exhausted");
const terminalBlockedItems = terminalProblemItems.filter((item) => item.evidenceStatus === "blocked");
const terminalFailedItems = terminalProblemItems.filter((item) => item.evidenceStatus === "fail");
if (counts.stale > 0) {
return {
status: "stale",
summary: `qcloop job ${job?.id || "unknown"} 仍在运行,${counts.stale} 个 item 疑似无进度。`,
nextAction: "不要中断进程;先导出 sidecar、记录卡点,等当前 worker 结束后再提交修正后的重跑 payload。",
};
}
if (counts.running > 0 || counts.pending > 0 || !TERMINAL_JOB_STATUSES.has(jobStatus)) {
return {
status: "running",
summary: `qcloop job ${job?.id || "unknown"} 尚未进入终态,running=${counts.running} pending=${counts.pending}。`,
nextAction: "继续观察 job/items;如果需要证据,只导出 sidecar,不覆盖官方 Evidence Pack。",
};
}
if (terminalFailedItems.length > 0 || (jobStatus === "failed" && terminalBlockedItems.length === 0)) {
return {
status: "fail",
summary: `qcloop job ${job?.id || "unknown"} 已终止但仍有 failed/exhausted item。`,
nextAction: "修复失败项或补足 verifier 可审查证据后,发起新的 qcloop 批次。",
};
}
if (terminalBlockedItems.length > 0) {
return {
status: "blocked",
summary: `qcloop job ${job?.id || "unknown"} 已终止但 ${terminalBlockedItems.length} 个 item 明确报告 worker 环境阻断。`,
nextAction: "先修复 qcloop worker 环境或权限,再用新批次重跑被阻断场景;不要覆盖官方 Evidence Pack。",
};
}
if (items.length > 0 && counts.success === items.length && jobStatus === "completed") {
return {
status: "complete",
summary: `qcloop job ${job?.id || "unknown"} 已完成,全部 item success。`,
nextAction: "可导出官方 Evidence Pack,并运行 release summary 与 completion audit。",
};
}
return {
status: "needs-human-review",
summary: `qcloop job ${job?.id || "unknown"} 状态无法自动归类。`,
nextAction: "人工审查 qcloop job/items 原始 JSON,再决定是否重跑或补 exporter 规则。",
};
}
function buildQCLoopStatusReport({ job, items, options = {} }) {
const generatedAt = options.generatedAt || new Date().toISOString();
const nowMs = parseTimestampMs(generatedAt) ?? Date.now();
const parsedStaleMinutes = Number(options.staleMinutes ?? 30);
const staleMinutes = Number.isFinite(parsedStaleMinutes) ? parsedStaleMinutes : 30;
const itemSummaries = asArray(items).map((item) =>
summarizeItem(item, {
nowMs,
staleMinutes,
}),
);
const counts = countByStatus(itemSummaries);
const verdict = buildVerdict(job, itemSummaries, counts);
return {
schemaVersion: "v1",
generatedAt,
staleMinutes,
job: {
id: job?.id || "unknown",
name: job?.name || "",
status: normalizeStatus(job?.status),
createdAt: job?.created_at || null,
finishedAt: job?.finished_at || null,
terminal: TERMINAL_JOB_STATUSES.has(normalizeStatus(job?.status)),
},
counts,
items: itemSummaries,
verdict,
};
}
function validateQCLoopStatusReport(report) {
const issues = [];
if (report?.schemaVersion !== "v1") {
issues.push("schemaVersion 必须是 v1。");
}
if (!report?.job || !isNonEmptyString(report.job.id)) {
issues.push("缺少 job.id。");
}
if (!Array.isArray(report?.items)) {
issues.push("items 必须是数组。");
}
if (!report?.counts || typeof report.counts.total !== "number") {
issues.push("缺少 counts.total。");
}
if (!report?.verdict || !isNonEmptyString(report.verdict.status)) {
issues.push("缺少 verdict.status。");
}
return {
valid: issues.length === 0,
issues,
};
}
export {
buildQCLoopStatusReport,
classifyItemStaleness,
normalizeStatus,
summarizeItem,
validateQCLoopStatusReport,
};
@@ -0,0 +1,168 @@
import { describe, expect, it } from "vitest";
import {
buildQCLoopStatusReport,
classifyItemStaleness,
validateQCLoopStatusReport,
} from "./agent-qc-qcloop-status-core.mjs";
const generatedAt = "2026-05-10T07:00:00.000Z";
describe("agent-qc-qcloop-status-core", () => {
it("应把长时间无输出的 running item 标记为 stale", () => {
const report = buildQCLoopStatusReport({
job: { id: "job-1", name: "p0", status: "running" },
items: [
{
id: "item-1",
item_value: '{"scenario_id":"skill-forge-register-bind-enable"}',
status: "running",
current_attempt_no: 1,
current_qc_no: 0,
attempts: [
{
id: "attempt-1",
status: "running",
started_at: "2026-05-10T06:00:00.000Z",
stdout: "",
stderr: "",
},
],
qc_rounds: [],
},
],
options: { generatedAt, staleMinutes: 30 },
});
expect(report.verdict.status).toBe("stale");
expect(report.counts.stale).toBe(1);
expect(report.items[0].staleSeconds).toBe(3600);
expect(report.items[0].worker.durationSeconds).toBe(3600);
expect(report.items[0].staleReasons.join("\n")).toContain("stdout/stderr 为空");
expect(validateQCLoopStatusReport(report).valid).toBe(true);
});
it("有输出的 running item 不应仅因运行时间长被标记为 stale", () => {
const result = classifyItemStaleness(
{ status: "running" },
{
status: "running",
startedAt: "2026-05-10T06:00:00.000Z",
durationMinutes: 60,
outputLength: 120,
},
{ staleMinutes: 30 },
);
expect(result.stale).toBe(false);
});
it("全部 success 且 job completed 时应给出 complete verdict", () => {
const report = buildQCLoopStatusReport({
job: { id: "job-2", name: "p0", status: "completed" },
items: [
{
id: "item-2",
item_value: "command-bridge-contract",
status: "success",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [
{
id: "attempt-2",
status: "success",
started_at: "2026-05-10T06:00:00.000Z",
finished_at: "2026-05-10T06:01:00.000Z",
stdout: "ok",
stderr: "",
},
],
qc_rounds: [{ id: "qc-2", status: "pass", feedback: "ok" }],
},
],
options: { generatedAt, staleMinutes: 30 },
});
expect(report.verdict.status).toBe("complete");
expect(report.counts.success).toBe(1);
});
it("failed/exhausted item 应保持 fail verdict", () => {
const report = buildQCLoopStatusReport({
job: { id: "job-3", name: "p0", status: "completed" },
items: [
{
id: "item-3",
item_value: "tool-approval-sandbox-boundary",
status: "exhausted",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [{ id: "attempt-3", status: "failed", stdout: "", stderr: "error" }],
qc_rounds: [{ id: "qc-3", status: "fail", feedback: "缺少证据" }],
},
],
options: { generatedAt, staleMinutes: 30 },
});
expect(report.verdict.status).toBe("fail");
expect(report.counts.exhausted).toBe(1);
expect(report.items[0].qc.feedback).toBe("缺少证据");
});
it("worker 明确报告 BLOCKED 的 exhausted item 应保持环境阻断语义", () => {
const report = buildQCLoopStatusReport({
job: { id: "job-4", name: "p0", status: "completed" },
items: [
{
id: "item-4",
item_value: "claw-chat-ready-streaming",
status: "exhausted",
current_attempt_no: 3,
current_qc_no: 3,
attempts: [
{
id: "attempt-4",
status: "success",
stdout: "QCLOOP_WORKER_RESULT=BLOCKED\nDevBridge preflight: BLOCKED",
stderr: "",
},
],
qc_rounds: [{ id: "qc-4", status: "fail", feedback: "DevBridge preflight blocked" }],
},
],
options: { generatedAt, staleMinutes: 30 },
});
expect(report.verdict.status).toBe("blocked");
expect(report.counts.exhausted).toBe(1);
expect(report.items[0].evidenceStatus).toBe("blocked");
});
it("内层 Codex CLI 环境错误应保持环境阻断语义", () => {
const report = buildQCLoopStatusReport({
job: { id: "job-5", name: "p0", status: "failed" },
items: [
{
id: "item-5",
item_value: "command-bridge-contract",
status: "failed",
current_attempt_no: 1,
current_qc_no: 1,
attempts: [
{
id: "attempt-5",
status: "failed",
stderr:
"QCLOOP_CODEX_BIN 不可用: /opt/homebrew/bin/codex: fork/exec /opt/homebrew/bin/codex: no such file or directory",
},
],
qc_rounds: [{ id: "qc-5", status: "fail", feedback: "verifier 输出格式错误" }],
},
],
options: { generatedAt, staleMinutes: 30 },
});
expect(report.verdict.status).toBe("blocked");
expect(report.items[0].evidenceStatus).toBe("blocked");
});
});
@@ -0,0 +1,194 @@
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function countByStatus(items, getStatus = (item) => item?.status) {
const counts = new Map();
for (const item of asArray(items)) {
const status = getStatus(item) || "unknown";
counts.set(status, (counts.get(status) ?? 0) + 1);
}
return Object.fromEntries(Array.from(counts.entries()).sort(([left], [right]) => left.localeCompare(right)));
}
function summarizeEvidencePack(pack, sourcePath = "") {
const scenarioResults = asArray(pack?.scenarioResults);
const laneResults = asArray(pack?.laneResults);
return {
sourcePath,
runId: pack?.runId || "unknown",
generatedAt: pack?.generatedAt || "",
status: pack?.verdict?.status || "unknown",
summary: pack?.verdict?.summary || "",
scenarioCount: scenarioResults.length,
scenarioIds: scenarioResults
.map((scenario) => String(scenario?.scenarioId || "").trim())
.filter(Boolean)
.sort(),
scenarioStatusCounts: countByStatus(scenarioResults),
laneStatusCounts: countByStatus(laneResults),
blockers: asArray(pack?.verdict?.blockers).filter(isNonEmptyString),
waivers: asArray(pack?.verdict?.waivers),
};
}
function normalizeScenarioIds(scenarioIds) {
return Array.from(
new Set(asArray(scenarioIds).map((scenarioId) => String(scenarioId || "").trim()).filter(Boolean)),
).sort();
}
function hasStructuredEvidenceRef(scenario) {
return asArray(scenario?.evidenceRefs).some((ref) => {
const normalized = String(ref || "").trim();
return normalized.length > 0 && !normalized.startsWith("qcloop:");
});
}
function summarizeHarnessReport(summaryReport, trendReport) {
return {
summaryGeneratedAt: summaryReport?.generatedAt || "",
trendGeneratedAt: trendReport?.generatedAt || "",
readyCount: summaryReport?.totals?.readyCount ?? null,
invalidCount: summaryReport?.totals?.invalidCount ?? null,
trendSampleCount: trendReport?.sampleCount ?? null,
signals: asArray(trendReport?.signals).filter(isNonEmptyString),
};
}
function buildAgentQcReleaseSummary({
evidencePacks = [],
harnessSummary = null,
harnessTrend = null,
requiredScenarioIds = [],
tag = "",
} = {}) {
const evidence = asArray(evidencePacks).map((entry) => summarizeEvidencePack(entry.pack, entry.sourcePath));
const statusCounts = countByStatus(evidence);
const blockers = evidence.flatMap((entry) => entry.blockers.map((blocker) => `${entry.runId}: ${blocker}`));
const waiverCount = evidence.reduce((sum, entry) => sum + entry.waivers.length, 0);
const hasFailingEvidence = evidence.some((entry) => entry.status !== "pass");
const weakEvidenceScenarioIds = normalizeScenarioIds(
asArray(evidencePacks).flatMap((entry) =>
asArray(entry?.pack?.scenarioResults)
.filter((scenario) => scenario?.status === "pass" && !hasStructuredEvidenceRef(scenario))
.map((scenario) => scenario?.scenarioId),
),
);
const structuredEvidenceBlockers = weakEvidenceScenarioIds.map(
(scenarioId) =>
`structured-evidence:${scenarioId}: pass scenario 缺少非 qcloop evidenceRefs,不能只凭 qcloop item id 作为发布证据。`,
);
const hasWeakStructuredEvidence = weakEvidenceScenarioIds.length > 0;
const status =
evidence.length === 0 ? "blocked" : hasFailingEvidence || hasWeakStructuredEvidence ? "fail" : "pass";
const coveredScenarioIds = normalizeScenarioIds(evidence.flatMap((entry) => entry.scenarioIds));
const normalizedRequiredScenarioIds = normalizeScenarioIds(requiredScenarioIds);
const coveredScenarioIdSet = new Set(coveredScenarioIds);
const missingRequiredScenarioIds = normalizedRequiredScenarioIds.filter(
(scenarioId) => !coveredScenarioIdSet.has(scenarioId),
);
return {
tag,
status,
evidenceCount: evidence.length,
statusCounts,
waiverCount,
blockers: [...blockers, ...structuredEvidenceBlockers],
coveredScenarioIds,
evidence,
harness: summarizeHarnessReport(harnessSummary, harnessTrend),
missingRequiredScenarioIds,
requiredScenarioIds: normalizedRequiredScenarioIds,
weakEvidenceScenarioIds,
};
}
function renderStatusCounts(counts) {
const entries = Object.entries(counts || {});
if (entries.length === 0) {
return "无";
}
return entries.map(([status, count]) => `${status}: ${count}`).join(" / ");
}
function renderAgentQcReleaseMarkdown(summary) {
const evidenceLines = summary.evidence.length
? summary.evidence
.map(
(entry) =>
`- ${entry.runId}: ${entry.status};场景 ${entry.scenarioCount};${renderStatusCounts(entry.scenarioStatusCounts)}${entry.sourcePath ? `;source ${entry.sourcePath}` : ""}`,
)
.join("\n")
: "- 无 Evidence Pack;发布应视为 blocked,除非有明确 waiver。";
const blockerLines = summary.blockers.length
? summary.blockers.map((blocker) => `- ${blocker}`).join("\n")
: "- 无";
const harnessLines = [
summary.harness.readyCount !== null ? `- Harness ready: ${summary.harness.readyCount}` : "- Harness ready: 未提供",
summary.harness.invalidCount !== null ? `- Harness invalid: ${summary.harness.invalidCount}` : "- Harness invalid: 未提供",
summary.harness.trendSampleCount !== null ? `- Harness trend samples: ${summary.harness.trendSampleCount}` : "- Harness trend samples: 未提供",
...summary.harness.signals.map((signal) => `- ${signal}`),
].join("\n");
return `## Agent QC Evidence${summary.tag ? ` (${summary.tag})` : ""}
- Verdict: ${summary.status}
- Evidence packs: ${summary.evidenceCount}
- Evidence status: ${renderStatusCounts(summary.statusCounts)}
- Scenario coverage: ${summary.requiredScenarioIds?.length ? `${summary.coveredScenarioIds.length}/${summary.requiredScenarioIds.length}` : `${summary.coveredScenarioIds?.length || 0} covered`}
- Waivers: ${summary.waiverCount}
### Evidence Packs
${evidenceLines}
### Harness Trend
${harnessLines}
### Blockers
${blockerLines}
`;
}
function validateReleaseSummary(summary, { requireEvidence = true } = {}) {
const issues = [];
if (requireEvidence && summary.evidenceCount === 0) {
issues.push("缺少 Agent QC Evidence Pack。请先导出 qcloop evidence 或显式记录 waiver。");
}
if (summary.status !== "pass") {
issues.push(`Agent QC release summary 状态为 ${summary.status},不能作为绿色发布证据。`);
}
if (asArray(summary.missingRequiredScenarioIds).length > 0) {
issues.push(`Agent QC Evidence Pack 未覆盖必需场景:${summary.missingRequiredScenarioIds.join(", ")}。`);
}
if (asArray(summary.weakEvidenceScenarioIds).length > 0) {
issues.push(
`Agent QC Evidence Pack 存在缺少结构化 evidenceRefs 的 pass 场景:${summary.weakEvidenceScenarioIds.join(", ")}。`,
);
}
return {
valid: issues.length === 0,
issues,
};
}
export {
buildAgentQcReleaseSummary,
countByStatus,
hasStructuredEvidenceRef,
normalizeScenarioIds,
renderAgentQcReleaseMarkdown,
summarizeEvidencePack,
summarizeHarnessReport,
validateReleaseSummary,
};
@@ -0,0 +1,124 @@
import { describe, expect, it } from "vitest";
import {
buildAgentQcReleaseSummary,
renderAgentQcReleaseMarkdown,
validateReleaseSummary,
} from "./agent-qc-release-summary-core.mjs";
const passPack = {
runId: "run-pass",
generatedAt: "2026-05-10T00:00:00.000Z",
verdict: { status: "pass", summary: "ok", blockers: [] },
laneResults: [{ laneId: "L4-behavior-eval", status: "pass" }],
scenarioResults: [
{
scenarioId: "command-bridge-contract",
status: "pass",
evidenceRefs: ["qcloop:item:item-1", ".lime/qc/contract-summary.json"],
},
{
scenarioId: "workspace-ready-session-restore",
status: "pass",
evidenceRefs: ["qcloop:item:item-2", ".lime/qc/gui-trace/workspace-ready.trace.zip"],
},
],
};
const failPack = {
runId: "run-fail",
generatedAt: "2026-05-10T00:00:00.000Z",
verdict: { status: "fail", summary: "bad", blockers: ["workspace: smoke failed"] },
laneResults: [{ laneId: "L4-behavior-eval", status: "fail" }],
scenarioResults: [{ scenarioId: "workspace-ready-session-restore", status: "fail", evidenceRefs: [] }],
};
describe("agent-qc-release-summary-core", () => {
it("应把全 pass Evidence Pack 汇总为 pass release summary", () => {
const summary = buildAgentQcReleaseSummary({
evidencePacks: [{ pack: passPack, sourcePath: "evidence.json" }],
harnessSummary: { generatedAt: "now", totals: { readyCount: 2, invalidCount: 0 } },
harnessTrend: { sampleCount: 3, signals: ["current gap 保持为 0。"] },
requiredScenarioIds: ["command-bridge-contract", "workspace-ready-session-restore"],
tag: "v1.2.3",
});
expect(summary.status).toBe("pass");
expect(summary.evidenceCount).toBe(1);
expect(summary.missingRequiredScenarioIds).toEqual([]);
expect(summary.harness.readyCount).toBe(2);
expect(validateReleaseSummary(summary).valid).toBe(true);
});
it("应把失败 Evidence Pack 汇总为 fail 并阻断 check", () => {
const summary = buildAgentQcReleaseSummary({
evidencePacks: [{ pack: failPack, sourcePath: "evidence.json" }],
});
expect(summary.status).toBe("fail");
expect(summary.blockers.join("\n")).toContain("workspace: smoke failed");
expect(validateReleaseSummary(summary).valid).toBe(false);
});
it("缺少 Evidence Pack 时默认 blocked", () => {
const summary = buildAgentQcReleaseSummary({ evidencePacks: [] });
expect(summary.status).toBe("blocked");
expect(validateReleaseSummary(summary).valid).toBe(false);
});
it("Evidence Pack pass 但缺必需 P0 场景时应阻断 release", () => {
const summary = buildAgentQcReleaseSummary({
evidencePacks: [{ pack: passPack, sourcePath: "evidence.json" }],
requiredScenarioIds: [
"command-bridge-contract",
"workspace-ready-session-restore",
"release-package-startup-smoke",
],
});
const validation = validateReleaseSummary(summary);
expect(summary.status).toBe("pass");
expect(summary.missingRequiredScenarioIds).toEqual(["release-package-startup-smoke"]);
expect(validation.valid).toBe(false);
expect(validation.issues.join("\n")).toContain("release-package-startup-smoke");
});
it("Evidence Pack pass 但 pass 场景只有 qcloop id 时应阻断 release", () => {
const weakPack = {
...passPack,
scenarioResults: [
{
scenarioId: "command-bridge-contract",
status: "pass",
evidenceRefs: ["qcloop:item:item-1", "qcloop:attempt:attempt-1"],
},
],
};
const summary = buildAgentQcReleaseSummary({
evidencePacks: [{ pack: weakPack, sourcePath: "evidence.json" }],
requiredScenarioIds: ["command-bridge-contract"],
});
const validation = validateReleaseSummary(summary);
expect(summary.status).toBe("fail");
expect(summary.weakEvidenceScenarioIds).toEqual(["command-bridge-contract"]);
expect(summary.blockers.join("\n")).toContain("structured-evidence:command-bridge-contract");
expect(validation.valid).toBe(false);
expect(validation.issues.join("\n")).toContain("结构化 evidenceRefs");
});
it("应渲染 release note 可用的 Markdown", () => {
const summary = buildAgentQcReleaseSummary({
evidencePacks: [{ pack: passPack, sourcePath: "evidence.json" }],
requiredScenarioIds: ["command-bridge-contract", "workspace-ready-session-restore"],
tag: "v1.2.3",
});
const markdown = renderAgentQcReleaseMarkdown(summary);
expect(markdown).toContain("Agent QC Evidence (v1.2.3)");
expect(markdown).toContain("Verdict: pass");
expect(markdown).toContain("Scenario coverage: 2/2");
expect(markdown).toContain("run-pass");
});
});
+505
View File
@@ -0,0 +1,505 @@
import fs from "node:fs";
const VALID_LANE_IDS = new Set([
"L0-static-unit",
"L1-contract-bridge",
"L2-agent-runtime",
"L3-product-surface",
"L4-behavior-eval",
"L5-release-ops",
]);
const VALID_RISKS = new Set(["P0", "P1", "P2"]);
const VALID_EXECUTORS = new Set([
"npm_command",
"rust_test",
"playwright_mcp",
"runtime_harness",
"qcloop",
"release_workflow",
"mixed",
]);
const VALID_STATUSES = new Set([
"pass",
"fail",
"blocked",
"needs-human-review",
"waived",
"skipped",
]);
const VALID_EVIDENCE_LAYERS = new Set([
"deterministic-smoke",
"gui-trace",
"runtime-transcript",
"release-artifact",
]);
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
}
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function isNonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function addIssue(issues, severity, path, message) {
issues.push({ severity, path, message });
}
function collectNpmScriptName(command) {
if (!isNonEmptyString(command)) {
return "";
}
const match = command.trim().match(/^npm\s+run\s+([^\s]+)/);
return match ? match[1] : "";
}
function validateNpmCommands(commands, packageScripts, pathPrefix, issues) {
for (const [index, command] of asArray(commands).entries()) {
if (!isNonEmptyString(command)) {
addIssue(issues, "error", `${pathPrefix}.commands[${index}]`, "命令必须是非空字符串。");
continue;
}
const scriptName = collectNpmScriptName(command);
if (scriptName && !Object.hasOwn(packageScripts, scriptName)) {
addIssue(
issues,
"error",
`${pathPrefix}.commands[${index}]`,
`package.json 中不存在 npm script: ${scriptName}`,
);
}
}
}
function validateQCLoopConfig(qcloop, issues) {
if (!qcloop || typeof qcloop !== "object") {
return;
}
const workerTemplate = qcloop.workerPromptTemplate;
if (!isNonEmptyString(workerTemplate)) {
addIssue(
issues,
"error",
"qcloop.workerPromptTemplate",
"qcloop workerPromptTemplate 必须存在,且要约束 worker 输出可审查证据。",
);
} else {
if (!workerTemplate.includes("evidence_required")) {
addIssue(
issues,
"error",
"qcloop.workerPromptTemplate",
"workerPromptTemplate 必须要求 worker 逐项列出 evidence_required,否则 repair 轮会再次缺证据。",
);
}
if (!workerTemplate.includes("failure_modes")) {
addIssue(
issues,
"error",
"qcloop.workerPromptTemplate",
"workerPromptTemplate 必须要求 worker 逐项说明 failure_modes 是否命中、覆盖或排除。",
);
}
if (!workerTemplate.includes("QCLOOP_WORKER_RESULT=BLOCKED")) {
addIssue(
issues,
"error",
"qcloop.workerPromptTemplate",
"workerPromptTemplate 必须显式约束环境阻断时输出 QCLOOP_WORKER_RESULT=BLOCKED。",
);
}
if (!workerTemplate.includes("QCLOOP_EVIDENCE_SUMMARY_JSON")) {
addIssue(
issues,
"error",
"qcloop.workerPromptTemplate",
"workerPromptTemplate 必须要求 worker 输出 QCLOOP_EVIDENCE_SUMMARY_JSON,避免 verifier 只能审查散文摘要。",
);
}
}
const template = qcloop.verifierPromptTemplate;
if (!isNonEmptyString(template)) {
addIssue(
issues,
"error",
"qcloop.verifierPromptTemplate",
"qcloop verifierPromptTemplate 必须存在,且要带 worker evidence 占位符。",
);
return;
}
if (!template.includes("{{stdout}}") && !template.includes("{{output}}")) {
addIssue(
issues,
"error",
"qcloop.verifierPromptTemplate",
"verifierPromptTemplate 必须包含 {{stdout}} 或 {{output}},否则 verifier 看不到 worker 输出。",
);
}
for (const placeholder of ["{{attempt_status}}", "{{exit_code}}"]) {
if (!template.includes(placeholder)) {
addIssue(
issues,
"error",
"qcloop.verifierPromptTemplate",
`verifierPromptTemplate 必须包含 ${placeholder},用于判定 worker 执行状态。`,
);
}
}
if (!template.includes("{{qc_history}}") && !template.includes("{{issue_ledger}}")) {
addIssue(
issues,
"error",
"qcloop.verifierPromptTemplate",
"verifierPromptTemplate 必须包含 {{qc_history}} 或 {{issue_ledger}},用于判断 repair 后旧问题是否仍开放。",
);
}
if (!template.includes("QCLOOP_EVIDENCE_SUMMARY_JSON")) {
addIssue(
issues,
"error",
"qcloop.verifierPromptTemplate",
"verifierPromptTemplate 必须要求审查 QCLOOP_EVIDENCE_SUMMARY_JSON,避免命令退出码被误判为证据完整。",
);
}
if (!template.includes('{"pass": true|false')) {
addIssue(
issues,
"error",
"qcloop.verifierPromptTemplate",
'verifierPromptTemplate 必须声明只输出 {"pass": true|false, "feedback": "..."} JSON,避免 qcloop 报“verifier 输出格式错误”。',
);
}
if (template.includes("{{stderr}}")) {
addIssue(
issues,
"warn",
"qcloop.verifierPromptTemplate",
"verifierPromptTemplate 包含 {{stderr}},Codex / GUI 场景可能产生超长 stderr;优先让 worker 在 stdout 输出摘要。",
);
}
}
function validateEvidenceSchema(schema, issues) {
if (!schema || typeof schema !== "object") {
addIssue(issues, "error", "evidenceSchema", "Evidence schema 必须是 JSON object。");
return;
}
for (const field of ["$schema", "$id", "type", "required", "properties"]) {
if (!Object.hasOwn(schema, field)) {
addIssue(issues, "error", `evidenceSchema.${field}`, `缺少 ${field}。`);
}
}
const required = asArray(schema.required);
for (const field of ["schemaVersion", "runId", "generatedAt", "subject", "laneResults", "scenarioResults", "verdict"]) {
if (!required.includes(field)) {
addIssue(issues, "error", "evidenceSchema.required", `缺少必填字段 ${field}。`);
}
}
const verdictStatus = schema?.properties?.verdict?.properties?.status?.enum;
if (Array.isArray(verdictStatus)) {
for (const status of ["pass", "fail", "blocked", "needs-human-review", "waived"]) {
if (!verdictStatus.includes(status)) {
addIssue(issues, "error", "evidenceSchema.verdict.status", `verdict.status 缺少 ${status}。`);
}
}
}
}
function validateLane(lane, index, packageScripts, issues) {
const pathPrefix = `lanes[${index}]`;
if (!isNonEmptyString(lane?.id)) {
addIssue(issues, "error", `${pathPrefix}.id`, "Lane 必须有 id。");
return;
}
if (!VALID_LANE_IDS.has(lane.id)) {
addIssue(issues, "warn", `${pathPrefix}.id`, `Lane id ${lane.id} 不在当前标准集合中。`);
}
for (const field of ["title", "objective", "gatePolicy"]) {
if (!isNonEmptyString(lane[field])) {
addIssue(issues, "error", `${pathPrefix}.${field}`, `Lane 缺少 ${field}。`);
}
}
validateNpmCommands(lane.defaultCommands, packageScripts, pathPrefix, issues);
}
function validateScenario(scenario, index, laneIds, packageScripts, issues) {
const pathPrefix = `scenarios[${index}]`;
for (const field of ["id", "title", "risk", "executor", "goal", "verifier"]) {
if (!isNonEmptyString(scenario?.[field])) {
addIssue(issues, "error", `${pathPrefix}.${field}`, `Scenario 缺少 ${field}。`);
}
}
if (isNonEmptyString(scenario?.risk) && !VALID_RISKS.has(scenario.risk)) {
addIssue(issues, "error", `${pathPrefix}.risk`, `未知风险等级 ${scenario.risk}。`);
}
if (isNonEmptyString(scenario?.executor) && !VALID_EXECUTORS.has(scenario.executor)) {
addIssue(issues, "error", `${pathPrefix}.executor`, `未知执行器 ${scenario.executor}。`);
}
const lanes = asArray(scenario?.lanes);
if (lanes.length === 0) {
addIssue(issues, "error", `${pathPrefix}.lanes`, "Scenario 至少要归属一个 lane。");
}
for (const laneId of lanes) {
if (!laneIds.has(laneId)) {
addIssue(issues, "error", `${pathPrefix}.lanes`, `引用了不存在的 lane: ${laneId}`);
}
}
if (asArray(scenario?.evidenceRequired).length === 0) {
addIssue(issues, "error", `${pathPrefix}.evidenceRequired`, "Scenario 必须声明证据要求。");
}
if (asArray(scenario?.failureModes).length === 0) {
addIssue(issues, "error", `${pathPrefix}.failureModes`, "Scenario 必须声明失败模式。");
}
const evidenceLayers = asArray(scenario?.evidenceLayers);
if (scenario?.risk === "P0" && evidenceLayers.length === 0) {
addIssue(
issues,
"error",
`${pathPrefix}.evidenceLayers`,
"P0 Scenario 必须声明 evidenceLayers,避免 deterministic smoke 被误读成 deep evidence。",
);
}
for (const [layerIndex, layer] of evidenceLayers.entries()) {
if (!VALID_EVIDENCE_LAYERS.has(layer)) {
addIssue(
issues,
"error",
`${pathPrefix}.evidenceLayers[${layerIndex}]`,
`未知 evidence layer: ${layer}`,
);
}
}
validateNpmCommands(scenario?.commands, packageScripts, pathPrefix, issues);
}
function validateAgentQcManifest(manifest, { packageScripts = {}, evidenceSchema = null } = {}) {
const issues = [];
if (!manifest || typeof manifest !== "object") {
return {
valid: false,
issues: [{ severity: "error", path: "manifest", message: "Manifest 必须是 JSON object。" }],
};
}
for (const field of ["manifestVersion", "title", "evidenceSchema", "lanes", "scenarios"]) {
if (!Object.hasOwn(manifest, field)) {
addIssue(issues, "error", field, `Manifest 缺少 ${field}。`);
}
}
if (manifest.manifestVersion !== "v1") {
addIssue(issues, "error", "manifestVersion", "当前只接受 manifestVersion=v1。");
}
validateQCLoopConfig(manifest.qcloop, issues);
const lanes = asArray(manifest.lanes);
const scenarios = asArray(manifest.scenarios);
const laneIds = new Set();
const scenarioIds = new Set();
if (lanes.length === 0) {
addIssue(issues, "error", "lanes", "至少需要一个测试 lane。");
}
for (const [index, lane] of lanes.entries()) {
validateLane(lane, index, packageScripts, issues);
if (isNonEmptyString(lane?.id)) {
if (laneIds.has(lane.id)) {
addIssue(issues, "error", `lanes[${index}].id`, `重复 lane id: ${lane.id}`);
}
laneIds.add(lane.id);
}
}
if (scenarios.length === 0) {
addIssue(issues, "error", "scenarios", "至少需要一个 Agent QC 场景。");
}
for (const [index, scenario] of scenarios.entries()) {
validateScenario(scenario, index, laneIds, packageScripts, issues);
if (isNonEmptyString(scenario?.id)) {
if (scenarioIds.has(scenario.id)) {
addIssue(issues, "error", `scenarios[${index}].id`, `重复 scenario id: ${scenario.id}`);
}
scenarioIds.add(scenario.id);
}
}
if (evidenceSchema) {
validateEvidenceSchema(evidenceSchema, issues);
}
const errors = issues.filter((issue) => issue.severity === "error");
return {
valid: errors.length === 0,
issues,
};
}
function countBy(items, getKey) {
const counts = new Map();
for (const item of items) {
const key = getKey(item);
counts.set(key, (counts.get(key) ?? 0) + 1);
}
return Array.from(counts.entries())
.map(([name, count]) => ({ name, count }))
.sort((left, right) => left.name.localeCompare(right.name));
}
function summarizeAgentQcManifest(manifest, validation) {
const lanes = asArray(manifest?.lanes);
const scenarios = asArray(manifest?.scenarios);
return {
title: manifest?.title ?? "Lime Agent QC",
manifestVersion: manifest?.manifestVersion ?? "unknown",
evidenceSchema: manifest?.evidenceSchema ?? "",
valid: validation.valid,
issueCount: validation.issues.length,
errorCount: validation.issues.filter((issue) => issue.severity === "error").length,
warnCount: validation.issues.filter((issue) => issue.severity === "warn").length,
laneCount: lanes.length,
scenarioCount: scenarios.length,
p0ScenarioCount: scenarios.filter((scenario) => scenario.risk === "P0").length,
lanes: lanes.map((lane) => ({
id: lane.id,
title: lane.title,
defaultCommandCount: asArray(lane.defaultCommands).length,
scenarioCount: scenarios.filter((scenario) => asArray(scenario.lanes).includes(lane.id)).length,
})),
scenarios: scenarios.map((scenario) => ({
id: scenario.id,
title: scenario.title,
risk: scenario.risk,
executor: scenario.executor,
lanes: asArray(scenario.lanes),
})),
riskBreakdown: countBy(scenarios, (scenario) => scenario.risk || "unknown"),
executorBreakdown: countBy(scenarios, (scenario) => scenario.executor || "unknown"),
issues: validation.issues,
};
}
function renderIssueList(issues) {
if (issues.length === 0) {
return "- 无";
}
return issues
.map((issue) => `- ${issue.severity.toUpperCase()} ${issue.path}: ${issue.message}`)
.join("\n");
}
function renderNameCountList(items) {
if (items.length === 0) {
return "- 无";
}
return items.map((item) => `- ${item.name}: ${item.count}`).join("\n");
}
function renderAgentQcMarkdownReport(summary) {
const laneLines = summary.lanes.length
? summary.lanes
.map(
(lane) =>
`- ${lane.id}: ${lane.title};场景 ${lane.scenarioCount};默认命令 ${lane.defaultCommandCount}`,
)
.join("\n")
: "- 无";
const scenarioLines = summary.scenarios.length
? summary.scenarios
.map(
(scenario) =>
`- ${scenario.id}: ${scenario.title};${scenario.risk};${scenario.executor}`,
)
.join("\n")
: "- 无";
return `# ${summary.title} 场景报告
## 结论
- Manifest: ${summary.manifestVersion}
- Evidence schema: ${summary.evidenceSchema || "未声明"}
- 状态: ${summary.valid ? "PASS" : "FAIL"}
- Lane 数: ${summary.laneCount}
- Scenario 数: ${summary.scenarioCount}
- P0 Scenario 数: ${summary.p0ScenarioCount}
- Issue: ${summary.issueCount}(error ${summary.errorCount} / warn ${summary.warnCount})
## Lane 覆盖
${laneLines}
## Scenario 清单
${scenarioLines}
## 风险分布
${renderNameCountList(summary.riskBreakdown)}
## 执行器分布
${renderNameCountList(summary.executorBreakdown)}
## 校验问题
${renderIssueList(summary.issues)}
`;
}
function createAgentQcReport({ manifest, packageJson = {}, evidenceSchema = null }) {
const validation = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts ?? {},
evidenceSchema,
});
return summarizeAgentQcManifest(manifest, validation);
}
export {
VALID_EVIDENCE_LAYERS,
VALID_EXECUTORS,
VALID_LANE_IDS,
VALID_RISKS,
VALID_STATUSES,
collectNpmScriptName,
createAgentQcReport,
readJsonFile,
renderAgentQcMarkdownReport,
summarizeAgentQcManifest,
validateAgentQcManifest,
};
+226
View File
@@ -0,0 +1,226 @@
import { describe, expect, it } from "vitest";
import {
collectNpmScriptName,
createAgentQcReport,
renderAgentQcMarkdownReport,
validateAgentQcManifest,
} from "./agent-qc-report-core.mjs";
const packageJson = {
scripts: {
"verify:local": "node scripts/local-ci.mjs",
"test:contracts": "node scripts/check-command-contracts.mjs",
"verify:gui-smoke": "node scripts/verify-gui-smoke.mjs",
},
};
const evidenceSchema = {
$schema: "https://json-schema.org/draft/2020-12/schema",
$id: "https://lime.local/agent-qc-evidence.schema.json",
type: "object",
required: [
"schemaVersion",
"runId",
"generatedAt",
"subject",
"laneResults",
"scenarioResults",
"verdict",
],
properties: {
verdict: {
type: "object",
properties: {
status: {
enum: ["pass", "fail", "blocked", "needs-human-review", "waived"],
},
},
},
},
};
const validManifest = {
manifestVersion: "v1",
title: "Lime Agent QC",
evidenceSchema: "docs/test/agent-qc-evidence.schema.json",
qcloop: {
workerPromptTemplate:
"worker {{item}} evidence_required failure_modes QCLOOP_WORKER_RESULT=BLOCKED QCLOOP_EVIDENCE_SUMMARY_JSON",
verifierPromptTemplate:
'verifier {{item}} status={{attempt_status}} code={{exit_code}} stdout={{stdout}} ledger={{issue_ledger}} QCLOOP_EVIDENCE_SUMMARY_JSON,只输出 {"pass": true|false, "feedback": "..."} JSON',
},
lanes: [
{
id: "L0-static-unit",
title: "快速卫生",
objective: "验证低级错误不会进入主链。",
gatePolicy: "所有 PR 默认执行。",
defaultCommands: ["npm run verify:local"],
},
{
id: "L1-contract-bridge",
title: "契约桥接",
objective: "验证命令和 Bridge 合同不漂移。",
gatePolicy: "命令和 Bridge 改动必跑。",
defaultCommands: ["npm run test:contracts"],
},
],
scenarios: [
{
id: "command-bridge-contract",
title: "命令桥接合同",
risk: "P0",
executor: "npm_command",
lanes: ["L1-contract-bridge"],
commands: ["npm run test:contracts"],
goal: "前端调用、Rust 注册、治理目录册、mock 四侧一致。",
verifier: "test:contracts 成功且没有新增 dead/compat 回流。",
evidenceRequired: ["command log", "contract summary"],
evidenceLayers: ["deterministic-smoke"],
failureModes: ["missing rust handler", "mock fallback drift"],
},
],
};
describe("agent-qc-report-core", () => {
it("能从 npm run 命令中提取 script 名称", () => {
expect(collectNpmScriptName("npm run verify:gui-smoke -- --reuse-running")).toBe(
"verify:gui-smoke",
);
expect(collectNpmScriptName("node scripts/foo.mjs")).toBe("");
});
it("应接受完整的 Agent QC manifest 和 evidence schema", () => {
const result = validateAgentQcManifest(validManifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(true);
expect(result.issues.filter((issue) => issue.severity === "error")).toEqual([]);
});
it("应阻止 qcloop verifier 缺少 worker stdout 占位符", () => {
const manifest = structuredClone(validManifest);
manifest.qcloop.verifierPromptTemplate =
"verifier {{item}} status={{attempt_status}} code={{exit_code}}";
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
"{{stdout}}",
);
});
it("应阻止 qcloop worker 缺少 evidence_required / failure_modes / BLOCKED / 结构化证据约束", () => {
const manifest = structuredClone(validManifest);
manifest.qcloop.workerPromptTemplate = "worker {{item}}";
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
const messages = result.issues.map((issue) => issue.message).join("\n");
expect(messages).toContain("evidence_required");
expect(messages).toContain("failure_modes");
expect(messages).toContain("QCLOOP_WORKER_RESULT=BLOCKED");
expect(messages).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON");
});
it("应阻止 qcloop verifier 缺少 attempt 状态占位符", () => {
const manifest = structuredClone(validManifest);
manifest.qcloop.verifierPromptTemplate = "verifier {{item}} stdout={{stdout}}";
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
"{{attempt_status}}",
);
});
it("应阻止 qcloop verifier 缺少 issue ledger 或结构化证据审查契约", () => {
const manifest = structuredClone(validManifest);
manifest.qcloop.verifierPromptTemplate =
"verifier {{item}} status={{attempt_status}} code={{exit_code}} stdout={{stdout}}";
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
const messages = result.issues.map((issue) => issue.message).join("\n");
expect(messages).toContain("{{qc_history}}");
expect(messages).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON");
expect(messages).toContain('{"pass": true|false');
});
it("应阻止引用不存在的 npm script", () => {
const manifest = structuredClone(validManifest);
manifest.scenarios[0].commands = ["npm run missing-script"];
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
"missing-script",
);
});
it("应阻止 P0 场景缺少 evidenceLayers", () => {
const manifest = structuredClone(validManifest);
delete manifest.scenarios[0].evidenceLayers;
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
"evidenceLayers",
);
});
it("应阻止未知 evidence layer", () => {
const manifest = structuredClone(validManifest);
manifest.scenarios[0].evidenceLayers = ["deterministic-smoke", "unknown-layer"];
const result = validateAgentQcManifest(manifest, {
packageScripts: packageJson.scripts,
evidenceSchema,
});
expect(result.valid).toBe(false);
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
"unknown-layer",
);
});
it("应把报告渲染成可读 Markdown", () => {
const report = createAgentQcReport({
manifest: validManifest,
packageJson,
evidenceSchema,
});
const markdown = renderAgentQcMarkdownReport(report);
expect(markdown).toContain("Lime Agent QC 场景报告");
expect(markdown).toContain("状态: PASS");
expect(markdown).toContain("command-bridge-contract");
});
});
@@ -0,0 +1,313 @@
function createTranscriptSteps() {
return [
{
id: "tool-request",
category: "tool request",
eventTypes: ["tool.started", "tool.args"],
summary:
"允许工具请求会投影为 tool.started/tool.args,保留 tool id、tool name 与输入参数摘要。",
details: {
toolCallId: "tool-1",
toolName: "read_file",
phase: "acting",
inputAvailable: true,
inputSummary: '{"path":"README.md"}',
},
sourceRefs: [
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:676",
],
},
{
id: "approval-decision",
category: "approval decision",
eventTypes: ["run.status", "permission.changed"],
summary:
"需要授权时会先进入 permission_review,并通过 permission.changed 暴露 decision、askProfileKeys 与 confirmation request。",
details: {
phase: "waiting",
permissionStatus: "requires_confirmation",
confirmationStatus: "not_requested",
confirmationRequestId: "approval-1",
decisionSource: "runtime",
decisionScope: "turn",
askProfileKeys: ["read_files"],
requiredProfileKeys: ["read_files", "write_artifacts"],
},
sourceRefs: [
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:85",
"src/components/agent/chat/components/AgentThreadTimeline.test.tsx:904",
"src/components/agent/chat/components/MessageList.test.tsx:1845",
],
},
{
id: "sandbox-policy",
category: "sandbox policy",
eventTypes: ["permission.changed"],
summary:
"turn_context 会透传 approvalPolicy 与 sandboxPolicy,确保 runtime policy 不会在 UI 证据链中丢失。",
details: {
phase: "preparing",
approvalPolicy: "on-request",
sandboxPolicy: "workspace-write",
sourceEvent: "turn_context",
},
sourceRefs: [
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:1480",
"src/components/agent/chat/hooks/agentStreamSubmitExecution.test.ts:1",
],
},
{
id: "tool-result",
category: "result",
eventTypes: ["tool.result"],
summary:
"成功工具结果会归档为 tool.result,并保留 artifact refs,避免出现 tool result missing。",
details: {
toolCallId: "tool-1",
success: true,
phase: "completed",
artifactPaths: [".lime/artifacts/demo.md"],
artifactIds: ["artifact-1"],
},
sourceRefs: [
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:676",
],
},
{
id: "tool-error",
category: "error",
eventTypes: ["tool.failed"],
summary:
"失败工具结果会归档为 tool.failed,并保留错误预览与 metadata,避免错误路径丢失。",
details: {
toolCallId: "tool-2",
success: false,
phase: "failed",
errorPreview: "Permission denied",
metadataKeys: ["sandboxed", "exit_code"],
},
sourceRefs: [
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:676",
"src/components/agent/chat/hooks/useAsterAgentChat.test.tsx:7801",
],
},
{
id: "timeout-recovery",
category: "recovery action",
eventTypes: ["timeout.recover", "timeout.defer", "timeout.fail"],
summary:
"首包超时与 inactivity timeout 都有明确恢复动作矩阵和用户可见文案,拒绝或超时不会把用户卡死。",
details: {
firstEventActions: {
recover: "recover",
defer: "defer",
fail: "fail",
},
inactivityActions: {
recover: "recover",
fail: "fail",
},
warnings: [
"[AsterChat] 首个运行时事件静默,已降级切换为会话快照同步: event-a",
"[AsterChat] 首个运行时事件暂未到达,已基于提交派发继续等待后续进度: event-a",
"[AsterChat] 运行时事件静默,已降级切换为会话快照同步: event-a",
],
failureMessage:
"执行已中断:运行时长时间没有返回新进度,请重试。",
},
sourceRefs: [
"src/components/agent/chat/hooks/agentStreamInactivityController.test.ts:1",
],
},
];
}
function createEvidenceChecks() {
return [
{
id: "tool timeline",
satisfied: true,
summary:
"由 tool-request、tool-result 与 tool-error 三段 transcript 组成完整 tool timeline。",
transcriptStepIds: ["tool-request", "tool-result", "tool-error"],
},
{
id: "approval decision",
satisfied: true,
summary:
"approval-decision transcript 暴露 permission.changed waiting 状态、decision source/scope 与 confirmation request。",
transcriptStepIds: ["approval-decision"],
},
{
id: "sandbox policy",
satisfied: true,
summary:
"sandbox-policy transcript 暴露 turn_context approvalPolicy/sandboxPolicy。",
transcriptStepIds: ["sandbox-policy"],
},
{
id: "error recovery transcript",
satisfied: true,
summary:
"timeout-recovery transcript 暴露 recover/defer/fail 动作矩阵与用户可见 warning/message。",
transcriptStepIds: ["timeout-recovery"],
},
];
}
function createFailureModeCoverage() {
return [
{
id: "approval bypass",
status: "covered",
summary:
"approval-decision 与 sandbox-policy transcript 证明 runtime 会先进入 waiting/policy gate,再继续工具执行。",
transcriptStepIds: ["approval-decision", "sandbox-policy"],
},
{
id: "tool result missing",
status: "covered",
summary:
"tool-result 与 tool-error transcript 同时存在,成功/失败路径都不会丢失结果。",
transcriptStepIds: ["tool-result", "tool-error"],
},
{
id: "timeout without recovery",
status: "covered",
summary:
"timeout-recovery transcript 暴露 recover/defer/fail 分支与用户可见 warning/message,不会让用户卡死。",
transcriptStepIds: ["timeout-recovery"],
},
{
id: "unsafe tool exposed",
status: "covered",
summary:
"工具权限与来源会进入 Harness 工具库存区块,危险工具暴露由 companion tool-surface smoke 共同覆盖。",
transcriptStepIds: ["approval-decision"],
companionCommand: "npm run smoke:agent-runtime-tool-surface",
},
];
}
function liveRuntimeTranscriptSatisfiesReleaseGate(liveRuntimeTranscript) {
const assertions = liveRuntimeTranscript?.assertions;
return Boolean(
assertions?.devBridgeHealthy &&
assertions?.permissionRequestCreatedBeforeModel &&
assertions?.deniedDecisionClearsPendingRequest &&
assertions?.resolvedDecisionClearsPendingRequest &&
assertions?.approvalPolicySubmitted &&
assertions?.sandboxPolicySubmitted,
);
}
function buildApprovalSandboxSmokeEvidence({
commandResults,
generatedAt,
liveRuntimeTranscript = null,
}) {
const transcriptSteps = createTranscriptSteps();
const evidenceChecks = createEvidenceChecks();
const failureModeCoverage = createFailureModeCoverage();
const liveRuntimeTranscriptVerified =
liveRuntimeTranscriptSatisfiesReleaseGate(liveRuntimeTranscript);
return {
schemaVersion: "v2",
scenarioId: "tool-approval-sandbox-boundary",
status: "pass",
generatedAt,
evidenceKind: "runtime-approval-sandbox-verified-transcript-smoke",
transcriptKind: liveRuntimeTranscriptVerified
? "verified_projection_and_live_runtime_transcript"
: "verified_projection_summary",
limitation: liveRuntimeTranscriptVerified
? "该 smoke 同时输出投影/组件事实源与 DevBridge live runtime permission confirmation transcript;tool result/error 和 timeout recovery 仍由定向回归覆盖。"
: "该 smoke 输出的是由前端提交、投影、UI 与超时恢复事实源生成的结构化 transcript summary;正式发布如需 live session 级证据,仍应由 qcloop 或 replay 采集。",
coverage: {
submitPreferences: true,
approvalDecisionProjection: true,
sandboxPolicyProjection: true,
toolRequestResultOrErrorProjection: true,
permissionRecoveryUi: true,
harnessPermissionInventory: true,
runtimeTranscriptSummary: true,
liveRuntimeTranscript: liveRuntimeTranscriptVerified,
},
liveRuntimeTranscript,
transcriptSteps,
evidenceChecks,
failureModeCoverage,
commandResults,
followUpForOfficialEvidence: [
liveRuntimeTranscriptVerified
? "qcloop worker stdout 应保留 live.transcript.* 行,供 release verifier 审计。"
: "如需 live session 级发布证据,将同样的 transcript 维度写入 qcloop worker stdout 或 replay artifact。",
"保留 tool request / approval decision / sandbox policy / result-or-error / recovery action 五类 transcript 字段。",
],
};
}
function renderApprovalSandboxTranscriptLines(evidence) {
const lines = [];
for (const step of evidence.transcriptSteps) {
const detailParts = [];
for (const [key, value] of Object.entries(step.details || {})) {
if (Array.isArray(value)) {
detailParts.push(`${key}=${value.join("|")}`);
} else if (value && typeof value === "object") {
detailParts.push(`${key}=${JSON.stringify(value)}`);
} else {
detailParts.push(`${key}=${String(value)}`);
}
}
lines.push(
`[smoke:agent-runtime-approval-sandbox] transcript.${step.id}: category=${step.category}; events=${step.eventTypes.join("/")}; ${step.summary}; ${detailParts.join("; ")}`,
);
}
for (const check of evidence.evidenceChecks) {
lines.push(
`[smoke:agent-runtime-approval-sandbox] evidence.${check.id}: ${
check.satisfied ? "satisfied" : "missing"
}; ${check.summary}; transcript=${check.transcriptStepIds.join(",")}`,
);
}
for (const mode of evidence.failureModeCoverage) {
lines.push(
`[smoke:agent-runtime-approval-sandbox] failure.${mode.id}: ${mode.status}; ${mode.summary}; transcript=${mode.transcriptStepIds.join(",")}${
mode.companionCommand ? `; companion=${mode.companionCommand}` : ""
}`,
);
}
const live = evidence.liveRuntimeTranscript;
if (live) {
lines.push(
`[smoke:agent-runtime-approval-sandbox] live.transcript.kind=${live.kind}; devBridgeStatus=${live.health?.status || "unknown"}; workspaceId=${live.workspaceId || ""}; liveRuntimeTranscript=${evidence.coverage.liveRuntimeTranscript ? "true" : "false"}`,
);
for (const flow of live.flows || []) {
lines.push(
`[smoke:agent-runtime-approval-sandbox] live.transcript.${flow.decision}.request: sessionId=${flow.sessionId}; turnId=${flow.turnId}; requestId=${flow.requestId}; permissionStatus=${flow.before?.permissionStatus}; confirmationStatus=${flow.before?.confirmationStatus}; pendingRequestCount=${flow.before?.pendingRequestCount}; latestTurnStatus=${flow.before?.latestTurnStatus}; approvalPolicy=${flow.submittedPolicies?.approvalPolicy}; sandboxPolicy=${flow.submittedPolicies?.sandboxPolicy}`,
);
lines.push(
`[smoke:agent-runtime-approval-sandbox] live.transcript.${flow.decision}.decision: confirmed=${flow.respond?.confirmed}; response=${flow.respond?.responseLabel}; afterConfirmationStatus=${flow.after?.confirmationStatus}; afterPendingRequestCount=${flow.after?.pendingRequestCount}; afterThreadStatus=${flow.after?.threadStatus}`,
);
}
for (const [key, value] of Object.entries(live.assertions || {})) {
lines.push(
`[smoke:agent-runtime-approval-sandbox] live.assertion.${key}: ${value ? "satisfied" : "missing"}`,
);
}
}
return lines;
}
export {
buildApprovalSandboxSmokeEvidence,
renderApprovalSandboxTranscriptLines,
};
@@ -0,0 +1,118 @@
import { describe, expect, it } from "vitest";
import {
buildApprovalSandboxSmokeEvidence,
renderApprovalSandboxTranscriptLines,
} from "./agent-runtime-approval-sandbox-smoke-core.mjs";
describe("agent-runtime-approval-sandbox-smoke-core", () => {
it("应生成覆盖 verifier 关键维度的 transcript 与 failure mode 摘要", () => {
const evidence = buildApprovalSandboxSmokeEvidence({
commandResults: [],
generatedAt: "2026-05-10T00:00:00.000Z",
});
expect(evidence.transcriptKind).toBe("verified_projection_summary");
expect(evidence.transcriptSteps.map((entry) => entry.category)).toEqual(
expect.arrayContaining([
"tool request",
"approval decision",
"sandbox policy",
"result",
"error",
"recovery action",
]),
);
expect(evidence.evidenceChecks.every((entry) => entry.satisfied)).toBe(true);
expect(evidence.failureModeCoverage.map((entry) => entry.id)).toEqual(
expect.arrayContaining([
"approval bypass",
"tool result missing",
"timeout without recovery",
"unsafe tool exposed",
]),
);
});
it("live runtime transcript 满足门禁时应标记为 liveRuntimeTranscript", () => {
const evidence = buildApprovalSandboxSmokeEvidence({
commandResults: [],
generatedAt: "2026-05-10T00:00:00.000Z",
liveRuntimeTranscript: {
kind: "devbridge-runtime-permission-confirmation",
health: { status: "ok" },
workspaceId: "workspace-1",
flows: [
{
decision: "denied",
sessionId: "session-denied",
turnId: "turn-denied",
requestId: "runtime_permission_confirmation:turn-denied",
submittedPolicies: {
approvalPolicy: "on-request",
sandboxPolicy: "workspace-write",
},
before: {
permissionStatus: "requires_confirmation",
confirmationStatus: "requested",
pendingRequestCount: 1,
latestTurnStatus: "failed",
},
respond: {
confirmed: false,
responseLabel: "拒绝",
},
after: {
confirmationStatus: "denied",
pendingRequestCount: 0,
threadStatus: "failed",
},
},
],
assertions: {
devBridgeHealthy: true,
permissionRequestCreatedBeforeModel: true,
deniedDecisionClearsPendingRequest: true,
resolvedDecisionClearsPendingRequest: true,
approvalPolicySubmitted: true,
sandboxPolicySubmitted: true,
},
},
});
expect(evidence.transcriptKind).toBe(
"verified_projection_and_live_runtime_transcript",
);
expect(evidence.coverage.liveRuntimeTranscript).toBe(true);
const lines = renderApprovalSandboxTranscriptLines(evidence).join("\n");
expect(lines).toContain("live.transcript.denied.request");
expect(lines).toContain(
"live.assertion.permissionRequestCreatedBeforeModel: satisfied",
);
});
it("应把 transcript、evidence 与 failure mode 渲染到 stdout 友好的行文本", () => {
const evidence = buildApprovalSandboxSmokeEvidence({
commandResults: [],
generatedAt: "2026-05-10T00:00:00.000Z",
});
const lines = renderApprovalSandboxTranscriptLines(evidence).join("\n");
expect(lines).toContain("transcript.tool-request");
expect(lines).toContain("transcript.approval-decision");
expect(lines).toContain("transcript.sandbox-policy");
expect(lines).toContain("transcript.tool-result");
expect(lines).toContain("transcript.tool-error");
expect(lines).toContain("transcript.timeout-recovery");
expect(lines).toContain("evidence.tool timeline: satisfied");
expect(lines).toContain("evidence.approval decision: satisfied");
expect(lines).toContain("evidence.sandbox policy: satisfied");
expect(lines).toContain("evidence.error recovery transcript: satisfied");
expect(lines).toContain("failure.approval bypass: covered");
expect(lines).toContain("failure.tool result missing: covered");
expect(lines).toContain("failure.timeout without recovery: covered");
expect(lines).toContain("failure.unsafe tool exposed: covered");
});
});
@@ -0,0 +1,221 @@
const REPORT_SCHEMA_VERSION = "lime.i18n.patchMetricsReport.v1";
function asFiniteNumber(value, fallback = 0) {
return Number.isFinite(Number(value)) ? Number(value) : fallback;
}
function asNonNegativeNumber(value, fallback = 0) {
return Math.max(0, asFiniteNumber(value, fallback));
}
function asArray(value) {
return Array.isArray(value) ? value : [];
}
function asString(value, fallback = "") {
return typeof value === "string" ? value : fallback;
}
function normalizeRun(run) {
return {
durationMs: asNonNegativeNumber(run?.durationMs),
language: asString(run?.language, "unknown"),
matchedSegments: asNonNegativeNumber(run?.matchedSegments),
replacedNodes: asNonNegativeNumber(run?.replacedNodes),
rootKind: asString(run?.rootKind, "unknown"),
timestamp: asNonNegativeNumber(run?.timestamp),
};
}
function sumRuns(runs, field) {
return runs.reduce((sum, run) => sum + asNonNegativeNumber(run[field]), 0);
}
function normalizeRuntimeMetrics(metrics) {
const recentRuns = asArray(metrics?.recentRuns ?? metrics?.runs).map(normalizeRun);
const patchTimes = asArray(metrics?.patchTimes).map((value) =>
asNonNegativeNumber(value),
);
const totalPatchTime = patchTimes.reduce((sum, value) => sum + value, 0);
const totalRuns = asNonNegativeNumber(
metrics?.totalRuns,
recentRuns.length,
);
const totalReplacedNodes = asNonNegativeNumber(
metrics?.totalReplacedNodes,
sumRuns(recentRuns, "replacedNodes"),
);
const totalMatchedSegments = asNonNegativeNumber(
metrics?.totalMatchedSegments,
sumRuns(recentRuns, "matchedSegments"),
);
const averagePatchTimeMs = asNonNegativeNumber(
metrics?.averagePatchTimeMs,
patchTimes.length > 0 ? totalPatchTime / patchTimes.length : 0,
);
const slowestPatchTimeMs = asNonNegativeNumber(
metrics?.slowestPatchTimeMs,
patchTimes.length > 0 ? Math.max(...patchTimes) : 0,
);
const lastRun = metrics?.lastRun
? normalizeRun(metrics.lastRun)
: (recentRuns.at(-1) ?? null);
return {
averagePatchTimeMs,
languageChanges: asNonNegativeNumber(metrics?.languageChanges),
lastRun,
recentRuns,
slowestPatchTimeMs,
totalMatchedSegments,
totalReplacedNodes,
totalRuns,
};
}
function resolveStatus(summary) {
if (summary.totalRuns === 0) {
return "missing-metrics";
}
if (
summary.totalMatchedSegments === 0 &&
summary.totalReplacedNodes === 0
) {
return "no-hit";
}
return "active-patch";
}
function buildRecommendations(status) {
if (status === "missing-metrics") {
return [
"先在 GUI smoke 或 Playwright 验证中导出 window.__I18N_METRICS__,再判断 Patch 退出进度。",
];
}
if (status === "no-hit") {
return [
"当前样本未命中 legacy Patch,可进入 current 主路径依赖审计,但还不能直接删除 DOM replacer。",
];
}
return [
"仍有 legacy Patch 命中,优先把 recentRuns 覆盖到的页面文案迁入 key-based namespace。",
];
}
function normalizeThresholds(thresholds = {}) {
return {
maxMatchedSegments:
thresholds.maxMatchedSegments === undefined
? null
: asNonNegativeNumber(thresholds.maxMatchedSegments),
maxReplacedNodes:
thresholds.maxReplacedNodes === undefined
? null
: asNonNegativeNumber(thresholds.maxReplacedNodes),
maxRuns:
thresholds.maxRuns === undefined
? null
: asNonNegativeNumber(thresholds.maxRuns),
};
}
function buildThresholdIssues(summary, thresholds) {
const issues = [];
const checks = [
["maxMatchedSegments", "totalMatchedSegments", "命中文本段数"],
["maxReplacedNodes", "totalReplacedNodes", "替换节点数"],
["maxRuns", "totalRuns", "Patch 运行次数"],
];
for (const [thresholdKey, summaryKey, label] of checks) {
const threshold = thresholds[thresholdKey];
if (threshold === null || threshold === undefined) {
continue;
}
const actual = summary[summaryKey];
if (actual > threshold) {
issues.push({
actual,
field: summaryKey,
message: `${label} ${actual} 超过门限 ${threshold}`,
threshold,
});
}
}
return issues;
}
export function createI18nPatchMetricsReport({
generatedAt = new Date().toISOString(),
metrics,
sourcePath = "",
thresholds,
} = {}) {
const normalizedMetrics = normalizeRuntimeMetrics(metrics ?? {});
const status = resolveStatus(normalizedMetrics);
const normalizedThresholds = normalizeThresholds(thresholds);
const thresholdIssues = buildThresholdIssues(
normalizedMetrics,
normalizedThresholds,
);
return {
schemaVersion: REPORT_SCHEMA_VERSION,
generatedAt,
sourcePath,
status,
retirementCandidate: status === "no-hit",
summary: {
averagePatchTimeMs: normalizedMetrics.averagePatchTimeMs,
languageChanges: normalizedMetrics.languageChanges,
lastRunAt: normalizedMetrics.lastRun?.timestamp ?? null,
lastRunLanguage: normalizedMetrics.lastRun?.language ?? null,
lastRunRootKind: normalizedMetrics.lastRun?.rootKind ?? null,
recentRunCount: normalizedMetrics.recentRuns.length,
slowestPatchTimeMs: normalizedMetrics.slowestPatchTimeMs,
totalMatchedSegments: normalizedMetrics.totalMatchedSegments,
totalReplacedNodes: normalizedMetrics.totalReplacedNodes,
totalRuns: normalizedMetrics.totalRuns,
},
thresholds: normalizedThresholds,
thresholdIssues,
recentRuns: normalizedMetrics.recentRuns,
recommendations: buildRecommendations(status),
};
}
export function renderI18nPatchMetricsTextReport(report) {
const lines = [
"Lime i18n Patch Metrics Report",
`状态: ${report.status}`,
`来源: ${report.sourcePath || "(未指定)"}`,
`生成时间: ${report.generatedAt}`,
`Patch 运行次数: ${report.summary.totalRuns}`,
`替换节点数: ${report.summary.totalReplacedNodes}`,
`命中文本段数: ${report.summary.totalMatchedSegments}`,
`语言切换次数: ${report.summary.languageChanges}`,
`平均耗时: ${report.summary.averagePatchTimeMs.toFixed(2)}ms`,
`最慢耗时: ${report.summary.slowestPatchTimeMs.toFixed(2)}ms`,
`退出候选: ${report.retirementCandidate ? "yes" : "no"}`,
];
if (report.thresholdIssues.length > 0) {
lines.push("", "门限问题:");
for (const issue of report.thresholdIssues) {
lines.push(`- ${issue.message}`);
}
}
lines.push("", "建议:");
for (const recommendation of report.recommendations) {
lines.push(`- ${recommendation}`);
}
return `${lines.join("\n")}\n`;
}
@@ -0,0 +1,84 @@
import { describe, expect, it } from "vitest";
import {
createI18nPatchMetricsReport,
renderI18nPatchMetricsTextReport,
} from "./i18n-patch-metrics-report-core.mjs";
describe("i18n-patch-metrics-report-core", () => {
it("应把 runtime metrics 转成稳定报告,并保留门限问题", () => {
const report = createI18nPatchMetricsReport({
generatedAt: "2026-05-10T00:00:00.000Z",
sourcePath: ".lime/i18n/patch-metrics.json",
thresholds: {
maxMatchedSegments: 1,
maxReplacedNodes: 1,
},
metrics: {
patchTimes: [10, 20],
languageChanges: 2,
totalRuns: 2,
totalReplacedNodes: 3,
totalMatchedSegments: 4,
runs: [
{
durationMs: 10,
language: "en",
matchedSegments: 4,
replacedNodes: 3,
rootKind: "document",
timestamp: 1778400000000,
},
],
},
});
expect(report.schemaVersion).toBe("lime.i18n.patchMetricsReport.v1");
expect(report.status).toBe("active-patch");
expect(report.retirementCandidate).toBe(false);
expect(report.summary.totalMatchedSegments).toBe(4);
expect(report.summary.averagePatchTimeMs).toBe(15);
expect(report.thresholdIssues.map((issue) => issue.field)).toEqual([
"totalMatchedSegments",
"totalReplacedNodes",
]);
});
it("应识别零命中样本为退出候选,但不等价于可直接删除", () => {
const report = createI18nPatchMetricsReport({
metrics: {
languageChanges: 1,
recentRuns: [
{
durationMs: 2,
language: "en",
matchedSegments: 0,
replacedNodes: 0,
rootKind: "document",
timestamp: 1778400000000,
},
],
totalMatchedSegments: 0,
totalReplacedNodes: 0,
totalRuns: 1,
},
});
expect(report.status).toBe("no-hit");
expect(report.retirementCandidate).toBe(true);
expect(report.recommendations.join("\n")).toContain("不能直接删除");
});
it("应渲染可读文本报告", () => {
const report = createI18nPatchMetricsReport({
generatedAt: "2026-05-10T00:00:00.000Z",
metrics: {},
sourcePath: "",
});
const text = renderI18nPatchMetricsTextReport(report);
expect(text).toContain("Lime i18n Patch Metrics Report");
expect(text).toContain("状态: missing-metrics");
expect(text).toContain("退出候选: no");
});
});
+111
View File
@@ -0,0 +1,111 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { spawnSync } from "node:child_process";
import { pathToFileURL } from "node:url";
const TauriAliasPatterns = [
["^@tauri-apps/api/core$", "core.ts"],
["^@tauri-apps/api/event$", "event.ts"],
["^@tauri-apps/api/window$", "window.ts"],
["^@tauri-apps/api/app$", "window.ts"],
["^@tauri-apps/api/path$", "window.ts"],
["^@tauri-apps/plugin-dialog$", "plugin-dialog.ts"],
["^@tauri-apps/plugin-shell$", "plugin-shell.ts"],
["^@tauri-apps/plugin-deep-link$", "plugin-deep-link.ts"],
["^@tauri-apps/plugin-global-shortcut$", "plugin-global-shortcut.ts"],
];
function toFileUrl(filePath) {
return pathToFileURL(filePath).href;
}
function createVitestSmokeConfig(rootDir) {
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lime-vitest-smoke-"));
const configPath = path.join(tempDir, "vitest.config.mjs");
const cacheDir = path.join(tempDir, "vite-cache");
const tauriMockDir = path.join(rootDir, "src/lib/tauri-mock");
const aliasSpecs = TauriAliasPatterns.map(([pattern, target]) => ({
pattern,
replacement: path.join(tauriMockDir, target),
}));
fs.writeFileSync(
configPath,
`import path from "node:path";\n` +
`import { defineConfig } from ${JSON.stringify(
toFileUrl(path.join(rootDir, "node_modules/vite/dist/node/index.js")),
)};\n` +
`import react from ${JSON.stringify(
toFileUrl(
path.join(rootDir, "node_modules/@vitejs/plugin-react/dist/index.js"),
),
)};\n` +
`import svgr from ${JSON.stringify(
toFileUrl(path.join(rootDir, "node_modules/vite-plugin-svgr/dist/index.js")),
)};\n` +
`const rootDir = ${JSON.stringify(rootDir)};\n` +
`const aliasSpecs = ${JSON.stringify(aliasSpecs)};\n` +
`export default defineConfig({\n` +
` root: rootDir,\n` +
` cacheDir: ${JSON.stringify(cacheDir)},\n` +
` define: { "import.meta.env.VITE_APP_VERSION": JSON.stringify(process.env.VITE_APP_VERSION || "test") },\n` +
` plugins: [react({ jsxRuntime: "automatic", jsxImportSource: "react", babel: { compact: true } }), svgr()],\n` +
` resolve: { alias: [\n` +
` { find: "@", replacement: path.resolve(rootDir, "src") },\n` +
` ...aliasSpecs.map((entry) => ({ find: new RegExp(entry.pattern), replacement: entry.replacement })),\n` +
` ] },\n` +
` test: {\n` +
` globals: true,\n` +
` environment: "jsdom",\n` +
` exclude: ["**/node_modules/**", "**/dist/**", "**/src-tauri/target/**"],\n` +
` },\n` +
`});\n`,
);
return {
configPath,
cleanup() {
fs.rmSync(tempDir, { recursive: true, force: true });
},
};
}
export function runVitestSmoke({ rootDir, label, args, logPrefix, env }) {
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
const config = createVitestSmokeConfig(rootDir);
const startedAt = Date.now();
console.log(`\n[${logPrefix}] > ${label}`);
try {
const result = spawnSync(
npmCommand,
["exec", "--", "vitest", "run", ...args, "--config", config.configPath],
{
cwd: rootDir,
stdio: "inherit",
env: { ...process.env, ...env },
},
);
if (result.error) {
throw result.error;
}
if (typeof result.status === "number" && result.status !== 0) {
const error = new Error(`[${logPrefix}] ${label} 失败`);
error.exitCode = result.status;
throw error;
}
return {
label,
status: "pass",
durationMs: Date.now() - startedAt,
args,
};
} finally {
config.cleanup();
}
}
+1
View File
@@ -11,6 +11,7 @@ const rootDir = process.cwd();
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
const cargoCommand = process.platform === "win32" ? "cargo.exe" : "cargo";
const BRIDGE_REASON_LABELS = {
agent_qc_contract: "Agent QC contract",
bridge_contracts: "bridge/contracts",
bridge_runtime: "DevBridge / mock / bridge runtime",
fallback_full_suite: "兜底全量",
+48 -1
View File
@@ -42,11 +42,26 @@ const FRONTEND_TOOLING_FILES = new Set([
const BRIDGE_FILES = new Set([
"vite.config.ts",
"scripts/agent-qc-completion-audit.mjs",
"docs/test/agent-qc-evidence.schema.json",
"docs/test/agent-qc-gui-flows.manifest.json",
"docs/test/agent-qc-scenarios.manifest.json",
"scripts/agent-qc-export-evidence.mjs",
"scripts/agent-qc-gui-flow-report.mjs",
"scripts/agent-qc-qcloop-job.mjs",
"scripts/agent-qc-release-summary.mjs",
"scripts/agent-qc-report.mjs",
"scripts/check-command-contracts.mjs",
"scripts/check-generated-slop-report.mjs",
"scripts/check-dev-bridge-health.mjs",
"scripts/harness-eval-history-record.mjs",
"scripts/harness-eval-trend-report.mjs",
"scripts/lib/agent-qc-completion-audit-core.mjs",
"scripts/lib/agent-qc-evidence-core.mjs",
"scripts/lib/agent-qc-gui-flow-core.mjs",
"scripts/lib/agent-qc-qcloop-job-core.mjs",
"scripts/lib/agent-qc-release-summary-core.mjs",
"scripts/lib/agent-qc-report-core.mjs",
"scripts/report-generated-slop.mjs",
"scripts/social-workbench-e2e-smoke.mjs",
"scripts/chrome-bridge-e2e.mjs",
@@ -65,6 +80,27 @@ const HARNESS_CLEANUP_CONTRACT_FILES = new Set([
"scripts/lib/harness-dashboard-core.mjs",
]);
const AGENT_QC_CONTRACT_FILES = new Set([
"docs/tests/agent-ops-qc.md",
"docs/tests/agent-qc-p0-scenarios.md",
"docs/tests/lime-agent-qc-rollout-plan.md",
"docs/test/agent-qc-evidence.schema.json",
"scripts/agent-qc-completion-audit.mjs",
"docs/test/agent-qc-gui-flows.manifest.json",
"docs/test/agent-qc-scenarios.manifest.json",
"scripts/agent-qc-export-evidence.mjs",
"scripts/agent-qc-gui-flow-report.mjs",
"scripts/agent-qc-qcloop-job.mjs",
"scripts/agent-qc-release-summary.mjs",
"scripts/agent-qc-report.mjs",
"scripts/lib/agent-qc-completion-audit-core.mjs",
"scripts/lib/agent-qc-evidence-core.mjs",
"scripts/lib/agent-qc-gui-flow-core.mjs",
"scripts/lib/agent-qc-qcloop-job-core.mjs",
"scripts/lib/agent-qc-release-summary-core.mjs",
"scripts/lib/agent-qc-report-core.mjs",
]);
const INTEGRITY_FILES = new Set([
"package.json",
"packages/lime-cli-npm/package.json",
@@ -264,7 +300,10 @@ function isDocsChange(file) {
}
function isDocsOnlyChange(files) {
return files.length > 0 && files.every((file) => isDocsChange(file));
return (
files.length > 0 &&
files.every((file) => isDocsChange(file) && !isAgentQcContractChange(file))
);
}
function isFrontendChange(file) {
@@ -291,6 +330,10 @@ function isHarnessCleanupContractChange(file) {
return HARNESS_CLEANUP_CONTRACT_FILES.has(file);
}
function isAgentQcContractChange(file) {
return AGENT_QC_CONTRACT_FILES.has(file);
}
function collectBridgeReasons(
changedFiles,
{ full = false, fallback = false, workflow = false } = {},
@@ -309,6 +352,10 @@ function collectBridgeReasons(
const reasons = [];
if (changedFiles.some(isAgentQcContractChange)) {
reasons.push("agent_qc_contract");
}
if (changedFiles.some(isHarnessCleanupContractChange)) {
reasons.push("harness_cleanup_contract");
}
+20
View File
@@ -3,6 +3,26 @@ import { describe, expect, it } from "vitest";
import { detectTasks } from "./quality-task-planner.mjs";
describe("quality-task-planner", () => {
it("应把 Agent QC manifest/schema 归到 bridge/contracts 风险", () => {
const tasks = detectTasks([
"docs/tests/agent-ops-qc.md",
"docs/tests/agent-qc-p0-scenarios.md",
"docs/tests/lime-agent-qc-rollout-plan.md",
"docs/test/agent-qc-scenarios.manifest.json",
"docs/test/agent-qc-evidence.schema.json",
"docs/test/agent-qc-gui-flows.manifest.json",
"scripts/lib/agent-qc-completion-audit-core.mjs",
"scripts/lib/agent-qc-evidence-core.mjs",
"scripts/lib/agent-qc-gui-flow-core.mjs",
"scripts/lib/agent-qc-qcloop-job-core.mjs",
"scripts/lib/agent-qc-release-summary-core.mjs",
]);
expect(tasks.bridge).toBe(true);
expect(tasks.bridgeReasons).toContain("agent_qc_contract");
expect(tasks.docsOnly).toBe(false);
});
it("应把 harness cleanup/report 主链文件归到 bridge/contracts 风险", () => {
const tasks = detectTasks([
"scripts/lib/generated-slop-report-core.mjs",
+123 -1
View File
@@ -6,6 +6,10 @@ import os from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
import {
createI18nPatchMetricsReport,
renderI18nPatchMetricsTextReport,
} from "./lib/i18n-patch-metrics-report-core.mjs";
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
@@ -40,6 +44,18 @@ const HEADLESS_TAURI_CONFIG_PATH = path.join(
"src-tauri",
"tauri.conf.headless.json",
);
const DEFAULT_I18N_PATCH_METRICS_OUTPUT = path.join(
rootDir,
".lime",
"i18n",
"patch-metrics.json",
);
const DEFAULT_I18N_PATCH_REPORT_OUTPUT = path.join(
rootDir,
".lime",
"i18n",
"patch-metrics-report.json",
);
const tauriCommand =
process.platform === "win32"
? path.join(rootDir, "node_modules", ".bin", "tauri.cmd")
@@ -173,6 +189,9 @@ const DEFAULTS = {
intervalMs: 1_000,
reuseRunning: false,
includeKnowledgeProductE2e: false,
i18nPatchMetricsOutput: DEFAULT_I18N_PATCH_METRICS_OUTPUT,
i18nPatchReportOutput: DEFAULT_I18N_PATCH_REPORT_OUTPUT,
skipI18nPatchMetrics: false,
sampleProjectName: "Lime Smoke Workspace",
};
DEFAULTS.timeoutMs = resolveDefaultTimeoutMs(DEFAULTS.cargoTargetDir);
@@ -213,6 +232,11 @@ Lime GUI 冒烟入口
--reuse-running 复用已启动的 headless Tauri,不主动拉起
--include-knowledge-product-e2e
额外执行项目资料产品 E2E 闭环验收
--i18n-patch-metrics-output <path>
导出 GUI 页面上的 window.__I18N_METRICS__,默认 .lime/i18n/patch-metrics.json
--i18n-patch-report-output <path>
写入 Patch metrics JSON 报告,默认 .lime/i18n/patch-metrics-report.json
--skip-i18n-patch-metrics 跳过 Patch metrics 导出与报告
-h, --help 显示帮助
`);
}
@@ -278,6 +302,27 @@ function parseArgs(argv) {
continue;
}
if (arg === "--i18n-patch-metrics-output" && argv[index + 1]) {
options.i18nPatchMetricsOutput = path.resolve(
String(argv[index + 1]).trim(),
);
index += 1;
continue;
}
if (arg === "--i18n-patch-report-output" && argv[index + 1]) {
options.i18nPatchReportOutput = path.resolve(
String(argv[index + 1]).trim(),
);
index += 1;
continue;
}
if (arg === "--skip-i18n-patch-metrics") {
options.skipI18nPatchMetrics = true;
continue;
}
if (arg === "--help" || arg === "-h") {
printHelp();
process.exit(0);
@@ -315,6 +360,15 @@ function parseArgs(argv) {
throw new Error("--cargo-target-dir 不能为空");
}
if (!options.skipI18nPatchMetrics) {
if (!options.i18nPatchMetricsOutput) {
throw new Error("--i18n-patch-metrics-output 不能为空");
}
if (!options.i18nPatchReportOutput) {
throw new Error("--i18n-patch-report-output 不能为空");
}
}
return options;
}
@@ -855,6 +909,45 @@ async function cleanupStaleGuiSmokeChromeProfiles(
}
}
function writeI18nPatchMetricsReport(options) {
if (options.skipI18nPatchMetrics) {
return;
}
if (!fs.existsSync(options.i18nPatchMetricsOutput)) {
console.warn(
`[verify:gui-smoke] 未发现 i18n Patch metrics: ${options.i18nPatchMetricsOutput}`,
);
return;
}
const metrics = JSON.parse(
fs.readFileSync(options.i18nPatchMetricsOutput, "utf8"),
);
const report = createI18nPatchMetricsReport({
metrics,
sourcePath: path.relative(rootDir, options.i18nPatchMetricsOutput),
});
fs.mkdirSync(path.dirname(options.i18nPatchReportOutput), {
recursive: true,
});
fs.writeFileSync(
options.i18nPatchReportOutput,
JSON.stringify(report, null, 2),
"utf8",
);
console.log(
renderI18nPatchMetricsTextReport(report)
.trim()
.split("\n")
.map((line) => `[verify:gui-smoke] ${line}`)
.join("\n"),
);
console.log(
`[verify:gui-smoke] i18n Patch metrics report: ${options.i18nPatchReportOutput}`,
);
}
function listListeningCommandsForPort(port) {
if (!port || process.platform === "win32") {
return [];
@@ -1481,7 +1574,7 @@ async function main() {
npmCommand,
[
"run",
"smoke:knowledge-gui",
"smoke:claw-chat-ready-streaming",
"--",
"--app-url",
options.appUrl,
@@ -1494,10 +1587,39 @@ async function main() {
"--interval-ms",
String(options.intervalMs),
],
"smoke:claw-chat-ready-streaming",
options.timeoutMs + 30_000,
);
await runCommand(
npmCommand,
[
"run",
"smoke:knowledge-gui",
"--",
"--app-url",
options.appUrl,
"--health-url",
options.healthUrl,
"--invoke-url",
options.invokeUrl,
"--timeout-ms",
String(options.timeoutMs),
"--interval-ms",
String(options.intervalMs),
...(options.skipI18nPatchMetrics
? []
: [
"--i18n-patch-metrics-output",
options.i18nPatchMetricsOutput,
]),
],
"smoke:knowledge-gui",
options.timeoutMs + 30_000,
);
writeI18nPatchMetricsReport(options);
if (options.includeKnowledgeProductE2e) {
await runCommand(
npmCommand,