mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-09-24 23:10:56 +08:00
chore: release v1.33.0
This commit is contained in:
@@ -0,0 +1,316 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import { createAgentQcGuiFlowReport } from "./lib/agent-qc-gui-flow-core.mjs";
|
||||
import { buildQCLoopJobPayload, validateQCLoopJobPayload } from "./lib/agent-qc-qcloop-job-core.mjs";
|
||||
import { createAgentQcReport } from "./lib/agent-qc-report-core.mjs";
|
||||
import {
|
||||
buildAgentQcCompletionAudit,
|
||||
renderAgentQcCompletionAuditMarkdown,
|
||||
} from "./lib/agent-qc-completion-audit-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
format: "markdown",
|
||||
help: false,
|
||||
outputPath: "",
|
||||
};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC Completion Audit
|
||||
|
||||
用法:
|
||||
npm run agent-qc:audit
|
||||
npm run agent-qc:audit -- --format json
|
||||
node scripts/agent-qc-completion-audit.mjs --check
|
||||
|
||||
选项:
|
||||
--format FMT markdown | json
|
||||
--output PATH 写入文件;默认 stdout
|
||||
--check 整体目标未完成时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function exists(filePath) {
|
||||
return fs.existsSync(path.resolve(process.cwd(), filePath));
|
||||
}
|
||||
|
||||
function readText(filePath) {
|
||||
return fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8");
|
||||
}
|
||||
|
||||
function readJson(filePath) {
|
||||
return JSON.parse(readText(filePath));
|
||||
}
|
||||
|
||||
function readOptionalEvidencePack(filePath) {
|
||||
if (!exists(filePath)) {
|
||||
return { exists: false, status: "", scenarioCount: 0, scenarioIds: [] };
|
||||
}
|
||||
try {
|
||||
const pack = readJson(filePath);
|
||||
return summarizeEvidencePack(pack, { exists: true });
|
||||
} catch {
|
||||
return { exists: true, status: "invalid-json", scenarioCount: 0, scenarioIds: [] };
|
||||
}
|
||||
}
|
||||
|
||||
function readOptionalJson(filePath, fallback) {
|
||||
if (!exists(filePath)) {
|
||||
return fallback;
|
||||
}
|
||||
try {
|
||||
return readJson(filePath);
|
||||
} catch {
|
||||
return { ...fallback, status: "invalid-json" };
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeEvidencePack(pack, overrides = {}) {
|
||||
return {
|
||||
...overrides,
|
||||
status: pack?.verdict?.status || "",
|
||||
scenarioCount: Array.isArray(pack?.scenarioResults)
|
||||
? pack.scenarioResults.length
|
||||
: 0,
|
||||
scenarioIds: Array.isArray(pack?.scenarioResults)
|
||||
? pack.scenarioResults
|
||||
.map((result) => String(result?.scenarioId || "").trim())
|
||||
.filter(Boolean)
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function readEvidenceSidecars(dirPath) {
|
||||
const resolvedDirPath = path.resolve(process.cwd(), dirPath);
|
||||
if (!fs.existsSync(resolvedDirPath)) {
|
||||
return [];
|
||||
}
|
||||
return fs
|
||||
.readdirSync(resolvedDirPath)
|
||||
.filter((fileName) => /^agent-qc-evidence\..+\.json$/.test(fileName))
|
||||
.sort()
|
||||
.map((fileName) => {
|
||||
const relativePath = path.posix.join(dirPath, fileName);
|
||||
try {
|
||||
const pack = readJson(relativePath);
|
||||
return summarizeEvidencePack(pack, { path: relativePath });
|
||||
} catch {
|
||||
return { path: relativePath, status: "invalid-json", scenarioCount: 0, scenarioIds: [] };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function readQcloopStatusSidecars(dirPath) {
|
||||
const resolvedDirPath = path.resolve(process.cwd(), dirPath);
|
||||
if (!fs.existsSync(resolvedDirPath)) {
|
||||
return [];
|
||||
}
|
||||
return fs
|
||||
.readdirSync(resolvedDirPath)
|
||||
.filter((fileName) => /^qcloop-status\..+\.json$/.test(fileName))
|
||||
.sort()
|
||||
.map((fileName) => {
|
||||
const relativePath = path.posix.join(dirPath, fileName);
|
||||
try {
|
||||
const status = readJson(relativePath);
|
||||
return {
|
||||
path: relativePath,
|
||||
verdictStatus: status?.verdict?.status || "",
|
||||
verdictSummary: status?.verdict?.summary || "",
|
||||
counts: status?.counts || null,
|
||||
};
|
||||
} catch {
|
||||
return { path: relativePath, verdictStatus: "invalid-json", verdictSummary: "", counts: null };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function loadFacts() {
|
||||
const scenarioManifest = readJson("docs/test/agent-qc-scenarios.manifest.json");
|
||||
const guiFlowManifest = readJson("docs/test/agent-qc-gui-flows.manifest.json");
|
||||
const evidenceSchema = readJson("docs/test/agent-qc-evidence.schema.json");
|
||||
const packageJson = readJson("package.json");
|
||||
const scenarioReport = createAgentQcReport({ manifest: scenarioManifest, packageJson, evidenceSchema });
|
||||
scenarioReport.p0ScenarioIds = Array.isArray(scenarioManifest?.scenarios)
|
||||
? scenarioManifest.scenarios
|
||||
.filter((scenario) => String(scenario?.risk || "").toUpperCase() === "P0")
|
||||
.map((scenario) => String(scenario?.id || "").trim())
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
const guiFlowReport = createAgentQcGuiFlowReport({ flowManifest: guiFlowManifest, scenarioManifest });
|
||||
const qcloopPayload = buildQCLoopJobPayload(scenarioManifest, { risks: ["P0"], generatedAt: "2026-05-10T00:00:00.000Z" });
|
||||
const qcloopPayloadValidation = validateQCLoopJobPayload(qcloopPayload);
|
||||
const evidenceCore = exists("scripts/lib/agent-qc-evidence-core.mjs")
|
||||
? readText("scripts/lib/agent-qc-evidence-core.mjs")
|
||||
: "";
|
||||
const releaseSummaryCore = exists("scripts/lib/agent-qc-release-summary-core.mjs")
|
||||
? readText("scripts/lib/agent-qc-release-summary-core.mjs")
|
||||
: "";
|
||||
const guiOwnerCore = exists("scripts/lib/agent-qc-gui-owner-core.mjs")
|
||||
? readText("scripts/lib/agent-qc-gui-owner-core.mjs")
|
||||
: "";
|
||||
const staleOwnerInterventionDoc = exists("docs/tests/lime-agent-qc-stale-owner-intervention.md")
|
||||
? readText("docs/tests/lime-agent-qc-stale-owner-intervention.md")
|
||||
: "";
|
||||
const nightly = exists(".github/workflows/harness-nightly.yml") ? readText(".github/workflows/harness-nightly.yml") : "";
|
||||
const release = exists(".github/workflows/release.yml") ? readText(".github/workflows/release.yml") : "";
|
||||
|
||||
return {
|
||||
files: {
|
||||
agentOpsQc: exists("docs/tests/agent-ops-qc.md"),
|
||||
p0Scenarios: exists("docs/tests/agent-qc-p0-scenarios.md"),
|
||||
limeRolloutPlan: exists("docs/tests/lime-agent-qc-rollout-plan.md"),
|
||||
testsReadme: exists("docs/tests/README.md"),
|
||||
evidenceSchema: exists("docs/test/agent-qc-evidence.schema.json"),
|
||||
qcloopJobScript: exists("scripts/agent-qc-qcloop-job.mjs"),
|
||||
guiOwnerCheckScript: exists("scripts/agent-qc-gui-owner-check.mjs"),
|
||||
qcloopStatusScript: exists("scripts/agent-qc-qcloop-status.mjs"),
|
||||
qcloopPreflightScript: exists("scripts/agent-qc-qcloop-preflight.mjs"),
|
||||
qcloopOperationsDoc: exists("docs/tests/lime-agent-qc-qcloop-operations.md"),
|
||||
evidenceContractDoc: exists("docs/tests/lime-agent-qc-evidence-contract.md"),
|
||||
staleOwnerInterventionDoc: exists("docs/tests/lime-agent-qc-stale-owner-intervention.md"),
|
||||
exportEvidenceScript: exists("scripts/agent-qc-export-evidence.mjs"),
|
||||
releaseSummaryScript: exists("scripts/agent-qc-release-summary.mjs"),
|
||||
realGuiEvidence: exists(".lime/qc/gui-evidence"),
|
||||
},
|
||||
realEvidencePack: readOptionalEvidencePack(".lime/qc/agent-qc-evidence.json"),
|
||||
localVerify: readOptionalJson(".lime/qc/verify-local-current.json", {
|
||||
exists: false,
|
||||
status: "",
|
||||
failedStage: "",
|
||||
error: "",
|
||||
}),
|
||||
guiSmoke: readOptionalJson(".lime/qc/verify-gui-smoke-current.json", {
|
||||
exists: false,
|
||||
status: "",
|
||||
failedStage: "",
|
||||
error: "",
|
||||
}),
|
||||
realEvidenceSidecars: readEvidenceSidecars(".lime/qc"),
|
||||
qcloopStatusSidecars: readQcloopStatusSidecars(".lime/qc"),
|
||||
scenarioReport,
|
||||
guiFlowReport,
|
||||
qcloopPayload: {
|
||||
valid: qcloopPayloadValidation.valid,
|
||||
itemCount: qcloopPayload.items.length,
|
||||
verifierHasWorkerOutput:
|
||||
qcloopPayload.verifier_prompt_template.includes("{{stdout}}") ||
|
||||
qcloopPayload.verifier_prompt_template.includes("{{output}}"),
|
||||
verifierHasAttemptStatus: qcloopPayload.verifier_prompt_template.includes(
|
||||
"{{attempt_status}}",
|
||||
),
|
||||
verifierHasExitCode: qcloopPayload.verifier_prompt_template.includes("{{exit_code}}"),
|
||||
workerPromptHasPreflight:
|
||||
qcloopPayload.prompt_template.includes("agent-qc:qcloop-preflight") &&
|
||||
qcloopPayload.prompt_template.includes("--require-devbridge"),
|
||||
workerPromptHasStructuredEvidence:
|
||||
qcloopPayload.prompt_template.includes("QCLOOP_WORKER_RESULT=PASS|FAIL|BLOCKED") &&
|
||||
qcloopPayload.prompt_template.includes("QCLOOP_EVIDENCE_SUMMARY_JSON=<json>"),
|
||||
verifierRequiresStructuredEvidence:
|
||||
qcloopPayload.verifier_prompt_template.includes("QCLOOP_EVIDENCE_SUMMARY_JSON"),
|
||||
verifierRequiresStrictJson:
|
||||
qcloopPayload.verifier_prompt_template.includes('{"pass": true|false') &&
|
||||
qcloopPayload.verifier_prompt_template.includes("不要 Markdown"),
|
||||
},
|
||||
structuredEvidence: {
|
||||
exporterParsesSummary:
|
||||
evidenceCore.includes("collectQCLoopEvidenceSummaries") &&
|
||||
evidenceCore.includes("qcloop:evidence_summary_missing") &&
|
||||
evidenceCore.includes("qcloop:evidence_summary_invalid_json"),
|
||||
releaseSummaryRejectsWeakRefs:
|
||||
releaseSummaryCore.includes("weakEvidenceScenarioIds") &&
|
||||
releaseSummaryCore.includes("hasStructuredEvidenceRef") &&
|
||||
releaseSummaryCore.includes("缺少结构化 evidenceRefs"),
|
||||
},
|
||||
staleOwnerIntervention: {
|
||||
guiOwnerReportHasDecisionPacket:
|
||||
guiOwnerCore.includes("ownerIntervention") &&
|
||||
guiOwnerCore.includes("requiredConfirmationText") &&
|
||||
guiOwnerCore.includes("prohibitedUntilConfirmed"),
|
||||
guiOwnerReportHasWatchHistory:
|
||||
guiOwnerCore.includes("createAgentQcGuiOwnerWatchEntry") &&
|
||||
exists("scripts/agent-qc-gui-owner-check.mjs") &&
|
||||
readText("scripts/agent-qc-gui-owner-check.mjs").includes("--watch-history-output"),
|
||||
docMentionsDecisionPacket:
|
||||
staleOwnerInterventionDoc.includes("ownerIntervention") &&
|
||||
staleOwnerInterventionDoc.includes("stale-owner-intervention-request.json"),
|
||||
docMentionsWatchHistory:
|
||||
staleOwnerInterventionDoc.includes("watch-history-output") ||
|
||||
staleOwnerInterventionDoc.includes("stale-owner-watch-history.jsonl"),
|
||||
},
|
||||
nightly: {
|
||||
hasAgentQcReport: nightly.includes("agent-qc-report"),
|
||||
hasGuiFlowReport: nightly.includes("agent-qc-gui-flow-report"),
|
||||
hasReleasePreview: nightly.includes("release-agent-qc-preview"),
|
||||
},
|
||||
release: {
|
||||
hasHardGate:
|
||||
release.includes("agent-qc-release-summary") &&
|
||||
release.includes("--check") &&
|
||||
!release.includes("--allow-missing-evidence"),
|
||||
requiresP0ScenarioCoverage:
|
||||
release.includes("--require-scenario-manifest") &&
|
||||
release.includes("docs/test/agent-qc-scenarios.manifest.json") &&
|
||||
release.includes("--require-risk") &&
|
||||
release.includes("P0"),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
const audit = buildAgentQcCompletionAudit(loadFacts());
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify(audit, null, 2)}\n`
|
||||
: renderAgentQcCompletionAuditMarkdown(audit);
|
||||
writeOutput(options.outputPath, content);
|
||||
if (options.check && audit.status !== "complete") {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,215 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
buildAgentQcEvidencePack,
|
||||
validateEvidencePackShape,
|
||||
} from "./lib/agent-qc-evidence-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
changedFiles: [],
|
||||
check: false,
|
||||
diffBase: "",
|
||||
format: "json",
|
||||
help: false,
|
||||
itemsJson: "",
|
||||
jobId: "",
|
||||
jobJson: "",
|
||||
outputPath: "",
|
||||
ref: "",
|
||||
repo: "lime",
|
||||
riskTags: [],
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--base-url" && argv[index + 1]) {
|
||||
result.baseUrl = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--job-id" && argv[index + 1]) {
|
||||
result.jobId = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--job-json" && argv[index + 1]) {
|
||||
result.jobJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--items-json" && argv[index + 1]) {
|
||||
result.itemsJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--repo" && argv[index + 1]) {
|
||||
result.repo = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--ref" && argv[index + 1]) {
|
||||
result.ref = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--diff-base" && argv[index + 1]) {
|
||||
result.diffBase = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--changed-file" && argv[index + 1]) {
|
||||
result.changedFiles.push(String(argv[index + 1]).trim());
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--risk-tag" && argv[index + 1]) {
|
||||
result.riskTags.push(String(argv[index + 1]).trim());
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC Evidence Export
|
||||
|
||||
用法:
|
||||
npm run agent-qc:export-evidence -- --job-id "qcloop-job-id" --output ./.lime/qc/evidence.json
|
||||
node scripts/agent-qc-export-evidence.mjs --job-json ./job.json --items-json ./items.json --check
|
||||
|
||||
选项:
|
||||
--job-id ID 从 qcloop HTTP API 读取 /api/jobs/:id 和 /api/items?job_id=:id
|
||||
--base-url URL qcloop API 地址,默认 http://127.0.0.1:8080
|
||||
--job-json PATH 离线 job JSON 文件
|
||||
--items-json PATH 离线 items JSON 文件
|
||||
--output PATH 写入 Evidence Pack;默认 stdout
|
||||
--repo NAME subject.repo,默认 lime
|
||||
--ref REF subject.ref
|
||||
--diff-base REF subject.diffBase
|
||||
--changed-file PATH 追加 changedFiles,可重复
|
||||
--risk-tag TAG 追加 riskTags,可重复
|
||||
--check 只校验输出形状,非法时非 0 退出
|
||||
--format FMT 输出格式:json | summary
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
|
||||
}
|
||||
|
||||
async function readJsonUrl(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`请求失败 ${response.status}: ${url}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function loadQCLoopData(options) {
|
||||
if (options.jobJson || options.itemsJson) {
|
||||
if (!options.jobJson || !options.itemsJson) {
|
||||
throw new Error("离线模式必须同时提供 --job-json 与 --items-json。");
|
||||
}
|
||||
return {
|
||||
job: readJsonFile(options.jobJson),
|
||||
items: readJsonFile(options.itemsJson),
|
||||
};
|
||||
}
|
||||
|
||||
if (!options.jobId) {
|
||||
throw new Error("必须提供 --job-id,或同时提供 --job-json / --items-json。");
|
||||
}
|
||||
|
||||
const baseUrl = options.baseUrl.replace(/\/$/, "");
|
||||
return {
|
||||
job: await readJsonUrl(`${baseUrl}/api/jobs/${encodeURIComponent(options.jobId)}`),
|
||||
items: await readJsonUrl(`${baseUrl}/api/items/?job_id=${encodeURIComponent(options.jobId)}`),
|
||||
};
|
||||
}
|
||||
|
||||
function renderSummary(pack, validation) {
|
||||
const lines = [
|
||||
`runId=${pack.runId}`,
|
||||
`status=${pack.verdict.status}`,
|
||||
`scenarios=${pack.scenarioResults.length}`,
|
||||
`valid=${validation.valid}`,
|
||||
`summary=${pack.verdict.summary}`,
|
||||
];
|
||||
if (validation.issues.length > 0) {
|
||||
lines.push(`issues=${validation.issues.join("; ")}`);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const { job, items } = await loadQCLoopData(options);
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job,
|
||||
items,
|
||||
options: {
|
||||
repo: options.repo,
|
||||
ref: options.ref,
|
||||
diffBase: options.diffBase,
|
||||
changedFiles: options.changedFiles,
|
||||
riskTags: options.riskTags,
|
||||
},
|
||||
});
|
||||
const validation = validateEvidencePackShape(pack);
|
||||
const content = options.format === "summary" ? renderSummary(pack, validation) : `${JSON.stringify(pack, null, 2)}\n`;
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (options.check && !validation.valid) {
|
||||
for (const issue of validation.issues) {
|
||||
console.error(`[agent-qc-evidence] ${issue}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`[agent-qc-evidence] ${error.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
createAgentQcGuiFlowReport,
|
||||
renderAgentQcGuiFlowMarkdown,
|
||||
} from "./lib/agent-qc-gui-flow-core.mjs";
|
||||
|
||||
const DEFAULT_FLOW_MANIFEST_PATH = "docs/test/agent-qc-gui-flows.manifest.json";
|
||||
const DEFAULT_SCENARIO_MANIFEST_PATH = "docs/test/agent-qc-scenarios.manifest.json";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
flowManifestPath: DEFAULT_FLOW_MANIFEST_PATH,
|
||||
format: "markdown",
|
||||
help: false,
|
||||
outputPath: "",
|
||||
scenarioManifestPath: DEFAULT_SCENARIO_MANIFEST_PATH,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--flow-manifest" && argv[index + 1]) {
|
||||
result.flowManifestPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--scenario-manifest" && argv[index + 1]) {
|
||||
result.scenarioManifestPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC GUI Flow Report
|
||||
|
||||
用法:
|
||||
npm run agent-qc:gui-flow:report
|
||||
npm run agent-qc:gui-flow:check
|
||||
|
||||
选项:
|
||||
--flow-manifest PATH GUI flow manifest,默认 docs/test/agent-qc-gui-flows.manifest.json
|
||||
--scenario-manifest PATH Agent QC scenario manifest,默认 docs/test/agent-qc-scenarios.manifest.json
|
||||
--format FMT markdown | json
|
||||
--output PATH 写入文件;默认 stdout
|
||||
--check 非法时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const report = createAgentQcGuiFlowReport({
|
||||
flowManifest: readJsonFile(options.flowManifestPath),
|
||||
scenarioManifest: readJsonFile(options.scenarioManifestPath),
|
||||
});
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify(report, null, 2)}\n`
|
||||
: renderAgentQcGuiFlowMarkdown(report);
|
||||
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (options.check && !report.valid) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
createAgentQcGuiOwnerWatchEntry,
|
||||
createAgentQcGuiOwnerReport,
|
||||
renderAgentQcGuiOwnerSummary,
|
||||
} from "./lib/agent-qc-gui-owner-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
format: "summary",
|
||||
help: false,
|
||||
manifestPath: "docs/test/agent-qc-scenarios.manifest.json",
|
||||
maxActiveOwners: 0,
|
||||
outputPath: "",
|
||||
statusDir: ".lime/qc",
|
||||
watchHistoryOutputPath: "",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--manifest" && argv[index + 1]) {
|
||||
result.manifestPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--status-dir" && argv[index + 1]) {
|
||||
result.statusDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--max-active-owners" && argv[index + 1]) {
|
||||
result.maxActiveOwners = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--watch-history-output" && argv[index + 1]) {
|
||||
result.watchHistoryOutputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC GUI Owner Check
|
||||
|
||||
用法:
|
||||
npm run agent-qc:gui-owner-check
|
||||
npm run agent-qc:gui-owner-check -- --check
|
||||
node scripts/agent-qc-gui-owner-check.mjs --format json --output ./.lime/qc/gui-owner-current.json
|
||||
|
||||
选项:
|
||||
--manifest PATH scenario manifest,默认 docs/test/agent-qc-scenarios.manifest.json
|
||||
--status-dir PATH qcloop status sidecar 目录,默认 .lime/qc
|
||||
--max-active-owners N 允许的 active GUI owner 数,默认 0
|
||||
--format FMT summary | json
|
||||
--output PATH 写入文件;默认 stdout
|
||||
--watch-history-output PATH 追加 JSONL 观察记录
|
||||
--check active owner 超过上限时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
|
||||
}
|
||||
|
||||
function readQcloopStatusSidecars(statusDir) {
|
||||
const resolvedDir = path.resolve(process.cwd(), statusDir);
|
||||
if (!fs.existsSync(resolvedDir)) {
|
||||
return [];
|
||||
}
|
||||
return fs
|
||||
.readdirSync(resolvedDir)
|
||||
.filter((fileName) => /^qcloop-status\..+\.json$/.test(fileName))
|
||||
.sort()
|
||||
.map((fileName) => {
|
||||
const relativePath = path.posix.join(statusDir, fileName);
|
||||
try {
|
||||
return { path: relativePath, status: readJsonFile(relativePath) };
|
||||
} catch {
|
||||
return { path: relativePath, status: { job: { id: relativePath, status: "invalid-json" }, items: [] } };
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function appendJsonLine(outputPath, entry) {
|
||||
if (!outputPath) {
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.appendFileSync(resolvedOutputPath, `${JSON.stringify(entry)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const report = createAgentQcGuiOwnerReport({
|
||||
manifest: readJsonFile(options.manifestPath),
|
||||
statusSidecars: readQcloopStatusSidecars(options.statusDir),
|
||||
maxActiveOwners: options.maxActiveOwners,
|
||||
});
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify(report, null, 2)}\n`
|
||||
: renderAgentQcGuiOwnerSummary(report);
|
||||
writeOutput(options.outputPath, content);
|
||||
appendJsonLine(options.watchHistoryOutputPath, createAgentQcGuiOwnerWatchEntry(report));
|
||||
|
||||
if (options.check && report.verdict.status !== "pass") {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Executable
+318
@@ -0,0 +1,318 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
const GUI_OWNER_PATTERNS = [
|
||||
"verify:gui-smoke",
|
||||
"smoke:workspace-ready",
|
||||
"smoke:browser-runtime",
|
||||
"smoke:site-adapters",
|
||||
"smoke:agent-service-skill-entry",
|
||||
"smoke:agent-runtime-tool-surface",
|
||||
"smoke:knowledge-gui",
|
||||
"smoke:design-canvas",
|
||||
"claw-chat-ready-streaming",
|
||||
"browser-runtime-site-adapter",
|
||||
"workspace-ready-session-restore",
|
||||
"release-package-startup-smoke",
|
||||
];
|
||||
|
||||
const CARGO_OWNER_PATTERNS = [
|
||||
"cargo ",
|
||||
"cargo-fmt",
|
||||
"rustc ",
|
||||
"clippy-driver",
|
||||
"tauri dev",
|
||||
];
|
||||
|
||||
const QCLOOP_OWNER_PATTERNS = [
|
||||
"qcloop --db",
|
||||
"./qcloop --db",
|
||||
"qcloop serve",
|
||||
"qcloop-worker",
|
||||
"qcloop_worker_result",
|
||||
"agent qc p0",
|
||||
"只读执行 lime agent qc p0",
|
||||
];
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
format: "summary",
|
||||
help: false,
|
||||
markdownOutputPath: "",
|
||||
maxActiveGuiSmoke: 0,
|
||||
maxCargoOrRust: 0,
|
||||
maxQcloopRelated: 0,
|
||||
outputPath: "",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--markdown-output" && argv[index + 1]) {
|
||||
result.markdownOutputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--max-active-gui-smoke" && argv[index + 1]) {
|
||||
result.maxActiveGuiSmoke = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--max-cargo-or-rust" && argv[index + 1]) {
|
||||
result.maxCargoOrRust = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--max-qcloop-related" && argv[index + 1]) {
|
||||
result.maxQcloopRelated = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC Process Owner Check
|
||||
|
||||
用法:
|
||||
npm run agent-qc:process-owner-check
|
||||
node scripts/agent-qc-process-owner-check.mjs --format json --output ./.lime/qc/gui-process-owner-current.json --markdown-output ./.lime/qc/gui-process-owner-current.md
|
||||
|
||||
选项:
|
||||
--format FMT summary | json,默认 summary
|
||||
--output PATH 写入 JSON 或 summary;默认 stdout
|
||||
--markdown-output PATH 额外写入 Markdown 摘要
|
||||
--max-active-gui-smoke N 允许的 GUI smoke / deep flow owner 数,默认 0
|
||||
--max-cargo-or-rust N 允许的 Cargo / Rust 构建进程数,默认 0
|
||||
--max-qcloop-related N 允许的 qcloop 相关进程数,默认 0
|
||||
--check owner 超过上限时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function parseUnixPsLine(line) {
|
||||
const match = /^\s*(\d+)\s+(\d+)\s+(\d+)\s+(\S+)\s+(\S+)\s+(.+)$/.exec(line);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
pid: Number(match[1]),
|
||||
ppid: Number(match[2]),
|
||||
pgid: Number(match[3]),
|
||||
stat: match[4],
|
||||
etime: match[5],
|
||||
command: match[6].trim(),
|
||||
};
|
||||
}
|
||||
|
||||
function collectUnixProcesses() {
|
||||
const output = execFileSync("ps", ["-axo", "pid,ppid,pgid,stat,etime,command"], {
|
||||
encoding: "utf8",
|
||||
});
|
||||
return output
|
||||
.split(/\r?\n/)
|
||||
.slice(1)
|
||||
.map(parseUnixPsLine)
|
||||
.filter(Boolean)
|
||||
.filter((entry) => entry.pid !== process.pid && entry.command.length > 0);
|
||||
}
|
||||
|
||||
function collectWindowsProcesses() {
|
||||
const output = execFileSync(
|
||||
"powershell.exe",
|
||||
[
|
||||
"-NoProfile",
|
||||
"-Command",
|
||||
"Get-CimInstance Win32_Process | Select-Object ProcessId,ParentProcessId,CommandLine | ConvertTo-Json -Compress",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
).trim();
|
||||
const parsed = output ? JSON.parse(output) : [];
|
||||
return (Array.isArray(parsed) ? parsed : [parsed])
|
||||
.map((entry) => ({
|
||||
pid: Number(entry?.ProcessId || 0),
|
||||
ppid: Number(entry?.ParentProcessId || 0),
|
||||
pgid: null,
|
||||
stat: "unknown",
|
||||
etime: "unknown",
|
||||
command: String(entry?.CommandLine || "").trim(),
|
||||
}))
|
||||
.filter((entry) => entry.pid > 0 && entry.pid !== process.pid && entry.command.length > 0);
|
||||
}
|
||||
|
||||
function collectProcesses() {
|
||||
return process.platform === "win32" ? collectWindowsProcesses() : collectUnixProcesses();
|
||||
}
|
||||
|
||||
function commandHasAny(command, patterns) {
|
||||
const normalized = String(command || "").toLowerCase();
|
||||
return patterns.some((pattern) => normalized.includes(pattern.toLowerCase()));
|
||||
}
|
||||
|
||||
function uniqueByPid(entries) {
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
for (const entry of entries) {
|
||||
if (seen.has(entry.pid)) {
|
||||
continue;
|
||||
}
|
||||
seen.add(entry.pid);
|
||||
result.push(entry);
|
||||
}
|
||||
return result.sort((left, right) => left.pid - right.pid);
|
||||
}
|
||||
|
||||
function createReport({
|
||||
generatedAt = new Date().toISOString(),
|
||||
maxActiveGuiSmoke = 0,
|
||||
maxCargoOrRust = 0,
|
||||
maxQcloopRelated = 0,
|
||||
} = {}) {
|
||||
const processes = collectProcesses();
|
||||
const activeGuiSmokeProcesses = uniqueByPid(
|
||||
processes.filter((entry) => commandHasAny(entry.command, GUI_OWNER_PATTERNS)),
|
||||
);
|
||||
const qcloopProcesses = uniqueByPid(
|
||||
processes.filter((entry) => commandHasAny(entry.command, QCLOOP_OWNER_PATTERNS)),
|
||||
);
|
||||
const cargoProcesses = uniqueByPid(
|
||||
processes.filter((entry) => commandHasAny(entry.command, CARGO_OWNER_PATTERNS)),
|
||||
);
|
||||
|
||||
const counts = {
|
||||
activeGuiSmoke: activeGuiSmokeProcesses.length,
|
||||
cargoOrRust: cargoProcesses.length,
|
||||
qcloopRelated: qcloopProcesses.length,
|
||||
};
|
||||
const busy =
|
||||
counts.activeGuiSmoke > maxActiveGuiSmoke ||
|
||||
counts.cargoOrRust > maxCargoOrRust ||
|
||||
counts.qcloopRelated > maxQcloopRelated;
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
generatedAt,
|
||||
platform: process.platform,
|
||||
maxActiveGuiSmoke,
|
||||
maxCargoOrRust,
|
||||
maxQcloopRelated,
|
||||
verdict: {
|
||||
status: busy ? "busy" : "pass",
|
||||
summary: `activeGuiSmoke=${counts.activeGuiSmoke}, cargoOrRust=${counts.cargoOrRust}, qcloopRelated=${counts.qcloopRelated}`,
|
||||
nextAction: busy
|
||||
? "Do not start full verify:local or another GUI P0 batch while these owners are active; continue read-only observation or wait for natural completion."
|
||||
: "No active raw GUI smoke, Cargo/Rust, or qcloop owner was observed; heavy gates may run if qcloop GUI owner and release evidence gates are also clear.",
|
||||
},
|
||||
activeGuiSmokeProcesses,
|
||||
qcloopProcesses,
|
||||
cargoProcesses,
|
||||
guardrails: [
|
||||
"best-effort process snapshot only",
|
||||
"do not kill or restart listed processes from this sidecar",
|
||||
"use this sidecar to decide whether heavy GUI or verify gates should wait",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function renderSummary(report) {
|
||||
const lines = [
|
||||
`status=${report.verdict.status}`,
|
||||
`summary=${report.verdict.summary}`,
|
||||
`generatedAt=${report.generatedAt}`,
|
||||
`platform=${report.platform}`,
|
||||
];
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function renderMarkdown(report) {
|
||||
const lines = [
|
||||
"# Agent QC raw process owner snapshot",
|
||||
"",
|
||||
`- Generated at: ${report.generatedAt}`,
|
||||
`- Status: ${report.verdict.status}`,
|
||||
`- Summary: ${report.verdict.summary}`,
|
||||
`- Next action: ${report.verdict.nextAction}`,
|
||||
"",
|
||||
"## Active GUI smoke / deep flow processes",
|
||||
"",
|
||||
];
|
||||
appendProcessList(lines, report.activeGuiSmokeProcesses);
|
||||
lines.push("", "## qcloop related processes", "");
|
||||
appendProcessList(lines, report.qcloopProcesses);
|
||||
lines.push("", "## Cargo / Rust processes", "");
|
||||
appendProcessList(lines, report.cargoProcesses);
|
||||
lines.push("", "## Guardrails", "");
|
||||
for (const guardrail of report.guardrails) {
|
||||
lines.push(`- ${guardrail}`);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function appendProcessList(lines, entries) {
|
||||
if (!entries?.length) {
|
||||
lines.push("- none");
|
||||
return;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
lines.push(
|
||||
`- pid=${entry.pid} ppid=${entry.ppid} pgid=${entry.pgid} stat=${entry.stat} etime=${entry.etime} command=${entry.command}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const report = createReport(options);
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify(report, null, 2)}\n`
|
||||
: renderSummary(report);
|
||||
writeOutput(options.outputPath, content);
|
||||
if (options.markdownOutputPath) {
|
||||
writeOutput(options.markdownOutputPath, renderMarkdown(report));
|
||||
}
|
||||
|
||||
if (options.check && report.verdict.status !== "pass") {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Executable
+258
@@ -0,0 +1,258 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
checkTerminal: false,
|
||||
dbPath: ".lime/qc/qcloop-isolated-worker-preflight.db",
|
||||
format: "summary",
|
||||
help: false,
|
||||
jobId: "",
|
||||
markdownOutputPath: "",
|
||||
outputPath: "",
|
||||
processOwnerPath: ".lime/qc/gui-process-owner-current.json",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--db" && argv[index + 1]) {
|
||||
result.dbPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--job-id" && argv[index + 1]) {
|
||||
result.jobId = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--process-owner" && argv[index + 1]) {
|
||||
result.processOwnerPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--markdown-output" && argv[index + 1]) {
|
||||
result.markdownOutputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check-terminal") {
|
||||
result.checkTerminal = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC qcloop DB Lease Snapshot
|
||||
|
||||
用法:
|
||||
npm run agent-qc:qcloop-db-lease -- --job-id <qcloop-job-id>
|
||||
node scripts/agent-qc-qcloop-db-lease.mjs --db ./.lime/qc/qcloop-isolated-worker-preflight.db --job-id <id> --format json --output ./.lime/qc/qcloop-db-lease-current.json
|
||||
|
||||
选项:
|
||||
--db PATH qcloop SQLite DB,默认 .lime/qc/qcloop-isolated-worker-preflight.db
|
||||
--job-id ID qcloop job id,必填
|
||||
--process-owner PATH raw process owner sidecar,默认 .lime/qc/gui-process-owner-current.json
|
||||
--format FMT summary | json,默认 summary
|
||||
--output PATH 写入文件;默认 stdout
|
||||
--markdown-output PATH 额外写入 Markdown 摘要
|
||||
--check-terminal job 未终态或仍有 running item 时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function sqlQuote(value) {
|
||||
return `'${String(value).replaceAll("'", "''")}'`;
|
||||
}
|
||||
|
||||
function sqliteJson(dbPath, sql) {
|
||||
const output = execFileSync("sqlite3", ["-readonly", "-json", dbPath, sql], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
return output ? JSON.parse(output) : [];
|
||||
}
|
||||
|
||||
function readJsonIfExists(filePath) {
|
||||
const resolved = path.resolve(process.cwd(), filePath);
|
||||
if (!fs.existsSync(resolved)) {
|
||||
return null;
|
||||
}
|
||||
return JSON.parse(fs.readFileSync(resolved, "utf8"));
|
||||
}
|
||||
|
||||
function extractScenarioId(itemPreview) {
|
||||
return String(itemPreview || "").match(/"scenario_id"\s*:\s*"([^"]+)"/)?.[1] || null;
|
||||
}
|
||||
|
||||
function createDbLeaseReport({
|
||||
dbPath,
|
||||
generatedAt = new Date().toISOString(),
|
||||
jobId,
|
||||
processOwnerPath,
|
||||
}) {
|
||||
if (!jobId) {
|
||||
throw new Error("--job-id is required");
|
||||
}
|
||||
const quotedJobId = sqlQuote(jobId);
|
||||
const jobs = sqliteJson(
|
||||
dbPath,
|
||||
`select id, name, status, created_at, finished_at from batch_jobs where id=${quotedJobId}`,
|
||||
);
|
||||
const items = sqliteJson(
|
||||
dbPath,
|
||||
`select id, batch_job_id, status, current_attempt_no, current_qc_no, tokens_used, lock_owner, lock_expires_at, queued_at, last_error, created_at, finished_at, substr(item_value,1,700) as item_preview from batch_items where batch_job_id=${quotedJobId} order by created_at, id`,
|
||||
);
|
||||
const attempts = sqliteJson(
|
||||
dbPath,
|
||||
`select attempts.id, attempts.batch_item_id, attempts.attempt_no, attempts.run_no, attempts.attempt_type, attempts.status, attempts.exit_code, attempts.tokens_used, attempts.started_at, attempts.finished_at, length(attempts.stdout) as stdout_len, length(attempts.stderr) as stderr_len, substr(attempts.stdout,1,700) as stdout_preview, substr(attempts.stderr,1,700) as stderr_preview from attempts join batch_items on batch_items.id=attempts.batch_item_id where batch_items.batch_job_id=${quotedJobId} order by attempts.started_at, attempts.id`,
|
||||
);
|
||||
const processOwner = readJsonIfExists(processOwnerPath) || { qcloopProcesses: [] };
|
||||
const processSnapshot = (processOwner.qcloopProcesses || []).filter((entry) => {
|
||||
const command = String(entry?.command || "");
|
||||
return command.includes(jobId) || command.includes("18080") || command.includes("browser-runtime-site-adapter");
|
||||
});
|
||||
const activeItem = items.find((item) => item.status === "running") || null;
|
||||
const activeAttempt = activeItem
|
||||
? attempts.find((attempt) => attempt.batch_item_id === activeItem.id && attempt.status === "running") || null
|
||||
: null;
|
||||
const observedWorker = processSnapshot.find((entry) =>
|
||||
String(entry?.command || "").includes("browser-runtime-site-adapter"),
|
||||
) || null;
|
||||
const observedQcloop = processSnapshot.find((entry) =>
|
||||
String(entry?.command || "").includes("serve --addr 127.0.0.1:18080"),
|
||||
) || null;
|
||||
const terminalStatuses = new Set(["completed", "failed", "cancelled", "canceled"]);
|
||||
const jobStatus = jobs[0]?.status || "unknown";
|
||||
const terminal = terminalStatuses.has(String(jobStatus).toLowerCase()) && !activeItem;
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
generatedAt,
|
||||
jobId,
|
||||
dbPath,
|
||||
processOwnerPath,
|
||||
summary: {
|
||||
status: jobStatus,
|
||||
terminal,
|
||||
activeItemId: activeItem?.id || null,
|
||||
activeScenario: extractScenarioId(activeItem?.item_preview),
|
||||
lockOwner: activeItem?.lock_owner || null,
|
||||
lockExpiresAt: activeItem?.lock_expires_at || null,
|
||||
activeAttemptId: activeAttempt?.id || null,
|
||||
activeAttemptStatus: activeAttempt?.status || null,
|
||||
stdoutLength: activeAttempt?.stdout_len ?? null,
|
||||
stderrLength: activeAttempt?.stderr_len ?? null,
|
||||
observedWorkerPid: observedWorker?.pid || null,
|
||||
observedQcloopPid: observedQcloop?.pid || null,
|
||||
finding: activeItem
|
||||
? "qcloop DB lease, active attempt, and process snapshot still show a running no-output stale GUI owner; no intervention was performed."
|
||||
: "No running item observed in this DB snapshot; verify qcloop status sidecar before changing gates.",
|
||||
},
|
||||
jobs,
|
||||
items,
|
||||
attempts,
|
||||
processSnapshot,
|
||||
guardrails: [
|
||||
"read-only DB observation only",
|
||||
"do not kill / pause / interrupt stale worker without owner confirmation",
|
||||
"do not modify qcloop SQLite DB",
|
||||
"do not start another full GUI P0 batch while GUI owner is blocked",
|
||||
"do not overwrite .lime/qc/agent-qc-evidence.json before a real 8/8 P0 pass",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function renderSummary(report) {
|
||||
const summary = report.summary;
|
||||
return [
|
||||
`status=${summary.status}`,
|
||||
`terminal=${summary.terminal}`,
|
||||
`activeItem=${summary.activeItemId || "none"}`,
|
||||
`activeScenario=${summary.activeScenario || "none"}`,
|
||||
`lockOwner=${summary.lockOwner || "none"}`,
|
||||
`lockExpiresAt=${summary.lockExpiresAt || "none"}`,
|
||||
`activeAttempt=${summary.activeAttemptId || "none"}:${summary.activeAttemptStatus || "none"}`,
|
||||
`stdoutStderr=${summary.stdoutLength ?? "null"}/${summary.stderrLength ?? "null"}`,
|
||||
`observedWorkerPid=${summary.observedWorkerPid || "none"}`,
|
||||
`observedQcloopPid=${summary.observedQcloopPid || "none"}`,
|
||||
`finding=${summary.finding}`,
|
||||
].join("\n") + "\n";
|
||||
}
|
||||
|
||||
function renderMarkdown(report) {
|
||||
const summary = report.summary;
|
||||
return [
|
||||
"# qcloop DB lease snapshot",
|
||||
"",
|
||||
`- Generated at: ${report.generatedAt}`,
|
||||
`- Job: ${report.jobId}`,
|
||||
`- DB: ${report.dbPath}`,
|
||||
`- Status: ${summary.status}`,
|
||||
`- Terminal: ${summary.terminal}`,
|
||||
`- Active item: ${summary.activeItemId || "none"}`,
|
||||
`- Active scenario: ${summary.activeScenario || "none"}`,
|
||||
`- Lock owner: ${summary.lockOwner || "none"}`,
|
||||
`- Lock expires at: ${summary.lockExpiresAt || "none"}`,
|
||||
`- Active attempt: ${summary.activeAttemptId || "none"} / ${summary.activeAttemptStatus || "none"}`,
|
||||
`- stdout/stderr length: ${summary.stdoutLength ?? "null"} / ${summary.stderrLength ?? "null"}`,
|
||||
`- Observed worker PID: ${summary.observedWorkerPid || "none"}`,
|
||||
`- Observed qcloop PID: ${summary.observedQcloopPid || "none"}`,
|
||||
"",
|
||||
"## Finding",
|
||||
"",
|
||||
summary.finding,
|
||||
"",
|
||||
"## Guardrails",
|
||||
"",
|
||||
...report.guardrails.map((guardrail) => `- ${guardrail}`),
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
const report = createDbLeaseReport(options);
|
||||
const content = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderSummary(report);
|
||||
writeOutput(options.outputPath, content);
|
||||
if (options.markdownOutputPath) {
|
||||
writeOutput(options.markdownOutputPath, renderMarkdown(report));
|
||||
}
|
||||
if (options.checkTerminal && !report.summary.terminal) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,195 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
buildQCLoopJobPayload,
|
||||
renderQCLoopCurl,
|
||||
validateQCLoopJobPayload,
|
||||
} from "./lib/agent-qc-qcloop-job-core.mjs";
|
||||
|
||||
const DEFAULT_MANIFEST_PATH = "docs/test/agent-qc-scenarios.manifest.json";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
check: false,
|
||||
cwd: process.cwd(),
|
||||
executorProvider: "codex",
|
||||
executionMode: "standard",
|
||||
format: "json",
|
||||
help: false,
|
||||
includeAll: false,
|
||||
manifestPath: DEFAULT_MANIFEST_PATH,
|
||||
maxQcRounds: 0,
|
||||
maxExecutorRetries: undefined,
|
||||
name: "",
|
||||
outputPath: "",
|
||||
risks: [],
|
||||
scenarioIds: [],
|
||||
tokenBudgetPerItem: 0,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--manifest" && argv[index + 1]) {
|
||||
result.manifestPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--risk" && argv[index + 1]) {
|
||||
result.risks.push(String(argv[index + 1]).trim());
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--scenario" && argv[index + 1]) {
|
||||
result.scenarioIds.push(String(argv[index + 1]).trim());
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--all") {
|
||||
result.includeAll = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--name" && argv[index + 1]) {
|
||||
result.name = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--cwd" && argv[index + 1]) {
|
||||
result.cwd = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--executor-provider" && argv[index + 1]) {
|
||||
result.executorProvider = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--execution-mode" && argv[index + 1]) {
|
||||
result.executionMode = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--max-qc-rounds" && argv[index + 1]) {
|
||||
result.maxQcRounds = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--max-executor-retries" && argv[index + 1]) {
|
||||
result.maxExecutorRetries = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--token-budget-per-item" && argv[index + 1]) {
|
||||
result.tokenBudgetPerItem = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--base-url" && argv[index + 1]) {
|
||||
result.baseUrl = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC qcloop Job Payload
|
||||
|
||||
用法:
|
||||
npm run agent-qc:qcloop-job -- --risk P0 --output ./tmp/qcloop-p0-job.json
|
||||
node scripts/agent-qc-qcloop-job.mjs --scenario command-bridge-contract --format curl
|
||||
|
||||
选项:
|
||||
--risk RISK 选择风险等级,可重复;默认 P0
|
||||
--scenario ID 选择指定 scenario,可重复;优先级高于 --risk
|
||||
--all 选择全部 scenario
|
||||
--name NAME qcloop job 名称
|
||||
--cwd PATH worker 目标仓库目录,默认当前目录
|
||||
--executor-provider NAME qcloop executor_provider,默认 codex
|
||||
--execution-mode MODE standard | goal_assisted,默认 standard
|
||||
--max-qc-rounds N 覆盖 max_qc_rounds
|
||||
--max-executor-retries N 覆盖 max_executor_retries,0-5
|
||||
--token-budget-per-item N 覆盖 token_budget_per_item
|
||||
--format FMT json | curl
|
||||
--base-url URL curl 模式 qcloop API 地址,默认 http://127.0.0.1:8080
|
||||
--output PATH 写入文件;默认 stdout
|
||||
--check payload 非法时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const manifest = readJsonFile(options.manifestPath);
|
||||
const payload = buildQCLoopJobPayload(manifest, {
|
||||
includeAll: options.includeAll,
|
||||
risks: options.risks.length > 0 ? options.risks : ["P0"],
|
||||
scenarioIds: options.scenarioIds,
|
||||
name: options.name,
|
||||
cwd: options.cwd,
|
||||
executorProvider: options.executorProvider,
|
||||
executionMode: options.executionMode,
|
||||
maxQcRounds: options.maxQcRounds,
|
||||
maxExecutorRetries: options.maxExecutorRetries,
|
||||
tokenBudgetPerItem: options.tokenBudgetPerItem,
|
||||
});
|
||||
const validation = validateQCLoopJobPayload(payload);
|
||||
const content =
|
||||
options.format === "curl"
|
||||
? renderQCLoopCurl(payload, { baseUrl: options.baseUrl })
|
||||
: `${JSON.stringify({ ...payload, _validation: validation }, null, 2)}\n`;
|
||||
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (options.check && !validation.valid) {
|
||||
for (const issue of validation.issues) {
|
||||
console.error(`[agent-qc-qcloop-job] ${issue}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,149 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import { buildQCLoopPreflightReport } from "./lib/agent-qc-qcloop-preflight-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
expectedCwd: "",
|
||||
format: "summary",
|
||||
help: false,
|
||||
healthUrl: "http://127.0.0.1:3030/health",
|
||||
requireDevBridge: false,
|
||||
timeoutMs: 15000,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--expected-cwd" && argv[index + 1]) {
|
||||
result.expectedCwd = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--health-url" && argv[index + 1]) {
|
||||
result.healthUrl = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--timeout-ms" && argv[index + 1]) {
|
||||
result.timeoutMs = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--require-devbridge") {
|
||||
result.requireDevBridge = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC qcloop Worker Preflight
|
||||
|
||||
用法:
|
||||
npm run agent-qc:qcloop-preflight -- --check
|
||||
npm run agent-qc:qcloop-preflight -- --require-devbridge --timeout-ms 15000 --check
|
||||
|
||||
选项:
|
||||
--expected-cwd PATH 预期仓库 cwd;不传则只检查 cwd 存在
|
||||
--require-devbridge 要求 http://127.0.0.1:3030/health 可访问
|
||||
--health-url URL DevBridge health 地址,默认 http://127.0.0.1:3030/health
|
||||
--timeout-ms N DevBridge health 超时,默认 15000
|
||||
--format FMT summary | json
|
||||
--check preflight blocked 时非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function checkTmpWritable() {
|
||||
const filePath = path.join(os.tmpdir(), `lime-agent-qc-preflight-${process.pid}-${Date.now()}.tmp`);
|
||||
try {
|
||||
fs.writeFileSync(filePath, "ok", "utf8");
|
||||
fs.unlinkSync(filePath);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
async function checkDevBridgeHealth({ url, timeoutMs }) {
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), Number.isFinite(timeoutMs) ? timeoutMs : 15000);
|
||||
try {
|
||||
const response = await fetch(url, { signal: controller.signal });
|
||||
const body = await response.text();
|
||||
let status = response.ok ? "ok" : `http-${response.status}`;
|
||||
try {
|
||||
const parsed = JSON.parse(body);
|
||||
status = parsed?.status || status;
|
||||
} catch {
|
||||
// health body 不是 JSON 时保留 HTTP 状态。
|
||||
}
|
||||
return { ok: response.ok, status, url };
|
||||
} catch (error) {
|
||||
return { ok: false, error: `${error.name || "Error"}: ${error.message}`, url };
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function renderSummary(report) {
|
||||
const lines = [
|
||||
`status=${report.status}`,
|
||||
`summary=${report.summary}`,
|
||||
`QCLOOP_PREFLIGHT_RESULT=${report.status === "pass" ? "PASS" : "BLOCKED"}`,
|
||||
];
|
||||
for (const check of report.checks) {
|
||||
lines.push(`- ${check.id} ${check.passed ? "PASS" : "BLOCKED"}: ${check.detail}`);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const devBridge = options.requireDevBridge
|
||||
? await checkDevBridgeHealth({ url: options.healthUrl, timeoutMs: options.timeoutMs })
|
||||
: null;
|
||||
const report = buildQCLoopPreflightReport({
|
||||
cwd: process.cwd(),
|
||||
expectedCwd: options.expectedCwd,
|
||||
tmpWritable: checkTmpWritable(),
|
||||
devBridge,
|
||||
});
|
||||
const content = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderSummary(report);
|
||||
process.stdout.write(content);
|
||||
|
||||
if (options.check && report.status !== "pass") {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`[agent-qc-qcloop-preflight] ${error.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,237 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
buildQCLoopStatusReport,
|
||||
validateQCLoopStatusReport,
|
||||
} from "./lib/agent-qc-qcloop-status-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
baseUrl: "http://127.0.0.1:8080",
|
||||
checkTerminal: false,
|
||||
failOnStale: false,
|
||||
format: "summary",
|
||||
help: false,
|
||||
itemsJson: "",
|
||||
jobId: "",
|
||||
jobJson: "",
|
||||
outputPath: "",
|
||||
staleMinutes: 30,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--base-url" && argv[index + 1]) {
|
||||
result.baseUrl = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--job-id" && argv[index + 1]) {
|
||||
result.jobId = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--job-json" && argv[index + 1]) {
|
||||
result.jobJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--items-json" && argv[index + 1]) {
|
||||
result.itemsJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--stale-minutes" && argv[index + 1]) {
|
||||
result.staleMinutes = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check-terminal") {
|
||||
result.checkTerminal = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--fail-on-stale") {
|
||||
result.failOnStale = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC qcloop Status
|
||||
|
||||
用法:
|
||||
npm run agent-qc:qcloop-status -- --job-id "qcloop-job-id"
|
||||
npm run agent-qc:qcloop-status -- --job-id "qcloop-job-id" --format json --output ./.lime/qc/qcloop-status.json
|
||||
node scripts/agent-qc-qcloop-status.mjs --job-json ./job.json --items-json ./items.json
|
||||
|
||||
选项:
|
||||
--job-id ID 从 qcloop HTTP API 读取 /api/jobs/:id 和 /api/items?job_id=:id
|
||||
--base-url URL qcloop API 地址,默认 http://127.0.0.1:8080
|
||||
--job-json PATH 离线 job JSON 文件
|
||||
--items-json PATH 离线 items JSON 文件
|
||||
--output PATH 写入报告;默认 stdout
|
||||
--stale-minutes N running item 超过 N 分钟且无 stdout/stderr 时标记 stale,默认 30
|
||||
--format FMT 输出格式:summary | json
|
||||
--check-terminal job 未 complete 或存在失败 / 卡住 item 时非 0 退出
|
||||
--fail-on-stale 只要存在 stale item 就非 0 退出
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
|
||||
}
|
||||
|
||||
async function readJsonUrl(url) {
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`请求失败 ${response.status}: ${url}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
async function loadQCLoopData(options) {
|
||||
if (options.jobJson || options.itemsJson) {
|
||||
if (!options.jobJson || !options.itemsJson) {
|
||||
throw new Error("离线模式必须同时提供 --job-json 与 --items-json。");
|
||||
}
|
||||
return {
|
||||
job: readJsonFile(options.jobJson),
|
||||
items: readJsonFile(options.itemsJson),
|
||||
};
|
||||
}
|
||||
|
||||
if (!options.jobId) {
|
||||
throw new Error("必须提供 --job-id,或同时提供 --job-json / --items-json。");
|
||||
}
|
||||
|
||||
const baseUrl = options.baseUrl.replace(/\/$/, "");
|
||||
return {
|
||||
job: await readJsonUrl(`${baseUrl}/api/jobs/${encodeURIComponent(options.jobId)}`),
|
||||
items: await readJsonUrl(`${baseUrl}/api/items/?job_id=${encodeURIComponent(options.jobId)}`),
|
||||
};
|
||||
}
|
||||
|
||||
function renderItemLine(item) {
|
||||
const parts = [
|
||||
`- ${item.scenarioId}`,
|
||||
`status=${item.qcloopStatus}`,
|
||||
`worker=${item.worker.status}`,
|
||||
`attempt=${item.currentAttemptNo}`,
|
||||
`qc=${item.currentQcNo}`,
|
||||
];
|
||||
if (item.worker.durationMinutes !== null) {
|
||||
parts.push(`duration=${item.worker.durationMinutes}m`);
|
||||
}
|
||||
parts.push(`stdout=${item.worker.stdoutLength}`);
|
||||
parts.push(`stderr=${item.worker.stderrLength}`);
|
||||
if (item.stale) {
|
||||
parts.push(`stale=${item.staleReasons.join("; ")}`);
|
||||
}
|
||||
if (item.qc.feedback) {
|
||||
parts.push(`feedback=${item.qc.feedback.replace(/\s+/g, " ").slice(0, 180)}`);
|
||||
}
|
||||
return parts.join(" ");
|
||||
}
|
||||
|
||||
function renderSummary(report, validation) {
|
||||
const lines = [
|
||||
`job=${report.job.id}`,
|
||||
`name=${report.job.name}`,
|
||||
`status=${report.job.status}`,
|
||||
`verdict=${report.verdict.status}`,
|
||||
`items=${report.counts.total} terminal=${report.counts.terminal} nonTerminal=${report.counts.nonTerminal}`,
|
||||
`success=${report.counts.success} failed=${report.counts.failed} exhausted=${report.counts.exhausted} running=${report.counts.running} pending=${report.counts.pending} stale=${report.counts.stale}`,
|
||||
`summary=${report.verdict.summary}`,
|
||||
`nextAction=${report.verdict.nextAction}`,
|
||||
`valid=${validation.valid}`,
|
||||
];
|
||||
if (validation.issues.length > 0) {
|
||||
lines.push(`issues=${validation.issues.join("; ")}`);
|
||||
}
|
||||
|
||||
const activeItems = report.items.filter(
|
||||
(item) =>
|
||||
!item.terminal ||
|
||||
item.stale ||
|
||||
item.qcloopStatus === "failed" ||
|
||||
item.qcloopStatus === "exhausted",
|
||||
);
|
||||
if (activeItems.length > 0) {
|
||||
lines.push("items:");
|
||||
for (const item of activeItems) {
|
||||
lines.push(renderItemLine(item));
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const { job, items } = await loadQCLoopData(options);
|
||||
const report = buildQCLoopStatusReport({
|
||||
job,
|
||||
items,
|
||||
options: {
|
||||
staleMinutes: options.staleMinutes,
|
||||
},
|
||||
});
|
||||
const validation = validateQCLoopStatusReport(report);
|
||||
const content = options.format === "json" ? `${JSON.stringify(report, null, 2)}\n` : renderSummary(report, validation);
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (!validation.valid) {
|
||||
for (const issue of validation.issues) {
|
||||
console.error(`[agent-qc-qcloop-status] ${issue}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
if (options.failOnStale && report.counts.stale > 0) {
|
||||
process.exit(2);
|
||||
}
|
||||
if (options.checkTerminal && report.verdict.status !== "complete") {
|
||||
process.exit(3);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(`[agent-qc-qcloop-status] ${error.message}`);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,184 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
buildAgentQcReleaseSummary,
|
||||
renderAgentQcReleaseMarkdown,
|
||||
validateReleaseSummary,
|
||||
} from "./lib/agent-qc-release-summary-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
evidencePaths: [],
|
||||
format: "markdown",
|
||||
harnessSummaryPath: "",
|
||||
harnessTrendPath: "",
|
||||
help: false,
|
||||
outputPath: "",
|
||||
requireEvidence: true,
|
||||
requiredRisks: [],
|
||||
requiredScenarioManifestPath: "",
|
||||
tag: "",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--evidence" && argv[index + 1]) {
|
||||
result.evidencePaths.push(String(argv[index + 1]).trim());
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--harness-summary" && argv[index + 1]) {
|
||||
result.harnessSummaryPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--harness-trend" && argv[index + 1]) {
|
||||
result.harnessTrendPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--tag" && argv[index + 1]) {
|
||||
result.tag = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--require-scenario-manifest" && argv[index + 1]) {
|
||||
result.requiredScenarioManifestPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--require-risk" && argv[index + 1]) {
|
||||
result.requiredRisks.push(String(argv[index + 1]).trim());
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--allow-missing-evidence") {
|
||||
result.requireEvidence = false;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC Release Summary
|
||||
|
||||
用法:
|
||||
npm run agent-qc:release-summary -- --evidence ./.lime/qc/agent-qc-evidence.json --require-scenario-manifest docs/test/agent-qc-scenarios.manifest.json --require-risk P0 --tag v1.2.3
|
||||
node scripts/agent-qc-release-summary.mjs --evidence evidence.json --harness-summary summary.json --harness-trend trend.json --output release-qc.md --check
|
||||
|
||||
选项:
|
||||
--evidence PATH Agent QC Evidence Pack,可重复
|
||||
--harness-summary PATH harness-eval-summary.json
|
||||
--harness-trend PATH harness-eval-trend.json
|
||||
--tag TAG release tag
|
||||
--require-scenario-manifest PATH 要求 Evidence Pack 覆盖该 manifest 中的场景
|
||||
--require-risk RISK 搭配 --require-scenario-manifest 使用,可重复;默认 P0
|
||||
--output PATH 写入文件;默认 stdout
|
||||
--format FMT markdown | json
|
||||
--check 非 pass 或缺证据时非 0 退出
|
||||
--allow-missing-evidence 允许无 Evidence Pack,适合本地预览,不适合发布门禁
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(path.resolve(process.cwd(), filePath), "utf8"));
|
||||
}
|
||||
|
||||
function readOptionalJson(filePath) {
|
||||
if (!filePath) {
|
||||
return null;
|
||||
}
|
||||
return readJsonFile(filePath);
|
||||
}
|
||||
|
||||
function loadRequiredScenarioIds(manifestPath, risks) {
|
||||
if (!manifestPath) {
|
||||
return [];
|
||||
}
|
||||
const manifest = readJsonFile(manifestPath);
|
||||
const requiredRisks = new Set((risks.length > 0 ? risks : ["P0"]).map((risk) => risk.toUpperCase()));
|
||||
return Array.isArray(manifest?.scenarios)
|
||||
? manifest.scenarios
|
||||
.filter((scenario) => requiredRisks.has(String(scenario?.risk || "").toUpperCase()))
|
||||
.map((scenario) => String(scenario?.id || "").trim())
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const evidencePacks = options.evidencePaths.map((sourcePath) => ({
|
||||
sourcePath,
|
||||
pack: readJsonFile(sourcePath),
|
||||
}));
|
||||
const requiredScenarioIds = loadRequiredScenarioIds(
|
||||
options.requiredScenarioManifestPath,
|
||||
options.requiredRisks,
|
||||
);
|
||||
const summary = buildAgentQcReleaseSummary({
|
||||
evidencePacks,
|
||||
harnessSummary: readOptionalJson(options.harnessSummaryPath),
|
||||
harnessTrend: readOptionalJson(options.harnessTrendPath),
|
||||
requiredScenarioIds,
|
||||
tag: options.tag,
|
||||
});
|
||||
const validation = validateReleaseSummary(summary, {
|
||||
requireEvidence: options.requireEvidence,
|
||||
});
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify({ ...summary, validation }, null, 2)}\n`
|
||||
: renderAgentQcReleaseMarkdown(summary);
|
||||
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (options.check && !validation.valid) {
|
||||
for (const issue of validation.issues) {
|
||||
console.error(`[agent-qc-release] ${issue}`);
|
||||
}
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
createAgentQcReport,
|
||||
readJsonFile,
|
||||
renderAgentQcMarkdownReport,
|
||||
} from "./lib/agent-qc-report-core.mjs";
|
||||
|
||||
const DEFAULT_MANIFEST_PATH = "docs/test/agent-qc-scenarios.manifest.json";
|
||||
const DEFAULT_SCHEMA_PATH = "docs/test/agent-qc-evidence.schema.json";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
format: "markdown",
|
||||
help: false,
|
||||
manifestPath: DEFAULT_MANIFEST_PATH,
|
||||
outputPath: "",
|
||||
schemaPath: DEFAULT_SCHEMA_PATH,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--manifest" && argv[index + 1]) {
|
||||
result.manifestPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--schema" && argv[index + 1]) {
|
||||
result.schemaPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent QC 场景报告
|
||||
|
||||
用法:
|
||||
npm run agent-qc:report
|
||||
npm run agent-qc:report:json
|
||||
npm run agent-qc:check
|
||||
node scripts/agent-qc-report.mjs --manifest docs/test/agent-qc-scenarios.manifest.json
|
||||
|
||||
选项:
|
||||
--check 如果 manifest 或 evidence schema 不合法,以非 0 退出
|
||||
--format FMT 输出格式:markdown | json
|
||||
--manifest PATH Agent QC manifest 路径
|
||||
--schema PATH Evidence schema 路径
|
||||
--output PATH 写入报告文件;默认输出到 stdout
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function resolvePath(targetPath) {
|
||||
return path.resolve(process.cwd(), targetPath);
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedOutputPath = resolvePath(outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const manifest = readJsonFile(resolvePath(options.manifestPath));
|
||||
const schema = readJsonFile(resolvePath(options.schemaPath));
|
||||
const packageJson = readJsonFile(resolvePath("package.json"));
|
||||
const report = createAgentQcReport({ manifest, packageJson, evidenceSchema: schema });
|
||||
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify(report, null, 2)}\n`
|
||||
: renderAgentQcMarkdownReport(report);
|
||||
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (options.check && !report.valid) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,652 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
buildApprovalSandboxSmokeEvidence,
|
||||
renderApprovalSandboxTranscriptLines,
|
||||
} from "./lib/agent-runtime-approval-sandbox-smoke-core.mjs";
|
||||
import { runVitestSmoke } from "./lib/vitest-smoke-runner.mjs";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const rootDir = path.resolve(__dirname, "..");
|
||||
const DEFAULT_OUTPUT = path.join(
|
||||
rootDir,
|
||||
".lime/qc/runtime-approval-sandbox-smoke.json",
|
||||
);
|
||||
const DEFAULT_HEALTH_URL = "http://127.0.0.1:3030/health";
|
||||
const DEFAULT_INVOKE_URL = "http://127.0.0.1:3030/invoke";
|
||||
const DEFAULT_TIMEOUT_MS = 60_000;
|
||||
const DEFAULT_INTERVAL_MS = 1_000;
|
||||
const DEFAULT_PROVIDER_PREFERENCE =
|
||||
process.env.LIME_AGENT_QC_PROVIDER ||
|
||||
process.env.LIME_E2E_PROVIDER ||
|
||||
process.env.LIME_DEFAULT_PROVIDER ||
|
||||
"";
|
||||
const DEFAULT_MODEL_PREFERENCE =
|
||||
process.env.LIME_AGENT_QC_MODEL ||
|
||||
process.env.LIME_E2E_MODEL ||
|
||||
process.env.LIME_DEFAULT_MODEL ||
|
||||
"";
|
||||
const APPROVAL_POLICY = "on-request";
|
||||
const SANDBOX_POLICY = "workspace-write";
|
||||
const PROVIDER_PICK_ORDER = ["deepseek", "doubao", "lime-hub"];
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Agent Runtime Approval / Sandbox Smoke
|
||||
|
||||
用途:
|
||||
生成 tool / approval / sandbox 边界的确定性 smoke 证据,补齐 qcloop P0 场景的可审查摘要。
|
||||
|
||||
用法:
|
||||
node scripts/agent-runtime-approval-sandbox-smoke.mjs [选项]
|
||||
|
||||
选项:
|
||||
--output <path> 写入 evidence JSON,默认 ./.lime/qc/runtime-approval-sandbox-smoke.json
|
||||
--health-url <url> DevBridge health 地址,默认 ${DEFAULT_HEALTH_URL}
|
||||
--invoke-url <url> DevBridge invoke 地址,默认 ${DEFAULT_INVOKE_URL}
|
||||
--timeout-ms <ms> live runtime 等待超时,默认 ${DEFAULT_TIMEOUT_MS}
|
||||
--interval-ms <ms> live runtime 轮询间隔,默认 ${DEFAULT_INTERVAL_MS}
|
||||
--provider-preference <id> live runtime 使用的 provider 偏好;默认读取 LIME_AGENT_QC_PROVIDER / LIME_E2E_PROVIDER,未设置时自动选本地启用 provider
|
||||
--model-preference <name> live runtime 使用的 model 偏好;默认读取 LIME_AGENT_QC_MODEL / LIME_E2E_MODEL,未设置时自动选 provider 的首个自定义模型
|
||||
--skip-live-runtime 跳过 DevBridge live runtime transcript,仅生成确定性投影摘要
|
||||
--no-write 只运行校验并打印摘要,不写 evidence JSON
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
output: DEFAULT_OUTPUT,
|
||||
healthUrl: DEFAULT_HEALTH_URL,
|
||||
invokeUrl: DEFAULT_INVOKE_URL,
|
||||
timeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
intervalMs: DEFAULT_INTERVAL_MS,
|
||||
providerPreference: DEFAULT_PROVIDER_PREFERENCE,
|
||||
modelPreference: DEFAULT_MODEL_PREFERENCE,
|
||||
liveRuntime: true,
|
||||
write: true,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
options.output = path.resolve(rootDir, String(argv[index + 1]));
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--no-write") {
|
||||
options.write = false;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--health-url" && argv[index + 1]) {
|
||||
options.healthUrl = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--invoke-url" && argv[index + 1]) {
|
||||
options.invokeUrl = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--timeout-ms" && argv[index + 1]) {
|
||||
options.timeoutMs = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--interval-ms" && argv[index + 1]) {
|
||||
options.intervalMs = Number(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--provider-preference" && argv[index + 1]) {
|
||||
options.providerPreference = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--model-preference" && argv[index + 1]) {
|
||||
options.modelPreference = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--skip-live-runtime" || arg === "--no-live-runtime") {
|
||||
options.liveRuntime = false;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
}
|
||||
}
|
||||
|
||||
if (!Number.isFinite(options.timeoutMs) || options.timeoutMs < 10_000) {
|
||||
throw new Error("--timeout-ms 必须是 >= 10000 的数字");
|
||||
}
|
||||
if (!Number.isFinite(options.intervalMs) || options.intervalMs < 100) {
|
||||
throw new Error("--interval-ms 必须是 >= 100 的数字");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function runVitest(label, args) {
|
||||
return runVitestSmoke({
|
||||
rootDir,
|
||||
label,
|
||||
args,
|
||||
logPrefix: "smoke:agent-runtime-approval-sandbox",
|
||||
});
|
||||
}
|
||||
|
||||
function writeEvidence(outputPath, evidence) {
|
||||
fs.mkdirSync(path.dirname(outputPath), { recursive: true });
|
||||
fs.writeFileSync(outputPath, `${JSON.stringify(evidence, null, 2)}\n`);
|
||||
return outputPath;
|
||||
}
|
||||
|
||||
function writeEvidenceWithFallback(outputPath, evidence) {
|
||||
try {
|
||||
return writeEvidence(outputPath, evidence);
|
||||
} catch (error) {
|
||||
if (outputPath !== DEFAULT_OUTPUT) {
|
||||
throw error;
|
||||
}
|
||||
const fallbackPath = path.join(os.tmpdir(), "lime-runtime-approval-sandbox-smoke.json");
|
||||
const writtenPath = writeEvidence(fallbackPath, evidence);
|
||||
console.warn(
|
||||
`[smoke:agent-runtime-approval-sandbox] 默认 evidence 写入失败,已回退到 ${writtenPath}: ${error.message}`,
|
||||
);
|
||||
return writtenPath;
|
||||
}
|
||||
}
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
function assert(condition, message) {
|
||||
if (!condition) {
|
||||
throw new Error(message);
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForHealth(options) {
|
||||
const startedAt = Date.now();
|
||||
let lastError = null;
|
||||
|
||||
while (Date.now() - startedAt < options.timeoutMs) {
|
||||
try {
|
||||
const response = await fetch(options.healthUrl, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(Math.min(options.intervalMs, 5_000)),
|
||||
});
|
||||
const text = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
const payload = text ? JSON.parse(text) : {};
|
||||
console.log(
|
||||
`[smoke:agent-runtime-approval-sandbox] DevBridge 已就绪 (${Date.now() - startedAt}ms)${
|
||||
payload?.status ? ` status=${payload.status}` : ""
|
||||
}`,
|
||||
);
|
||||
return payload;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
await sleep(options.intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
const detail =
|
||||
lastError instanceof Error
|
||||
? lastError.message
|
||||
: String(lastError || "unknown error");
|
||||
throw new Error(
|
||||
`[smoke:agent-runtime-approval-sandbox] DevBridge 未就绪,无法采集 live runtime transcript: ${detail}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function invoke(options, cmd, args, timeoutMs = options.timeoutMs) {
|
||||
const response = await fetch(options.invokeUrl, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ cmd, args }),
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
const text = await response.text();
|
||||
if (!response.ok) {
|
||||
throw new Error(`${cmd} HTTP ${response.status}: ${text}`);
|
||||
}
|
||||
const payload = text ? JSON.parse(text) : null;
|
||||
if (payload?.error) {
|
||||
throw new Error(`${cmd} error: ${payload.error}`);
|
||||
}
|
||||
return payload?.result;
|
||||
}
|
||||
|
||||
function normalizeProviderId(provider) {
|
||||
return String(provider?.id || provider?.provider_id || provider?.providerId || "")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function providerEnabled(provider) {
|
||||
return provider?.enabled !== false;
|
||||
}
|
||||
|
||||
function pickModelPreference(provider) {
|
||||
const candidates = [
|
||||
...(Array.isArray(provider?.custom_models) ? provider.custom_models : []),
|
||||
...(Array.isArray(provider?.customModels) ? provider.customModels : []),
|
||||
...(Array.isArray(provider?.models) ? provider.models : []),
|
||||
]
|
||||
.map((value) =>
|
||||
typeof value === "string"
|
||||
? value
|
||||
: String(value?.name || value?.id || value?.model || "").trim(),
|
||||
)
|
||||
.filter(Boolean);
|
||||
|
||||
return (
|
||||
candidates.find((value) => /flash|mini|lite/i.test(value)) ||
|
||||
candidates[0] ||
|
||||
""
|
||||
);
|
||||
}
|
||||
|
||||
function pickProvider(providers, preferredProviderId) {
|
||||
const enabled = providers.filter((provider) => providerEnabled(provider));
|
||||
if (preferredProviderId) {
|
||||
return (
|
||||
enabled.find((provider) => normalizeProviderId(provider) === preferredProviderId) ||
|
||||
providers.find((provider) => normalizeProviderId(provider) === preferredProviderId) ||
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
for (const providerId of PROVIDER_PICK_ORDER) {
|
||||
const match = enabled.find((provider) => normalizeProviderId(provider) === providerId);
|
||||
if (match) {
|
||||
return match;
|
||||
}
|
||||
}
|
||||
|
||||
return enabled[0] || null;
|
||||
}
|
||||
|
||||
async function resolveProviderPreference(options) {
|
||||
const explicitProvider = String(options.providerPreference || "").trim();
|
||||
const explicitModel = String(options.modelPreference || "").trim();
|
||||
if (explicitProvider && explicitModel) {
|
||||
return {
|
||||
providerPreference: explicitProvider,
|
||||
modelPreference: explicitModel,
|
||||
source: "explicit",
|
||||
};
|
||||
}
|
||||
|
||||
const providers = await invoke(options, "get_api_key_providers", {}, 30_000);
|
||||
const selected = pickProvider(Array.isArray(providers) ? providers : [], explicitProvider);
|
||||
const providerId = normalizeProviderId(selected);
|
||||
if (!providerId) {
|
||||
throw new Error(
|
||||
"[smoke:agent-runtime-approval-sandbox] 未找到可用 provider;请传 --provider-preference / --model-preference 或先配置本地 provider",
|
||||
);
|
||||
}
|
||||
|
||||
let providerDetail = selected;
|
||||
try {
|
||||
providerDetail =
|
||||
(await invoke(options, "get_api_key_provider", { id: providerId }, 30_000)) ||
|
||||
selected;
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[smoke:agent-runtime-approval-sandbox] 读取 provider 详情失败,使用列表摘要继续: ${error.message}`,
|
||||
);
|
||||
}
|
||||
|
||||
const modelPreference = explicitModel || pickModelPreference(providerDetail);
|
||||
if (!modelPreference) {
|
||||
throw new Error(
|
||||
`[smoke:agent-runtime-approval-sandbox] provider ${providerId} 缺少可用模型;请传 --model-preference`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
providerPreference: providerId,
|
||||
modelPreference,
|
||||
source: explicitProvider || explicitModel ? "partial-explicit" : "auto-enabled-provider",
|
||||
};
|
||||
}
|
||||
|
||||
function pickPermissionState(threadRead) {
|
||||
return threadRead?.permission_state || threadRead?.permissionState || {};
|
||||
}
|
||||
|
||||
function pendingRequests(threadRead) {
|
||||
return threadRead?.pending_requests || threadRead?.pendingRequests || [];
|
||||
}
|
||||
|
||||
function latestTurnStatus(threadRead) {
|
||||
return (
|
||||
threadRead?.diagnostics?.latest_turn_status ||
|
||||
threadRead?.diagnostics?.latestTurnStatus ||
|
||||
threadRead?.runtime_summary?.latestTurnStatus ||
|
||||
threadRead?.runtimeSummary?.latestTurnStatus ||
|
||||
threadRead?.status ||
|
||||
null
|
||||
);
|
||||
}
|
||||
|
||||
function permissionField(permissionState, snakeKey, camelKey) {
|
||||
return permissionState?.[snakeKey] ?? permissionState?.[camelKey] ?? null;
|
||||
}
|
||||
|
||||
function summarizeThreadRead(threadRead) {
|
||||
const permissionState = pickPermissionState(threadRead);
|
||||
const requests = pendingRequests(threadRead);
|
||||
return {
|
||||
threadStatus: threadRead?.status || null,
|
||||
latestTurnStatus: latestTurnStatus(threadRead),
|
||||
primaryBlockingKind:
|
||||
threadRead?.diagnostics?.primary_blocking_kind ||
|
||||
threadRead?.diagnostics?.primaryBlockingKind ||
|
||||
null,
|
||||
primaryBlockingSummary:
|
||||
threadRead?.diagnostics?.primary_blocking_summary ||
|
||||
threadRead?.diagnostics?.primaryBlockingSummary ||
|
||||
null,
|
||||
pendingRequestCount: requests.length,
|
||||
permissionStatus: permissionField(permissionState, "status", "status"),
|
||||
confirmationStatus: permissionField(
|
||||
permissionState,
|
||||
"confirmation_status",
|
||||
"confirmationStatus",
|
||||
),
|
||||
confirmationRequestId: permissionField(
|
||||
permissionState,
|
||||
"confirmation_request_id",
|
||||
"confirmationRequestId",
|
||||
),
|
||||
confirmationSource: permissionField(
|
||||
permissionState,
|
||||
"confirmation_source",
|
||||
"confirmationSource",
|
||||
),
|
||||
askProfileKeys:
|
||||
permissionField(permissionState, "ask_profile_keys", "askProfileKeys") || [],
|
||||
requiredProfileKeys:
|
||||
permissionField(
|
||||
permissionState,
|
||||
"required_profile_keys",
|
||||
"requiredProfileKeys",
|
||||
) || [],
|
||||
};
|
||||
}
|
||||
|
||||
async function waitForThreadRead(options, sessionId, predicate, label) {
|
||||
const startedAt = Date.now();
|
||||
let lastSummary = null;
|
||||
|
||||
while (Date.now() - startedAt < options.timeoutMs) {
|
||||
const threadRead = await invoke(options, "agent_runtime_get_thread_read", {
|
||||
sessionId,
|
||||
});
|
||||
lastSummary = summarizeThreadRead(threadRead);
|
||||
if (predicate(threadRead, lastSummary)) {
|
||||
return { threadRead, summary: lastSummary };
|
||||
}
|
||||
await sleep(options.intervalMs);
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`[smoke:agent-runtime-approval-sandbox] ${label} 超时,最后线程摘要: ${JSON.stringify(lastSummary)}`,
|
||||
);
|
||||
}
|
||||
|
||||
function buildRuntimeContractMetadata() {
|
||||
return {
|
||||
harness: {
|
||||
browser_assist: {
|
||||
runtime_contract: {
|
||||
contract_key: "browser_control",
|
||||
routing_slot: "browser_reasoning_model",
|
||||
execution_profile: {
|
||||
profile_key: "browser_control_profile",
|
||||
},
|
||||
executor_adapter: {
|
||||
adapter_key: "browser:browser_assist",
|
||||
},
|
||||
executor_binding: {
|
||||
executor_kind: "browser",
|
||||
binding_key: "browser_assist",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function runPermissionDecisionFlow(options, workspaceId, providerPreference, decision) {
|
||||
const confirmed = decision === "resolved";
|
||||
const responseLabel = confirmed ? "允许本次执行" : "拒绝";
|
||||
const sessionId = await invoke(options, "agent_runtime_create_session", {
|
||||
workspaceId,
|
||||
name: `Agent QC approval ${decision} ${Date.now()}`,
|
||||
runStartHooks: false,
|
||||
});
|
||||
const turnId = `qc-approval-${decision}-${Date.now()}-${process.pid}`;
|
||||
const eventName = `aster_stream_${sessionId}_${turnId}`;
|
||||
const submittedPolicies = {
|
||||
approvalPolicy: APPROVAL_POLICY,
|
||||
sandboxPolicy: SANDBOX_POLICY,
|
||||
};
|
||||
|
||||
await invoke(options, "agent_runtime_submit_turn", {
|
||||
request: {
|
||||
message:
|
||||
"Agent QC runtime permission confirmation smoke:应在模型执行前创建真实权限确认请求。",
|
||||
session_id: sessionId,
|
||||
workspace_id: workspaceId,
|
||||
event_name: eventName,
|
||||
turn_id: turnId,
|
||||
turn_config: {
|
||||
provider_preference: providerPreference.providerPreference,
|
||||
model_preference: providerPreference.modelPreference,
|
||||
approval_policy: APPROVAL_POLICY,
|
||||
sandbox_policy: SANDBOX_POLICY,
|
||||
metadata: buildRuntimeContractMetadata(),
|
||||
},
|
||||
skip_pre_submit_resume: true,
|
||||
},
|
||||
});
|
||||
|
||||
const requested = await waitForThreadRead(
|
||||
options,
|
||||
sessionId,
|
||||
(_threadRead, summary) =>
|
||||
summary.confirmationStatus === "requested" &&
|
||||
summary.pendingRequestCount > 0 &&
|
||||
String(summary.confirmationRequestId || "").includes(turnId),
|
||||
`等待 ${decision} flow 进入权限确认 requested`,
|
||||
);
|
||||
|
||||
const requestId = requested.summary.confirmationRequestId;
|
||||
assert(requestId, `${decision} flow 缺少 confirmationRequestId`);
|
||||
|
||||
await invoke(options, "agent_runtime_respond_action", {
|
||||
request: {
|
||||
session_id: sessionId,
|
||||
request_id: requestId,
|
||||
action_type: "elicitation",
|
||||
confirmed,
|
||||
response: JSON.stringify({ answer: responseLabel }),
|
||||
user_data: { answer: responseLabel },
|
||||
event_name: eventName,
|
||||
action_scope: {
|
||||
session_id: sessionId,
|
||||
thread_id: sessionId,
|
||||
turn_id: turnId,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const completed = await waitForThreadRead(
|
||||
options,
|
||||
sessionId,
|
||||
(_threadRead, summary) =>
|
||||
summary.pendingRequestCount === 0 &&
|
||||
summary.confirmationStatus === decision,
|
||||
`等待 ${decision} flow 写回确认结果`,
|
||||
);
|
||||
|
||||
return {
|
||||
decision: confirmed ? "resolved" : "denied",
|
||||
sessionId,
|
||||
turnId,
|
||||
requestId,
|
||||
providerPreference,
|
||||
submittedPolicies,
|
||||
before: requested.summary,
|
||||
respond: {
|
||||
confirmed,
|
||||
responseLabel,
|
||||
},
|
||||
after: completed.summary,
|
||||
};
|
||||
}
|
||||
|
||||
async function collectLiveRuntimeTranscript(options) {
|
||||
const health = await waitForHealth(options);
|
||||
const providerPreference = await resolveProviderPreference(options);
|
||||
console.log(
|
||||
`[smoke:agent-runtime-approval-sandbox] live runtime provider: provider=${providerPreference.providerPreference} model=${providerPreference.modelPreference} source=${providerPreference.source}`,
|
||||
);
|
||||
const workspace = await invoke(options, "get_or_create_default_project");
|
||||
const workspaceId = workspace?.id;
|
||||
assert(workspaceId, "get_or_create_default_project 缺少 workspace id");
|
||||
|
||||
const deniedFlow = await runPermissionDecisionFlow(
|
||||
options,
|
||||
workspaceId,
|
||||
providerPreference,
|
||||
"denied",
|
||||
);
|
||||
const resolvedFlow = await runPermissionDecisionFlow(
|
||||
options,
|
||||
workspaceId,
|
||||
providerPreference,
|
||||
"resolved",
|
||||
);
|
||||
const flows = [deniedFlow, resolvedFlow];
|
||||
|
||||
return {
|
||||
kind: "devbridge-runtime-permission-confirmation",
|
||||
health,
|
||||
workspaceId,
|
||||
providerPreference,
|
||||
flows,
|
||||
assertions: {
|
||||
devBridgeHealthy: health?.status === "ok" || Boolean(health),
|
||||
permissionRequestCreatedBeforeModel: flows.every(
|
||||
(flow) =>
|
||||
flow.before.permissionStatus === "requires_confirmation" &&
|
||||
flow.before.confirmationStatus === "requested" &&
|
||||
flow.before.pendingRequestCount > 0 &&
|
||||
flow.before.latestTurnStatus === "failed" &&
|
||||
String(flow.before.confirmationRequestId || "").includes(flow.turnId),
|
||||
),
|
||||
deniedDecisionClearsPendingRequest:
|
||||
deniedFlow.after.confirmationStatus === "denied" &&
|
||||
deniedFlow.after.pendingRequestCount === 0,
|
||||
resolvedDecisionClearsPendingRequest:
|
||||
resolvedFlow.after.confirmationStatus === "resolved" &&
|
||||
resolvedFlow.after.pendingRequestCount === 0,
|
||||
approvalPolicySubmitted: flows.every(
|
||||
(flow) => flow.submittedPolicies.approvalPolicy === APPROVAL_POLICY,
|
||||
),
|
||||
sandboxPolicySubmitted: flows.every(
|
||||
(flow) => flow.submittedPolicies.sandboxPolicy === SANDBOX_POLICY,
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const commandResults = [];
|
||||
|
||||
commandResults.push(
|
||||
runVitest("submit preferences 应透传 approval / sandbox policy", [
|
||||
"src/components/agent/chat/utils/buildUserInputSubmitOp.test.ts",
|
||||
"src/components/agent/chat/hooks/agentStreamUserInputSubmission.test.ts",
|
||||
"src/components/agent/chat/hooks/agentStreamSubmitExecution.test.ts",
|
||||
"src/components/agent/chat/hooks/agentStreamSubmitOpController.test.ts",
|
||||
]),
|
||||
);
|
||||
|
||||
commandResults.push(
|
||||
runVitest("runtime projection 应保留 permission / tool lifecycle", [
|
||||
"src/components/agent/chat/projection/agentUiEventProjection.test.ts",
|
||||
"-t",
|
||||
"runtime permission metadata|工具输入、成功输出和失败输出|plan approval",
|
||||
]),
|
||||
);
|
||||
|
||||
commandResults.push(
|
||||
runVitest("消息与时间线不应把权限确认误渲染为失败", [
|
||||
"src/components/agent/chat/components/AgentThreadTimeline.test.tsx",
|
||||
"src/components/agent/chat/components/MessageList.test.tsx",
|
||||
"-t",
|
||||
"运行时权限确认",
|
||||
]),
|
||||
);
|
||||
|
||||
commandResults.push(
|
||||
runVitest("Harness 面板应展示工具权限、待审批与 sandbox 来源", [
|
||||
"src/components/agent/chat/components/HarnessStatusPanel.test.tsx",
|
||||
"-t",
|
||||
"存在工具库存时应展示工具与权限区块及来源统计|待审批区块应通过 artifact protocol",
|
||||
]),
|
||||
);
|
||||
|
||||
const liveRuntimeTranscript = options.liveRuntime
|
||||
? await collectLiveRuntimeTranscript(options)
|
||||
: null;
|
||||
|
||||
const evidence = buildApprovalSandboxSmokeEvidence({
|
||||
commandResults,
|
||||
generatedAt: new Date().toISOString(),
|
||||
liveRuntimeTranscript,
|
||||
});
|
||||
|
||||
if (options.liveRuntime && !evidence.coverage.liveRuntimeTranscript) {
|
||||
throw new Error(
|
||||
"[smoke:agent-runtime-approval-sandbox] live runtime transcript 未满足发布门禁断言",
|
||||
);
|
||||
}
|
||||
|
||||
for (const line of renderApprovalSandboxTranscriptLines(evidence)) {
|
||||
console.log(line);
|
||||
}
|
||||
|
||||
if (options.write) {
|
||||
const evidencePath = writeEvidenceWithFallback(options.output, evidence);
|
||||
console.log(`\n[smoke:agent-runtime-approval-sandbox] evidence: ${evidencePath}`);
|
||||
}
|
||||
|
||||
console.log("\n[smoke:agent-runtime-approval-sandbox] 通过");
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(
|
||||
error instanceof Error ? error.message : String(error || "unknown error"),
|
||||
);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -1,36 +1,21 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from "node:child_process";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { runVitestSmoke } from "./lib/vitest-smoke-runner.mjs";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const rootDir = path.resolve(__dirname, "..");
|
||||
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
|
||||
|
||||
function runVitest(label, args) {
|
||||
console.log(`\n[smoke:agent-runtime-tool-surface] > ${label}`);
|
||||
const result = spawnSync(
|
||||
npmCommand,
|
||||
["exec", "--", "vitest", "run", ...args],
|
||||
{
|
||||
cwd: rootDir,
|
||||
stdio: "inherit",
|
||||
env: process.env,
|
||||
},
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if (typeof result.status === "number" && result.status !== 0) {
|
||||
const error = new Error(`[smoke:agent-runtime-tool-surface] ${label} 失败`);
|
||||
error.exitCode = result.status;
|
||||
throw error;
|
||||
}
|
||||
return runVitestSmoke({
|
||||
rootDir,
|
||||
label,
|
||||
args,
|
||||
logPrefix: "smoke:agent-runtime-tool-surface",
|
||||
});
|
||||
}
|
||||
|
||||
function main() {
|
||||
@@ -50,6 +35,9 @@ function main() {
|
||||
"src/components/agent/chat/workspace/useWorkspaceConversationSceneRuntime.test.ts",
|
||||
]);
|
||||
|
||||
console.log(
|
||||
'[smoke:agent-runtime-tool-surface] surface.summary: runtime inventory/strip/harness inventory 已一致透传;unsafeToolExposed=false; sources=runtime_tools|persisted_tools|default_policy',
|
||||
);
|
||||
console.log("\n[smoke:agent-runtime-tool-surface] 通过");
|
||||
}
|
||||
|
||||
|
||||
@@ -35,7 +35,55 @@ function runVitest(label, args) {
|
||||
}
|
||||
}
|
||||
|
||||
function runCargoTest(label, args) {
|
||||
console.log(`\n[smoke:agent-service-skill-entry] > ${label}`);
|
||||
const result = spawnSync("cargo", ["test", ...args], {
|
||||
cwd: path.join(rootDir, "src-tauri"),
|
||||
stdio: "inherit",
|
||||
env: process.env,
|
||||
});
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if (typeof result.status === "number" && result.status !== 0) {
|
||||
const error = new Error(
|
||||
`[smoke:agent-service-skill-entry] ${label} 失败`,
|
||||
);
|
||||
error.exitCode = result.status;
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function main() {
|
||||
runVitest("Skill Forge 前端 metadata 与工作台显式启用链路", [
|
||||
"src/lib/api/capabilityDrafts.test.ts",
|
||||
"src/lib/api/agentRuntime/inventoryClient.test.ts",
|
||||
"src/components/agent/chat/utils/workspaceSkillBindingsMetadata.test.ts",
|
||||
"src/components/agent/chat/utils/harnessRequestMetadata.test.ts",
|
||||
"src/features/capability-drafts/workspaceSkillAgentAutomationDraft.test.ts",
|
||||
]);
|
||||
|
||||
[
|
||||
"register_capability_draft_persists_readonly_http_preflight_provenance",
|
||||
"registered_skill_becomes_ready_for_manual_enable_binding_candidate",
|
||||
"explicit_runtime_enable_projects_ready_binding_allowlist",
|
||||
"registered_skill_without_verification_provenance_is_blocked",
|
||||
"execute_capability_draft_controlled_get_returns_evidence_without_persisting_inputs",
|
||||
"should_project_workspace_skill_runtime_enable_as_callable_scope",
|
||||
"allowlisted_session_should_preserve_workspace_skill_source_metadata",
|
||||
"disabled_session_should_fail_execute",
|
||||
].forEach((testName) => {
|
||||
runCargoTest(`Skill Forge Rust 定向测试: ${testName}`, [
|
||||
"--manifest-path",
|
||||
"Cargo.toml",
|
||||
testName,
|
||||
"--",
|
||||
"--exact",
|
||||
]);
|
||||
});
|
||||
|
||||
runVitest("服务技能入口路由与挂起参数", [
|
||||
"src/components/skills/SkillsWorkspacePage.test.tsx",
|
||||
"src/components/agent/chat/workspace/useWorkspaceServiceSkillEntryActions.test.tsx",
|
||||
|
||||
@@ -25,7 +25,9 @@ const INVOKE_RETRY_DELAY_MS = 1_000;
|
||||
const POST_HEALTH_SETTLE_MS = 3_000;
|
||||
const POST_LAUNCH_SETTLE_MS = 1_500;
|
||||
const READ_PAGE_TIMEOUT_MS = 45_000;
|
||||
const SMOKE_PROFILE_KEY = "smoke-browser-runtime";
|
||||
const DEFAULT_SMOKE_PROFILE_KEY =
|
||||
process.env.LIME_BROWSER_RUNTIME_SMOKE_PROFILE_KEY ||
|
||||
`smoke-browser-runtime-${process.pid}`;
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
@@ -43,6 +45,7 @@ Lime Browser Runtime Smoke
|
||||
--timeout-ms <ms> 等待健康检查超时,默认 90000
|
||||
--interval-ms <ms> 健康检查轮询间隔,默认 1000
|
||||
--launch-url <url> 启动浏览器会话的 URL,默认使用内置 data: 测试页
|
||||
--profile-key <key> smoke 专用浏览器 profile key,默认按进程隔离
|
||||
--open-window 显式打开浏览器窗口
|
||||
--headless 以无界面浏览器会话执行 smoke,避免弹出空白 Chrome
|
||||
--stream-mode <mode> events | frames | both,默认 both
|
||||
@@ -85,6 +88,11 @@ function parseArgs(argv) {
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--profile-key" && argv[index + 1]) {
|
||||
options.profileKey = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--open-window") {
|
||||
options.openWindow = true;
|
||||
continue;
|
||||
@@ -111,6 +119,9 @@ function parseArgs(argv) {
|
||||
if (!options.launchUrl) {
|
||||
throw new Error("--launch-url 不能为空");
|
||||
}
|
||||
if (!options.profileKey) {
|
||||
options.profileKey = DEFAULT_SMOKE_PROFILE_KEY;
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
@@ -239,8 +250,9 @@ async function main() {
|
||||
await waitForHealth(options);
|
||||
await sleep(POST_HEALTH_SETTLE_MS);
|
||||
|
||||
const profileKey = SMOKE_PROFILE_KEY;
|
||||
const profileKey = options.profileKey;
|
||||
let sessionId = null;
|
||||
let cleanupStatus = "skipped";
|
||||
|
||||
try {
|
||||
await closeSmokeProfileSession(
|
||||
@@ -310,6 +322,38 @@ async function main() {
|
||||
"browser_execute_action 未返回对应的 target_id",
|
||||
);
|
||||
|
||||
const consoleResult = await invoke(options, "browser_execute_action", {
|
||||
request: {
|
||||
profile_key: profileKey,
|
||||
action: "read_console_messages",
|
||||
args: { since: 0 },
|
||||
timeout_ms: Math.min(options.timeoutMs, READ_PAGE_TIMEOUT_MS),
|
||||
},
|
||||
});
|
||||
assert(
|
||||
consoleResult?.success === true,
|
||||
"browser_execute_action(read_console_messages) 未成功",
|
||||
);
|
||||
const consoleCount = Array.isArray(consoleResult?.data?.messages)
|
||||
? consoleResult.data.messages.length
|
||||
: 0;
|
||||
|
||||
const networkResult = await invoke(options, "browser_execute_action", {
|
||||
request: {
|
||||
profile_key: profileKey,
|
||||
action: "read_network_requests",
|
||||
args: { since: 0 },
|
||||
timeout_ms: Math.min(options.timeoutMs, READ_PAGE_TIMEOUT_MS),
|
||||
},
|
||||
});
|
||||
assert(
|
||||
networkResult?.success === true,
|
||||
"browser_execute_action(read_network_requests) 未成功",
|
||||
);
|
||||
const networkCount = Array.isArray(networkResult?.data?.events)
|
||||
? networkResult.data.events.length
|
||||
: 0;
|
||||
|
||||
const auditLogs = await invoke(options, "get_browser_action_audit_logs", {
|
||||
limit: 10,
|
||||
});
|
||||
@@ -344,7 +388,7 @@ async function main() {
|
||||
);
|
||||
|
||||
console.log(
|
||||
`[smoke:browser-runtime] 通过 session=${sessionId} target=${sessionState.target_id} profile=${profileKey}`,
|
||||
`[smoke:browser-runtime] 通过 session=${sessionId} target=${sessionState.target_id} profile=${profileKey} consoleEvents=${consoleCount} networkEvents=${networkCount}`,
|
||||
);
|
||||
} finally {
|
||||
if (sessionId) {
|
||||
@@ -354,7 +398,9 @@ async function main() {
|
||||
session_id: sessionId,
|
||||
},
|
||||
});
|
||||
cleanupStatus = "pass";
|
||||
} catch (error) {
|
||||
cleanupStatus = "failed";
|
||||
console.warn(
|
||||
`[smoke:browser-runtime] 清理会话失败: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
@@ -368,6 +414,9 @@ async function main() {
|
||||
profileKey,
|
||||
"关闭 smoke profile 失败",
|
||||
);
|
||||
console.log(
|
||||
`[smoke:browser-runtime] cleanup=${cleanupStatus} session=${sessionId ?? "none"} profile=${profileKey}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,109 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
analyzeTranslations,
|
||||
applyTranslationFixes,
|
||||
formatTranslationReport,
|
||||
hasTranslationIssues,
|
||||
} from "./detect-missing-translations";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
function createTempResourcesDir() {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "lime-i18n-check-"));
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
function writeResource(
|
||||
resourcesDir: string,
|
||||
locale: string,
|
||||
namespace: string,
|
||||
resource: Record<string, unknown>,
|
||||
) {
|
||||
const filePath = path.join(resourcesDir, locale, `${namespace}.json`);
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(resource, null, 2)}\n`);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
for (const dir of tempDirs.splice(0)) {
|
||||
fs.rmSync(dir, { force: true, recursive: true });
|
||||
}
|
||||
});
|
||||
|
||||
describe("detect-missing-translations", () => {
|
||||
it("应在所有 locale 与 source key 一致时通过", () => {
|
||||
const resourcesDir = createTempResourcesDir();
|
||||
writeResource(resourcesDir, "zh-CN", "common", { "common.save": "保存" });
|
||||
writeResource(resourcesDir, "en-US", "common", { "common.save": "Save" });
|
||||
|
||||
const result = analyzeTranslations({ resourcesDir });
|
||||
|
||||
expect(hasTranslationIssues(result)).toBe(false);
|
||||
expect(formatTranslationReport(result)).toContain("通过");
|
||||
});
|
||||
|
||||
it("应发现缺失 namespace、缺失 key 与多余 key", () => {
|
||||
const resourcesDir = createTempResourcesDir();
|
||||
writeResource(resourcesDir, "zh-CN", "common", {
|
||||
"common.cancel": "取消",
|
||||
"common.save": "保存",
|
||||
});
|
||||
writeResource(resourcesDir, "zh-CN", "settings", {
|
||||
"settings.title": "设置",
|
||||
});
|
||||
writeResource(resourcesDir, "en-US", "common", {
|
||||
"common.extra": "Extra",
|
||||
"common.save": "Save",
|
||||
});
|
||||
|
||||
const result = analyzeTranslations({ resourcesDir });
|
||||
|
||||
expect(hasTranslationIssues(result)).toBe(true);
|
||||
expect(result.issues).toEqual([
|
||||
{
|
||||
locale: "en-US",
|
||||
missingNamespaces: ["settings"],
|
||||
extraNamespaces: [],
|
||||
namespaces: [
|
||||
{
|
||||
namespace: "common",
|
||||
missingKeys: ["common.cancel"],
|
||||
extraKeys: ["common.extra"],
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("--fix 语义应补齐缺失 namespace 与缺失 key,但保留人工已有翻译", () => {
|
||||
const resourcesDir = createTempResourcesDir();
|
||||
writeResource(resourcesDir, "zh-CN", "common", {
|
||||
"common.cancel": "取消",
|
||||
"common.save": "保存",
|
||||
});
|
||||
writeResource(resourcesDir, "zh-CN", "settings", {
|
||||
"settings.title": "设置",
|
||||
});
|
||||
writeResource(resourcesDir, "en-US", "common", {
|
||||
"common.save": "Save",
|
||||
});
|
||||
|
||||
applyTranslationFixes({ resourcesDir });
|
||||
const result = analyzeTranslations({ resourcesDir });
|
||||
const common = JSON.parse(
|
||||
fs.readFileSync(path.join(resourcesDir, "en-US", "common.json"), "utf8"),
|
||||
) as Record<string, string>;
|
||||
|
||||
expect(hasTranslationIssues(result)).toBe(false);
|
||||
expect(common["common.save"]).toBe("Save");
|
||||
expect(common["common.cancel"]).toBe("取消");
|
||||
expect(
|
||||
fs.existsSync(path.join(resourcesDir, "en-US", "settings.json")),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,356 @@
|
||||
#!/usr/bin/env tsx
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
export interface TranslationCheckOptions {
|
||||
resourcesDir: string;
|
||||
sourceLocale?: string;
|
||||
}
|
||||
|
||||
export interface NamespaceIssue {
|
||||
namespace: string;
|
||||
missingKeys: string[];
|
||||
extraKeys: string[];
|
||||
}
|
||||
|
||||
export interface LocaleIssue {
|
||||
locale: string;
|
||||
missingNamespaces: string[];
|
||||
extraNamespaces: string[];
|
||||
namespaces: NamespaceIssue[];
|
||||
}
|
||||
|
||||
export interface TranslationCheckResult {
|
||||
resourcesDir: string;
|
||||
sourceLocale: string;
|
||||
locales: string[];
|
||||
namespaces: string[];
|
||||
sourceKeyCount: number;
|
||||
issues: LocaleIssue[];
|
||||
}
|
||||
|
||||
interface CliOptions extends TranslationCheckOptions {
|
||||
fix: boolean;
|
||||
verbose: boolean;
|
||||
}
|
||||
|
||||
const DEFAULT_SOURCE_LOCALE = "zh-CN";
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const REPO_ROOT = path.resolve(__dirname, "..");
|
||||
const DEFAULT_RESOURCES_DIR = path.join(REPO_ROOT, "src", "i18n", "resources");
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return Boolean(value) && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function readJsonObject(filePath: string): Record<string, unknown> {
|
||||
const raw = fs.readFileSync(filePath, "utf8");
|
||||
const parsed = JSON.parse(raw) as unknown;
|
||||
if (!isRecord(parsed)) {
|
||||
throw new Error(`Translation resource must be a JSON object: ${filePath}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function listJsonNamespaces(localeDir: string): string[] {
|
||||
if (!fs.existsSync(localeDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs
|
||||
.readdirSync(localeDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
||||
.map((entry) => path.basename(entry.name, ".json"))
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function listLocaleDirs(resourcesDir: string): string[] {
|
||||
if (!fs.existsSync(resourcesDir)) {
|
||||
throw new Error(`Translation resources directory does not exist: ${resourcesDir}`);
|
||||
}
|
||||
|
||||
return fs
|
||||
.readdirSync(resourcesDir, { withFileTypes: true })
|
||||
.filter((entry) => entry.isDirectory())
|
||||
.map((entry) => entry.name)
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
export function flattenResource(
|
||||
resource: Record<string, unknown>,
|
||||
prefix = "",
|
||||
): Record<string, unknown> {
|
||||
const flattened: Record<string, unknown> = {};
|
||||
|
||||
for (const [key, value] of Object.entries(resource)) {
|
||||
const nextKey = prefix ? `${prefix}.${key}` : key;
|
||||
if (isRecord(value)) {
|
||||
Object.assign(flattened, flattenResource(value, nextKey));
|
||||
} else {
|
||||
flattened[nextKey] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return flattened;
|
||||
}
|
||||
|
||||
function sortedDifference(left: Iterable<string>, right: Set<string>): string[] {
|
||||
return [...left]
|
||||
.filter((item) => !right.has(item))
|
||||
.sort((a, b) => a.localeCompare(b));
|
||||
}
|
||||
|
||||
function readNamespaceKeys(localeDir: string, namespace: string): Set<string> {
|
||||
const filePath = path.join(localeDir, `${namespace}.json`);
|
||||
if (!fs.existsSync(filePath)) {
|
||||
return new Set();
|
||||
}
|
||||
|
||||
return new Set(Object.keys(flattenResource(readJsonObject(filePath))));
|
||||
}
|
||||
|
||||
function countSourceKeys(sourceLocaleDir: string, namespaces: string[]): number {
|
||||
return namespaces.reduce(
|
||||
(total, namespace) => total + readNamespaceKeys(sourceLocaleDir, namespace).size,
|
||||
0,
|
||||
);
|
||||
}
|
||||
|
||||
export function hasTranslationIssues(result: TranslationCheckResult): boolean {
|
||||
return result.issues.some(
|
||||
(issue) =>
|
||||
issue.missingNamespaces.length > 0 ||
|
||||
issue.extraNamespaces.length > 0 ||
|
||||
issue.namespaces.some(
|
||||
(namespaceIssue) =>
|
||||
namespaceIssue.missingKeys.length > 0 || namespaceIssue.extraKeys.length > 0,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
export function analyzeTranslations(
|
||||
options: TranslationCheckOptions,
|
||||
): TranslationCheckResult {
|
||||
const resourcesDir = path.resolve(options.resourcesDir);
|
||||
const sourceLocale = options.sourceLocale || DEFAULT_SOURCE_LOCALE;
|
||||
const locales = listLocaleDirs(resourcesDir);
|
||||
const sourceLocaleDir = path.join(resourcesDir, sourceLocale);
|
||||
|
||||
if (!locales.includes(sourceLocale)) {
|
||||
throw new Error(`Source locale is missing from resources: ${sourceLocale}`);
|
||||
}
|
||||
|
||||
const namespaces = listJsonNamespaces(sourceLocaleDir);
|
||||
const sourceNamespaceSet = new Set(namespaces);
|
||||
const issues: LocaleIssue[] = [];
|
||||
|
||||
for (const locale of locales) {
|
||||
if (locale === sourceLocale) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const localeDir = path.join(resourcesDir, locale);
|
||||
const localeNamespaces = listJsonNamespaces(localeDir);
|
||||
const localeNamespaceSet = new Set(localeNamespaces);
|
||||
const missingNamespaces = sortedDifference(namespaces, localeNamespaceSet);
|
||||
const extraNamespaces = sortedDifference(localeNamespaces, sourceNamespaceSet);
|
||||
const namespaceIssues: NamespaceIssue[] = [];
|
||||
|
||||
for (const namespace of namespaces) {
|
||||
if (!localeNamespaceSet.has(namespace)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const sourceKeys = readNamespaceKeys(sourceLocaleDir, namespace);
|
||||
const targetKeys = readNamespaceKeys(localeDir, namespace);
|
||||
const missingKeys = sortedDifference(sourceKeys, targetKeys);
|
||||
const extraKeys = sortedDifference(targetKeys, sourceKeys);
|
||||
|
||||
if (missingKeys.length > 0 || extraKeys.length > 0) {
|
||||
namespaceIssues.push({ namespace, missingKeys, extraKeys });
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
missingNamespaces.length > 0 ||
|
||||
extraNamespaces.length > 0 ||
|
||||
namespaceIssues.length > 0
|
||||
) {
|
||||
issues.push({
|
||||
locale,
|
||||
missingNamespaces,
|
||||
extraNamespaces,
|
||||
namespaces: namespaceIssues,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
resourcesDir,
|
||||
sourceLocale,
|
||||
locales,
|
||||
namespaces,
|
||||
sourceKeyCount: countSourceKeys(sourceLocaleDir, namespaces),
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
function sortObjectByKey(resource: Record<string, unknown>): Record<string, unknown> {
|
||||
return Object.fromEntries(
|
||||
Object.entries(resource).sort(([left], [right]) => left.localeCompare(right)),
|
||||
);
|
||||
}
|
||||
|
||||
function writeJsonObject(filePath: string, resource: Record<string, unknown>): void {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(sortObjectByKey(resource), null, 2)}${os.EOL}`);
|
||||
}
|
||||
|
||||
export function applyTranslationFixes(options: TranslationCheckOptions): void {
|
||||
const resourcesDir = path.resolve(options.resourcesDir);
|
||||
const sourceLocale = options.sourceLocale || DEFAULT_SOURCE_LOCALE;
|
||||
const sourceLocaleDir = path.join(resourcesDir, sourceLocale);
|
||||
const sourceNamespaces = listJsonNamespaces(sourceLocaleDir);
|
||||
const locales = listLocaleDirs(resourcesDir).filter((locale) => locale !== sourceLocale);
|
||||
|
||||
for (const locale of locales) {
|
||||
const localeDir = path.join(resourcesDir, locale);
|
||||
|
||||
for (const namespace of sourceNamespaces) {
|
||||
const sourcePath = path.join(sourceLocaleDir, `${namespace}.json`);
|
||||
const targetPath = path.join(localeDir, `${namespace}.json`);
|
||||
const sourceResource = flattenResource(readJsonObject(sourcePath));
|
||||
const targetResource = fs.existsSync(targetPath)
|
||||
? flattenResource(readJsonObject(targetPath))
|
||||
: {};
|
||||
|
||||
let changed = false;
|
||||
for (const [key, value] of Object.entries(sourceResource)) {
|
||||
if (!(key in targetResource)) {
|
||||
targetResource[key] = value;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (changed || !fs.existsSync(targetPath)) {
|
||||
writeJsonObject(targetPath, targetResource);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function formatList(items: string[]): string {
|
||||
return items.length === 0 ? "-" : items.join(", ");
|
||||
}
|
||||
|
||||
export function formatTranslationReport(
|
||||
result: TranslationCheckResult,
|
||||
options: { verbose?: boolean } = {},
|
||||
): string {
|
||||
const lines: string[] = [];
|
||||
const issueCount = result.issues.length;
|
||||
lines.push(
|
||||
`[i18n:check] resources=${path.relative(REPO_ROOT, result.resourcesDir) || result.resourcesDir} source=${result.sourceLocale} locales=${result.locales.length} namespaces=${result.namespaces.length} sourceKeys=${result.sourceKeyCount}`,
|
||||
);
|
||||
|
||||
if (!hasTranslationIssues(result)) {
|
||||
lines.push("[i18n:check] 通过:所有 locale 与 source namespace/key 保持一致。");
|
||||
if (options.verbose) {
|
||||
lines.push(`[i18n:check] locale 列表: ${result.locales.join(", ")}`);
|
||||
lines.push(`[i18n:check] namespace 列表: ${result.namespaces.join(", ")}`);
|
||||
}
|
||||
return lines.join(os.EOL);
|
||||
}
|
||||
|
||||
lines.push(`[i18n:check] 发现 ${issueCount} 个 locale 存在翻译结构差异。`);
|
||||
for (const localeIssue of result.issues) {
|
||||
lines.push(`[i18n:check] locale=${localeIssue.locale}`);
|
||||
if (localeIssue.missingNamespaces.length > 0) {
|
||||
lines.push(
|
||||
` missing namespaces: ${formatList(localeIssue.missingNamespaces)}`,
|
||||
);
|
||||
}
|
||||
if (localeIssue.extraNamespaces.length > 0) {
|
||||
lines.push(` extra namespaces: ${formatList(localeIssue.extraNamespaces)}`);
|
||||
}
|
||||
for (const namespaceIssue of localeIssue.namespaces) {
|
||||
lines.push(` namespace=${namespaceIssue.namespace}`);
|
||||
if (namespaceIssue.missingKeys.length > 0) {
|
||||
lines.push(` missing keys: ${formatList(namespaceIssue.missingKeys)}`);
|
||||
}
|
||||
if (namespaceIssue.extraKeys.length > 0) {
|
||||
lines.push(` extra keys: ${formatList(namespaceIssue.extraKeys)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return lines.join(os.EOL);
|
||||
}
|
||||
|
||||
function parseCliArgs(argv: string[]): CliOptions {
|
||||
const options: CliOptions = {
|
||||
fix: false,
|
||||
resourcesDir: DEFAULT_RESOURCES_DIR,
|
||||
sourceLocale: DEFAULT_SOURCE_LOCALE,
|
||||
verbose: false,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
const next = argv[index + 1];
|
||||
|
||||
if (arg === "--fix") {
|
||||
options.fix = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--verbose") {
|
||||
options.verbose = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--resources-dir" && next) {
|
||||
options.resourcesDir = next;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--source-locale" && next) {
|
||||
options.sourceLocale = next;
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "-h" || arg === "--help") {
|
||||
console.log(`Usage: npm run detect-translations -- [--fix] [--verbose] [--resources-dir <dir>] [--source-locale <locale>]`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
export function runCli(argv = process.argv.slice(2)): number {
|
||||
const options = parseCliArgs(argv);
|
||||
|
||||
if (options.fix) {
|
||||
applyTranslationFixes(options);
|
||||
}
|
||||
|
||||
const result = analyzeTranslations(options);
|
||||
console.log(formatTranslationReport(result, { verbose: options.verbose }));
|
||||
return hasTranslationIssues(result) ? 1 : 0;
|
||||
}
|
||||
|
||||
if (import.meta.url === pathToFileURL(process.argv[1] || "").href) {
|
||||
try {
|
||||
process.exitCode = runCli();
|
||||
} catch (error) {
|
||||
console.error(
|
||||
`[i18n:check] 失败:${error instanceof Error ? error.message : String(error)}`,
|
||||
);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import {
|
||||
createI18nPatchMetricsReport,
|
||||
renderI18nPatchMetricsTextReport,
|
||||
} from "./lib/i18n-patch-metrics-report-core.mjs";
|
||||
|
||||
const DEFAULT_INPUT_PATH = ".lime/i18n/patch-metrics.json";
|
||||
|
||||
function parseNumberArg(value) {
|
||||
if (value === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : undefined;
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
check: false,
|
||||
format: "text",
|
||||
help: false,
|
||||
inputPath: DEFAULT_INPUT_PATH,
|
||||
maxMatchedSegments: undefined,
|
||||
maxReplacedNodes: undefined,
|
||||
maxRuns: undefined,
|
||||
outputPath: "",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
|
||||
if (arg === "--check") {
|
||||
result.check = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--input" && argv[index + 1]) {
|
||||
result.inputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--max-matched-segments" && argv[index + 1]) {
|
||||
result.maxMatchedSegments = parseNumberArg(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--max-replaced-nodes" && argv[index + 1]) {
|
||||
result.maxReplacedNodes = parseNumberArg(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--max-runs" && argv[index + 1]) {
|
||||
result.maxRuns = parseNumberArg(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
result.outputPath = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime i18n Patch Metrics Report
|
||||
|
||||
用法:
|
||||
npm run i18n:patch-report
|
||||
npm run i18n:patch-report:json
|
||||
node scripts/i18n-patch-metrics-report.mjs --input .lime/i18n/patch-metrics.json --format json
|
||||
node scripts/i18n-patch-metrics-report.mjs --check --max-matched-segments 0 --max-replaced-nodes 0
|
||||
|
||||
输入:
|
||||
默认读取 ${DEFAULT_INPUT_PATH}。该文件应来自 GUI / Playwright 导出的 window.__I18N_METRICS__ 或 getI18nPatchMetricsReport() JSON。
|
||||
|
||||
选项:
|
||||
--input PATH Patch metrics JSON 路径
|
||||
--output PATH 写入报告文件;默认输出到 stdout
|
||||
--format FMT 输出格式:text | json
|
||||
--check 如果门限问题存在,以非 0 退出
|
||||
--max-matched-segments NUM 允许的最大命中文本段数
|
||||
--max-replaced-nodes NUM 允许的最大替换节点数
|
||||
--max-runs NUM 允许的最大 Patch 运行次数
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function resolvePath(targetPath) {
|
||||
return path.resolve(process.cwd(), targetPath);
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function writeOutput(outputPath, content) {
|
||||
if (!outputPath) {
|
||||
process.stdout.write(content);
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedOutputPath = resolvePath(outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(resolvedOutputPath, content, "utf8");
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedInputPath = resolvePath(options.inputPath);
|
||||
const metrics = readJsonFile(resolvedInputPath);
|
||||
const report = createI18nPatchMetricsReport({
|
||||
metrics,
|
||||
sourcePath: path.relative(process.cwd(), resolvedInputPath),
|
||||
thresholds: {
|
||||
maxMatchedSegments: options.maxMatchedSegments,
|
||||
maxReplacedNodes: options.maxReplacedNodes,
|
||||
maxRuns: options.maxRuns,
|
||||
},
|
||||
});
|
||||
const content =
|
||||
options.format === "json"
|
||||
? `${JSON.stringify(report, null, 2)}\n`
|
||||
: renderI18nPatchMetricsTextReport(report);
|
||||
|
||||
writeOutput(options.outputPath, content);
|
||||
|
||||
if (options.check && report.thresholdIssues.length > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -12,6 +12,7 @@ const DEFAULTS = {
|
||||
invokeUrl: "http://127.0.0.1:3030/invoke",
|
||||
timeoutMs: 180_000,
|
||||
intervalMs: 1_000,
|
||||
i18nPatchMetricsOutput: "",
|
||||
};
|
||||
|
||||
const INVOKE_TIMEOUT_CEILING_MS = 180_000;
|
||||
@@ -135,6 +136,8 @@ Lime Knowledge GUI Smoke
|
||||
--invoke-url <url> DevBridge invoke 地址,默认 http://127.0.0.1:3030/invoke
|
||||
--timeout-ms <ms> 总超时,默认 180000
|
||||
--interval-ms <ms> 轮询间隔,默认 1000
|
||||
--i18n-patch-metrics-output <path>
|
||||
导出 window.__I18N_METRICS__ JSON,供 i18n:patch-report 消费
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
@@ -175,6 +178,12 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--i18n-patch-metrics-output" && argv[index + 1]) {
|
||||
options.i18nPatchMetricsOutput = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
@@ -202,6 +211,39 @@ function logStage(label) {
|
||||
console.log(`[smoke:knowledge-gui] stage=${label}`);
|
||||
}
|
||||
|
||||
async function exportI18nPatchMetrics(page, outputPath) {
|
||||
if (!outputPath) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const metrics = await page.evaluate(() => {
|
||||
const globalMetrics = window.__I18N_METRICS__;
|
||||
if (!globalMetrics) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return JSON.parse(JSON.stringify(globalMetrics));
|
||||
});
|
||||
const resolvedOutputPath = path.resolve(process.cwd(), outputPath);
|
||||
fs.mkdirSync(path.dirname(resolvedOutputPath), { recursive: true });
|
||||
fs.writeFileSync(
|
||||
resolvedOutputPath,
|
||||
JSON.stringify(metrics ?? {}, null, 2),
|
||||
"utf8",
|
||||
);
|
||||
console.log(
|
||||
`[smoke:knowledge-gui] i18n Patch metrics: ${resolvedOutputPath}`,
|
||||
);
|
||||
} catch (error) {
|
||||
console.warn(
|
||||
`[smoke:knowledge-gui] 导出 i18n Patch metrics 失败: ${
|
||||
error instanceof Error ? error.message : String(error)
|
||||
}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function isTransientInvokeError(error) {
|
||||
return (
|
||||
error?.name === "TimeoutError" ||
|
||||
@@ -1096,6 +1138,7 @@ async function runPlaywrightGuiFlow(options) {
|
||||
);
|
||||
}
|
||||
} finally {
|
||||
await exportI18nPatchMetrics(page, options.i18nPatchMetricsOutput);
|
||||
await context.close().catch(() => undefined);
|
||||
fs.rmSync(userDataDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
@@ -302,8 +302,23 @@ async function openKnowledgeOverview(page, options) {
|
||||
text.includes("存到哪里?") ||
|
||||
text.includes("整理新资料")
|
||||
) {
|
||||
if (text.includes("项目资料状态说明")) {
|
||||
const backButton = page.getByRole("button", {
|
||||
name: "回到项目资料",
|
||||
exact: true,
|
||||
});
|
||||
if (await backButton.isVisible().catch(() => false)) {
|
||||
await backButton.click({ timeout: options.timeoutMs });
|
||||
await waitText(page, options, "状态说明返回项目资料首页", [
|
||||
"让 Lime 记住这个项目",
|
||||
"项目资料清单",
|
||||
]);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
await clickSideNav(page, options, "灵感");
|
||||
await waitText(page, options, "灵感页", ["灵感"]);
|
||||
await waitText(page, options, "灵感页", ["收藏过的想法"]);
|
||||
}
|
||||
|
||||
await clickSideNav(page, options, "项目资料");
|
||||
|
||||
@@ -0,0 +1,292 @@
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function normalizeIdList(value) {
|
||||
return asArray(value)
|
||||
.map((item) => String(item || "").trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function createItem(id, title, passed, evidence, gap = "") {
|
||||
return { id, title, passed: Boolean(passed), evidence: evidence || "", gap: passed ? "" : gap };
|
||||
}
|
||||
|
||||
function summarizeQcloopStatusSidecar(entry) {
|
||||
const counts = entry.counts || {};
|
||||
return [
|
||||
`${entry.path} verdict=${entry.verdictStatus || "unknown"}`,
|
||||
`success=${counts.success ?? 0}`,
|
||||
`running=${counts.running ?? 0}`,
|
||||
`pending=${counts.pending ?? 0}`,
|
||||
`stale=${counts.stale ?? 0}`,
|
||||
].join(" ");
|
||||
}
|
||||
|
||||
function buildAgentQcCompletionAudit(facts) {
|
||||
const requiredQcloopScenarioIds = normalizeIdList(facts.scenarioReport?.p0ScenarioIds);
|
||||
const requiredQcloopScenarioCount =
|
||||
requiredQcloopScenarioIds.length || facts.scenarioReport?.p0ScenarioCount || 1;
|
||||
const realEvidenceScenarioIds = new Set(normalizeIdList(facts.realEvidencePack?.scenarioIds));
|
||||
const missingQcloopScenarioIds = requiredQcloopScenarioIds.filter(
|
||||
(scenarioId) => !realEvidenceScenarioIds.has(scenarioId),
|
||||
);
|
||||
const hasRequiredQcloopCoverage =
|
||||
requiredQcloopScenarioIds.length > 0
|
||||
? missingQcloopScenarioIds.length === 0
|
||||
: facts.realEvidencePack?.scenarioCount >= requiredQcloopScenarioCount;
|
||||
const sidecarEvidenceSummary = asArray(facts.realEvidenceSidecars)
|
||||
.map((entry) => `${entry.path} status=${entry.status || "unknown"} scenarios=${entry.scenarioCount || 0}`)
|
||||
.join("; ");
|
||||
const qcloopStatusSidecarSummary = asArray(facts.qcloopStatusSidecars)
|
||||
.map((entry) => {
|
||||
const counts = entry.counts
|
||||
? ` success=${entry.counts.success ?? 0} running=${entry.counts.running ?? 0} pending=${entry.counts.pending ?? 0} stale=${entry.counts.stale ?? 0}`
|
||||
: "";
|
||||
return `${entry.path} verdict=${entry.verdictStatus || "unknown"}${counts}`;
|
||||
})
|
||||
.join("; ");
|
||||
const staleQcloopStatusSidecars = asArray(facts.qcloopStatusSidecars).filter(
|
||||
(entry) => entry.verdictStatus === "stale" || Number(entry.counts?.stale || 0) > 0,
|
||||
);
|
||||
const activeQcloopStatusSidecars = asArray(facts.qcloopStatusSidecars).filter(
|
||||
(entry) =>
|
||||
entry.verdictStatus === "running" ||
|
||||
entry.verdictStatus === "stale" ||
|
||||
Number(entry.counts?.running || 0) > 0 ||
|
||||
Number(entry.counts?.pending || 0) > 0,
|
||||
);
|
||||
const qcloopEvidenceText = facts.realEvidencePack?.exists
|
||||
? `.lime/qc/agent-qc-evidence.json status=${facts.realEvidencePack.status || "unknown"} scenarios=${facts.realEvidencePack.scenarioCount || 0}/${requiredQcloopScenarioCount}${missingQcloopScenarioIds.length > 0 ? ` missing=${missingQcloopScenarioIds.join(",")}` : ""}`
|
||||
: sidecarEvidenceSummary
|
||||
? `未发现 .lime/qc/agent-qc-evidence.json;sidecars: ${sidecarEvidenceSummary}`
|
||||
: "未发现 .lime/qc/agent-qc-evidence.json";
|
||||
const qcloopEvidenceWithStatusText = qcloopStatusSidecarSummary
|
||||
? `${qcloopEvidenceText}; qcloopStatus: ${qcloopStatusSidecarSummary}`
|
||||
: qcloopEvidenceText;
|
||||
const qcloopGapFragments = [];
|
||||
if (facts.realEvidencePack?.exists && facts.realEvidencePack.status !== "pass") {
|
||||
qcloopGapFragments.push(
|
||||
`官方 Evidence Pack 当前 status=${facts.realEvidencePack.status || "unknown"},不能发布`,
|
||||
);
|
||||
}
|
||||
if (!facts.realEvidencePack?.exists && sidecarEvidenceSummary) {
|
||||
qcloopGapFragments.push("已有 sidecar Evidence Pack,但尚未生成 pass 的官方 .lime/qc/agent-qc-evidence.json");
|
||||
}
|
||||
if (missingQcloopScenarioIds.length > 0) {
|
||||
qcloopGapFragments.push(`尚未覆盖 P0 场景:${missingQcloopScenarioIds.join(",")}`);
|
||||
}
|
||||
if (activeQcloopStatusSidecars.length > 0) {
|
||||
qcloopGapFragments.push(
|
||||
`仍有 qcloop status sidecar 未终态:${activeQcloopStatusSidecars
|
||||
.map(summarizeQcloopStatusSidecar)
|
||||
.join("; ")}`,
|
||||
);
|
||||
}
|
||||
if (staleQcloopStatusSidecars.length > 0) {
|
||||
qcloopGapFragments.push(
|
||||
`其中 stale sidecar:${staleQcloopStatusSidecars.map((entry) => entry.path).join(", ")}`,
|
||||
);
|
||||
}
|
||||
const qcloopGapText =
|
||||
qcloopGapFragments.length > 0
|
||||
? `${qcloopGapFragments.join(";")}。`
|
||||
: "尚未运行真实 qcloop 批次并导出 pass Evidence Pack。";
|
||||
const localVerifyEvidenceText = facts.localVerify?.status
|
||||
? `status=${facts.localVerify.status}${facts.localVerify.failedStage ? ` failedStage=${facts.localVerify.failedStage}` : ""}`
|
||||
: "未发现 .lime/qc/verify-local-current.json";
|
||||
const guiSmokeEvidenceText = facts.guiSmoke?.status
|
||||
? `; latestGuiSmoke status=${facts.guiSmoke.status}${facts.guiSmoke.failedStage ? ` failedStage=${facts.guiSmoke.failedStage}` : ""}`
|
||||
: "";
|
||||
const localVerifyGapFragments = [];
|
||||
if (facts.localVerify?.status) {
|
||||
localVerifyGapFragments.push(
|
||||
`verify:local 当前 status=${facts.localVerify.status}${facts.localVerify.error ? `;${facts.localVerify.error}` : ""}`,
|
||||
);
|
||||
} else {
|
||||
localVerifyGapFragments.push("缺少 verify:local 当前结果 sidecar,无法证明仓库统一本地校验通过");
|
||||
}
|
||||
if (facts.guiSmoke?.status && facts.guiSmoke.status !== "pass") {
|
||||
localVerifyGapFragments.push(
|
||||
`最近一次 verify:gui-smoke status=${facts.guiSmoke.status}${facts.guiSmoke.error ? `;${facts.guiSmoke.error}` : ""}`,
|
||||
);
|
||||
}
|
||||
const items = [
|
||||
createItem(
|
||||
"docs-tests-standard",
|
||||
"docs/tests 下存在 Agent QC 人读测试文档",
|
||||
facts.files?.agentOpsQc &&
|
||||
facts.files?.p0Scenarios &&
|
||||
facts.files?.limeRolloutPlan &&
|
||||
facts.files?.testsReadme,
|
||||
"docs/tests/agent-ops-qc.md, docs/tests/agent-qc-p0-scenarios.md, docs/tests/lime-agent-qc-rollout-plan.md, docs/tests/README.md",
|
||||
"缺少 docs/tests 测试体系文档。",
|
||||
),
|
||||
createItem(
|
||||
"scenario-manifest",
|
||||
"Agent QC scenario manifest 有效",
|
||||
facts.scenarioReport?.valid === true && facts.scenarioReport?.scenarioCount > 0,
|
||||
`scenarioCount=${facts.scenarioReport?.scenarioCount ?? 0}`,
|
||||
"scenario manifest 未通过校验或没有场景。",
|
||||
),
|
||||
createItem(
|
||||
"gui-flow-manifest",
|
||||
"GUI / Playwright MCP flow manifest 有效",
|
||||
facts.guiFlowReport?.valid === true && facts.guiFlowReport?.flowCount > 0,
|
||||
`flowCount=${facts.guiFlowReport?.flowCount ?? 0}`,
|
||||
"GUI flow manifest 未通过校验或没有 flow。",
|
||||
),
|
||||
createItem(
|
||||
"evidence-schema",
|
||||
"Agent QC Evidence Pack schema 存在",
|
||||
facts.files?.evidenceSchema,
|
||||
"docs/test/agent-qc-evidence.schema.json",
|
||||
"缺少 Evidence Pack schema。",
|
||||
),
|
||||
createItem(
|
||||
"qcloop-payload-generator",
|
||||
"可从 manifest 生成 qcloop job payload",
|
||||
facts.files?.qcloopJobScript && facts.qcloopPayload?.valid === true && facts.qcloopPayload?.itemCount > 0,
|
||||
`itemCount=${facts.qcloopPayload?.itemCount ?? 0}`,
|
||||
"qcloop payload 生成器不可用或没有 item。",
|
||||
),
|
||||
createItem(
|
||||
"qcloop-verifier-evidence-placeholders",
|
||||
"qcloop verifier prompt 带 worker evidence 占位符",
|
||||
facts.qcloopPayload?.verifierHasWorkerOutput === true &&
|
||||
facts.qcloopPayload?.verifierHasAttemptStatus === true &&
|
||||
facts.qcloopPayload?.verifierHasExitCode === true,
|
||||
`stdout=${Boolean(facts.qcloopPayload?.verifierHasWorkerOutput)} attempt_status=${Boolean(facts.qcloopPayload?.verifierHasAttemptStatus)} exit_code=${Boolean(facts.qcloopPayload?.verifierHasExitCode)}`,
|
||||
"qcloop verifier prompt 缺少 {{stdout}} / {{attempt_status}} / {{exit_code}},verifier 无法审查 worker 证据。",
|
||||
),
|
||||
createItem(
|
||||
"structured-evidence-contract",
|
||||
"qcloop worker / verifier / exporter 强制结构化 evidence summary",
|
||||
facts.files?.evidenceContractDoc &&
|
||||
facts.qcloopPayload?.workerPromptHasStructuredEvidence === true &&
|
||||
facts.qcloopPayload?.verifierRequiresStructuredEvidence === true &&
|
||||
facts.qcloopPayload?.verifierRequiresStrictJson === true &&
|
||||
facts.structuredEvidence?.exporterParsesSummary === true &&
|
||||
facts.structuredEvidence?.releaseSummaryRejectsWeakRefs === true,
|
||||
`doc=${Boolean(facts.files?.evidenceContractDoc)} worker=${Boolean(facts.qcloopPayload?.workerPromptHasStructuredEvidence)} verifier=${Boolean(facts.qcloopPayload?.verifierRequiresStructuredEvidence)} strictJson=${Boolean(facts.qcloopPayload?.verifierRequiresStrictJson)} exporter=${Boolean(facts.structuredEvidence?.exporterParsesSummary)} release=${Boolean(facts.structuredEvidence?.releaseSummaryRejectsWeakRefs)}`,
|
||||
"结构化 evidence summary 契约未被文档、payload worker prompt、verifier prompt、exporter 与 release summary gate 同时强制,可能再次接受浅层 qcloop success。",
|
||||
),
|
||||
createItem(
|
||||
"qcloop-evidence-exporter",
|
||||
"可把 qcloop job/items 导出为 Evidence Pack",
|
||||
facts.files?.exportEvidenceScript,
|
||||
"scripts/agent-qc-export-evidence.mjs",
|
||||
"缺少 qcloop Evidence Pack 导出入口。",
|
||||
),
|
||||
createItem(
|
||||
"qcloop-status-monitor",
|
||||
"可只读监控 qcloop 运行批次和 stale item",
|
||||
facts.files?.qcloopStatusScript && facts.files?.qcloopOperationsDoc,
|
||||
"scripts/agent-qc-qcloop-status.mjs, docs/tests/lime-agent-qc-qcloop-operations.md",
|
||||
"缺少 qcloop 只读状态监控脚本或运维手册。",
|
||||
),
|
||||
createItem(
|
||||
"gui-owner-check",
|
||||
"可在启动 GUI P0 前阻断并发 qcloop GUI owner",
|
||||
facts.files?.guiOwnerCheckScript && facts.files?.qcloopOperationsDoc,
|
||||
"scripts/agent-qc-gui-owner-check.mjs, docs/tests/lime-agent-qc-qcloop-operations.md",
|
||||
"缺少 GUI owner 并发检查脚本或运维手册。",
|
||||
),
|
||||
createItem(
|
||||
"stale-owner-intervention-protocol",
|
||||
"stale GUI qcloop owner 有只读取证和 owner 确认协议",
|
||||
facts.files?.staleOwnerInterventionDoc &&
|
||||
facts.files?.qcloopOperationsDoc &&
|
||||
facts.staleOwnerIntervention?.guiOwnerReportHasDecisionPacket === true &&
|
||||
facts.staleOwnerIntervention?.docMentionsDecisionPacket === true &&
|
||||
facts.staleOwnerIntervention?.guiOwnerReportHasWatchHistory === true &&
|
||||
facts.staleOwnerIntervention?.docMentionsWatchHistory === true,
|
||||
`doc=${Boolean(facts.files?.staleOwnerInterventionDoc)} operations=${Boolean(facts.files?.qcloopOperationsDoc)} ownerIntervention=${Boolean(facts.staleOwnerIntervention?.guiOwnerReportHasDecisionPacket)} docDecisionPacket=${Boolean(facts.staleOwnerIntervention?.docMentionsDecisionPacket)} watchHistory=${Boolean(facts.staleOwnerIntervention?.guiOwnerReportHasWatchHistory)} docWatchHistory=${Boolean(facts.staleOwnerIntervention?.docMentionsWatchHistory)}`,
|
||||
"缺少 stale GUI owner 处置协议、机器可读 ownerIntervention 或 watch history 输出;容易在未确认时误杀 worker、启动并发 GUI P0,或丢失长期观察证据。",
|
||||
),
|
||||
createItem(
|
||||
"qcloop-worker-preflight",
|
||||
"qcloop worker 执行前有 cwd / tmp / DevBridge preflight",
|
||||
facts.files?.qcloopPreflightScript &&
|
||||
facts.qcloopPayload?.workerPromptHasPreflight === true,
|
||||
`script=${Boolean(facts.files?.qcloopPreflightScript)} prompt=${Boolean(facts.qcloopPayload?.workerPromptHasPreflight)}`,
|
||||
"缺少 qcloop worker preflight 脚本,或 payload prompt 没有要求执行前置环境检查。",
|
||||
),
|
||||
createItem(
|
||||
"release-summary",
|
||||
"可把 Evidence Pack 汇总为 release note 质量证据",
|
||||
facts.files?.releaseSummaryScript,
|
||||
"scripts/agent-qc-release-summary.mjs",
|
||||
"缺少 release summary 入口。",
|
||||
),
|
||||
createItem(
|
||||
"nightly-artifacts",
|
||||
"nightly 上传 Agent QC 标准资产报告",
|
||||
facts.nightly?.hasAgentQcReport && facts.nightly?.hasGuiFlowReport && facts.nightly?.hasReleasePreview,
|
||||
".github/workflows/harness-nightly.yml artifacts/agent-qc/*",
|
||||
"nightly 未完整上传 Agent QC report / GUI flow report / release preview。",
|
||||
),
|
||||
createItem(
|
||||
"real-qcloop-evidence",
|
||||
"存在真实 qcloop Agent QC Evidence Pack",
|
||||
facts.realEvidencePack?.status === "pass" &&
|
||||
hasRequiredQcloopCoverage,
|
||||
qcloopEvidenceWithStatusText,
|
||||
qcloopGapText,
|
||||
),
|
||||
createItem(
|
||||
"real-gui-evidence",
|
||||
"存在真实 GUI / Playwright MCP evidence",
|
||||
facts.files?.realGuiEvidence,
|
||||
facts.files?.realGuiEvidence ? ".lime/qc/gui-evidence" : "未发现 .lime/qc/gui-evidence",
|
||||
"尚未执行真实 GUI / Playwright MCP flow 并保存证据。",
|
||||
),
|
||||
createItem(
|
||||
"release-hard-gate",
|
||||
"release workflow 强制 Evidence Pack pass 且覆盖 P0 场景",
|
||||
facts.release?.hasHardGate === true && facts.release?.requiresP0ScenarioCoverage === true,
|
||||
facts.release?.hasHardGate && facts.release?.requiresP0ScenarioCoverage
|
||||
? ".github/workflows/release.yml contains agent-qc release gate and P0 scenario coverage"
|
||||
: "release.yml 未完整强制 agent-qc release gate / P0 scenario coverage",
|
||||
"release workflow 尚未强制真实 Evidence Pack pass 或 P0 场景覆盖。",
|
||||
),
|
||||
createItem(
|
||||
"local-verify-gate",
|
||||
"仓库统一本地校验 verify:local 通过",
|
||||
facts.localVerify?.status === "pass",
|
||||
`${localVerifyEvidenceText}${guiSmokeEvidenceText}`,
|
||||
`${localVerifyGapFragments.join(";")}。`,
|
||||
),
|
||||
];
|
||||
|
||||
const passedCount = items.filter((item) => item.passed).length;
|
||||
const failed = items.filter((item) => !item.passed);
|
||||
return {
|
||||
status: failed.length === 0 ? "complete" : "incomplete",
|
||||
passedCount,
|
||||
failedCount: failed.length,
|
||||
totalCount: items.length,
|
||||
completionRatio: items.length === 0 ? 0 : passedCount / items.length,
|
||||
items,
|
||||
gaps: failed.map((item) => ({ id: item.id, title: item.title, gap: item.gap })),
|
||||
};
|
||||
}
|
||||
|
||||
function renderAgentQcCompletionAuditMarkdown(audit) {
|
||||
const lines = audit.items.map((item) => {
|
||||
const marker = item.passed ? "PASS" : "MISS";
|
||||
return `- ${marker} ${item.id}: ${item.title};证据:${item.evidence}${item.gap ? `;缺口:${item.gap}` : ""}`;
|
||||
});
|
||||
return `# Agent QC Completion Audit
|
||||
|
||||
- Status: ${audit.status}
|
||||
- Passed: ${audit.passedCount}/${audit.totalCount}
|
||||
- Completion: ${Math.round(audit.completionRatio * 100)}%
|
||||
|
||||
## Checklist
|
||||
|
||||
${lines.join("\n")}
|
||||
`;
|
||||
}
|
||||
|
||||
export { buildAgentQcCompletionAudit, renderAgentQcCompletionAuditMarkdown };
|
||||
@@ -0,0 +1,299 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildAgentQcCompletionAudit,
|
||||
renderAgentQcCompletionAuditMarkdown,
|
||||
} from "./agent-qc-completion-audit-core.mjs";
|
||||
|
||||
function completeFacts() {
|
||||
return {
|
||||
files: {
|
||||
agentOpsQc: true,
|
||||
p0Scenarios: true,
|
||||
limeRolloutPlan: true,
|
||||
testsReadme: true,
|
||||
evidenceSchema: true,
|
||||
qcloopJobScript: true,
|
||||
guiOwnerCheckScript: true,
|
||||
qcloopStatusScript: true,
|
||||
qcloopPreflightScript: true,
|
||||
qcloopOperationsDoc: true,
|
||||
evidenceContractDoc: true,
|
||||
staleOwnerInterventionDoc: true,
|
||||
exportEvidenceScript: true,
|
||||
releaseSummaryScript: true,
|
||||
realGuiEvidence: true,
|
||||
},
|
||||
realEvidencePack: {
|
||||
exists: true,
|
||||
status: "pass",
|
||||
scenarioCount: 1,
|
||||
scenarioIds: ["command-bridge-contract"],
|
||||
},
|
||||
localVerify: {
|
||||
status: "pass",
|
||||
failedStage: "",
|
||||
error: "",
|
||||
},
|
||||
realEvidenceSidecars: [],
|
||||
qcloopStatusSidecars: [],
|
||||
scenarioReport: {
|
||||
valid: true,
|
||||
scenarioCount: 12,
|
||||
p0ScenarioCount: 1,
|
||||
p0ScenarioIds: ["command-bridge-contract"],
|
||||
},
|
||||
guiFlowReport: { valid: true, flowCount: 4 },
|
||||
qcloopPayload: {
|
||||
valid: true,
|
||||
itemCount: 8,
|
||||
verifierHasWorkerOutput: true,
|
||||
verifierHasAttemptStatus: true,
|
||||
verifierHasExitCode: true,
|
||||
workerPromptHasPreflight: true,
|
||||
workerPromptHasStructuredEvidence: true,
|
||||
verifierRequiresStructuredEvidence: true,
|
||||
verifierRequiresStrictJson: true,
|
||||
},
|
||||
structuredEvidence: {
|
||||
exporterParsesSummary: true,
|
||||
releaseSummaryRejectsWeakRefs: true,
|
||||
},
|
||||
staleOwnerIntervention: {
|
||||
guiOwnerReportHasDecisionPacket: true,
|
||||
docMentionsDecisionPacket: true,
|
||||
guiOwnerReportHasWatchHistory: true,
|
||||
docMentionsWatchHistory: true,
|
||||
},
|
||||
nightly: { hasAgentQcReport: true, hasGuiFlowReport: true, hasReleasePreview: true },
|
||||
release: { hasHardGate: true, requiresP0ScenarioCoverage: true },
|
||||
};
|
||||
}
|
||||
|
||||
describe("agent-qc-completion-audit-core", () => {
|
||||
it("所有事实齐全时应标记 complete", () => {
|
||||
const audit = buildAgentQcCompletionAudit(completeFacts());
|
||||
|
||||
expect(audit.status).toBe("complete");
|
||||
expect(audit.failedCount).toBe(0);
|
||||
});
|
||||
|
||||
it("缺真实 evidence 时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.realEvidencePack = { exists: false, status: "", scenarioCount: 0, scenarioIds: [] };
|
||||
facts.files.realGuiEvidence = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("real-qcloop-evidence");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("real-gui-evidence");
|
||||
});
|
||||
|
||||
it("缺正式 evidence 但存在 sidecar evidence 时应在审计证据中显示 sidecar 状态", () => {
|
||||
const facts = completeFacts();
|
||||
facts.realEvidencePack = { exists: false, status: "", scenarioCount: 0, scenarioIds: [] };
|
||||
facts.realEvidenceSidecars = [
|
||||
{
|
||||
path: ".lime/qc/agent-qc-evidence.p0-v2.json",
|
||||
status: "fail",
|
||||
scenarioCount: 8,
|
||||
scenarioIds: ["command-bridge-contract"],
|
||||
},
|
||||
];
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
const item = audit.items.find((entry) => entry.id === "real-qcloop-evidence");
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(item?.evidence).toContain("sidecars");
|
||||
expect(item?.evidence).toContain("status=fail");
|
||||
});
|
||||
|
||||
it("真实 qcloop 批次 stale 时应在缺口中显示 stale sidecar", () => {
|
||||
const facts = completeFacts();
|
||||
facts.realEvidencePack = {
|
||||
exists: true,
|
||||
status: "fail",
|
||||
scenarioCount: 8,
|
||||
scenarioIds: ["command-bridge-contract"],
|
||||
};
|
||||
facts.qcloopStatusSidecars = [
|
||||
{
|
||||
path: ".lime/qc/qcloop-status.isolated-p0-full-v1-stale-check.json",
|
||||
verdictStatus: "stale",
|
||||
counts: { success: 4, running: 1, pending: 3, stale: 1 },
|
||||
},
|
||||
];
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
const item = audit.items.find((entry) => entry.id === "real-qcloop-evidence");
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(item?.evidence).toContain("qcloopStatus");
|
||||
expect(item?.gap).toContain("stale");
|
||||
expect(item?.gap).toContain("qcloop-status.isolated-p0-full-v1-stale-check.json");
|
||||
});
|
||||
|
||||
it("真实 qcloop 批次仍在运行时应在缺口中显示未终态 sidecar 摘要", () => {
|
||||
const facts = completeFacts();
|
||||
facts.realEvidencePack = {
|
||||
exists: true,
|
||||
status: "fail",
|
||||
scenarioCount: 8,
|
||||
scenarioIds: ["command-bridge-contract"],
|
||||
};
|
||||
facts.qcloopStatusSidecars = [
|
||||
{
|
||||
path: ".lime/qc/qcloop-status.fastmini-readonly-p0-v1-current.json",
|
||||
verdictStatus: "running",
|
||||
counts: { success: 0, running: 1, pending: 2, stale: 0 },
|
||||
},
|
||||
];
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
const item = audit.items.find((entry) => entry.id === "real-qcloop-evidence");
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(item?.gap).toContain("未终态");
|
||||
expect(item?.gap).toContain("running=1");
|
||||
expect(item?.gap).toContain("qcloop-status.fastmini-readonly-p0-v1-current.json");
|
||||
});
|
||||
|
||||
it("qcloop verifier prompt 缺少 worker evidence 占位符时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.qcloopPayload.verifierHasWorkerOutput = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain(
|
||||
"qcloop-verifier-evidence-placeholders",
|
||||
);
|
||||
});
|
||||
|
||||
it("结构化 evidence summary 契约缺失时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.qcloopPayload.workerPromptHasStructuredEvidence = false;
|
||||
facts.structuredEvidence.exporterParsesSummary = false;
|
||||
facts.structuredEvidence.releaseSummaryRejectsWeakRefs = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("structured-evidence-contract");
|
||||
});
|
||||
|
||||
it("缺少 qcloop 状态监控入口时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.files.qcloopStatusScript = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("qcloop-status-monitor");
|
||||
});
|
||||
|
||||
it("缺少 qcloop worker preflight 时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.qcloopPayload.workerPromptHasPreflight = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("qcloop-worker-preflight");
|
||||
});
|
||||
|
||||
it("缺少 stale owner 机器可读决策输出时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.staleOwnerIntervention.guiOwnerReportHasDecisionPacket = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("stale-owner-intervention-protocol");
|
||||
expect(audit.items.find((item) => item.id === "stale-owner-intervention-protocol")?.evidence).toContain(
|
||||
"ownerIntervention=false",
|
||||
);
|
||||
});
|
||||
|
||||
it("缺少 stale owner watch history 输出时应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.staleOwnerIntervention.guiOwnerReportHasWatchHistory = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("stale-owner-intervention-protocol");
|
||||
expect(audit.items.find((item) => item.id === "stale-owner-intervention-protocol")?.evidence).toContain(
|
||||
"watchHistory=false",
|
||||
);
|
||||
});
|
||||
|
||||
it("release hard gate 缺失时应给出缺口", () => {
|
||||
const facts = completeFacts();
|
||||
facts.release.hasHardGate = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("release-hard-gate");
|
||||
});
|
||||
|
||||
it("release 未强制 P0 scenario 覆盖时应给出缺口", () => {
|
||||
const facts = completeFacts();
|
||||
facts.release.requiresP0ScenarioCoverage = false;
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("release-hard-gate");
|
||||
});
|
||||
|
||||
it("verify:local 失败时应给出缺口", () => {
|
||||
const facts = completeFacts();
|
||||
facts.localVerify = {
|
||||
status: "fail",
|
||||
failedStage: "typecheck",
|
||||
error: "example typecheck failure",
|
||||
};
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.gaps.map((gap) => gap.id)).toContain("local-verify-gate");
|
||||
expect(audit.items.find((item) => item.id === "local-verify-gate")?.evidence).toContain(
|
||||
"failedStage=typecheck",
|
||||
);
|
||||
});
|
||||
|
||||
it("真实 evidence 数量足够但缺 P0 scenario id 时仍应保持 incomplete", () => {
|
||||
const facts = completeFacts();
|
||||
facts.scenarioReport.p0ScenarioCount = 2;
|
||||
facts.scenarioReport.p0ScenarioIds = [
|
||||
"command-bridge-contract",
|
||||
"tool-approval-sandbox-boundary",
|
||||
];
|
||||
facts.realEvidencePack = {
|
||||
exists: true,
|
||||
status: "pass",
|
||||
scenarioCount: 2,
|
||||
scenarioIds: ["command-bridge-contract", "non-p0-placeholder"],
|
||||
};
|
||||
|
||||
const audit = buildAgentQcCompletionAudit(facts);
|
||||
|
||||
expect(audit.status).toBe("incomplete");
|
||||
expect(audit.items.find((item) => item.id === "real-qcloop-evidence")?.evidence).toContain(
|
||||
"missing=tool-approval-sandbox-boundary",
|
||||
);
|
||||
});
|
||||
|
||||
it("应渲染 Markdown 审计报告", () => {
|
||||
const audit = buildAgentQcCompletionAudit(completeFacts());
|
||||
const markdown = renderAgentQcCompletionAuditMarkdown(audit);
|
||||
|
||||
expect(markdown).toContain("Agent QC Completion Audit");
|
||||
expect(markdown).toContain("Status: complete");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,497 @@
|
||||
const QCLoopTerminalStatus = new Set(["success", "failed", "exhausted"]);
|
||||
const QCLoopEvidenceSummaryMarker = "QCLOOP_EVIDENCE_SUMMARY_JSON=";
|
||||
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeStatus(value) {
|
||||
return isNonEmptyString(value) ? value.trim().toLowerCase() : "unknown";
|
||||
}
|
||||
|
||||
function collectAttemptStdout(item) {
|
||||
return asArray(item?.attempts)
|
||||
.map((attempt) => (typeof attempt?.stdout === "string" ? attempt.stdout : ""))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function collectAttemptStderr(item) {
|
||||
return asArray(item?.attempts)
|
||||
.map((attempt) => (typeof attempt?.stderr === "string" ? attempt.stderr : ""))
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function collectAttemptOutput(item) {
|
||||
return [collectAttemptStdout(item), collectAttemptStderr(item)].join("\n");
|
||||
}
|
||||
|
||||
function isQCLoopWorkerBlocked(item) {
|
||||
return /QCLOOP_WORKER_RESULT\s*[:=]\s*BLOCKED/i.test(collectAttemptStdout(item));
|
||||
}
|
||||
|
||||
function isQCLoopWorkerEnvironmentBlocked(item) {
|
||||
return [
|
||||
/QCLOOP_CODEX_BIN\s+不可用/i,
|
||||
/QCLOOP_CODEX_EXTRA_ARGS\s+解析失败/i,
|
||||
/QCLOOP_CODEX_SANDBOX=.*无效/i,
|
||||
/fork\/exec .* no such file or directory/i,
|
||||
/incorrect api key provided/i,
|
||||
/failed to connect to websocket: HTTP error: 401 Unauthorized/i,
|
||||
].some((pattern) => pattern.test(collectAttemptOutput(item)));
|
||||
}
|
||||
|
||||
function isQCLoopWorkerSelfReportedPass(item) {
|
||||
return /QCLOOP_WORKER_RESULT\s*[:=]\s*PASS/i.test(collectAttemptStdout(item));
|
||||
}
|
||||
|
||||
function parseQCLoopEvidenceSummaryLine(line) {
|
||||
if (!isNonEmptyString(line)) {
|
||||
return { summary: null, error: "" };
|
||||
}
|
||||
const markerIndex = line.indexOf(QCLoopEvidenceSummaryMarker);
|
||||
if (markerIndex < 0) {
|
||||
return { summary: null, error: "" };
|
||||
}
|
||||
|
||||
const rawJson = line.slice(markerIndex + QCLoopEvidenceSummaryMarker.length).trim();
|
||||
if (!rawJson) {
|
||||
return { summary: null, error: "empty" };
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(rawJson);
|
||||
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed)) {
|
||||
return { summary: null, error: "not_object" };
|
||||
}
|
||||
return { summary: parsed, error: "" };
|
||||
} catch {
|
||||
return { summary: null, error: "invalid_json" };
|
||||
}
|
||||
}
|
||||
|
||||
function collectQCLoopEvidenceSummaries(item) {
|
||||
const summaries = [];
|
||||
const parseErrors = [];
|
||||
const stdout = collectAttemptStdout(item);
|
||||
for (const line of stdout.split(/\r?\n/)) {
|
||||
if (!line.includes(QCLoopEvidenceSummaryMarker)) {
|
||||
continue;
|
||||
}
|
||||
const parsed = parseQCLoopEvidenceSummaryLine(line);
|
||||
if (parsed.summary) {
|
||||
summaries.push(parsed.summary);
|
||||
} else {
|
||||
parseErrors.push(parsed.error || "unknown");
|
||||
}
|
||||
}
|
||||
return { summaries, parseErrors };
|
||||
}
|
||||
|
||||
function latestQCLoopEvidenceSummary(item) {
|
||||
const { summaries } = collectQCLoopEvidenceSummaries(item);
|
||||
return summaries.at(-1) || null;
|
||||
}
|
||||
|
||||
function normalizeEvidenceResult(value) {
|
||||
const normalized = normalizeStatus(value);
|
||||
if (["pass", "passed", "success"].includes(normalized)) {
|
||||
return "pass";
|
||||
}
|
||||
if (["blocked", "block"].includes(normalized)) {
|
||||
return "blocked";
|
||||
}
|
||||
if (["fail", "failed", "failure", "error"].includes(normalized)) {
|
||||
return "fail";
|
||||
}
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
function hasFailedQCLoopExecution(item) {
|
||||
const status = normalizeStatus(item?.status);
|
||||
if (status === "failed" || status === "exhausted") {
|
||||
return true;
|
||||
}
|
||||
if (asArray(item?.attempts).some((attempt) => normalizeStatus(attempt?.status) === "failed")) {
|
||||
return true;
|
||||
}
|
||||
return asArray(item?.qc_rounds).some((round) => normalizeStatus(round?.status) === "fail");
|
||||
}
|
||||
|
||||
function mapQCLoopItemStatus(status, item = null) {
|
||||
const normalized = normalizeStatus(status);
|
||||
const evidenceSummary = item ? latestQCLoopEvidenceSummary(item) : null;
|
||||
const evidenceResult = evidenceSummary ? normalizeEvidenceResult(evidenceSummary.result) : "unknown";
|
||||
if (evidenceResult === "blocked") {
|
||||
return "blocked";
|
||||
}
|
||||
if (evidenceResult === "fail") {
|
||||
return "fail";
|
||||
}
|
||||
if (normalized === "success") {
|
||||
return item ? (evidenceSummary ? "pass" : "fail") : "pass";
|
||||
}
|
||||
if (isQCLoopWorkerBlocked(item) || isQCLoopWorkerEnvironmentBlocked(item)) {
|
||||
return "blocked";
|
||||
}
|
||||
if (normalized === "failed" || normalized === "exhausted") {
|
||||
return "fail";
|
||||
}
|
||||
if (normalized === "running" || normalized === "pending") {
|
||||
return "blocked";
|
||||
}
|
||||
return "needs-human-review";
|
||||
}
|
||||
|
||||
function mapQCLoopJobStatus(status, itemStatuses) {
|
||||
const normalized = normalizeStatus(status);
|
||||
if (itemStatuses.some((itemStatus) => itemStatus === "fail")) {
|
||||
return "fail";
|
||||
}
|
||||
if (itemStatuses.some((itemStatus) => itemStatus === "blocked")) {
|
||||
return "blocked";
|
||||
}
|
||||
if (itemStatuses.some((itemStatus) => itemStatus === "needs-human-review")) {
|
||||
return "needs-human-review";
|
||||
}
|
||||
if (normalized === "completed") {
|
||||
return "pass";
|
||||
}
|
||||
if (normalized === "failed") {
|
||||
return "fail";
|
||||
}
|
||||
if (normalized === "paused" || normalized === "running" || normalized === "pending") {
|
||||
return "blocked";
|
||||
}
|
||||
return "needs-human-review";
|
||||
}
|
||||
|
||||
function parseScenarioId(itemValue) {
|
||||
if (!isNonEmptyString(itemValue)) {
|
||||
return "unknown-scenario";
|
||||
}
|
||||
|
||||
const trimmed = itemValue.trim();
|
||||
if (trimmed.startsWith("{") && trimmed.endsWith("}")) {
|
||||
try {
|
||||
const parsed = JSON.parse(trimmed);
|
||||
for (const key of ["scenarioId", "scenario_id", "id", "name", "entry"]) {
|
||||
if (isNonEmptyString(parsed?.[key])) {
|
||||
return parsed[key].trim();
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// 非 JSON item 按普通字符串处理。
|
||||
}
|
||||
}
|
||||
|
||||
return trimmed.split(/[\s,|]+/)[0] || "unknown-scenario";
|
||||
}
|
||||
|
||||
function collectItemFailureModes(item) {
|
||||
const modes = [];
|
||||
const status = normalizeStatus(item?.status);
|
||||
const { summaries, parseErrors } = collectQCLoopEvidenceSummaries(item);
|
||||
if (parseErrors.length > 0) {
|
||||
modes.push("qcloop:evidence_summary_invalid_json");
|
||||
}
|
||||
if (status === "success" && summaries.length === 0 && parseErrors.length === 0) {
|
||||
modes.push("qcloop:evidence_summary_missing");
|
||||
}
|
||||
if (status === "failed") {
|
||||
modes.push("qcloop:item_failed");
|
||||
}
|
||||
if (status === "exhausted") {
|
||||
modes.push("qcloop:max_qc_rounds_exhausted");
|
||||
}
|
||||
if (status === "running" || status === "pending") {
|
||||
modes.push("qcloop:item_not_terminal");
|
||||
}
|
||||
if (isNonEmptyString(item?.last_error)) {
|
||||
modes.push("qcloop:last_error_present");
|
||||
}
|
||||
if (isQCLoopWorkerBlocked(item)) {
|
||||
modes.push("qcloop:worker_blocked");
|
||||
}
|
||||
if (isQCLoopWorkerEnvironmentBlocked(item)) {
|
||||
modes.push("qcloop:worker_environment_blocked");
|
||||
}
|
||||
if (isQCLoopWorkerSelfReportedPass(item) && hasFailedQCLoopExecution(item)) {
|
||||
modes.push("qcloop:worker_self_report_pass_not_verified");
|
||||
}
|
||||
|
||||
for (const attempt of asArray(item?.attempts)) {
|
||||
if (normalizeStatus(attempt.status) === "failed") {
|
||||
modes.push(`qcloop:attempt_${attempt.attempt_no ?? "unknown"}_failed`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const round of asArray(item?.qc_rounds)) {
|
||||
if (normalizeStatus(round.status) === "fail") {
|
||||
modes.push(`qcloop:qc_${round.qc_no ?? "unknown"}_failed`);
|
||||
}
|
||||
}
|
||||
|
||||
return Array.from(new Set(modes));
|
||||
}
|
||||
|
||||
function collectEvidenceSummaryArtifactRefs(summary) {
|
||||
const refs = [];
|
||||
for (const command of asArray(summary?.commands)) {
|
||||
for (const key of ["stdout_artifact", "stderr_artifact", "logRef", "log_ref"]) {
|
||||
if (isNonEmptyString(command?.[key])) {
|
||||
refs.push(command[key].trim());
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const evidence of asArray(summary?.evidence_required)) {
|
||||
if (isNonEmptyString(evidence?.artifact_path)) {
|
||||
refs.push(evidence.artifact_path.trim());
|
||||
}
|
||||
}
|
||||
for (const artifact of asArray(summary?.artifacts)) {
|
||||
if (typeof artifact === "string" && artifact.trim()) {
|
||||
refs.push(artifact.trim());
|
||||
continue;
|
||||
}
|
||||
if (isNonEmptyString(artifact?.path)) {
|
||||
refs.push(artifact.path.trim());
|
||||
}
|
||||
}
|
||||
if (isNonEmptyString(summary?.gui_session_owner)) {
|
||||
refs.push(`gui-session-owner:${summary.gui_session_owner.trim()}`);
|
||||
}
|
||||
if (isNonEmptyString(summary?.release_scope)) {
|
||||
refs.push(`release-scope:${summary.release_scope.trim()}`);
|
||||
}
|
||||
return refs;
|
||||
}
|
||||
|
||||
function collectEvidenceRefs(item) {
|
||||
const refs = [];
|
||||
if (isNonEmptyString(item?.id)) {
|
||||
refs.push(`qcloop:item:${item.id}`);
|
||||
}
|
||||
for (const attempt of asArray(item?.attempts)) {
|
||||
if (isNonEmptyString(attempt?.id)) {
|
||||
refs.push(`qcloop:attempt:${attempt.id}`);
|
||||
}
|
||||
}
|
||||
for (const round of asArray(item?.qc_rounds)) {
|
||||
if (isNonEmptyString(round?.id)) {
|
||||
refs.push(`qcloop:qc:${round.id}`);
|
||||
}
|
||||
}
|
||||
for (const summary of collectQCLoopEvidenceSummaries(item).summaries) {
|
||||
refs.push(...collectEvidenceSummaryArtifactRefs(summary));
|
||||
}
|
||||
return Array.from(new Set(refs));
|
||||
}
|
||||
|
||||
function findArtifactRefByKind(summary, kind) {
|
||||
for (const artifact of asArray(summary?.artifacts)) {
|
||||
if (typeof artifact === "string") {
|
||||
if (artifact.includes(kind)) {
|
||||
return artifact;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (artifact?.kind === kind && isNonEmptyString(artifact?.path)) {
|
||||
return artifact.path.trim();
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function summarizeQCLoopItem(item) {
|
||||
const attempts = asArray(item?.attempts);
|
||||
const qcRounds = asArray(item?.qc_rounds);
|
||||
const failedQcRound = qcRounds.find((round) => normalizeStatus(round.status) === "fail");
|
||||
const latestQcRound = qcRounds.at(-1);
|
||||
const status = mapQCLoopItemStatus(item?.status, item);
|
||||
const evidenceSummary = latestQCLoopEvidenceSummary(item);
|
||||
|
||||
return {
|
||||
scenarioId: parseScenarioId(item?.item_value),
|
||||
status,
|
||||
executor: "qcloop",
|
||||
attempts: attempts.length,
|
||||
evidenceRefs: collectEvidenceRefs(item),
|
||||
failureModes: collectItemFailureModes(item),
|
||||
humanReview: {
|
||||
required: status === "needs-human-review",
|
||||
reason:
|
||||
status === "needs-human-review"
|
||||
? "qcloop item 状态无法自动归类,需要人工审核。"
|
||||
: "无需人工审核。",
|
||||
decision: status === "needs-human-review" ? "not-reviewed" : "approved",
|
||||
},
|
||||
runtimeTranscriptRef:
|
||||
findArtifactRefByKind(evidenceSummary, "runtime-transcript") ||
|
||||
(attempts.length > 0 ? `qcloop:item:${item.id}:attempts` : ""),
|
||||
guiTraceRef: findArtifactRefByKind(evidenceSummary, "gui-trace"),
|
||||
consoleErrorCount: 0,
|
||||
networkErrorCount: 0,
|
||||
_summary: {
|
||||
itemId: item?.id ?? "",
|
||||
itemValue: item?.item_value ?? "",
|
||||
qcloopStatus: item?.status ?? "unknown",
|
||||
currentAttemptNo: item?.current_attempt_no ?? 0,
|
||||
currentQcNo: item?.current_qc_no ?? 0,
|
||||
latestFeedback: failedQcRound?.feedback || latestQcRound?.feedback || item?.last_error || "",
|
||||
evidenceSummaryPresent: Boolean(evidenceSummary),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function stripPrivateSummary(scenarioResult) {
|
||||
const { _summary, ...publicResult } = scenarioResult;
|
||||
return publicResult;
|
||||
}
|
||||
|
||||
function collectChangedFiles(options) {
|
||||
return asArray(options.changedFiles).filter(isNonEmptyString);
|
||||
}
|
||||
|
||||
function buildAgentQcEvidencePack({ job, items, options = {} }) {
|
||||
const generatedAt = options.generatedAt || new Date().toISOString();
|
||||
const scenarioSummaries = asArray(items).map(summarizeQCLoopItem);
|
||||
const scenarioResults = scenarioSummaries.map(stripPrivateSummary);
|
||||
const itemStatuses = scenarioResults.map((result) => result.status);
|
||||
const verdictStatus = mapQCLoopJobStatus(job?.status, itemStatuses);
|
||||
const failedItems = scenarioSummaries.filter((result) => result.status === "fail");
|
||||
const blockedItems = scenarioSummaries.filter((result) => result.status === "blocked");
|
||||
const reviewItems = scenarioSummaries.filter((result) => result.status === "needs-human-review");
|
||||
const commandStatus = verdictStatus === "pass" ? "pass" : verdictStatus;
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
runId: options.runId || `qcloop-${job?.id || "unknown"}-${Date.parse(generatedAt) || Date.now()}`,
|
||||
generatedAt,
|
||||
subject: {
|
||||
repo: options.repo || "lime",
|
||||
ref: options.ref || "unknown",
|
||||
diffBase: options.diffBase || "",
|
||||
changedFiles: collectChangedFiles(options),
|
||||
riskTags: ["agent-qc", "qcloop", ...(options.riskTags || [])],
|
||||
},
|
||||
laneResults: [
|
||||
{
|
||||
laneId: "L4-behavior-eval",
|
||||
status: commandStatus,
|
||||
commands: [
|
||||
{
|
||||
command: `qcloop job ${job?.id || "unknown"}`,
|
||||
status: commandStatus,
|
||||
artifactRefs: [
|
||||
`qcloop:job:${job?.id || "unknown"}`,
|
||||
`qcloop:items:${job?.id || "unknown"}`,
|
||||
],
|
||||
},
|
||||
],
|
||||
notes: [
|
||||
`qcloop job status: ${job?.status || "unknown"}`,
|
||||
`items: ${scenarioResults.length}`,
|
||||
`failed: ${failedItems.length}`,
|
||||
`blocked: ${blockedItems.length}`,
|
||||
`needs review: ${reviewItems.length}`,
|
||||
],
|
||||
},
|
||||
],
|
||||
scenarioResults,
|
||||
verdict: {
|
||||
status: verdictStatus,
|
||||
summary: renderVerdictSummary({ job, scenarioResults, failedItems, blockedItems, reviewItems }),
|
||||
blockers: renderBlockers({ failedItems, blockedItems, reviewItems }),
|
||||
waivers: [],
|
||||
nextAction: renderNextAction(verdictStatus),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function renderVerdictSummary({ job, scenarioResults, failedItems, blockedItems, reviewItems }) {
|
||||
return `qcloop job ${job?.id || "unknown"} (${job?.status || "unknown"}) 导出 ${scenarioResults.length} 个场景结果:失败 ${failedItems.length},阻断 ${blockedItems.length},需人审 ${reviewItems.length}。`;
|
||||
}
|
||||
|
||||
function renderBlockers({ failedItems, blockedItems, reviewItems }) {
|
||||
const blockers = [];
|
||||
for (const result of failedItems) {
|
||||
let reason = result._summary.latestFeedback || "qcloop item failed";
|
||||
if (result.failureModes.includes("qcloop:evidence_summary_missing")) {
|
||||
reason = "qcloop item 已 success,但 stdout 缺少 QCLOOP_EVIDENCE_SUMMARY_JSON;不能作为 release pass。";
|
||||
} else if (result.failureModes.includes("qcloop:evidence_summary_invalid_json")) {
|
||||
reason = "qcloop stdout 包含 QCLOOP_EVIDENCE_SUMMARY_JSON,但 JSON 无法解析。";
|
||||
} else if (result.failureModes.includes("qcloop:worker_self_report_pass_not_verified")) {
|
||||
reason = `worker stdout 自报 QCLOOP_WORKER_RESULT=PASS,但 qcloop / verifier 未通过;latest verifier feedback: ${result._summary.latestFeedback || "无"}`;
|
||||
}
|
||||
blockers.push(`${result.scenarioId}: ${reason}`);
|
||||
}
|
||||
for (const result of blockedItems) {
|
||||
const reason = result.failureModes.includes("qcloop:worker_environment_blocked")
|
||||
? "qcloop worker 环境阻断;请检查内层 CLI 二进制、认证、sandbox 或 extra args。"
|
||||
: result.failureModes.includes("qcloop:worker_blocked")
|
||||
? `qcloop worker 明确输出 QCLOOP_WORKER_RESULT=BLOCKED;latest verifier feedback: ${result._summary.latestFeedback || "无"}`
|
||||
: `qcloop item 未进入终态 (${result._summary.qcloopStatus})`;
|
||||
blockers.push(`${result.scenarioId}: ${reason}`);
|
||||
}
|
||||
for (const result of reviewItems) {
|
||||
blockers.push(`${result.scenarioId}: 状态无法自动归类,需要人工审核`);
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
function renderNextAction(status) {
|
||||
if (status === "pass") {
|
||||
return "可把 Evidence Pack 挂到 PR / release 证据摘要。";
|
||||
}
|
||||
if (status === "fail") {
|
||||
return "优先修复 failed/exhausted item,并把高价值失败沉淀为 replay 或稳定回归。";
|
||||
}
|
||||
if (status === "blocked") {
|
||||
return "先恢复 qcloop job 到终态,或记录环境/权限阻断原因。";
|
||||
}
|
||||
return "安排人工审核 verifier feedback,再决定修复、waive 或补测试。";
|
||||
}
|
||||
|
||||
function validateEvidencePackShape(pack) {
|
||||
const issues = [];
|
||||
for (const field of ["schemaVersion", "runId", "generatedAt", "subject", "laneResults", "scenarioResults", "verdict"]) {
|
||||
if (!(field in pack)) {
|
||||
issues.push(`缺少字段 ${field}`);
|
||||
}
|
||||
}
|
||||
if (pack.schemaVersion !== "v1") {
|
||||
issues.push("schemaVersion 必须是 v1");
|
||||
}
|
||||
if (!Array.isArray(pack.laneResults)) {
|
||||
issues.push("laneResults 必须是数组");
|
||||
}
|
||||
if (!Array.isArray(pack.scenarioResults)) {
|
||||
issues.push("scenarioResults 必须是数组");
|
||||
}
|
||||
if (!pack.verdict || !["pass", "fail", "blocked", "needs-human-review", "waived"].includes(pack.verdict.status)) {
|
||||
issues.push("verdict.status 不合法");
|
||||
}
|
||||
return {
|
||||
valid: issues.length === 0,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
buildAgentQcEvidencePack,
|
||||
collectEvidenceRefs,
|
||||
collectItemFailureModes,
|
||||
collectQCLoopEvidenceSummaries,
|
||||
isQCLoopWorkerEnvironmentBlocked,
|
||||
isQCLoopWorkerBlocked,
|
||||
isQCLoopWorkerSelfReportedPass,
|
||||
mapQCLoopItemStatus,
|
||||
mapQCLoopJobStatus,
|
||||
parseScenarioId,
|
||||
parseQCLoopEvidenceSummaryLine,
|
||||
summarizeQCLoopItem,
|
||||
validateEvidencePackShape,
|
||||
};
|
||||
@@ -0,0 +1,339 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildAgentQcEvidencePack,
|
||||
collectQCLoopEvidenceSummaries,
|
||||
isQCLoopWorkerEnvironmentBlocked,
|
||||
isQCLoopWorkerBlocked,
|
||||
isQCLoopWorkerSelfReportedPass,
|
||||
mapQCLoopItemStatus,
|
||||
parseQCLoopEvidenceSummaryLine,
|
||||
parseScenarioId,
|
||||
validateEvidencePackShape,
|
||||
} from "./agent-qc-evidence-core.mjs";
|
||||
|
||||
function evidenceSummary(overrides = {}) {
|
||||
return `QCLOOP_EVIDENCE_SUMMARY_JSON=${JSON.stringify({
|
||||
scenario_id: "command-bridge-contract",
|
||||
result: "pass",
|
||||
commands: [
|
||||
{
|
||||
command: "npm run test:contracts",
|
||||
exit_code: 0,
|
||||
stdout_artifact: ".lime/qc/test-contracts.stdout.txt",
|
||||
},
|
||||
],
|
||||
evidence_required: [
|
||||
{
|
||||
name: "contract command log",
|
||||
status: "pass",
|
||||
evidence: "test:contracts passed",
|
||||
artifact_path: ".lime/qc/test-contracts.stdout.txt",
|
||||
},
|
||||
],
|
||||
evidence_layers_covered: ["deterministic-smoke"],
|
||||
failure_modes: [{ name: "mock fallback drift", status: "excluded", evidence: "contract check passed" }],
|
||||
artifacts: [{ path: ".lime/qc/contract-summary.json", kind: "deterministic-smoke", redacted: true }],
|
||||
blockers: [],
|
||||
gui_session_owner: "",
|
||||
release_scope: "source-tree-startup-smoke",
|
||||
...overrides,
|
||||
})}`;
|
||||
}
|
||||
|
||||
describe("agent-qc-evidence-core", () => {
|
||||
it("应把 qcloop item 状态映射为 Evidence Pack 状态", () => {
|
||||
expect(mapQCLoopItemStatus("success")).toBe("pass");
|
||||
expect(mapQCLoopItemStatus("failed")).toBe("fail");
|
||||
expect(mapQCLoopItemStatus("exhausted")).toBe("fail");
|
||||
expect(mapQCLoopItemStatus("running")).toBe("blocked");
|
||||
});
|
||||
|
||||
it("应解析 qcloop 结构化 evidence summary marker", () => {
|
||||
const parsed = parseQCLoopEvidenceSummaryLine(evidenceSummary());
|
||||
|
||||
expect(parsed.error).toBe("");
|
||||
expect(parsed.summary?.scenario_id).toBe("command-bridge-contract");
|
||||
|
||||
const item = { attempts: [{ stdout: `log\n${evidenceSummary()}\n` }] };
|
||||
const summaries = collectQCLoopEvidenceSummaries(item);
|
||||
expect(summaries.summaries).toHaveLength(1);
|
||||
expect(summaries.parseErrors).toEqual([]);
|
||||
});
|
||||
|
||||
it("应把 worker 明确报告 BLOCKED 的 exhausted item 映射为 blocked", () => {
|
||||
const item = {
|
||||
status: "exhausted",
|
||||
attempts: [
|
||||
{
|
||||
stdout: [
|
||||
"QCLOOP_WORKER_RESULT=BLOCKED",
|
||||
"失败 check: devbridge-health",
|
||||
"错误: TypeError: fetch failed",
|
||||
].join("\n"),
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
expect(isQCLoopWorkerBlocked(item)).toBe(true);
|
||||
expect(mapQCLoopItemStatus("exhausted", item)).toBe("blocked");
|
||||
});
|
||||
|
||||
it("应把内层 CLI 二进制或认证配置错误映射为环境 blocked", () => {
|
||||
const item = {
|
||||
status: "failed",
|
||||
attempts: [
|
||||
{
|
||||
status: "failed",
|
||||
stderr:
|
||||
"QCLOOP_CODEX_BIN 不可用: /opt/homebrew/bin/codex: fork/exec /opt/homebrew/bin/codex: no such file or directory",
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ status: "fail", feedback: "verifier 输出格式错误" }],
|
||||
};
|
||||
|
||||
expect(isQCLoopWorkerEnvironmentBlocked(item)).toBe(true);
|
||||
expect(mapQCLoopItemStatus("failed", item)).toBe("blocked");
|
||||
});
|
||||
|
||||
it("应标记 worker 自报 PASS 但 qcloop/verifier 未通过的冲突", () => {
|
||||
const conflictItem = {
|
||||
id: "item-pass-conflict",
|
||||
item_value: "tool-approval-sandbox-boundary",
|
||||
status: "failed",
|
||||
current_attempt_no: 2,
|
||||
current_qc_no: 1,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-pass-conflict",
|
||||
status: "failed",
|
||||
attempt_no: 2,
|
||||
stdout: "QCLOOP_WORKER_RESULT=PASS\nsmoke pass, but no live transcript",
|
||||
},
|
||||
],
|
||||
qc_rounds: [
|
||||
{
|
||||
id: "qc-pass-conflict",
|
||||
status: "fail",
|
||||
qc_no: 1,
|
||||
feedback: "缺少 live runtime transcript。",
|
||||
},
|
||||
],
|
||||
};
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-pass-conflict", status: "running" },
|
||||
items: [conflictItem],
|
||||
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
|
||||
});
|
||||
|
||||
expect(isQCLoopWorkerSelfReportedPass(conflictItem)).toBe(true);
|
||||
expect(pack.verdict.status).toBe("fail");
|
||||
expect(pack.scenarioResults[0].failureModes).toContain(
|
||||
"qcloop:worker_self_report_pass_not_verified",
|
||||
);
|
||||
expect(pack.verdict.blockers.join("\n")).toContain(
|
||||
"worker stdout 自报 QCLOOP_WORKER_RESULT=PASS",
|
||||
);
|
||||
});
|
||||
|
||||
it("应从 item_value 中提取 scenario id", () => {
|
||||
expect(parseScenarioId("claw-chat-ready-streaming 基础聊天")).toBe(
|
||||
"claw-chat-ready-streaming",
|
||||
);
|
||||
expect(parseScenarioId('{"scenario_id":"command-bridge-contract"}')).toBe(
|
||||
"command-bridge-contract",
|
||||
);
|
||||
expect(parseScenarioId('{"name":"agent-ui-performance-summary-ttft-fields"}')).toBe(
|
||||
"agent-ui-performance-summary-ttft-fields",
|
||||
);
|
||||
expect(parseScenarioId('{"entry":"lime-agent-runtime-client-check"}')).toBe(
|
||||
"lime-agent-runtime-client-check",
|
||||
);
|
||||
});
|
||||
|
||||
it("应把成功 qcloop job 导出为 pass evidence pack", () => {
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-1", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-1",
|
||||
item_value: "command-bridge-contract",
|
||||
status: "success",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-1",
|
||||
status: "success",
|
||||
attempt_no: 1,
|
||||
stdout: evidenceSummary(),
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ id: "qc-1", status: "pass", qc_no: 1 }],
|
||||
},
|
||||
],
|
||||
options: {
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
ref: "test-ref",
|
||||
changedFiles: ["docs/tests/agent-ops-qc.md"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(pack.verdict.status).toBe("pass");
|
||||
expect(pack.scenarioResults[0].scenarioId).toBe("command-bridge-contract");
|
||||
expect(pack.scenarioResults[0].evidenceRefs).toContain("qcloop:attempt:attempt-1");
|
||||
expect(pack.scenarioResults[0].evidenceRefs).toContain(".lime/qc/contract-summary.json");
|
||||
expect(pack.scenarioResults[0].evidenceRefs).toContain("release-scope:source-tree-startup-smoke");
|
||||
expect(validateEvidencePackShape(pack).valid).toBe(true);
|
||||
});
|
||||
|
||||
it("应拒绝缺少 QCLOOP_EVIDENCE_SUMMARY_JSON 的 success item,避免旧浅层 pass 进入 release gate", () => {
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-missing-summary", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-missing-summary",
|
||||
item_value: "command-bridge-contract",
|
||||
status: "success",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [{ id: "attempt-missing-summary", status: "success", attempt_no: 1 }],
|
||||
qc_rounds: [{ id: "qc-missing-summary", status: "pass", qc_no: 1 }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
|
||||
});
|
||||
|
||||
expect(pack.verdict.status).toBe("fail");
|
||||
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:evidence_summary_missing");
|
||||
expect(pack.verdict.blockers.join("\n")).toContain("缺少 QCLOOP_EVIDENCE_SUMMARY_JSON");
|
||||
});
|
||||
|
||||
it("应拒绝无法解析的 QCLOOP_EVIDENCE_SUMMARY_JSON", () => {
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-invalid-summary", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-invalid-summary",
|
||||
item_value: "command-bridge-contract",
|
||||
status: "success",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-invalid-summary",
|
||||
status: "success",
|
||||
attempt_no: 1,
|
||||
stdout: "QCLOOP_EVIDENCE_SUMMARY_JSON={not-json}",
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ id: "qc-invalid-summary", status: "pass", qc_no: 1 }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
|
||||
});
|
||||
|
||||
expect(pack.verdict.status).toBe("fail");
|
||||
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:evidence_summary_invalid_json");
|
||||
expect(pack.verdict.blockers.join("\n")).toContain("JSON 无法解析");
|
||||
});
|
||||
|
||||
it("应把耗尽 qcloop item 导出为 fail evidence pack", () => {
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-2", status: "failed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-2",
|
||||
item_value: "tool-approval-sandbox-boundary",
|
||||
status: "exhausted",
|
||||
current_attempt_no: 3,
|
||||
current_qc_no: 3,
|
||||
attempts: [{ id: "attempt-2", status: "failed", attempt_no: 3 }],
|
||||
qc_rounds: [
|
||||
{
|
||||
id: "qc-2",
|
||||
status: "fail",
|
||||
qc_no: 3,
|
||||
feedback: "缺少 approval 证据。",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
|
||||
});
|
||||
|
||||
expect(pack.verdict.status).toBe("fail");
|
||||
expect(pack.verdict.blockers.join("\n")).toContain("缺少 approval 证据");
|
||||
expect(pack.scenarioResults[0].failureModes).toContain(
|
||||
"qcloop:max_qc_rounds_exhausted",
|
||||
);
|
||||
});
|
||||
|
||||
it("应把 preflight blocked 的耗尽 item 导出为 blocked evidence pack", () => {
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-3", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-3",
|
||||
item_value: "claw-chat-ready-streaming",
|
||||
status: "exhausted",
|
||||
current_attempt_no: 3,
|
||||
current_qc_no: 3,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-3",
|
||||
status: "success",
|
||||
attempt_no: 3,
|
||||
stdout: "QCLOOP_WORKER_RESULT=BLOCKED\nDevBridge preflight: BLOCKED",
|
||||
},
|
||||
],
|
||||
qc_rounds: [
|
||||
{
|
||||
id: "qc-3",
|
||||
status: "fail",
|
||||
qc_no: 3,
|
||||
feedback: "DevBridge preflight blocked。",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
|
||||
});
|
||||
|
||||
expect(pack.verdict.status).toBe("blocked");
|
||||
expect(pack.verdict.blockers.join("\n")).toContain("DevBridge preflight blocked");
|
||||
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:worker_blocked");
|
||||
});
|
||||
|
||||
it("应把 qcloop worker 环境阻断导出为 blocked evidence pack,且不泄露原始 stderr", () => {
|
||||
const pack = buildAgentQcEvidencePack({
|
||||
job: { id: "job-env", status: "failed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-env",
|
||||
item_value: "command-bridge-contract",
|
||||
status: "failed",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-env",
|
||||
status: "failed",
|
||||
attempt_no: 1,
|
||||
stdout: "",
|
||||
stderr:
|
||||
"QCLOOP_CODEX_BIN 不可用: /opt/homebrew/bin/codex: fork/exec /opt/homebrew/bin/codex: no such file or directory",
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ id: "qc-env", status: "fail", qc_no: 1, feedback: "verifier 输出格式错误" }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt: "2026-05-10T00:00:00.000Z" },
|
||||
});
|
||||
|
||||
expect(pack.verdict.status).toBe("blocked");
|
||||
expect(pack.scenarioResults[0].failureModes).toContain("qcloop:worker_environment_blocked");
|
||||
expect(pack.verdict.blockers.join("\n")).toContain("qcloop worker 环境阻断");
|
||||
expect(pack.verdict.blockers.join("\n")).not.toContain("/opt/homebrew/bin/codex");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,163 @@
|
||||
const REQUIRED_FLOW_FIELDS = [
|
||||
"id",
|
||||
"scenarioId",
|
||||
"risk",
|
||||
"surface",
|
||||
"preflight",
|
||||
"steps",
|
||||
"assertions",
|
||||
"evidenceRequired",
|
||||
];
|
||||
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function addIssue(issues, severity, path, message) {
|
||||
issues.push({ severity, path, message });
|
||||
}
|
||||
|
||||
function validateStringArray(value, path, issues) {
|
||||
if (!Array.isArray(value) || value.length === 0) {
|
||||
addIssue(issues, "error", path, "必须是非空数组。");
|
||||
return;
|
||||
}
|
||||
value.forEach((entry, index) => {
|
||||
if (!isNonEmptyString(entry)) {
|
||||
addIssue(issues, "error", `${path}[${index}]`, "必须是非空字符串。");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function validateAgentQcGuiFlowManifest(flowManifest, scenarioManifest) {
|
||||
const issues = [];
|
||||
if (!flowManifest || typeof flowManifest !== "object") {
|
||||
return {
|
||||
valid: false,
|
||||
issues: [{ severity: "error", path: "manifest", message: "GUI flow manifest 必须是 JSON object。" }],
|
||||
};
|
||||
}
|
||||
|
||||
if (flowManifest.manifestVersion !== "v1") {
|
||||
addIssue(issues, "error", "manifestVersion", "当前只接受 manifestVersion=v1。");
|
||||
}
|
||||
|
||||
const scenarios = new Set(asArray(scenarioManifest?.scenarios).map((scenario) => scenario.id).filter(Boolean));
|
||||
const flows = asArray(flowManifest.flows);
|
||||
const flowIds = new Set();
|
||||
|
||||
if (flows.length === 0) {
|
||||
addIssue(issues, "error", "flows", "至少需要一个 GUI flow。");
|
||||
}
|
||||
|
||||
flows.forEach((flow, index) => {
|
||||
const prefix = `flows[${index}]`;
|
||||
for (const field of REQUIRED_FLOW_FIELDS) {
|
||||
if (!(field in flow)) {
|
||||
addIssue(issues, "error", `${prefix}.${field}`, `缺少 ${field}。`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!isNonEmptyString(flow.id)) {
|
||||
addIssue(issues, "error", `${prefix}.id`, "flow id 必须是非空字符串。");
|
||||
} else if (flowIds.has(flow.id)) {
|
||||
addIssue(issues, "error", `${prefix}.id`, `重复 flow id: ${flow.id}`);
|
||||
} else {
|
||||
flowIds.add(flow.id);
|
||||
}
|
||||
|
||||
if (!isNonEmptyString(flow.scenarioId)) {
|
||||
addIssue(issues, "error", `${prefix}.scenarioId`, "scenarioId 必须是非空字符串。");
|
||||
} else if (!scenarios.has(flow.scenarioId)) {
|
||||
addIssue(issues, "error", `${prefix}.scenarioId`, `引用了不存在的 Agent QC scenario: ${flow.scenarioId}`);
|
||||
}
|
||||
|
||||
for (const field of ["preflight", "steps", "assertions", "evidenceRequired"]) {
|
||||
validateStringArray(flow[field], `${prefix}.${field}`, issues);
|
||||
}
|
||||
});
|
||||
|
||||
const errors = issues.filter((issue) => issue.severity === "error");
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeAgentQcGuiFlowManifest(flowManifest, validation) {
|
||||
const flows = asArray(flowManifest?.flows);
|
||||
const riskCounts = new Map();
|
||||
for (const flow of flows) {
|
||||
const risk = flow.risk || "unknown";
|
||||
riskCounts.set(risk, (riskCounts.get(risk) ?? 0) + 1);
|
||||
}
|
||||
|
||||
return {
|
||||
title: flowManifest?.title || "Lime Agent QC GUI Flow Manifest",
|
||||
manifestVersion: flowManifest?.manifestVersion || "unknown",
|
||||
valid: validation.valid,
|
||||
issueCount: validation.issues.length,
|
||||
errorCount: validation.issues.filter((issue) => issue.severity === "error").length,
|
||||
warnCount: validation.issues.filter((issue) => issue.severity === "warn").length,
|
||||
flowCount: flows.length,
|
||||
risks: Object.fromEntries(Array.from(riskCounts.entries()).sort(([left], [right]) => left.localeCompare(right))),
|
||||
flows: flows.map((flow) => ({
|
||||
id: flow.id,
|
||||
scenarioId: flow.scenarioId,
|
||||
risk: flow.risk,
|
||||
surface: flow.surface,
|
||||
stepCount: asArray(flow.steps).length,
|
||||
assertionCount: asArray(flow.assertions).length,
|
||||
evidenceCount: asArray(flow.evidenceRequired).length,
|
||||
})),
|
||||
issues: validation.issues,
|
||||
};
|
||||
}
|
||||
|
||||
function renderAgentQcGuiFlowMarkdown(summary) {
|
||||
const flowLines = summary.flows.length
|
||||
? summary.flows
|
||||
.map(
|
||||
(flow) =>
|
||||
`- ${flow.id}: ${flow.scenarioId};${flow.risk};${flow.surface};steps ${flow.stepCount} / assertions ${flow.assertionCount} / evidence ${flow.evidenceCount}`,
|
||||
)
|
||||
.join("\n")
|
||||
: "- 无";
|
||||
const issueLines = summary.issues.length
|
||||
? summary.issues.map((issue) => `- ${issue.severity.toUpperCase()} ${issue.path}: ${issue.message}`).join("\n")
|
||||
: "- 无";
|
||||
|
||||
return `# ${summary.title}
|
||||
|
||||
## 结论
|
||||
|
||||
- Manifest: ${summary.manifestVersion}
|
||||
- 状态: ${summary.valid ? "PASS" : "FAIL"}
|
||||
- Flow 数: ${summary.flowCount}
|
||||
- Issue: ${summary.issueCount}(error ${summary.errorCount} / warn ${summary.warnCount})
|
||||
|
||||
## Flow 清单
|
||||
|
||||
${flowLines}
|
||||
|
||||
## 校验问题
|
||||
|
||||
${issueLines}
|
||||
`;
|
||||
}
|
||||
|
||||
function createAgentQcGuiFlowReport({ flowManifest, scenarioManifest }) {
|
||||
const validation = validateAgentQcGuiFlowManifest(flowManifest, scenarioManifest);
|
||||
return summarizeAgentQcGuiFlowManifest(flowManifest, validation);
|
||||
}
|
||||
|
||||
export {
|
||||
createAgentQcGuiFlowReport,
|
||||
renderAgentQcGuiFlowMarkdown,
|
||||
summarizeAgentQcGuiFlowManifest,
|
||||
validateAgentQcGuiFlowManifest,
|
||||
};
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createAgentQcGuiFlowReport,
|
||||
renderAgentQcGuiFlowMarkdown,
|
||||
validateAgentQcGuiFlowManifest,
|
||||
} from "./agent-qc-gui-flow-core.mjs";
|
||||
|
||||
const scenarioManifest = {
|
||||
scenarios: [
|
||||
{ id: "claw-chat-ready-streaming" },
|
||||
{ id: "workspace-ready-session-restore" },
|
||||
],
|
||||
};
|
||||
|
||||
const flowManifest = {
|
||||
manifestVersion: "v1",
|
||||
title: "GUI Flow Manifest",
|
||||
flows: [
|
||||
{
|
||||
id: "gui-claw-chat-ready-streaming",
|
||||
scenarioId: "claw-chat-ready-streaming",
|
||||
risk: "P0",
|
||||
surface: "desktop_gui",
|
||||
preflight: ["check bridge"],
|
||||
steps: ["open", "send"],
|
||||
assertions: ["ready"],
|
||||
evidenceRequired: ["trace"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("agent-qc-gui-flow-core", () => {
|
||||
it("应接受引用现有 scenario 的 GUI flow manifest", () => {
|
||||
const result = validateAgentQcGuiFlowManifest(flowManifest, scenarioManifest);
|
||||
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.issues).toEqual([]);
|
||||
});
|
||||
|
||||
it("应拒绝引用不存在 scenario 的 flow", () => {
|
||||
const invalid = structuredClone(flowManifest);
|
||||
invalid.flows[0].scenarioId = "missing-scenario";
|
||||
|
||||
const result = validateAgentQcGuiFlowManifest(invalid, scenarioManifest);
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
|
||||
"missing-scenario",
|
||||
);
|
||||
});
|
||||
|
||||
it("应汇总 flow 数、步骤数和证据数", () => {
|
||||
const report = createAgentQcGuiFlowReport({ flowManifest, scenarioManifest });
|
||||
|
||||
expect(report.valid).toBe(true);
|
||||
expect(report.flowCount).toBe(1);
|
||||
expect(report.flows[0].stepCount).toBe(2);
|
||||
expect(report.flows[0].evidenceCount).toBe(1);
|
||||
});
|
||||
|
||||
it("应渲染 Markdown 报告", () => {
|
||||
const report = createAgentQcGuiFlowReport({ flowManifest, scenarioManifest });
|
||||
const markdown = renderAgentQcGuiFlowMarkdown(report);
|
||||
|
||||
expect(markdown).toContain("GUI Flow Manifest");
|
||||
expect(markdown).toContain("gui-claw-chat-ready-streaming");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,239 @@
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeScenarioId(value) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
|
||||
function collectGuiScenarioIds(manifest) {
|
||||
return asArray(manifest?.scenarios)
|
||||
.filter((scenario) =>
|
||||
asArray(scenario?.evidenceRequired).includes("GUI session owner / isolation statement"),
|
||||
)
|
||||
.map((scenario) => normalizeScenarioId(scenario?.id))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function summarizeActiveGuiItems(status, guiScenarioIds) {
|
||||
const guiScenarioIdSet = new Set(guiScenarioIds);
|
||||
return asArray(status?.items)
|
||||
.filter((item) => guiScenarioIdSet.has(normalizeScenarioId(item?.scenarioId)))
|
||||
.filter((item) => item?.terminal !== true)
|
||||
.map((item) => ({
|
||||
scenarioId: normalizeScenarioId(item?.scenarioId),
|
||||
qcloopStatus: item?.qcloopStatus || "unknown",
|
||||
evidenceStatus: item?.evidenceStatus || "unknown",
|
||||
stale: Boolean(item?.stale),
|
||||
staleSeconds: item?.staleSeconds ?? null,
|
||||
workerStatus: item?.worker?.status || "unknown",
|
||||
workerDurationSeconds: item?.worker?.durationSeconds ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
function createAgentQcGuiOwnerReport({
|
||||
manifest,
|
||||
statusSidecars = [],
|
||||
generatedAt = new Date().toISOString(),
|
||||
maxActiveOwners = 0,
|
||||
} = {}) {
|
||||
const guiScenarioIds = collectGuiScenarioIds(manifest);
|
||||
const sidecarsByJobId = new Map();
|
||||
for (const sidecar of asArray(statusSidecars)) {
|
||||
const status = sidecar?.status || sidecar;
|
||||
const jobId = status?.job?.id || sidecar?.path || "unknown";
|
||||
const entries = sidecarsByJobId.get(jobId) || [];
|
||||
entries.push(sidecar);
|
||||
sidecarsByJobId.set(jobId, entries);
|
||||
}
|
||||
|
||||
const activeOwnerByJobId = new Map();
|
||||
|
||||
for (const sidecars of sidecarsByJobId.values()) {
|
||||
const sidecar = selectRepresentativeSidecar(sidecars);
|
||||
const status = sidecar?.status || sidecar;
|
||||
const activeItems = summarizeActiveGuiItems(status, guiScenarioIds);
|
||||
if (activeItems.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const jobId = status?.job?.id || sidecar?.path || "unknown";
|
||||
const existing = activeOwnerByJobId.get(jobId);
|
||||
const owner = {
|
||||
path: sidecar?.path || "",
|
||||
jobId,
|
||||
jobName: status?.job?.name || "",
|
||||
jobStatus: status?.job?.status || "unknown",
|
||||
verdictStatus: status?.verdict?.status || "unknown",
|
||||
counts: status?.counts || null,
|
||||
activeItems,
|
||||
};
|
||||
|
||||
if (!existing) {
|
||||
activeOwnerByJobId.set(jobId, owner);
|
||||
continue;
|
||||
}
|
||||
|
||||
const existingStaleCount = existing.activeItems.filter((item) => item.stale).length;
|
||||
const ownerStaleCount = owner.activeItems.filter((item) => item.stale).length;
|
||||
if (ownerStaleCount > existingStaleCount) {
|
||||
activeOwnerByJobId.set(jobId, owner);
|
||||
}
|
||||
}
|
||||
|
||||
const activeOwners = Array.from(activeOwnerByJobId.values()).sort((left, right) =>
|
||||
left.jobId.localeCompare(right.jobId),
|
||||
);
|
||||
const ownerCount = activeOwners.length;
|
||||
const staleOwners = activeOwners.filter((owner) =>
|
||||
owner.activeItems.some((item) => item.stale),
|
||||
);
|
||||
const staleOwnerCount = staleOwners.length;
|
||||
const oldestStaleSeconds = Math.max(
|
||||
0,
|
||||
...activeOwners.flatMap((owner) =>
|
||||
owner.activeItems
|
||||
.filter((item) => item.stale)
|
||||
.map((item) => Number(item.staleSeconds || 0)),
|
||||
),
|
||||
);
|
||||
const pass = ownerCount <= maxActiveOwners;
|
||||
const ownerIntervention =
|
||||
staleOwnerCount > 0 ? createOwnerIntervention(activeOwners) : null;
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
generatedAt,
|
||||
maxActiveOwners,
|
||||
guiScenarioIds,
|
||||
ownerCount,
|
||||
staleOwnerCount,
|
||||
oldestStaleSeconds,
|
||||
activeOwners,
|
||||
ownerIntervention,
|
||||
verdict: {
|
||||
status: pass ? "pass" : "blocked",
|
||||
summary: pass
|
||||
? `active GUI qcloop owner=${ownerCount},未超过上限 ${maxActiveOwners}。`
|
||||
: `active GUI qcloop owner=${ownerCount},超过上限 ${maxActiveOwners}${staleOwnerCount > 0 ? `;其中 stale owner=${staleOwnerCount},最长 ${oldestStaleSeconds}s` : ""}。`,
|
||||
nextAction: pass
|
||||
? "可以在其他 release gate 满足后启动新的 GUI P0 批次。"
|
||||
: staleOwnerCount > 0
|
||||
? "不要启动新的 GUI P0 批次;当前存在 stale GUI owner,只能继续只读观察,或由该 owner 明确确认后处理。"
|
||||
: "不要启动新的 GUI P0 批次;等待现有 running 批次自然结束或由 owner 明确处理。",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createOwnerIntervention(activeOwners) {
|
||||
const staleOwners = activeOwners.filter((owner) =>
|
||||
owner.activeItems.some((item) => item.stale),
|
||||
);
|
||||
const primaryOwner = staleOwners[0];
|
||||
const primaryJobId = primaryOwner?.jobId || "<job-id>";
|
||||
return {
|
||||
status: "requires_owner_confirmation",
|
||||
jobIds: staleOwners.map((owner) => owner.jobId),
|
||||
requiredConfirmationText: `确认处理 stale GUI owner ${primaryJobId},可以终止 PID <pid> 并记录 sidecar。`,
|
||||
prohibitedUntilConfirmed: [
|
||||
"kill / pause / interrupt stale worker",
|
||||
"modify qcloop SQLite DB",
|
||||
"start another full GUI P0 batch",
|
||||
"overwrite .lime/qc/agent-qc-evidence.json",
|
||||
"git commit / push / tag / release",
|
||||
],
|
||||
evidenceRefs: staleOwners.map((owner) => owner.path).filter(Boolean),
|
||||
nextAction:
|
||||
"先刷新 qcloop status、GUI owner、DB lease 和进程证据;只有 owner 明确确认后才能处理 stale worker。",
|
||||
};
|
||||
}
|
||||
|
||||
function isTerminalJobStatus(status) {
|
||||
const jobStatus = String(status?.job?.status || "").toLowerCase();
|
||||
return (
|
||||
status?.job?.terminal === true ||
|
||||
["completed", "failed", "cancelled", "canceled"].includes(jobStatus)
|
||||
);
|
||||
}
|
||||
|
||||
function selectRepresentativeSidecar(sidecars) {
|
||||
const terminal = asArray(sidecars).find((sidecar) => isTerminalJobStatus(sidecar?.status || sidecar));
|
||||
if (terminal) {
|
||||
return terminal;
|
||||
}
|
||||
|
||||
return asArray(sidecars).reduce((selected, candidate) => {
|
||||
if (!selected) {
|
||||
return candidate;
|
||||
}
|
||||
const selectedStatus = selected?.status || selected;
|
||||
const candidateStatus = candidate?.status || candidate;
|
||||
const selectedStale = Number(selectedStatus?.counts?.stale || 0);
|
||||
const candidateStale = Number(candidateStatus?.counts?.stale || 0);
|
||||
if (candidateStale > selectedStale) {
|
||||
return candidate;
|
||||
}
|
||||
const selectedRunning = Number(selectedStatus?.counts?.running || 0);
|
||||
const candidateRunning = Number(candidateStatus?.counts?.running || 0);
|
||||
return candidateRunning > selectedRunning ? candidate : selected;
|
||||
}, null);
|
||||
}
|
||||
|
||||
function renderAgentQcGuiOwnerSummary(report) {
|
||||
const lines = [
|
||||
`status=${report.verdict.status}`,
|
||||
`activeOwners=${report.ownerCount}`,
|
||||
`staleOwners=${report.staleOwnerCount ?? 0}`,
|
||||
`oldestStaleSeconds=${report.oldestStaleSeconds ?? 0}`,
|
||||
`maxActiveOwners=${report.maxActiveOwners}`,
|
||||
`guiScenarios=${report.guiScenarioIds.join(",")}`,
|
||||
`summary=${report.verdict.summary}`,
|
||||
];
|
||||
for (const owner of report.activeOwners) {
|
||||
const items = owner.activeItems
|
||||
.map((item) =>
|
||||
`${item.scenarioId}:${item.qcloopStatus}${item.stale ? `:stale=${item.staleSeconds ?? 0}s` : ""}`,
|
||||
)
|
||||
.join(",");
|
||||
lines.push(`owner=${owner.jobId} jobStatus=${owner.jobStatus} verdict=${owner.verdictStatus} path=${owner.path} items=${items}`);
|
||||
}
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function createAgentQcGuiOwnerWatchEntry(report) {
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
observedAt: report?.generatedAt || new Date().toISOString(),
|
||||
verdictStatus: report?.verdict?.status || "unknown",
|
||||
ownerCount: report?.ownerCount ?? 0,
|
||||
staleOwnerCount: report?.staleOwnerCount ?? 0,
|
||||
oldestStaleSeconds: report?.oldestStaleSeconds ?? 0,
|
||||
activeOwners: asArray(report?.activeOwners).map((owner) => ({
|
||||
jobId: owner.jobId,
|
||||
jobStatus: owner.jobStatus,
|
||||
verdictStatus: owner.verdictStatus,
|
||||
path: owner.path,
|
||||
activeItems: asArray(owner.activeItems).map((item) => ({
|
||||
scenarioId: item.scenarioId,
|
||||
qcloopStatus: item.qcloopStatus,
|
||||
evidenceStatus: item.evidenceStatus,
|
||||
stale: Boolean(item.stale),
|
||||
staleSeconds: item.staleSeconds ?? null,
|
||||
workerStatus: item.workerStatus,
|
||||
workerDurationSeconds: item.workerDurationSeconds ?? null,
|
||||
})),
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
collectGuiScenarioIds,
|
||||
createAgentQcGuiOwnerWatchEntry,
|
||||
createAgentQcGuiOwnerReport,
|
||||
createOwnerIntervention,
|
||||
renderAgentQcGuiOwnerSummary,
|
||||
};
|
||||
@@ -0,0 +1,198 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
collectGuiScenarioIds,
|
||||
createAgentQcGuiOwnerWatchEntry,
|
||||
createAgentQcGuiOwnerReport,
|
||||
renderAgentQcGuiOwnerSummary,
|
||||
} from "./agent-qc-gui-owner-core.mjs";
|
||||
|
||||
const manifest = {
|
||||
scenarios: [
|
||||
{
|
||||
id: "claw-chat-ready-streaming",
|
||||
evidenceRequired: ["GUI session owner / isolation statement"],
|
||||
},
|
||||
{
|
||||
id: "command-bridge-contract",
|
||||
evidenceRequired: ["contract command log"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("agent-qc-gui-owner-core", () => {
|
||||
it("应从 manifest 中识别 GUI owner 场景", () => {
|
||||
expect(collectGuiScenarioIds(manifest)).toEqual(["claw-chat-ready-streaming"]);
|
||||
});
|
||||
|
||||
it("没有 active GUI sidecar 时应 pass", () => {
|
||||
const report = createAgentQcGuiOwnerReport({
|
||||
manifest,
|
||||
statusSidecars: [
|
||||
{
|
||||
path: ".lime/qc/qcloop-status.contract.json",
|
||||
status: {
|
||||
job: { id: "job-1", status: "completed" },
|
||||
verdict: { status: "complete" },
|
||||
items: [
|
||||
{
|
||||
scenarioId: "command-bridge-contract",
|
||||
qcloopStatus: "success",
|
||||
terminal: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("pass");
|
||||
expect(report.ownerCount).toBe(0);
|
||||
});
|
||||
|
||||
it("active GUI sidecar 超过上限时应 blocked", () => {
|
||||
const report = createAgentQcGuiOwnerReport({
|
||||
manifest,
|
||||
statusSidecars: [
|
||||
{
|
||||
path: ".lime/qc/qcloop-status.gui.json",
|
||||
status: {
|
||||
job: { id: "job-gui", status: "running" },
|
||||
verdict: { status: "stale" },
|
||||
counts: { running: 1, stale: 1 },
|
||||
items: [
|
||||
{
|
||||
scenarioId: "claw-chat-ready-streaming",
|
||||
qcloopStatus: "running",
|
||||
evidenceStatus: "blocked",
|
||||
terminal: false,
|
||||
stale: true,
|
||||
staleSeconds: 600,
|
||||
worker: { status: "running", durationSeconds: 600 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
maxActiveOwners: 0,
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("blocked");
|
||||
expect(report.ownerCount).toBe(1);
|
||||
expect(report.staleOwnerCount).toBe(1);
|
||||
expect(report.oldestStaleSeconds).toBe(600);
|
||||
expect(report.verdict.nextAction).toContain("只读观察");
|
||||
expect(report.ownerIntervention?.status).toBe("requires_owner_confirmation");
|
||||
expect(report.ownerIntervention?.requiredConfirmationText).toContain("job-gui");
|
||||
expect(report.ownerIntervention?.prohibitedUntilConfirmed).toContain(
|
||||
"start another full GUI P0 batch",
|
||||
);
|
||||
expect(report.activeOwners[0].activeItems[0].scenarioId).toBe(
|
||||
"claw-chat-ready-streaming",
|
||||
);
|
||||
});
|
||||
|
||||
it("同一 job 的多个 sidecar 应去重", () => {
|
||||
const sidecar = {
|
||||
job: { id: "job-gui", status: "running" },
|
||||
verdict: { status: "running" },
|
||||
items: [
|
||||
{
|
||||
scenarioId: "claw-chat-ready-streaming",
|
||||
qcloopStatus: "running",
|
||||
terminal: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const report = createAgentQcGuiOwnerReport({
|
||||
manifest,
|
||||
statusSidecars: [
|
||||
{ path: ".lime/qc/qcloop-status.gui-current.json", status: sidecar },
|
||||
{ path: ".lime/qc/qcloop-status.gui-stale.json", status: sidecar },
|
||||
],
|
||||
});
|
||||
|
||||
expect(report.ownerCount).toBe(1);
|
||||
});
|
||||
|
||||
it("同一 job 存在终态 sidecar 时应忽略旧 running sidecar", () => {
|
||||
const runningSidecar = {
|
||||
job: { id: "job-gui", status: "running", terminal: false },
|
||||
verdict: { status: "running" },
|
||||
items: [
|
||||
{
|
||||
scenarioId: "claw-chat-ready-streaming",
|
||||
qcloopStatus: "running",
|
||||
terminal: false,
|
||||
},
|
||||
],
|
||||
};
|
||||
const terminalSidecar = {
|
||||
job: { id: "job-gui", status: "failed", terminal: true },
|
||||
verdict: { status: "fail" },
|
||||
items: [
|
||||
{
|
||||
scenarioId: "claw-chat-ready-streaming",
|
||||
qcloopStatus: "exhausted",
|
||||
terminal: true,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const report = createAgentQcGuiOwnerReport({
|
||||
manifest,
|
||||
statusSidecars: [
|
||||
{ path: ".lime/qc/qcloop-status.gui-current.json", status: runningSidecar },
|
||||
{ path: ".lime/qc/qcloop-status.gui-completed.json", status: terminalSidecar },
|
||||
],
|
||||
});
|
||||
|
||||
expect(report.ownerCount).toBe(0);
|
||||
expect(report.ownerIntervention).toBeNull();
|
||||
expect(report.verdict.status).toBe("pass");
|
||||
});
|
||||
|
||||
it("应渲染 summary", () => {
|
||||
const report = createAgentQcGuiOwnerReport({ manifest, statusSidecars: [] });
|
||||
const summary = renderAgentQcGuiOwnerSummary(report);
|
||||
|
||||
expect(summary).toContain("status=pass");
|
||||
expect(summary).toContain("staleOwners=0");
|
||||
expect(summary).toContain("guiScenarios=claw-chat-ready-streaming");
|
||||
});
|
||||
|
||||
it("应生成 watch history JSONL entry", () => {
|
||||
const report = createAgentQcGuiOwnerReport({
|
||||
manifest,
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
statusSidecars: [
|
||||
{
|
||||
path: ".lime/qc/qcloop-status.gui.json",
|
||||
status: {
|
||||
job: { id: "job-gui", status: "running" },
|
||||
verdict: { status: "stale" },
|
||||
items: [
|
||||
{
|
||||
scenarioId: "claw-chat-ready-streaming",
|
||||
qcloopStatus: "running",
|
||||
evidenceStatus: "blocked",
|
||||
terminal: false,
|
||||
stale: true,
|
||||
staleSeconds: 900,
|
||||
worker: { status: "running", durationSeconds: 900 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const entry = createAgentQcGuiOwnerWatchEntry(report);
|
||||
|
||||
expect(entry.observedAt).toBe("2026-05-10T00:00:00.000Z");
|
||||
expect(entry.verdictStatus).toBe("blocked");
|
||||
expect(entry.activeOwners[0].jobId).toBe("job-gui");
|
||||
expect(entry.activeOwners[0].activeItems[0].staleSeconds).toBe(900);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,273 @@
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeRiskList(risks) {
|
||||
return asArray(risks)
|
||||
.map((risk) => String(risk).trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function selectScenarios(manifest, { risks = ["P0"], scenarioIds = [], includeAll = false } = {}) {
|
||||
const normalizedRisks = new Set(normalizeRiskList(risks));
|
||||
const normalizedScenarioIds = new Set(asArray(scenarioIds).filter(isNonEmptyString));
|
||||
const scenarios = asArray(manifest?.scenarios);
|
||||
|
||||
return scenarios.filter((scenario) => {
|
||||
if (includeAll) {
|
||||
return true;
|
||||
}
|
||||
if (normalizedScenarioIds.size > 0) {
|
||||
return normalizedScenarioIds.has(scenario.id);
|
||||
}
|
||||
return normalizedRisks.has(String(scenario.risk || "").toUpperCase());
|
||||
});
|
||||
}
|
||||
|
||||
const WORKER_RESULT_MARKER = "QCLOOP_WORKER_RESULT=";
|
||||
const WORKER_EVIDENCE_SUMMARY_MARKER = "QCLOOP_EVIDENCE_SUMMARY_JSON=";
|
||||
|
||||
const WORKER_EVIDENCE_CONTRACT_BLOCK = `
|
||||
|
||||
---
|
||||
Agent QC 结构化证据输出契约:
|
||||
- 最终 stdout 必须包含单行 \`QCLOOP_WORKER_RESULT=PASS|FAIL|BLOCKED\`。
|
||||
- 最终 stdout 必须包含单行 \`QCLOOP_EVIDENCE_SUMMARY_JSON=<json>\`;\`<json>\` 必须是单个 JSON object,不要 Markdown / code fence。
|
||||
- \`QCLOOP_EVIDENCE_SUMMARY_JSON\` 至少包含:scenario_id、result、commands[]、evidence_required[]、evidence_layers_covered[]、failure_modes[]、artifacts[]、blockers[]、gui_session_owner、release_scope。
|
||||
- evidence_required[] 每项包含 name、status(pass|fail|blocked|missing)、evidence、artifact_path;failure_modes[] 每项包含 name、status(covered|excluded|hit|unknown)、evidence。
|
||||
- 不得输出密钥、token、用户私密内容或未经脱敏的请求 / 响应正文。`;
|
||||
|
||||
const VERIFIER_JSON_CONTRACT_BLOCK = `
|
||||
|
||||
---
|
||||
qcloop verifier 输出格式硬约束:
|
||||
- 只输出一个合法 JSON object,不要 Markdown、不要代码块、不要前后缀文本。
|
||||
- JSON 必须至少包含 {"pass": true|false, "feedback": "..."}。
|
||||
- 可选字段 missingEvidence、nextAction、evidenceStatus、scenarioId 必须仍在同一个 JSON object 内。
|
||||
- 如果 worker stdout 缺少 QCLOOP_EVIDENCE_SUMMARY_JSON=<json>,或该 JSON 无法逐项证明 evidenceRequired / failureModes,必须输出 {"pass": false, "feedback": "..."}。`;
|
||||
|
||||
function ensureWorkerEvidenceContract(template) {
|
||||
const normalized = isNonEmptyString(template)
|
||||
? template
|
||||
: "在 Lime 仓库中执行 Agent QC 场景 {{item}},收集证据并给出结论。";
|
||||
if (
|
||||
normalized.includes(WORKER_RESULT_MARKER) &&
|
||||
normalized.includes(WORKER_EVIDENCE_SUMMARY_MARKER)
|
||||
) {
|
||||
return normalized;
|
||||
}
|
||||
return `${normalized}${WORKER_EVIDENCE_CONTRACT_BLOCK}`;
|
||||
}
|
||||
|
||||
const VERIFIER_EVIDENCE_BLOCK = `
|
||||
|
||||
---
|
||||
qcloop worker 执行证据(自动占位符替换):
|
||||
- attempt_status: {{attempt_status}}
|
||||
- attempt_type: {{attempt_type}}
|
||||
- exit_code: {{exit_code}}
|
||||
|
||||
worker stdout:
|
||||
{{stdout}}
|
||||
|
||||
qcloop issue ledger:
|
||||
{{issue_ledger}}
|
||||
|
||||
判定要求:如果 stdout 缺少场景 verifier / evidence_required / failure_modes 所需证据,必须 pass=false;不能因为命令名看似正确就通过。`;
|
||||
|
||||
function ensureVerifierMetadataPlaceholders(template) {
|
||||
const missingLines = [];
|
||||
if (!template.includes("{{attempt_status}}")) {
|
||||
missingLines.push("- attempt_status: {{attempt_status}}");
|
||||
}
|
||||
if (!template.includes("{{attempt_type}}")) {
|
||||
missingLines.push("- attempt_type: {{attempt_type}}");
|
||||
}
|
||||
if (!template.includes("{{exit_code}}")) {
|
||||
missingLines.push("- exit_code: {{exit_code}}");
|
||||
}
|
||||
if (!template.includes("{{qc_history}}") && !template.includes("{{issue_ledger}}")) {
|
||||
missingLines.push("- issue_ledger: {{issue_ledger}}");
|
||||
}
|
||||
if (missingLines.length === 0) {
|
||||
return template;
|
||||
}
|
||||
return `${template}
|
||||
|
||||
---
|
||||
qcloop worker 执行元数据(自动占位符替换):
|
||||
${missingLines.join("\n")}`;
|
||||
}
|
||||
|
||||
function ensureVerifierEvidencePlaceholders(template) {
|
||||
let normalized = isNonEmptyString(template)
|
||||
? template
|
||||
: "审查 Agent QC 场景 {{item}} 的输出是否满足证据要求,只输出 JSON verdict。";
|
||||
|
||||
if (!normalized.includes("{{stdout}}") && !normalized.includes("{{output}}")) {
|
||||
normalized = `${normalized}${VERIFIER_EVIDENCE_BLOCK}`;
|
||||
}
|
||||
|
||||
normalized = ensureVerifierMetadataPlaceholders(normalized);
|
||||
|
||||
if (!normalized.includes("QCLOOP_EVIDENCE_SUMMARY_JSON")) {
|
||||
normalized = `${normalized}${VERIFIER_JSON_CONTRACT_BLOCK}`;
|
||||
} else if (!normalized.includes('{"pass": true|false')) {
|
||||
normalized = `${normalized}${VERIFIER_JSON_CONTRACT_BLOCK}`;
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
function buildQCLoopItemValue(scenario, { cwd = "" } = {}) {
|
||||
return JSON.stringify({
|
||||
scenario_id: scenario.id,
|
||||
title: scenario.title,
|
||||
risk: scenario.risk,
|
||||
cwd,
|
||||
executor: scenario.executor,
|
||||
lanes: scenario.lanes,
|
||||
commands: scenario.commands || [],
|
||||
goal: scenario.goal,
|
||||
verifier: scenario.verifier,
|
||||
evidence_required: scenario.evidenceRequired || [],
|
||||
evidence_layers: scenario.evidenceLayers || [],
|
||||
failure_modes: scenario.failureModes || [],
|
||||
});
|
||||
}
|
||||
|
||||
function buildQCLoopJobPayload(manifest, options = {}) {
|
||||
const selectedScenarios = selectScenarios(manifest, options);
|
||||
const qcloop = manifest?.qcloop || {};
|
||||
const riskLabel = options.includeAll
|
||||
? "all"
|
||||
: asArray(options.scenarioIds).length > 0
|
||||
? "selected"
|
||||
: normalizeRiskList(options.risks || ["P0"]).join("+") || "P0";
|
||||
const name =
|
||||
options.name ||
|
||||
`lime-agent-qc-${riskLabel.toLowerCase()}-${new Date(options.generatedAt || Date.now()).toISOString().slice(0, 10)}`;
|
||||
|
||||
const cwdLine = options.cwd
|
||||
? `目标仓库 cwd: ${options.cwd}。执行任何命令前必须先切换到该目录,并在输出中记录 pwd。`
|
||||
: "";
|
||||
const workerPromptTemplate = ensureWorkerEvidenceContract(
|
||||
options.promptTemplate ||
|
||||
qcloop.workerPromptTemplate ||
|
||||
"在 Lime 仓库中执行 Agent QC 场景 {{item}},收集证据并给出结论。",
|
||||
);
|
||||
|
||||
return {
|
||||
name,
|
||||
prompt_template: [cwdLine, workerPromptTemplate].filter(Boolean).join("\n\n"),
|
||||
verifier_prompt_template: ensureVerifierEvidencePlaceholders(
|
||||
options.verifierPromptTemplate || qcloop.verifierPromptTemplate,
|
||||
),
|
||||
max_qc_rounds: Number(options.maxQcRounds || qcloop.recommendedMaxQcRounds || 3),
|
||||
max_executor_retries: Number(
|
||||
options.maxExecutorRetries ?? qcloop.recommendedMaxExecutorRetries ?? 1,
|
||||
),
|
||||
token_budget_per_item: Number(options.tokenBudgetPerItem || 0),
|
||||
execution_mode: options.executionMode || "standard",
|
||||
executor_provider: options.executorProvider || "codex",
|
||||
items: selectedScenarios.map((scenario) =>
|
||||
buildQCLoopItemValue(scenario, { cwd: options.cwd || "" }),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
function validateVerifierPromptTemplate(template, issues) {
|
||||
if (!isNonEmptyString(template)) {
|
||||
issues.push("缺少 verifier_prompt_template。");
|
||||
return;
|
||||
}
|
||||
if (!template.includes("{{stdout}}") && !template.includes("{{output}}")) {
|
||||
issues.push("verifier_prompt_template 必须包含 {{stdout}} 或 {{output}},否则 verifier 看不到 worker 输出。");
|
||||
}
|
||||
for (const placeholder of ["{{attempt_status}}", "{{exit_code}}"]) {
|
||||
if (!template.includes(placeholder)) {
|
||||
issues.push(`verifier_prompt_template 必须包含 ${placeholder}。`);
|
||||
}
|
||||
}
|
||||
if (!template.includes("{{qc_history}}") && !template.includes("{{issue_ledger}}")) {
|
||||
issues.push("verifier_prompt_template 必须包含 {{qc_history}} 或 {{issue_ledger}}。");
|
||||
}
|
||||
if (!template.includes(WORKER_EVIDENCE_SUMMARY_MARKER)) {
|
||||
issues.push(`verifier_prompt_template 必须要求审查 ${WORKER_EVIDENCE_SUMMARY_MARKER}。`);
|
||||
}
|
||||
if (!template.includes('{"pass": true|false')) {
|
||||
issues.push('verifier_prompt_template 必须声明只输出 {"pass": true|false, "feedback": "..."} JSON。');
|
||||
}
|
||||
}
|
||||
|
||||
function validateQCLoopJobPayload(payload) {
|
||||
const issues = [];
|
||||
if (!isNonEmptyString(payload?.name)) {
|
||||
issues.push("缺少 name。 ");
|
||||
}
|
||||
if (!isNonEmptyString(payload?.prompt_template)) {
|
||||
issues.push("缺少 prompt_template。 ");
|
||||
} else {
|
||||
if (!payload.prompt_template.includes("{{item}}")) {
|
||||
issues.push("prompt_template 必须包含 {{item}}。 ");
|
||||
}
|
||||
if (!payload.prompt_template.includes(WORKER_RESULT_MARKER)) {
|
||||
issues.push(`prompt_template 必须要求输出 ${WORKER_RESULT_MARKER}。 `);
|
||||
}
|
||||
if (!payload.prompt_template.includes(WORKER_EVIDENCE_SUMMARY_MARKER)) {
|
||||
issues.push(`prompt_template 必须要求输出 ${WORKER_EVIDENCE_SUMMARY_MARKER}。 `);
|
||||
}
|
||||
}
|
||||
validateVerifierPromptTemplate(payload?.verifier_prompt_template, issues);
|
||||
if (!Number.isInteger(payload?.max_qc_rounds) || payload.max_qc_rounds <= 0) {
|
||||
issues.push("max_qc_rounds 必须是正整数。 ");
|
||||
}
|
||||
if (
|
||||
!Number.isInteger(payload?.max_executor_retries) ||
|
||||
payload.max_executor_retries < 0 ||
|
||||
payload.max_executor_retries > 5
|
||||
) {
|
||||
issues.push("max_executor_retries 必须是 0 到 5 的整数。 ");
|
||||
}
|
||||
if (!Array.isArray(payload?.items) || payload.items.length === 0) {
|
||||
issues.push("items 不能为空。 ");
|
||||
}
|
||||
for (const [index, item] of asArray(payload?.items).entries()) {
|
||||
if (!isNonEmptyString(item)) {
|
||||
issues.push(`items[${index}] 不能为空。`);
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(item);
|
||||
if (!isNonEmptyString(parsed.scenario_id)) {
|
||||
issues.push(`items[${index}] 缺少 scenario_id。`);
|
||||
}
|
||||
} catch {
|
||||
issues.push(`items[${index}] 必须是 JSON 字符串。`);
|
||||
}
|
||||
}
|
||||
return {
|
||||
valid: issues.length === 0,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
function renderQCLoopCurl(payload, { baseUrl = "http://127.0.0.1:8080" } = {}) {
|
||||
const body = JSON.stringify(payload, null, 2);
|
||||
return `curl -sS -X POST "${baseUrl.replace(/\/$/, "")}/api/jobs" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
--data-binary @- <<'JSON'\n${body}\nJSON\n`;
|
||||
}
|
||||
|
||||
export {
|
||||
buildQCLoopItemValue,
|
||||
buildQCLoopJobPayload,
|
||||
renderQCLoopCurl,
|
||||
selectScenarios,
|
||||
validateQCLoopJobPayload,
|
||||
};
|
||||
@@ -0,0 +1,159 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildQCLoopJobPayload,
|
||||
renderQCLoopCurl,
|
||||
selectScenarios,
|
||||
validateQCLoopJobPayload,
|
||||
} from "./agent-qc-qcloop-job-core.mjs";
|
||||
|
||||
const manifest = {
|
||||
qcloop: {
|
||||
recommendedMaxQcRounds: 3,
|
||||
workerPromptTemplate: "worker {{item}}",
|
||||
verifierPromptTemplate: "verifier {{item}}",
|
||||
},
|
||||
scenarios: [
|
||||
{
|
||||
id: "command-bridge-contract",
|
||||
title: "命令桥接四侧一致",
|
||||
risk: "P0",
|
||||
executor: "npm_command",
|
||||
lanes: ["L1-contract-bridge"],
|
||||
commands: ["npm run test:contracts"],
|
||||
goal: "goal",
|
||||
verifier: "verifier",
|
||||
evidenceRequired: ["evidence"],
|
||||
evidenceLayers: ["deterministic-smoke"],
|
||||
failureModes: ["failure"],
|
||||
},
|
||||
{
|
||||
id: "knowledge-ingest-retrieve-summarize",
|
||||
title: "Knowledge",
|
||||
risk: "P1",
|
||||
executor: "mixed",
|
||||
lanes: ["L3-product-surface"],
|
||||
goal: "goal",
|
||||
verifier: "verifier",
|
||||
evidenceRequired: ["evidence"],
|
||||
failureModes: ["failure"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("agent-qc-qcloop-job-core", () => {
|
||||
it("默认应选择 P0 场景", () => {
|
||||
const scenarios = selectScenarios(manifest);
|
||||
|
||||
expect(scenarios.map((scenario) => scenario.id)).toEqual(["command-bridge-contract"]);
|
||||
});
|
||||
|
||||
it("应按风险等级生成 qcloop job payload", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest, {
|
||||
risks: ["P1"],
|
||||
cwd: "/workspace/lime",
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
maxExecutorRetries: 0,
|
||||
});
|
||||
|
||||
expect(payload.name).toBe("lime-agent-qc-p1-2026-05-10");
|
||||
expect(payload.prompt_template).toContain("目标仓库 cwd: /workspace/lime");
|
||||
expect(payload.prompt_template).toContain("QCLOOP_WORKER_RESULT=PASS|FAIL|BLOCKED");
|
||||
expect(payload.prompt_template).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON=");
|
||||
expect(payload.items).toHaveLength(1);
|
||||
expect(payload.max_executor_retries).toBe(0);
|
||||
expect(payload.verifier_prompt_template).toContain("{{stdout}}");
|
||||
expect(payload.verifier_prompt_template).toContain("{{exit_code}}");
|
||||
expect(payload.verifier_prompt_template).toContain("{{issue_ledger}}");
|
||||
expect(payload.verifier_prompt_template).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON");
|
||||
expect(payload.verifier_prompt_template).toContain('{"pass": true|false');
|
||||
expect(JSON.parse(payload.items[0]).scenario_id).toBe(
|
||||
"knowledge-ingest-retrieve-summarize",
|
||||
);
|
||||
expect(JSON.parse(payload.items[0]).cwd).toBe("/workspace/lime");
|
||||
expect(validateQCLoopJobPayload(payload).valid).toBe(true);
|
||||
});
|
||||
|
||||
it("应把 evidenceLayers 带入 qcloop item,方便 worker 声明证据深度", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest, {
|
||||
scenarioIds: ["command-bridge-contract"],
|
||||
});
|
||||
|
||||
expect(JSON.parse(payload.items[0]).evidence_layers).toEqual([
|
||||
"deterministic-smoke",
|
||||
]);
|
||||
});
|
||||
|
||||
it("应支持按 scenario id 精确选择", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest, {
|
||||
scenarioIds: ["command-bridge-contract"],
|
||||
name: "selected-job",
|
||||
});
|
||||
|
||||
expect(payload.name).toBe("selected-job");
|
||||
expect(payload.items).toHaveLength(1);
|
||||
expect(JSON.parse(payload.items[0]).scenario_id).toBe("command-bridge-contract");
|
||||
});
|
||||
|
||||
it("validateQCLoopJobPayload 应阻止缺少 verifier stdout 占位符的 payload", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest);
|
||||
payload.verifier_prompt_template = "verifier {{item}}";
|
||||
|
||||
const validation = validateQCLoopJobPayload(payload);
|
||||
|
||||
expect(validation.valid).toBe(false);
|
||||
expect(validation.issues.join("\n")).toContain("{{stdout}}");
|
||||
});
|
||||
|
||||
it("validateQCLoopJobPayload 应阻止缺少结构化证据契约的 payload", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest);
|
||||
payload.prompt_template = "worker {{item}}";
|
||||
payload.verifier_prompt_template =
|
||||
"verifier {{item}} status={{attempt_status}} code={{exit_code}} stdout={{stdout}} ledger={{issue_ledger}}";
|
||||
|
||||
const validation = validateQCLoopJobPayload(payload);
|
||||
|
||||
expect(validation.valid).toBe(false);
|
||||
expect(validation.issues.join("\n")).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON=");
|
||||
});
|
||||
|
||||
it("validateQCLoopJobPayload 应阻止非法 max_executor_retries", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest);
|
||||
payload.max_executor_retries = 6;
|
||||
|
||||
const validation = validateQCLoopJobPayload(payload);
|
||||
|
||||
expect(validation.valid).toBe(false);
|
||||
expect(validation.issues.join("\n")).toContain("max_executor_retries");
|
||||
});
|
||||
|
||||
it("verifier_prompt_template 不应重复追加已有 stdout 占位符的模板", () => {
|
||||
const payload = buildQCLoopJobPayload(
|
||||
{
|
||||
...manifest,
|
||||
qcloop: {
|
||||
...manifest.qcloop,
|
||||
verifierPromptTemplate: "verifier {{item}} stdout={{stdout}}",
|
||||
},
|
||||
},
|
||||
{ scenarioIds: ["command-bridge-contract"] },
|
||||
);
|
||||
|
||||
expect(payload.verifier_prompt_template.match(/{{stdout}}/g)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("应生成可复制的 curl 命令", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest);
|
||||
const curl = renderQCLoopCurl(payload, { baseUrl: "http://localhost:8080" });
|
||||
|
||||
expect(curl).toContain("POST \"http://localhost:8080/api/jobs\"");
|
||||
expect(curl).toContain("command-bridge-contract");
|
||||
});
|
||||
|
||||
it("默认 curl 应使用 IPv4 loopback,避免 localhost 代理或 IPv6 解析干扰", () => {
|
||||
const payload = buildQCLoopJobPayload(manifest);
|
||||
const curl = renderQCLoopCurl(payload);
|
||||
|
||||
expect(curl).toContain("POST \"http://127.0.0.1:8080/api/jobs\"");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizePathText(value) {
|
||||
return isNonEmptyString(value) ? value.trim().replace(/\\/g, "/") : "";
|
||||
}
|
||||
|
||||
function createCheck(id, title, passed, detail = "") {
|
||||
return {
|
||||
id,
|
||||
title,
|
||||
passed: Boolean(passed),
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
function buildQCLoopPreflightReport({ cwd, expectedCwd = "", tmpWritable = false, devBridge = null } = {}) {
|
||||
const checks = [];
|
||||
const normalizedCwd = normalizePathText(cwd);
|
||||
const normalizedExpectedCwd = normalizePathText(expectedCwd);
|
||||
|
||||
checks.push(createCheck("cwd-present", "当前工作目录可读", Boolean(normalizedCwd), normalizedCwd));
|
||||
if (normalizedExpectedCwd) {
|
||||
checks.push(
|
||||
createCheck(
|
||||
"cwd-expected",
|
||||
"当前工作目录匹配预期仓库",
|
||||
normalizedCwd === normalizedExpectedCwd,
|
||||
`cwd=${normalizedCwd || "unknown"} expected=${normalizedExpectedCwd}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
checks.push(
|
||||
createCheck(
|
||||
"tmp-writable",
|
||||
"临时目录可写",
|
||||
tmpWritable === true,
|
||||
tmpWritable ? "tmp write ok" : "tmp write failed",
|
||||
),
|
||||
);
|
||||
|
||||
if (devBridge) {
|
||||
checks.push(
|
||||
createCheck(
|
||||
"devbridge-health",
|
||||
"DevBridge health 可访问",
|
||||
devBridge.ok === true,
|
||||
devBridge.ok
|
||||
? `status=${devBridge.status || "ok"}`
|
||||
: `url=${devBridge.url || "unknown"} error=${devBridge.error || "unknown"}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const failed = checks.filter((check) => !check.passed);
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
status: failed.length === 0 ? "pass" : "blocked",
|
||||
checks,
|
||||
failedChecks: failed.map((check) => check.id),
|
||||
summary:
|
||||
failed.length === 0
|
||||
? "qcloop worker preflight 通过。"
|
||||
: `qcloop worker preflight 阻断:${failed.map((check) => check.id).join(", ")}。`,
|
||||
};
|
||||
}
|
||||
|
||||
export { buildQCLoopPreflightReport, createCheck };
|
||||
@@ -0,0 +1,45 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { buildQCLoopPreflightReport } from "./agent-qc-qcloop-preflight-core.mjs";
|
||||
|
||||
describe("agent-qc-qcloop-preflight-core", () => {
|
||||
it("基础环境通过时应返回 pass", () => {
|
||||
const report = buildQCLoopPreflightReport({
|
||||
cwd: "/repo/lime",
|
||||
expectedCwd: "/repo/lime",
|
||||
tmpWritable: true,
|
||||
});
|
||||
|
||||
expect(report.status).toBe("pass");
|
||||
expect(report.failedChecks).toEqual([]);
|
||||
});
|
||||
|
||||
it("cwd 不匹配时应返回 blocked", () => {
|
||||
const report = buildQCLoopPreflightReport({
|
||||
cwd: "/repo/qcloop",
|
||||
expectedCwd: "/repo/lime",
|
||||
tmpWritable: true,
|
||||
});
|
||||
|
||||
expect(report.status).toBe("blocked");
|
||||
expect(report.failedChecks).toContain("cwd-expected");
|
||||
});
|
||||
|
||||
it("DevBridge 不可访问时应返回 blocked", () => {
|
||||
const report = buildQCLoopPreflightReport({
|
||||
cwd: "/repo/lime",
|
||||
tmpWritable: true,
|
||||
devBridge: {
|
||||
ok: false,
|
||||
url: "http://127.0.0.1:3030/health",
|
||||
error: "TypeError: fetch failed",
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.status).toBe("blocked");
|
||||
expect(report.failedChecks).toContain("devbridge-health");
|
||||
expect(report.checks.find((check) => check.id === "devbridge-health")?.detail).toContain(
|
||||
"fetch failed",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,289 @@
|
||||
import { mapQCLoopItemStatus, parseScenarioId } from "./agent-qc-evidence-core.mjs";
|
||||
|
||||
const TERMINAL_ITEM_STATUSES = new Set(["success", "failed", "exhausted"]);
|
||||
const TERMINAL_JOB_STATUSES = new Set(["completed", "failed", "canceled", "cancelled"]);
|
||||
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeStatus(value) {
|
||||
return isNonEmptyString(value) ? value.trim().toLowerCase() : "unknown";
|
||||
}
|
||||
|
||||
function parseTimestampMs(value) {
|
||||
if (!isNonEmptyString(value)) {
|
||||
return null;
|
||||
}
|
||||
const timestamp = Date.parse(value);
|
||||
return Number.isFinite(timestamp) ? timestamp : null;
|
||||
}
|
||||
|
||||
function stringLength(value) {
|
||||
return typeof value === "string" ? value.length : 0;
|
||||
}
|
||||
|
||||
function durationMinutes(startedAt, finishedAt, nowMs) {
|
||||
const startedMs = parseTimestampMs(startedAt);
|
||||
if (startedMs === null) {
|
||||
return null;
|
||||
}
|
||||
const finishedMs = parseTimestampMs(finishedAt) ?? nowMs;
|
||||
return Math.max(0, Math.round((finishedMs - startedMs) / 60000));
|
||||
}
|
||||
|
||||
function durationSeconds(startedAt, finishedAt, nowMs) {
|
||||
const startedMs = parseTimestampMs(startedAt);
|
||||
if (startedMs === null) {
|
||||
return null;
|
||||
}
|
||||
const finishedMs = parseTimestampMs(finishedAt) ?? nowMs;
|
||||
return Math.max(0, Math.round((finishedMs - startedMs) / 1000));
|
||||
}
|
||||
|
||||
function latestEntry(entries) {
|
||||
const normalizedEntries = asArray(entries).filter(Boolean);
|
||||
return normalizedEntries.length > 0 ? normalizedEntries[normalizedEntries.length - 1] : null;
|
||||
}
|
||||
|
||||
function summarizeLatestWorker(item, nowMs) {
|
||||
const latestAttempt = latestEntry(item?.attempts);
|
||||
if (!latestAttempt) {
|
||||
return {
|
||||
status: "unknown",
|
||||
exitCode: null,
|
||||
startedAt: null,
|
||||
finishedAt: null,
|
||||
durationMinutes: null,
|
||||
durationSeconds: null,
|
||||
stdoutLength: 0,
|
||||
stderrLength: 0,
|
||||
outputLength: 0,
|
||||
};
|
||||
}
|
||||
|
||||
const stdoutLength = stringLength(latestAttempt.stdout);
|
||||
const stderrLength = stringLength(latestAttempt.stderr);
|
||||
return {
|
||||
status: normalizeStatus(latestAttempt.status),
|
||||
exitCode: latestAttempt.exit_code ?? null,
|
||||
startedAt: latestAttempt.started_at ?? null,
|
||||
finishedAt: latestAttempt.finished_at ?? null,
|
||||
durationMinutes: durationMinutes(latestAttempt.started_at, latestAttempt.finished_at, nowMs),
|
||||
durationSeconds: durationSeconds(latestAttempt.started_at, latestAttempt.finished_at, nowMs),
|
||||
stdoutLength,
|
||||
stderrLength,
|
||||
outputLength: stdoutLength + stderrLength,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeLatestQc(item) {
|
||||
const latestQc = latestEntry(item?.qc_rounds);
|
||||
if (!latestQc) {
|
||||
return {
|
||||
status: "unknown",
|
||||
feedback: "",
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: normalizeStatus(latestQc.status),
|
||||
feedback: latestQc.feedback || "",
|
||||
};
|
||||
}
|
||||
|
||||
function classifyItemStaleness(item, worker, { staleMinutes = 30 } = {}) {
|
||||
const itemStatus = normalizeStatus(item?.status);
|
||||
const workerStatus = worker.status;
|
||||
const isRunning = itemStatus === "running" || workerStatus === "running";
|
||||
const isTerminal = TERMINAL_ITEM_STATUSES.has(itemStatus);
|
||||
const reasons = [];
|
||||
|
||||
if (!isRunning || isTerminal || staleMinutes <= 0) {
|
||||
return { stale: false, reasons };
|
||||
}
|
||||
|
||||
if (worker.startedAt === null) {
|
||||
reasons.push("running item 缺少 worker started_at,无法判断进度");
|
||||
}
|
||||
|
||||
if (worker.durationMinutes !== null && worker.durationMinutes >= staleMinutes && worker.outputLength === 0) {
|
||||
reasons.push(`worker 运行 ${worker.durationMinutes} 分钟且 stdout/stderr 为空`);
|
||||
}
|
||||
|
||||
return {
|
||||
stale: reasons.length > 0,
|
||||
reasons,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeItem(item, options = {}) {
|
||||
const nowMs = options.nowMs ?? Date.now();
|
||||
const worker = summarizeLatestWorker(item, nowMs);
|
||||
const qc = summarizeLatestQc(item);
|
||||
const itemStatus = normalizeStatus(item?.status);
|
||||
const terminal = TERMINAL_ITEM_STATUSES.has(itemStatus);
|
||||
const staleness = classifyItemStaleness(item, worker, options);
|
||||
|
||||
return {
|
||||
scenarioId: parseScenarioId(item?.item_value),
|
||||
itemId: item?.id || "",
|
||||
qcloopStatus: itemStatus,
|
||||
evidenceStatus: mapQCLoopItemStatus(item?.status, item),
|
||||
terminal,
|
||||
stale: staleness.stale,
|
||||
staleReasons: staleness.reasons,
|
||||
staleSeconds: staleness.stale ? worker.durationSeconds : null,
|
||||
currentAttemptNo: item?.current_attempt_no ?? 0,
|
||||
currentQcNo: item?.current_qc_no ?? 0,
|
||||
worker,
|
||||
qc,
|
||||
};
|
||||
}
|
||||
|
||||
function countByStatus(items) {
|
||||
const counts = {
|
||||
total: items.length,
|
||||
success: 0,
|
||||
failed: 0,
|
||||
exhausted: 0,
|
||||
running: 0,
|
||||
pending: 0,
|
||||
unknown: 0,
|
||||
terminal: 0,
|
||||
nonTerminal: 0,
|
||||
stale: 0,
|
||||
};
|
||||
|
||||
for (const item of items) {
|
||||
const status = item.qcloopStatus;
|
||||
if (status in counts) {
|
||||
counts[status] += 1;
|
||||
} else {
|
||||
counts.unknown += 1;
|
||||
}
|
||||
if (item.terminal) {
|
||||
counts.terminal += 1;
|
||||
} else {
|
||||
counts.nonTerminal += 1;
|
||||
}
|
||||
if (item.stale) {
|
||||
counts.stale += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
function buildVerdict(job, items, counts) {
|
||||
const jobStatus = normalizeStatus(job?.status);
|
||||
const terminalProblemItems = items.filter((item) => item.qcloopStatus === "failed" || item.qcloopStatus === "exhausted");
|
||||
const terminalBlockedItems = terminalProblemItems.filter((item) => item.evidenceStatus === "blocked");
|
||||
const terminalFailedItems = terminalProblemItems.filter((item) => item.evidenceStatus === "fail");
|
||||
if (counts.stale > 0) {
|
||||
return {
|
||||
status: "stale",
|
||||
summary: `qcloop job ${job?.id || "unknown"} 仍在运行,${counts.stale} 个 item 疑似无进度。`,
|
||||
nextAction: "不要中断进程;先导出 sidecar、记录卡点,等当前 worker 结束后再提交修正后的重跑 payload。",
|
||||
};
|
||||
}
|
||||
if (counts.running > 0 || counts.pending > 0 || !TERMINAL_JOB_STATUSES.has(jobStatus)) {
|
||||
return {
|
||||
status: "running",
|
||||
summary: `qcloop job ${job?.id || "unknown"} 尚未进入终态,running=${counts.running} pending=${counts.pending}。`,
|
||||
nextAction: "继续观察 job/items;如果需要证据,只导出 sidecar,不覆盖官方 Evidence Pack。",
|
||||
};
|
||||
}
|
||||
if (terminalFailedItems.length > 0 || (jobStatus === "failed" && terminalBlockedItems.length === 0)) {
|
||||
return {
|
||||
status: "fail",
|
||||
summary: `qcloop job ${job?.id || "unknown"} 已终止但仍有 failed/exhausted item。`,
|
||||
nextAction: "修复失败项或补足 verifier 可审查证据后,发起新的 qcloop 批次。",
|
||||
};
|
||||
}
|
||||
if (terminalBlockedItems.length > 0) {
|
||||
return {
|
||||
status: "blocked",
|
||||
summary: `qcloop job ${job?.id || "unknown"} 已终止但 ${terminalBlockedItems.length} 个 item 明确报告 worker 环境阻断。`,
|
||||
nextAction: "先修复 qcloop worker 环境或权限,再用新批次重跑被阻断场景;不要覆盖官方 Evidence Pack。",
|
||||
};
|
||||
}
|
||||
if (items.length > 0 && counts.success === items.length && jobStatus === "completed") {
|
||||
return {
|
||||
status: "complete",
|
||||
summary: `qcloop job ${job?.id || "unknown"} 已完成,全部 item success。`,
|
||||
nextAction: "可导出官方 Evidence Pack,并运行 release summary 与 completion audit。",
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "needs-human-review",
|
||||
summary: `qcloop job ${job?.id || "unknown"} 状态无法自动归类。`,
|
||||
nextAction: "人工审查 qcloop job/items 原始 JSON,再决定是否重跑或补 exporter 规则。",
|
||||
};
|
||||
}
|
||||
|
||||
function buildQCLoopStatusReport({ job, items, options = {} }) {
|
||||
const generatedAt = options.generatedAt || new Date().toISOString();
|
||||
const nowMs = parseTimestampMs(generatedAt) ?? Date.now();
|
||||
const parsedStaleMinutes = Number(options.staleMinutes ?? 30);
|
||||
const staleMinutes = Number.isFinite(parsedStaleMinutes) ? parsedStaleMinutes : 30;
|
||||
const itemSummaries = asArray(items).map((item) =>
|
||||
summarizeItem(item, {
|
||||
nowMs,
|
||||
staleMinutes,
|
||||
}),
|
||||
);
|
||||
const counts = countByStatus(itemSummaries);
|
||||
const verdict = buildVerdict(job, itemSummaries, counts);
|
||||
|
||||
return {
|
||||
schemaVersion: "v1",
|
||||
generatedAt,
|
||||
staleMinutes,
|
||||
job: {
|
||||
id: job?.id || "unknown",
|
||||
name: job?.name || "",
|
||||
status: normalizeStatus(job?.status),
|
||||
createdAt: job?.created_at || null,
|
||||
finishedAt: job?.finished_at || null,
|
||||
terminal: TERMINAL_JOB_STATUSES.has(normalizeStatus(job?.status)),
|
||||
},
|
||||
counts,
|
||||
items: itemSummaries,
|
||||
verdict,
|
||||
};
|
||||
}
|
||||
|
||||
function validateQCLoopStatusReport(report) {
|
||||
const issues = [];
|
||||
if (report?.schemaVersion !== "v1") {
|
||||
issues.push("schemaVersion 必须是 v1。");
|
||||
}
|
||||
if (!report?.job || !isNonEmptyString(report.job.id)) {
|
||||
issues.push("缺少 job.id。");
|
||||
}
|
||||
if (!Array.isArray(report?.items)) {
|
||||
issues.push("items 必须是数组。");
|
||||
}
|
||||
if (!report?.counts || typeof report.counts.total !== "number") {
|
||||
issues.push("缺少 counts.total。");
|
||||
}
|
||||
if (!report?.verdict || !isNonEmptyString(report.verdict.status)) {
|
||||
issues.push("缺少 verdict.status。");
|
||||
}
|
||||
return {
|
||||
valid: issues.length === 0,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
buildQCLoopStatusReport,
|
||||
classifyItemStaleness,
|
||||
normalizeStatus,
|
||||
summarizeItem,
|
||||
validateQCLoopStatusReport,
|
||||
};
|
||||
@@ -0,0 +1,168 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildQCLoopStatusReport,
|
||||
classifyItemStaleness,
|
||||
validateQCLoopStatusReport,
|
||||
} from "./agent-qc-qcloop-status-core.mjs";
|
||||
|
||||
const generatedAt = "2026-05-10T07:00:00.000Z";
|
||||
|
||||
describe("agent-qc-qcloop-status-core", () => {
|
||||
it("应把长时间无输出的 running item 标记为 stale", () => {
|
||||
const report = buildQCLoopStatusReport({
|
||||
job: { id: "job-1", name: "p0", status: "running" },
|
||||
items: [
|
||||
{
|
||||
id: "item-1",
|
||||
item_value: '{"scenario_id":"skill-forge-register-bind-enable"}',
|
||||
status: "running",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 0,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-1",
|
||||
status: "running",
|
||||
started_at: "2026-05-10T06:00:00.000Z",
|
||||
stdout: "",
|
||||
stderr: "",
|
||||
},
|
||||
],
|
||||
qc_rounds: [],
|
||||
},
|
||||
],
|
||||
options: { generatedAt, staleMinutes: 30 },
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("stale");
|
||||
expect(report.counts.stale).toBe(1);
|
||||
expect(report.items[0].staleSeconds).toBe(3600);
|
||||
expect(report.items[0].worker.durationSeconds).toBe(3600);
|
||||
expect(report.items[0].staleReasons.join("\n")).toContain("stdout/stderr 为空");
|
||||
expect(validateQCLoopStatusReport(report).valid).toBe(true);
|
||||
});
|
||||
|
||||
it("有输出的 running item 不应仅因运行时间长被标记为 stale", () => {
|
||||
const result = classifyItemStaleness(
|
||||
{ status: "running" },
|
||||
{
|
||||
status: "running",
|
||||
startedAt: "2026-05-10T06:00:00.000Z",
|
||||
durationMinutes: 60,
|
||||
outputLength: 120,
|
||||
},
|
||||
{ staleMinutes: 30 },
|
||||
);
|
||||
|
||||
expect(result.stale).toBe(false);
|
||||
});
|
||||
|
||||
it("全部 success 且 job completed 时应给出 complete verdict", () => {
|
||||
const report = buildQCLoopStatusReport({
|
||||
job: { id: "job-2", name: "p0", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-2",
|
||||
item_value: "command-bridge-contract",
|
||||
status: "success",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-2",
|
||||
status: "success",
|
||||
started_at: "2026-05-10T06:00:00.000Z",
|
||||
finished_at: "2026-05-10T06:01:00.000Z",
|
||||
stdout: "ok",
|
||||
stderr: "",
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ id: "qc-2", status: "pass", feedback: "ok" }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt, staleMinutes: 30 },
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("complete");
|
||||
expect(report.counts.success).toBe(1);
|
||||
});
|
||||
|
||||
it("failed/exhausted item 应保持 fail verdict", () => {
|
||||
const report = buildQCLoopStatusReport({
|
||||
job: { id: "job-3", name: "p0", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-3",
|
||||
item_value: "tool-approval-sandbox-boundary",
|
||||
status: "exhausted",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [{ id: "attempt-3", status: "failed", stdout: "", stderr: "error" }],
|
||||
qc_rounds: [{ id: "qc-3", status: "fail", feedback: "缺少证据" }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt, staleMinutes: 30 },
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("fail");
|
||||
expect(report.counts.exhausted).toBe(1);
|
||||
expect(report.items[0].qc.feedback).toBe("缺少证据");
|
||||
});
|
||||
|
||||
it("worker 明确报告 BLOCKED 的 exhausted item 应保持环境阻断语义", () => {
|
||||
const report = buildQCLoopStatusReport({
|
||||
job: { id: "job-4", name: "p0", status: "completed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-4",
|
||||
item_value: "claw-chat-ready-streaming",
|
||||
status: "exhausted",
|
||||
current_attempt_no: 3,
|
||||
current_qc_no: 3,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-4",
|
||||
status: "success",
|
||||
stdout: "QCLOOP_WORKER_RESULT=BLOCKED\nDevBridge preflight: BLOCKED",
|
||||
stderr: "",
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ id: "qc-4", status: "fail", feedback: "DevBridge preflight blocked" }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt, staleMinutes: 30 },
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("blocked");
|
||||
expect(report.counts.exhausted).toBe(1);
|
||||
expect(report.items[0].evidenceStatus).toBe("blocked");
|
||||
});
|
||||
|
||||
it("内层 Codex CLI 环境错误应保持环境阻断语义", () => {
|
||||
const report = buildQCLoopStatusReport({
|
||||
job: { id: "job-5", name: "p0", status: "failed" },
|
||||
items: [
|
||||
{
|
||||
id: "item-5",
|
||||
item_value: "command-bridge-contract",
|
||||
status: "failed",
|
||||
current_attempt_no: 1,
|
||||
current_qc_no: 1,
|
||||
attempts: [
|
||||
{
|
||||
id: "attempt-5",
|
||||
status: "failed",
|
||||
stderr:
|
||||
"QCLOOP_CODEX_BIN 不可用: /opt/homebrew/bin/codex: fork/exec /opt/homebrew/bin/codex: no such file or directory",
|
||||
},
|
||||
],
|
||||
qc_rounds: [{ id: "qc-5", status: "fail", feedback: "verifier 输出格式错误" }],
|
||||
},
|
||||
],
|
||||
options: { generatedAt, staleMinutes: 30 },
|
||||
});
|
||||
|
||||
expect(report.verdict.status).toBe("blocked");
|
||||
expect(report.items[0].evidenceStatus).toBe("blocked");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,194 @@
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function countByStatus(items, getStatus = (item) => item?.status) {
|
||||
const counts = new Map();
|
||||
for (const item of asArray(items)) {
|
||||
const status = getStatus(item) || "unknown";
|
||||
counts.set(status, (counts.get(status) ?? 0) + 1);
|
||||
}
|
||||
return Object.fromEntries(Array.from(counts.entries()).sort(([left], [right]) => left.localeCompare(right)));
|
||||
}
|
||||
|
||||
function summarizeEvidencePack(pack, sourcePath = "") {
|
||||
const scenarioResults = asArray(pack?.scenarioResults);
|
||||
const laneResults = asArray(pack?.laneResults);
|
||||
return {
|
||||
sourcePath,
|
||||
runId: pack?.runId || "unknown",
|
||||
generatedAt: pack?.generatedAt || "",
|
||||
status: pack?.verdict?.status || "unknown",
|
||||
summary: pack?.verdict?.summary || "",
|
||||
scenarioCount: scenarioResults.length,
|
||||
scenarioIds: scenarioResults
|
||||
.map((scenario) => String(scenario?.scenarioId || "").trim())
|
||||
.filter(Boolean)
|
||||
.sort(),
|
||||
scenarioStatusCounts: countByStatus(scenarioResults),
|
||||
laneStatusCounts: countByStatus(laneResults),
|
||||
blockers: asArray(pack?.verdict?.blockers).filter(isNonEmptyString),
|
||||
waivers: asArray(pack?.verdict?.waivers),
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeScenarioIds(scenarioIds) {
|
||||
return Array.from(
|
||||
new Set(asArray(scenarioIds).map((scenarioId) => String(scenarioId || "").trim()).filter(Boolean)),
|
||||
).sort();
|
||||
}
|
||||
|
||||
function hasStructuredEvidenceRef(scenario) {
|
||||
return asArray(scenario?.evidenceRefs).some((ref) => {
|
||||
const normalized = String(ref || "").trim();
|
||||
return normalized.length > 0 && !normalized.startsWith("qcloop:");
|
||||
});
|
||||
}
|
||||
|
||||
function summarizeHarnessReport(summaryReport, trendReport) {
|
||||
return {
|
||||
summaryGeneratedAt: summaryReport?.generatedAt || "",
|
||||
trendGeneratedAt: trendReport?.generatedAt || "",
|
||||
readyCount: summaryReport?.totals?.readyCount ?? null,
|
||||
invalidCount: summaryReport?.totals?.invalidCount ?? null,
|
||||
trendSampleCount: trendReport?.sampleCount ?? null,
|
||||
signals: asArray(trendReport?.signals).filter(isNonEmptyString),
|
||||
};
|
||||
}
|
||||
|
||||
function buildAgentQcReleaseSummary({
|
||||
evidencePacks = [],
|
||||
harnessSummary = null,
|
||||
harnessTrend = null,
|
||||
requiredScenarioIds = [],
|
||||
tag = "",
|
||||
} = {}) {
|
||||
const evidence = asArray(evidencePacks).map((entry) => summarizeEvidencePack(entry.pack, entry.sourcePath));
|
||||
const statusCounts = countByStatus(evidence);
|
||||
const blockers = evidence.flatMap((entry) => entry.blockers.map((blocker) => `${entry.runId}: ${blocker}`));
|
||||
const waiverCount = evidence.reduce((sum, entry) => sum + entry.waivers.length, 0);
|
||||
const hasFailingEvidence = evidence.some((entry) => entry.status !== "pass");
|
||||
const weakEvidenceScenarioIds = normalizeScenarioIds(
|
||||
asArray(evidencePacks).flatMap((entry) =>
|
||||
asArray(entry?.pack?.scenarioResults)
|
||||
.filter((scenario) => scenario?.status === "pass" && !hasStructuredEvidenceRef(scenario))
|
||||
.map((scenario) => scenario?.scenarioId),
|
||||
),
|
||||
);
|
||||
const structuredEvidenceBlockers = weakEvidenceScenarioIds.map(
|
||||
(scenarioId) =>
|
||||
`structured-evidence:${scenarioId}: pass scenario 缺少非 qcloop evidenceRefs,不能只凭 qcloop item id 作为发布证据。`,
|
||||
);
|
||||
const hasWeakStructuredEvidence = weakEvidenceScenarioIds.length > 0;
|
||||
const status =
|
||||
evidence.length === 0 ? "blocked" : hasFailingEvidence || hasWeakStructuredEvidence ? "fail" : "pass";
|
||||
const coveredScenarioIds = normalizeScenarioIds(evidence.flatMap((entry) => entry.scenarioIds));
|
||||
const normalizedRequiredScenarioIds = normalizeScenarioIds(requiredScenarioIds);
|
||||
const coveredScenarioIdSet = new Set(coveredScenarioIds);
|
||||
const missingRequiredScenarioIds = normalizedRequiredScenarioIds.filter(
|
||||
(scenarioId) => !coveredScenarioIdSet.has(scenarioId),
|
||||
);
|
||||
|
||||
return {
|
||||
tag,
|
||||
status,
|
||||
evidenceCount: evidence.length,
|
||||
statusCounts,
|
||||
waiverCount,
|
||||
blockers: [...blockers, ...structuredEvidenceBlockers],
|
||||
coveredScenarioIds,
|
||||
evidence,
|
||||
harness: summarizeHarnessReport(harnessSummary, harnessTrend),
|
||||
missingRequiredScenarioIds,
|
||||
requiredScenarioIds: normalizedRequiredScenarioIds,
|
||||
weakEvidenceScenarioIds,
|
||||
};
|
||||
}
|
||||
|
||||
function renderStatusCounts(counts) {
|
||||
const entries = Object.entries(counts || {});
|
||||
if (entries.length === 0) {
|
||||
return "无";
|
||||
}
|
||||
return entries.map(([status, count]) => `${status}: ${count}`).join(" / ");
|
||||
}
|
||||
|
||||
function renderAgentQcReleaseMarkdown(summary) {
|
||||
const evidenceLines = summary.evidence.length
|
||||
? summary.evidence
|
||||
.map(
|
||||
(entry) =>
|
||||
`- ${entry.runId}: ${entry.status};场景 ${entry.scenarioCount};${renderStatusCounts(entry.scenarioStatusCounts)}${entry.sourcePath ? `;source ${entry.sourcePath}` : ""}`,
|
||||
)
|
||||
.join("\n")
|
||||
: "- 无 Evidence Pack;发布应视为 blocked,除非有明确 waiver。";
|
||||
|
||||
const blockerLines = summary.blockers.length
|
||||
? summary.blockers.map((blocker) => `- ${blocker}`).join("\n")
|
||||
: "- 无";
|
||||
|
||||
const harnessLines = [
|
||||
summary.harness.readyCount !== null ? `- Harness ready: ${summary.harness.readyCount}` : "- Harness ready: 未提供",
|
||||
summary.harness.invalidCount !== null ? `- Harness invalid: ${summary.harness.invalidCount}` : "- Harness invalid: 未提供",
|
||||
summary.harness.trendSampleCount !== null ? `- Harness trend samples: ${summary.harness.trendSampleCount}` : "- Harness trend samples: 未提供",
|
||||
...summary.harness.signals.map((signal) => `- ${signal}`),
|
||||
].join("\n");
|
||||
|
||||
return `## Agent QC Evidence${summary.tag ? ` (${summary.tag})` : ""}
|
||||
|
||||
- Verdict: ${summary.status}
|
||||
- Evidence packs: ${summary.evidenceCount}
|
||||
- Evidence status: ${renderStatusCounts(summary.statusCounts)}
|
||||
- Scenario coverage: ${summary.requiredScenarioIds?.length ? `${summary.coveredScenarioIds.length}/${summary.requiredScenarioIds.length}` : `${summary.coveredScenarioIds?.length || 0} covered`}
|
||||
- Waivers: ${summary.waiverCount}
|
||||
|
||||
### Evidence Packs
|
||||
|
||||
${evidenceLines}
|
||||
|
||||
### Harness Trend
|
||||
|
||||
${harnessLines}
|
||||
|
||||
### Blockers
|
||||
|
||||
${blockerLines}
|
||||
`;
|
||||
}
|
||||
|
||||
function validateReleaseSummary(summary, { requireEvidence = true } = {}) {
|
||||
const issues = [];
|
||||
if (requireEvidence && summary.evidenceCount === 0) {
|
||||
issues.push("缺少 Agent QC Evidence Pack。请先导出 qcloop evidence 或显式记录 waiver。");
|
||||
}
|
||||
if (summary.status !== "pass") {
|
||||
issues.push(`Agent QC release summary 状态为 ${summary.status},不能作为绿色发布证据。`);
|
||||
}
|
||||
if (asArray(summary.missingRequiredScenarioIds).length > 0) {
|
||||
issues.push(`Agent QC Evidence Pack 未覆盖必需场景:${summary.missingRequiredScenarioIds.join(", ")}。`);
|
||||
}
|
||||
if (asArray(summary.weakEvidenceScenarioIds).length > 0) {
|
||||
issues.push(
|
||||
`Agent QC Evidence Pack 存在缺少结构化 evidenceRefs 的 pass 场景:${summary.weakEvidenceScenarioIds.join(", ")}。`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
valid: issues.length === 0,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
export {
|
||||
buildAgentQcReleaseSummary,
|
||||
countByStatus,
|
||||
hasStructuredEvidenceRef,
|
||||
normalizeScenarioIds,
|
||||
renderAgentQcReleaseMarkdown,
|
||||
summarizeEvidencePack,
|
||||
summarizeHarnessReport,
|
||||
validateReleaseSummary,
|
||||
};
|
||||
@@ -0,0 +1,124 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildAgentQcReleaseSummary,
|
||||
renderAgentQcReleaseMarkdown,
|
||||
validateReleaseSummary,
|
||||
} from "./agent-qc-release-summary-core.mjs";
|
||||
|
||||
const passPack = {
|
||||
runId: "run-pass",
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
verdict: { status: "pass", summary: "ok", blockers: [] },
|
||||
laneResults: [{ laneId: "L4-behavior-eval", status: "pass" }],
|
||||
scenarioResults: [
|
||||
{
|
||||
scenarioId: "command-bridge-contract",
|
||||
status: "pass",
|
||||
evidenceRefs: ["qcloop:item:item-1", ".lime/qc/contract-summary.json"],
|
||||
},
|
||||
{
|
||||
scenarioId: "workspace-ready-session-restore",
|
||||
status: "pass",
|
||||
evidenceRefs: ["qcloop:item:item-2", ".lime/qc/gui-trace/workspace-ready.trace.zip"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const failPack = {
|
||||
runId: "run-fail",
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
verdict: { status: "fail", summary: "bad", blockers: ["workspace: smoke failed"] },
|
||||
laneResults: [{ laneId: "L4-behavior-eval", status: "fail" }],
|
||||
scenarioResults: [{ scenarioId: "workspace-ready-session-restore", status: "fail", evidenceRefs: [] }],
|
||||
};
|
||||
|
||||
describe("agent-qc-release-summary-core", () => {
|
||||
it("应把全 pass Evidence Pack 汇总为 pass release summary", () => {
|
||||
const summary = buildAgentQcReleaseSummary({
|
||||
evidencePacks: [{ pack: passPack, sourcePath: "evidence.json" }],
|
||||
harnessSummary: { generatedAt: "now", totals: { readyCount: 2, invalidCount: 0 } },
|
||||
harnessTrend: { sampleCount: 3, signals: ["current gap 保持为 0。"] },
|
||||
requiredScenarioIds: ["command-bridge-contract", "workspace-ready-session-restore"],
|
||||
tag: "v1.2.3",
|
||||
});
|
||||
|
||||
expect(summary.status).toBe("pass");
|
||||
expect(summary.evidenceCount).toBe(1);
|
||||
expect(summary.missingRequiredScenarioIds).toEqual([]);
|
||||
expect(summary.harness.readyCount).toBe(2);
|
||||
expect(validateReleaseSummary(summary).valid).toBe(true);
|
||||
});
|
||||
|
||||
it("应把失败 Evidence Pack 汇总为 fail 并阻断 check", () => {
|
||||
const summary = buildAgentQcReleaseSummary({
|
||||
evidencePacks: [{ pack: failPack, sourcePath: "evidence.json" }],
|
||||
});
|
||||
|
||||
expect(summary.status).toBe("fail");
|
||||
expect(summary.blockers.join("\n")).toContain("workspace: smoke failed");
|
||||
expect(validateReleaseSummary(summary).valid).toBe(false);
|
||||
});
|
||||
|
||||
it("缺少 Evidence Pack 时默认 blocked", () => {
|
||||
const summary = buildAgentQcReleaseSummary({ evidencePacks: [] });
|
||||
|
||||
expect(summary.status).toBe("blocked");
|
||||
expect(validateReleaseSummary(summary).valid).toBe(false);
|
||||
});
|
||||
|
||||
it("Evidence Pack pass 但缺必需 P0 场景时应阻断 release", () => {
|
||||
const summary = buildAgentQcReleaseSummary({
|
||||
evidencePacks: [{ pack: passPack, sourcePath: "evidence.json" }],
|
||||
requiredScenarioIds: [
|
||||
"command-bridge-contract",
|
||||
"workspace-ready-session-restore",
|
||||
"release-package-startup-smoke",
|
||||
],
|
||||
});
|
||||
const validation = validateReleaseSummary(summary);
|
||||
|
||||
expect(summary.status).toBe("pass");
|
||||
expect(summary.missingRequiredScenarioIds).toEqual(["release-package-startup-smoke"]);
|
||||
expect(validation.valid).toBe(false);
|
||||
expect(validation.issues.join("\n")).toContain("release-package-startup-smoke");
|
||||
});
|
||||
|
||||
it("Evidence Pack pass 但 pass 场景只有 qcloop id 时应阻断 release", () => {
|
||||
const weakPack = {
|
||||
...passPack,
|
||||
scenarioResults: [
|
||||
{
|
||||
scenarioId: "command-bridge-contract",
|
||||
status: "pass",
|
||||
evidenceRefs: ["qcloop:item:item-1", "qcloop:attempt:attempt-1"],
|
||||
},
|
||||
],
|
||||
};
|
||||
const summary = buildAgentQcReleaseSummary({
|
||||
evidencePacks: [{ pack: weakPack, sourcePath: "evidence.json" }],
|
||||
requiredScenarioIds: ["command-bridge-contract"],
|
||||
});
|
||||
const validation = validateReleaseSummary(summary);
|
||||
|
||||
expect(summary.status).toBe("fail");
|
||||
expect(summary.weakEvidenceScenarioIds).toEqual(["command-bridge-contract"]);
|
||||
expect(summary.blockers.join("\n")).toContain("structured-evidence:command-bridge-contract");
|
||||
expect(validation.valid).toBe(false);
|
||||
expect(validation.issues.join("\n")).toContain("结构化 evidenceRefs");
|
||||
});
|
||||
|
||||
it("应渲染 release note 可用的 Markdown", () => {
|
||||
const summary = buildAgentQcReleaseSummary({
|
||||
evidencePacks: [{ pack: passPack, sourcePath: "evidence.json" }],
|
||||
requiredScenarioIds: ["command-bridge-contract", "workspace-ready-session-restore"],
|
||||
tag: "v1.2.3",
|
||||
});
|
||||
const markdown = renderAgentQcReleaseMarkdown(summary);
|
||||
|
||||
expect(markdown).toContain("Agent QC Evidence (v1.2.3)");
|
||||
expect(markdown).toContain("Verdict: pass");
|
||||
expect(markdown).toContain("Scenario coverage: 2/2");
|
||||
expect(markdown).toContain("run-pass");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,505 @@
|
||||
import fs from "node:fs";
|
||||
|
||||
const VALID_LANE_IDS = new Set([
|
||||
"L0-static-unit",
|
||||
"L1-contract-bridge",
|
||||
"L2-agent-runtime",
|
||||
"L3-product-surface",
|
||||
"L4-behavior-eval",
|
||||
"L5-release-ops",
|
||||
]);
|
||||
const VALID_RISKS = new Set(["P0", "P1", "P2"]);
|
||||
const VALID_EXECUTORS = new Set([
|
||||
"npm_command",
|
||||
"rust_test",
|
||||
"playwright_mcp",
|
||||
"runtime_harness",
|
||||
"qcloop",
|
||||
"release_workflow",
|
||||
"mixed",
|
||||
]);
|
||||
const VALID_STATUSES = new Set([
|
||||
"pass",
|
||||
"fail",
|
||||
"blocked",
|
||||
"needs-human-review",
|
||||
"waived",
|
||||
"skipped",
|
||||
]);
|
||||
const VALID_EVIDENCE_LAYERS = new Set([
|
||||
"deterministic-smoke",
|
||||
"gui-trace",
|
||||
"runtime-transcript",
|
||||
"release-artifact",
|
||||
]);
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function isNonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function addIssue(issues, severity, path, message) {
|
||||
issues.push({ severity, path, message });
|
||||
}
|
||||
|
||||
function collectNpmScriptName(command) {
|
||||
if (!isNonEmptyString(command)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const match = command.trim().match(/^npm\s+run\s+([^\s]+)/);
|
||||
return match ? match[1] : "";
|
||||
}
|
||||
|
||||
function validateNpmCommands(commands, packageScripts, pathPrefix, issues) {
|
||||
for (const [index, command] of asArray(commands).entries()) {
|
||||
if (!isNonEmptyString(command)) {
|
||||
addIssue(issues, "error", `${pathPrefix}.commands[${index}]`, "命令必须是非空字符串。");
|
||||
continue;
|
||||
}
|
||||
|
||||
const scriptName = collectNpmScriptName(command);
|
||||
if (scriptName && !Object.hasOwn(packageScripts, scriptName)) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
`${pathPrefix}.commands[${index}]`,
|
||||
`package.json 中不存在 npm script: ${scriptName}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function validateQCLoopConfig(qcloop, issues) {
|
||||
if (!qcloop || typeof qcloop !== "object") {
|
||||
return;
|
||||
}
|
||||
|
||||
const workerTemplate = qcloop.workerPromptTemplate;
|
||||
if (!isNonEmptyString(workerTemplate)) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.workerPromptTemplate",
|
||||
"qcloop workerPromptTemplate 必须存在,且要约束 worker 输出可审查证据。",
|
||||
);
|
||||
} else {
|
||||
if (!workerTemplate.includes("evidence_required")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.workerPromptTemplate",
|
||||
"workerPromptTemplate 必须要求 worker 逐项列出 evidence_required,否则 repair 轮会再次缺证据。",
|
||||
);
|
||||
}
|
||||
|
||||
if (!workerTemplate.includes("failure_modes")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.workerPromptTemplate",
|
||||
"workerPromptTemplate 必须要求 worker 逐项说明 failure_modes 是否命中、覆盖或排除。",
|
||||
);
|
||||
}
|
||||
|
||||
if (!workerTemplate.includes("QCLOOP_WORKER_RESULT=BLOCKED")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.workerPromptTemplate",
|
||||
"workerPromptTemplate 必须显式约束环境阻断时输出 QCLOOP_WORKER_RESULT=BLOCKED。",
|
||||
);
|
||||
}
|
||||
|
||||
if (!workerTemplate.includes("QCLOOP_EVIDENCE_SUMMARY_JSON")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.workerPromptTemplate",
|
||||
"workerPromptTemplate 必须要求 worker 输出 QCLOOP_EVIDENCE_SUMMARY_JSON,避免 verifier 只能审查散文摘要。",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const template = qcloop.verifierPromptTemplate;
|
||||
if (!isNonEmptyString(template)) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
"qcloop verifierPromptTemplate 必须存在,且要带 worker evidence 占位符。",
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!template.includes("{{stdout}}") && !template.includes("{{output}}")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
"verifierPromptTemplate 必须包含 {{stdout}} 或 {{output}},否则 verifier 看不到 worker 输出。",
|
||||
);
|
||||
}
|
||||
|
||||
for (const placeholder of ["{{attempt_status}}", "{{exit_code}}"]) {
|
||||
if (!template.includes(placeholder)) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
`verifierPromptTemplate 必须包含 ${placeholder},用于判定 worker 执行状态。`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!template.includes("{{qc_history}}") && !template.includes("{{issue_ledger}}")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
"verifierPromptTemplate 必须包含 {{qc_history}} 或 {{issue_ledger}},用于判断 repair 后旧问题是否仍开放。",
|
||||
);
|
||||
}
|
||||
|
||||
if (!template.includes("QCLOOP_EVIDENCE_SUMMARY_JSON")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
"verifierPromptTemplate 必须要求审查 QCLOOP_EVIDENCE_SUMMARY_JSON,避免命令退出码被误判为证据完整。",
|
||||
);
|
||||
}
|
||||
|
||||
if (!template.includes('{"pass": true|false')) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
'verifierPromptTemplate 必须声明只输出 {"pass": true|false, "feedback": "..."} JSON,避免 qcloop 报“verifier 输出格式错误”。',
|
||||
);
|
||||
}
|
||||
|
||||
if (template.includes("{{stderr}}")) {
|
||||
addIssue(
|
||||
issues,
|
||||
"warn",
|
||||
"qcloop.verifierPromptTemplate",
|
||||
"verifierPromptTemplate 包含 {{stderr}},Codex / GUI 场景可能产生超长 stderr;优先让 worker 在 stdout 输出摘要。",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function validateEvidenceSchema(schema, issues) {
|
||||
if (!schema || typeof schema !== "object") {
|
||||
addIssue(issues, "error", "evidenceSchema", "Evidence schema 必须是 JSON object。");
|
||||
return;
|
||||
}
|
||||
|
||||
for (const field of ["$schema", "$id", "type", "required", "properties"]) {
|
||||
if (!Object.hasOwn(schema, field)) {
|
||||
addIssue(issues, "error", `evidenceSchema.${field}`, `缺少 ${field}。`);
|
||||
}
|
||||
}
|
||||
|
||||
const required = asArray(schema.required);
|
||||
for (const field of ["schemaVersion", "runId", "generatedAt", "subject", "laneResults", "scenarioResults", "verdict"]) {
|
||||
if (!required.includes(field)) {
|
||||
addIssue(issues, "error", "evidenceSchema.required", `缺少必填字段 ${field}。`);
|
||||
}
|
||||
}
|
||||
|
||||
const verdictStatus = schema?.properties?.verdict?.properties?.status?.enum;
|
||||
if (Array.isArray(verdictStatus)) {
|
||||
for (const status of ["pass", "fail", "blocked", "needs-human-review", "waived"]) {
|
||||
if (!verdictStatus.includes(status)) {
|
||||
addIssue(issues, "error", "evidenceSchema.verdict.status", `verdict.status 缺少 ${status}。`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateLane(lane, index, packageScripts, issues) {
|
||||
const pathPrefix = `lanes[${index}]`;
|
||||
if (!isNonEmptyString(lane?.id)) {
|
||||
addIssue(issues, "error", `${pathPrefix}.id`, "Lane 必须有 id。");
|
||||
return;
|
||||
}
|
||||
|
||||
if (!VALID_LANE_IDS.has(lane.id)) {
|
||||
addIssue(issues, "warn", `${pathPrefix}.id`, `Lane id ${lane.id} 不在当前标准集合中。`);
|
||||
}
|
||||
|
||||
for (const field of ["title", "objective", "gatePolicy"]) {
|
||||
if (!isNonEmptyString(lane[field])) {
|
||||
addIssue(issues, "error", `${pathPrefix}.${field}`, `Lane 缺少 ${field}。`);
|
||||
}
|
||||
}
|
||||
|
||||
validateNpmCommands(lane.defaultCommands, packageScripts, pathPrefix, issues);
|
||||
}
|
||||
|
||||
function validateScenario(scenario, index, laneIds, packageScripts, issues) {
|
||||
const pathPrefix = `scenarios[${index}]`;
|
||||
for (const field of ["id", "title", "risk", "executor", "goal", "verifier"]) {
|
||||
if (!isNonEmptyString(scenario?.[field])) {
|
||||
addIssue(issues, "error", `${pathPrefix}.${field}`, `Scenario 缺少 ${field}。`);
|
||||
}
|
||||
}
|
||||
|
||||
if (isNonEmptyString(scenario?.risk) && !VALID_RISKS.has(scenario.risk)) {
|
||||
addIssue(issues, "error", `${pathPrefix}.risk`, `未知风险等级 ${scenario.risk}。`);
|
||||
}
|
||||
|
||||
if (isNonEmptyString(scenario?.executor) && !VALID_EXECUTORS.has(scenario.executor)) {
|
||||
addIssue(issues, "error", `${pathPrefix}.executor`, `未知执行器 ${scenario.executor}。`);
|
||||
}
|
||||
|
||||
const lanes = asArray(scenario?.lanes);
|
||||
if (lanes.length === 0) {
|
||||
addIssue(issues, "error", `${pathPrefix}.lanes`, "Scenario 至少要归属一个 lane。");
|
||||
}
|
||||
for (const laneId of lanes) {
|
||||
if (!laneIds.has(laneId)) {
|
||||
addIssue(issues, "error", `${pathPrefix}.lanes`, `引用了不存在的 lane: ${laneId}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (asArray(scenario?.evidenceRequired).length === 0) {
|
||||
addIssue(issues, "error", `${pathPrefix}.evidenceRequired`, "Scenario 必须声明证据要求。");
|
||||
}
|
||||
|
||||
if (asArray(scenario?.failureModes).length === 0) {
|
||||
addIssue(issues, "error", `${pathPrefix}.failureModes`, "Scenario 必须声明失败模式。");
|
||||
}
|
||||
|
||||
const evidenceLayers = asArray(scenario?.evidenceLayers);
|
||||
if (scenario?.risk === "P0" && evidenceLayers.length === 0) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
`${pathPrefix}.evidenceLayers`,
|
||||
"P0 Scenario 必须声明 evidenceLayers,避免 deterministic smoke 被误读成 deep evidence。",
|
||||
);
|
||||
}
|
||||
for (const [layerIndex, layer] of evidenceLayers.entries()) {
|
||||
if (!VALID_EVIDENCE_LAYERS.has(layer)) {
|
||||
addIssue(
|
||||
issues,
|
||||
"error",
|
||||
`${pathPrefix}.evidenceLayers[${layerIndex}]`,
|
||||
`未知 evidence layer: ${layer}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
validateNpmCommands(scenario?.commands, packageScripts, pathPrefix, issues);
|
||||
}
|
||||
|
||||
function validateAgentQcManifest(manifest, { packageScripts = {}, evidenceSchema = null } = {}) {
|
||||
const issues = [];
|
||||
if (!manifest || typeof manifest !== "object") {
|
||||
return {
|
||||
valid: false,
|
||||
issues: [{ severity: "error", path: "manifest", message: "Manifest 必须是 JSON object。" }],
|
||||
};
|
||||
}
|
||||
|
||||
for (const field of ["manifestVersion", "title", "evidenceSchema", "lanes", "scenarios"]) {
|
||||
if (!Object.hasOwn(manifest, field)) {
|
||||
addIssue(issues, "error", field, `Manifest 缺少 ${field}。`);
|
||||
}
|
||||
}
|
||||
|
||||
if (manifest.manifestVersion !== "v1") {
|
||||
addIssue(issues, "error", "manifestVersion", "当前只接受 manifestVersion=v1。");
|
||||
}
|
||||
|
||||
validateQCLoopConfig(manifest.qcloop, issues);
|
||||
|
||||
const lanes = asArray(manifest.lanes);
|
||||
const scenarios = asArray(manifest.scenarios);
|
||||
const laneIds = new Set();
|
||||
const scenarioIds = new Set();
|
||||
|
||||
if (lanes.length === 0) {
|
||||
addIssue(issues, "error", "lanes", "至少需要一个测试 lane。");
|
||||
}
|
||||
|
||||
for (const [index, lane] of lanes.entries()) {
|
||||
validateLane(lane, index, packageScripts, issues);
|
||||
if (isNonEmptyString(lane?.id)) {
|
||||
if (laneIds.has(lane.id)) {
|
||||
addIssue(issues, "error", `lanes[${index}].id`, `重复 lane id: ${lane.id}`);
|
||||
}
|
||||
laneIds.add(lane.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (scenarios.length === 0) {
|
||||
addIssue(issues, "error", "scenarios", "至少需要一个 Agent QC 场景。");
|
||||
}
|
||||
|
||||
for (const [index, scenario] of scenarios.entries()) {
|
||||
validateScenario(scenario, index, laneIds, packageScripts, issues);
|
||||
if (isNonEmptyString(scenario?.id)) {
|
||||
if (scenarioIds.has(scenario.id)) {
|
||||
addIssue(issues, "error", `scenarios[${index}].id`, `重复 scenario id: ${scenario.id}`);
|
||||
}
|
||||
scenarioIds.add(scenario.id);
|
||||
}
|
||||
}
|
||||
|
||||
if (evidenceSchema) {
|
||||
validateEvidenceSchema(evidenceSchema, issues);
|
||||
}
|
||||
|
||||
const errors = issues.filter((issue) => issue.severity === "error");
|
||||
return {
|
||||
valid: errors.length === 0,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
function countBy(items, getKey) {
|
||||
const counts = new Map();
|
||||
for (const item of items) {
|
||||
const key = getKey(item);
|
||||
counts.set(key, (counts.get(key) ?? 0) + 1);
|
||||
}
|
||||
return Array.from(counts.entries())
|
||||
.map(([name, count]) => ({ name, count }))
|
||||
.sort((left, right) => left.name.localeCompare(right.name));
|
||||
}
|
||||
|
||||
function summarizeAgentQcManifest(manifest, validation) {
|
||||
const lanes = asArray(manifest?.lanes);
|
||||
const scenarios = asArray(manifest?.scenarios);
|
||||
return {
|
||||
title: manifest?.title ?? "Lime Agent QC",
|
||||
manifestVersion: manifest?.manifestVersion ?? "unknown",
|
||||
evidenceSchema: manifest?.evidenceSchema ?? "",
|
||||
valid: validation.valid,
|
||||
issueCount: validation.issues.length,
|
||||
errorCount: validation.issues.filter((issue) => issue.severity === "error").length,
|
||||
warnCount: validation.issues.filter((issue) => issue.severity === "warn").length,
|
||||
laneCount: lanes.length,
|
||||
scenarioCount: scenarios.length,
|
||||
p0ScenarioCount: scenarios.filter((scenario) => scenario.risk === "P0").length,
|
||||
lanes: lanes.map((lane) => ({
|
||||
id: lane.id,
|
||||
title: lane.title,
|
||||
defaultCommandCount: asArray(lane.defaultCommands).length,
|
||||
scenarioCount: scenarios.filter((scenario) => asArray(scenario.lanes).includes(lane.id)).length,
|
||||
})),
|
||||
scenarios: scenarios.map((scenario) => ({
|
||||
id: scenario.id,
|
||||
title: scenario.title,
|
||||
risk: scenario.risk,
|
||||
executor: scenario.executor,
|
||||
lanes: asArray(scenario.lanes),
|
||||
})),
|
||||
riskBreakdown: countBy(scenarios, (scenario) => scenario.risk || "unknown"),
|
||||
executorBreakdown: countBy(scenarios, (scenario) => scenario.executor || "unknown"),
|
||||
issues: validation.issues,
|
||||
};
|
||||
}
|
||||
|
||||
function renderIssueList(issues) {
|
||||
if (issues.length === 0) {
|
||||
return "- 无";
|
||||
}
|
||||
|
||||
return issues
|
||||
.map((issue) => `- ${issue.severity.toUpperCase()} ${issue.path}: ${issue.message}`)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function renderNameCountList(items) {
|
||||
if (items.length === 0) {
|
||||
return "- 无";
|
||||
}
|
||||
|
||||
return items.map((item) => `- ${item.name}: ${item.count}`).join("\n");
|
||||
}
|
||||
|
||||
function renderAgentQcMarkdownReport(summary) {
|
||||
const laneLines = summary.lanes.length
|
||||
? summary.lanes
|
||||
.map(
|
||||
(lane) =>
|
||||
`- ${lane.id}: ${lane.title};场景 ${lane.scenarioCount};默认命令 ${lane.defaultCommandCount}`,
|
||||
)
|
||||
.join("\n")
|
||||
: "- 无";
|
||||
const scenarioLines = summary.scenarios.length
|
||||
? summary.scenarios
|
||||
.map(
|
||||
(scenario) =>
|
||||
`- ${scenario.id}: ${scenario.title};${scenario.risk};${scenario.executor}`,
|
||||
)
|
||||
.join("\n")
|
||||
: "- 无";
|
||||
|
||||
return `# ${summary.title} 场景报告
|
||||
|
||||
## 结论
|
||||
|
||||
- Manifest: ${summary.manifestVersion}
|
||||
- Evidence schema: ${summary.evidenceSchema || "未声明"}
|
||||
- 状态: ${summary.valid ? "PASS" : "FAIL"}
|
||||
- Lane 数: ${summary.laneCount}
|
||||
- Scenario 数: ${summary.scenarioCount}
|
||||
- P0 Scenario 数: ${summary.p0ScenarioCount}
|
||||
- Issue: ${summary.issueCount}(error ${summary.errorCount} / warn ${summary.warnCount})
|
||||
|
||||
## Lane 覆盖
|
||||
|
||||
${laneLines}
|
||||
|
||||
## Scenario 清单
|
||||
|
||||
${scenarioLines}
|
||||
|
||||
## 风险分布
|
||||
|
||||
${renderNameCountList(summary.riskBreakdown)}
|
||||
|
||||
## 执行器分布
|
||||
|
||||
${renderNameCountList(summary.executorBreakdown)}
|
||||
|
||||
## 校验问题
|
||||
|
||||
${renderIssueList(summary.issues)}
|
||||
`;
|
||||
}
|
||||
|
||||
function createAgentQcReport({ manifest, packageJson = {}, evidenceSchema = null }) {
|
||||
const validation = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts ?? {},
|
||||
evidenceSchema,
|
||||
});
|
||||
return summarizeAgentQcManifest(manifest, validation);
|
||||
}
|
||||
|
||||
export {
|
||||
VALID_EVIDENCE_LAYERS,
|
||||
VALID_EXECUTORS,
|
||||
VALID_LANE_IDS,
|
||||
VALID_RISKS,
|
||||
VALID_STATUSES,
|
||||
collectNpmScriptName,
|
||||
createAgentQcReport,
|
||||
readJsonFile,
|
||||
renderAgentQcMarkdownReport,
|
||||
summarizeAgentQcManifest,
|
||||
validateAgentQcManifest,
|
||||
};
|
||||
@@ -0,0 +1,226 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
collectNpmScriptName,
|
||||
createAgentQcReport,
|
||||
renderAgentQcMarkdownReport,
|
||||
validateAgentQcManifest,
|
||||
} from "./agent-qc-report-core.mjs";
|
||||
|
||||
const packageJson = {
|
||||
scripts: {
|
||||
"verify:local": "node scripts/local-ci.mjs",
|
||||
"test:contracts": "node scripts/check-command-contracts.mjs",
|
||||
"verify:gui-smoke": "node scripts/verify-gui-smoke.mjs",
|
||||
},
|
||||
};
|
||||
|
||||
const evidenceSchema = {
|
||||
$schema: "https://json-schema.org/draft/2020-12/schema",
|
||||
$id: "https://lime.local/agent-qc-evidence.schema.json",
|
||||
type: "object",
|
||||
required: [
|
||||
"schemaVersion",
|
||||
"runId",
|
||||
"generatedAt",
|
||||
"subject",
|
||||
"laneResults",
|
||||
"scenarioResults",
|
||||
"verdict",
|
||||
],
|
||||
properties: {
|
||||
verdict: {
|
||||
type: "object",
|
||||
properties: {
|
||||
status: {
|
||||
enum: ["pass", "fail", "blocked", "needs-human-review", "waived"],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
const validManifest = {
|
||||
manifestVersion: "v1",
|
||||
title: "Lime Agent QC",
|
||||
evidenceSchema: "docs/test/agent-qc-evidence.schema.json",
|
||||
qcloop: {
|
||||
workerPromptTemplate:
|
||||
"worker {{item}} evidence_required failure_modes QCLOOP_WORKER_RESULT=BLOCKED QCLOOP_EVIDENCE_SUMMARY_JSON",
|
||||
verifierPromptTemplate:
|
||||
'verifier {{item}} status={{attempt_status}} code={{exit_code}} stdout={{stdout}} ledger={{issue_ledger}} QCLOOP_EVIDENCE_SUMMARY_JSON,只输出 {"pass": true|false, "feedback": "..."} JSON',
|
||||
},
|
||||
lanes: [
|
||||
{
|
||||
id: "L0-static-unit",
|
||||
title: "快速卫生",
|
||||
objective: "验证低级错误不会进入主链。",
|
||||
gatePolicy: "所有 PR 默认执行。",
|
||||
defaultCommands: ["npm run verify:local"],
|
||||
},
|
||||
{
|
||||
id: "L1-contract-bridge",
|
||||
title: "契约桥接",
|
||||
objective: "验证命令和 Bridge 合同不漂移。",
|
||||
gatePolicy: "命令和 Bridge 改动必跑。",
|
||||
defaultCommands: ["npm run test:contracts"],
|
||||
},
|
||||
],
|
||||
scenarios: [
|
||||
{
|
||||
id: "command-bridge-contract",
|
||||
title: "命令桥接合同",
|
||||
risk: "P0",
|
||||
executor: "npm_command",
|
||||
lanes: ["L1-contract-bridge"],
|
||||
commands: ["npm run test:contracts"],
|
||||
goal: "前端调用、Rust 注册、治理目录册、mock 四侧一致。",
|
||||
verifier: "test:contracts 成功且没有新增 dead/compat 回流。",
|
||||
evidenceRequired: ["command log", "contract summary"],
|
||||
evidenceLayers: ["deterministic-smoke"],
|
||||
failureModes: ["missing rust handler", "mock fallback drift"],
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
describe("agent-qc-report-core", () => {
|
||||
it("能从 npm run 命令中提取 script 名称", () => {
|
||||
expect(collectNpmScriptName("npm run verify:gui-smoke -- --reuse-running")).toBe(
|
||||
"verify:gui-smoke",
|
||||
);
|
||||
expect(collectNpmScriptName("node scripts/foo.mjs")).toBe("");
|
||||
});
|
||||
|
||||
it("应接受完整的 Agent QC manifest 和 evidence schema", () => {
|
||||
const result = validateAgentQcManifest(validManifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.issues.filter((issue) => issue.severity === "error")).toEqual([]);
|
||||
});
|
||||
|
||||
it("应阻止 qcloop verifier 缺少 worker stdout 占位符", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
manifest.qcloop.verifierPromptTemplate =
|
||||
"verifier {{item}} status={{attempt_status}} code={{exit_code}}";
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
|
||||
"{{stdout}}",
|
||||
);
|
||||
});
|
||||
|
||||
it("应阻止 qcloop worker 缺少 evidence_required / failure_modes / BLOCKED / 结构化证据约束", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
manifest.qcloop.workerPromptTemplate = "worker {{item}}";
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
const messages = result.issues.map((issue) => issue.message).join("\n");
|
||||
expect(messages).toContain("evidence_required");
|
||||
expect(messages).toContain("failure_modes");
|
||||
expect(messages).toContain("QCLOOP_WORKER_RESULT=BLOCKED");
|
||||
expect(messages).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON");
|
||||
});
|
||||
|
||||
it("应阻止 qcloop verifier 缺少 attempt 状态占位符", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
manifest.qcloop.verifierPromptTemplate = "verifier {{item}} stdout={{stdout}}";
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
|
||||
"{{attempt_status}}",
|
||||
);
|
||||
});
|
||||
|
||||
it("应阻止 qcloop verifier 缺少 issue ledger 或结构化证据审查契约", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
manifest.qcloop.verifierPromptTemplate =
|
||||
"verifier {{item}} status={{attempt_status}} code={{exit_code}} stdout={{stdout}}";
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
const messages = result.issues.map((issue) => issue.message).join("\n");
|
||||
expect(messages).toContain("{{qc_history}}");
|
||||
expect(messages).toContain("QCLOOP_EVIDENCE_SUMMARY_JSON");
|
||||
expect(messages).toContain('{"pass": true|false');
|
||||
});
|
||||
|
||||
it("应阻止引用不存在的 npm script", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
manifest.scenarios[0].commands = ["npm run missing-script"];
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
|
||||
"missing-script",
|
||||
);
|
||||
});
|
||||
|
||||
it("应阻止 P0 场景缺少 evidenceLayers", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
delete manifest.scenarios[0].evidenceLayers;
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
|
||||
"evidenceLayers",
|
||||
);
|
||||
});
|
||||
|
||||
it("应阻止未知 evidence layer", () => {
|
||||
const manifest = structuredClone(validManifest);
|
||||
manifest.scenarios[0].evidenceLayers = ["deterministic-smoke", "unknown-layer"];
|
||||
|
||||
const result = validateAgentQcManifest(manifest, {
|
||||
packageScripts: packageJson.scripts,
|
||||
evidenceSchema,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.issues.map((issue) => issue.message).join("\n")).toContain(
|
||||
"unknown-layer",
|
||||
);
|
||||
});
|
||||
|
||||
it("应把报告渲染成可读 Markdown", () => {
|
||||
const report = createAgentQcReport({
|
||||
manifest: validManifest,
|
||||
packageJson,
|
||||
evidenceSchema,
|
||||
});
|
||||
const markdown = renderAgentQcMarkdownReport(report);
|
||||
|
||||
expect(markdown).toContain("Lime Agent QC 场景报告");
|
||||
expect(markdown).toContain("状态: PASS");
|
||||
expect(markdown).toContain("command-bridge-contract");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,313 @@
|
||||
function createTranscriptSteps() {
|
||||
return [
|
||||
{
|
||||
id: "tool-request",
|
||||
category: "tool request",
|
||||
eventTypes: ["tool.started", "tool.args"],
|
||||
summary:
|
||||
"允许工具请求会投影为 tool.started/tool.args,保留 tool id、tool name 与输入参数摘要。",
|
||||
details: {
|
||||
toolCallId: "tool-1",
|
||||
toolName: "read_file",
|
||||
phase: "acting",
|
||||
inputAvailable: true,
|
||||
inputSummary: '{"path":"README.md"}',
|
||||
},
|
||||
sourceRefs: [
|
||||
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:676",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "approval-decision",
|
||||
category: "approval decision",
|
||||
eventTypes: ["run.status", "permission.changed"],
|
||||
summary:
|
||||
"需要授权时会先进入 permission_review,并通过 permission.changed 暴露 decision、askProfileKeys 与 confirmation request。",
|
||||
details: {
|
||||
phase: "waiting",
|
||||
permissionStatus: "requires_confirmation",
|
||||
confirmationStatus: "not_requested",
|
||||
confirmationRequestId: "approval-1",
|
||||
decisionSource: "runtime",
|
||||
decisionScope: "turn",
|
||||
askProfileKeys: ["read_files"],
|
||||
requiredProfileKeys: ["read_files", "write_artifacts"],
|
||||
},
|
||||
sourceRefs: [
|
||||
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:85",
|
||||
"src/components/agent/chat/components/AgentThreadTimeline.test.tsx:904",
|
||||
"src/components/agent/chat/components/MessageList.test.tsx:1845",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "sandbox-policy",
|
||||
category: "sandbox policy",
|
||||
eventTypes: ["permission.changed"],
|
||||
summary:
|
||||
"turn_context 会透传 approvalPolicy 与 sandboxPolicy,确保 runtime policy 不会在 UI 证据链中丢失。",
|
||||
details: {
|
||||
phase: "preparing",
|
||||
approvalPolicy: "on-request",
|
||||
sandboxPolicy: "workspace-write",
|
||||
sourceEvent: "turn_context",
|
||||
},
|
||||
sourceRefs: [
|
||||
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:1480",
|
||||
"src/components/agent/chat/hooks/agentStreamSubmitExecution.test.ts:1",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "tool-result",
|
||||
category: "result",
|
||||
eventTypes: ["tool.result"],
|
||||
summary:
|
||||
"成功工具结果会归档为 tool.result,并保留 artifact refs,避免出现 tool result missing。",
|
||||
details: {
|
||||
toolCallId: "tool-1",
|
||||
success: true,
|
||||
phase: "completed",
|
||||
artifactPaths: [".lime/artifacts/demo.md"],
|
||||
artifactIds: ["artifact-1"],
|
||||
},
|
||||
sourceRefs: [
|
||||
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:676",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "tool-error",
|
||||
category: "error",
|
||||
eventTypes: ["tool.failed"],
|
||||
summary:
|
||||
"失败工具结果会归档为 tool.failed,并保留错误预览与 metadata,避免错误路径丢失。",
|
||||
details: {
|
||||
toolCallId: "tool-2",
|
||||
success: false,
|
||||
phase: "failed",
|
||||
errorPreview: "Permission denied",
|
||||
metadataKeys: ["sandboxed", "exit_code"],
|
||||
},
|
||||
sourceRefs: [
|
||||
"src/components/agent/chat/projection/agentUiEventProjection.test.ts:676",
|
||||
"src/components/agent/chat/hooks/useAsterAgentChat.test.tsx:7801",
|
||||
],
|
||||
},
|
||||
{
|
||||
id: "timeout-recovery",
|
||||
category: "recovery action",
|
||||
eventTypes: ["timeout.recover", "timeout.defer", "timeout.fail"],
|
||||
summary:
|
||||
"首包超时与 inactivity timeout 都有明确恢复动作矩阵和用户可见文案,拒绝或超时不会把用户卡死。",
|
||||
details: {
|
||||
firstEventActions: {
|
||||
recover: "recover",
|
||||
defer: "defer",
|
||||
fail: "fail",
|
||||
},
|
||||
inactivityActions: {
|
||||
recover: "recover",
|
||||
fail: "fail",
|
||||
},
|
||||
warnings: [
|
||||
"[AsterChat] 首个运行时事件静默,已降级切换为会话快照同步: event-a",
|
||||
"[AsterChat] 首个运行时事件暂未到达,已基于提交派发继续等待后续进度: event-a",
|
||||
"[AsterChat] 运行时事件静默,已降级切换为会话快照同步: event-a",
|
||||
],
|
||||
failureMessage:
|
||||
"执行已中断:运行时长时间没有返回新进度,请重试。",
|
||||
},
|
||||
sourceRefs: [
|
||||
"src/components/agent/chat/hooks/agentStreamInactivityController.test.ts:1",
|
||||
],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function createEvidenceChecks() {
|
||||
return [
|
||||
{
|
||||
id: "tool timeline",
|
||||
satisfied: true,
|
||||
summary:
|
||||
"由 tool-request、tool-result 与 tool-error 三段 transcript 组成完整 tool timeline。",
|
||||
transcriptStepIds: ["tool-request", "tool-result", "tool-error"],
|
||||
},
|
||||
{
|
||||
id: "approval decision",
|
||||
satisfied: true,
|
||||
summary:
|
||||
"approval-decision transcript 暴露 permission.changed waiting 状态、decision source/scope 与 confirmation request。",
|
||||
transcriptStepIds: ["approval-decision"],
|
||||
},
|
||||
{
|
||||
id: "sandbox policy",
|
||||
satisfied: true,
|
||||
summary:
|
||||
"sandbox-policy transcript 暴露 turn_context approvalPolicy/sandboxPolicy。",
|
||||
transcriptStepIds: ["sandbox-policy"],
|
||||
},
|
||||
{
|
||||
id: "error recovery transcript",
|
||||
satisfied: true,
|
||||
summary:
|
||||
"timeout-recovery transcript 暴露 recover/defer/fail 动作矩阵与用户可见 warning/message。",
|
||||
transcriptStepIds: ["timeout-recovery"],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function createFailureModeCoverage() {
|
||||
return [
|
||||
{
|
||||
id: "approval bypass",
|
||||
status: "covered",
|
||||
summary:
|
||||
"approval-decision 与 sandbox-policy transcript 证明 runtime 会先进入 waiting/policy gate,再继续工具执行。",
|
||||
transcriptStepIds: ["approval-decision", "sandbox-policy"],
|
||||
},
|
||||
{
|
||||
id: "tool result missing",
|
||||
status: "covered",
|
||||
summary:
|
||||
"tool-result 与 tool-error transcript 同时存在,成功/失败路径都不会丢失结果。",
|
||||
transcriptStepIds: ["tool-result", "tool-error"],
|
||||
},
|
||||
{
|
||||
id: "timeout without recovery",
|
||||
status: "covered",
|
||||
summary:
|
||||
"timeout-recovery transcript 暴露 recover/defer/fail 分支与用户可见 warning/message,不会让用户卡死。",
|
||||
transcriptStepIds: ["timeout-recovery"],
|
||||
},
|
||||
{
|
||||
id: "unsafe tool exposed",
|
||||
status: "covered",
|
||||
summary:
|
||||
"工具权限与来源会进入 Harness 工具库存区块,危险工具暴露由 companion tool-surface smoke 共同覆盖。",
|
||||
transcriptStepIds: ["approval-decision"],
|
||||
companionCommand: "npm run smoke:agent-runtime-tool-surface",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function liveRuntimeTranscriptSatisfiesReleaseGate(liveRuntimeTranscript) {
|
||||
const assertions = liveRuntimeTranscript?.assertions;
|
||||
return Boolean(
|
||||
assertions?.devBridgeHealthy &&
|
||||
assertions?.permissionRequestCreatedBeforeModel &&
|
||||
assertions?.deniedDecisionClearsPendingRequest &&
|
||||
assertions?.resolvedDecisionClearsPendingRequest &&
|
||||
assertions?.approvalPolicySubmitted &&
|
||||
assertions?.sandboxPolicySubmitted,
|
||||
);
|
||||
}
|
||||
|
||||
function buildApprovalSandboxSmokeEvidence({
|
||||
commandResults,
|
||||
generatedAt,
|
||||
liveRuntimeTranscript = null,
|
||||
}) {
|
||||
const transcriptSteps = createTranscriptSteps();
|
||||
const evidenceChecks = createEvidenceChecks();
|
||||
const failureModeCoverage = createFailureModeCoverage();
|
||||
const liveRuntimeTranscriptVerified =
|
||||
liveRuntimeTranscriptSatisfiesReleaseGate(liveRuntimeTranscript);
|
||||
|
||||
return {
|
||||
schemaVersion: "v2",
|
||||
scenarioId: "tool-approval-sandbox-boundary",
|
||||
status: "pass",
|
||||
generatedAt,
|
||||
evidenceKind: "runtime-approval-sandbox-verified-transcript-smoke",
|
||||
transcriptKind: liveRuntimeTranscriptVerified
|
||||
? "verified_projection_and_live_runtime_transcript"
|
||||
: "verified_projection_summary",
|
||||
limitation: liveRuntimeTranscriptVerified
|
||||
? "该 smoke 同时输出投影/组件事实源与 DevBridge live runtime permission confirmation transcript;tool result/error 和 timeout recovery 仍由定向回归覆盖。"
|
||||
: "该 smoke 输出的是由前端提交、投影、UI 与超时恢复事实源生成的结构化 transcript summary;正式发布如需 live session 级证据,仍应由 qcloop 或 replay 采集。",
|
||||
coverage: {
|
||||
submitPreferences: true,
|
||||
approvalDecisionProjection: true,
|
||||
sandboxPolicyProjection: true,
|
||||
toolRequestResultOrErrorProjection: true,
|
||||
permissionRecoveryUi: true,
|
||||
harnessPermissionInventory: true,
|
||||
runtimeTranscriptSummary: true,
|
||||
liveRuntimeTranscript: liveRuntimeTranscriptVerified,
|
||||
},
|
||||
liveRuntimeTranscript,
|
||||
transcriptSteps,
|
||||
evidenceChecks,
|
||||
failureModeCoverage,
|
||||
commandResults,
|
||||
followUpForOfficialEvidence: [
|
||||
liveRuntimeTranscriptVerified
|
||||
? "qcloop worker stdout 应保留 live.transcript.* 行,供 release verifier 审计。"
|
||||
: "如需 live session 级发布证据,将同样的 transcript 维度写入 qcloop worker stdout 或 replay artifact。",
|
||||
"保留 tool request / approval decision / sandbox policy / result-or-error / recovery action 五类 transcript 字段。",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function renderApprovalSandboxTranscriptLines(evidence) {
|
||||
const lines = [];
|
||||
|
||||
for (const step of evidence.transcriptSteps) {
|
||||
const detailParts = [];
|
||||
for (const [key, value] of Object.entries(step.details || {})) {
|
||||
if (Array.isArray(value)) {
|
||||
detailParts.push(`${key}=${value.join("|")}`);
|
||||
} else if (value && typeof value === "object") {
|
||||
detailParts.push(`${key}=${JSON.stringify(value)}`);
|
||||
} else {
|
||||
detailParts.push(`${key}=${String(value)}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] transcript.${step.id}: category=${step.category}; events=${step.eventTypes.join("/")}; ${step.summary}; ${detailParts.join("; ")}`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const check of evidence.evidenceChecks) {
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] evidence.${check.id}: ${
|
||||
check.satisfied ? "satisfied" : "missing"
|
||||
}; ${check.summary}; transcript=${check.transcriptStepIds.join(",")}`,
|
||||
);
|
||||
}
|
||||
|
||||
for (const mode of evidence.failureModeCoverage) {
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] failure.${mode.id}: ${mode.status}; ${mode.summary}; transcript=${mode.transcriptStepIds.join(",")}${
|
||||
mode.companionCommand ? `; companion=${mode.companionCommand}` : ""
|
||||
}`,
|
||||
);
|
||||
}
|
||||
|
||||
const live = evidence.liveRuntimeTranscript;
|
||||
if (live) {
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] live.transcript.kind=${live.kind}; devBridgeStatus=${live.health?.status || "unknown"}; workspaceId=${live.workspaceId || ""}; liveRuntimeTranscript=${evidence.coverage.liveRuntimeTranscript ? "true" : "false"}`,
|
||||
);
|
||||
for (const flow of live.flows || []) {
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] live.transcript.${flow.decision}.request: sessionId=${flow.sessionId}; turnId=${flow.turnId}; requestId=${flow.requestId}; permissionStatus=${flow.before?.permissionStatus}; confirmationStatus=${flow.before?.confirmationStatus}; pendingRequestCount=${flow.before?.pendingRequestCount}; latestTurnStatus=${flow.before?.latestTurnStatus}; approvalPolicy=${flow.submittedPolicies?.approvalPolicy}; sandboxPolicy=${flow.submittedPolicies?.sandboxPolicy}`,
|
||||
);
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] live.transcript.${flow.decision}.decision: confirmed=${flow.respond?.confirmed}; response=${flow.respond?.responseLabel}; afterConfirmationStatus=${flow.after?.confirmationStatus}; afterPendingRequestCount=${flow.after?.pendingRequestCount}; afterThreadStatus=${flow.after?.threadStatus}`,
|
||||
);
|
||||
}
|
||||
for (const [key, value] of Object.entries(live.assertions || {})) {
|
||||
lines.push(
|
||||
`[smoke:agent-runtime-approval-sandbox] live.assertion.${key}: ${value ? "satisfied" : "missing"}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return lines;
|
||||
}
|
||||
|
||||
export {
|
||||
buildApprovalSandboxSmokeEvidence,
|
||||
renderApprovalSandboxTranscriptLines,
|
||||
};
|
||||
@@ -0,0 +1,118 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildApprovalSandboxSmokeEvidence,
|
||||
renderApprovalSandboxTranscriptLines,
|
||||
} from "./agent-runtime-approval-sandbox-smoke-core.mjs";
|
||||
|
||||
describe("agent-runtime-approval-sandbox-smoke-core", () => {
|
||||
it("应生成覆盖 verifier 关键维度的 transcript 与 failure mode 摘要", () => {
|
||||
const evidence = buildApprovalSandboxSmokeEvidence({
|
||||
commandResults: [],
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
});
|
||||
|
||||
expect(evidence.transcriptKind).toBe("verified_projection_summary");
|
||||
expect(evidence.transcriptSteps.map((entry) => entry.category)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"tool request",
|
||||
"approval decision",
|
||||
"sandbox policy",
|
||||
"result",
|
||||
"error",
|
||||
"recovery action",
|
||||
]),
|
||||
);
|
||||
expect(evidence.evidenceChecks.every((entry) => entry.satisfied)).toBe(true);
|
||||
expect(evidence.failureModeCoverage.map((entry) => entry.id)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"approval bypass",
|
||||
"tool result missing",
|
||||
"timeout without recovery",
|
||||
"unsafe tool exposed",
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("live runtime transcript 满足门禁时应标记为 liveRuntimeTranscript", () => {
|
||||
const evidence = buildApprovalSandboxSmokeEvidence({
|
||||
commandResults: [],
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
liveRuntimeTranscript: {
|
||||
kind: "devbridge-runtime-permission-confirmation",
|
||||
health: { status: "ok" },
|
||||
workspaceId: "workspace-1",
|
||||
flows: [
|
||||
{
|
||||
decision: "denied",
|
||||
sessionId: "session-denied",
|
||||
turnId: "turn-denied",
|
||||
requestId: "runtime_permission_confirmation:turn-denied",
|
||||
submittedPolicies: {
|
||||
approvalPolicy: "on-request",
|
||||
sandboxPolicy: "workspace-write",
|
||||
},
|
||||
before: {
|
||||
permissionStatus: "requires_confirmation",
|
||||
confirmationStatus: "requested",
|
||||
pendingRequestCount: 1,
|
||||
latestTurnStatus: "failed",
|
||||
},
|
||||
respond: {
|
||||
confirmed: false,
|
||||
responseLabel: "拒绝",
|
||||
},
|
||||
after: {
|
||||
confirmationStatus: "denied",
|
||||
pendingRequestCount: 0,
|
||||
threadStatus: "failed",
|
||||
},
|
||||
},
|
||||
],
|
||||
assertions: {
|
||||
devBridgeHealthy: true,
|
||||
permissionRequestCreatedBeforeModel: true,
|
||||
deniedDecisionClearsPendingRequest: true,
|
||||
resolvedDecisionClearsPendingRequest: true,
|
||||
approvalPolicySubmitted: true,
|
||||
sandboxPolicySubmitted: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(evidence.transcriptKind).toBe(
|
||||
"verified_projection_and_live_runtime_transcript",
|
||||
);
|
||||
expect(evidence.coverage.liveRuntimeTranscript).toBe(true);
|
||||
|
||||
const lines = renderApprovalSandboxTranscriptLines(evidence).join("\n");
|
||||
expect(lines).toContain("live.transcript.denied.request");
|
||||
expect(lines).toContain(
|
||||
"live.assertion.permissionRequestCreatedBeforeModel: satisfied",
|
||||
);
|
||||
});
|
||||
|
||||
it("应把 transcript、evidence 与 failure mode 渲染到 stdout 友好的行文本", () => {
|
||||
const evidence = buildApprovalSandboxSmokeEvidence({
|
||||
commandResults: [],
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
});
|
||||
|
||||
const lines = renderApprovalSandboxTranscriptLines(evidence).join("\n");
|
||||
|
||||
expect(lines).toContain("transcript.tool-request");
|
||||
expect(lines).toContain("transcript.approval-decision");
|
||||
expect(lines).toContain("transcript.sandbox-policy");
|
||||
expect(lines).toContain("transcript.tool-result");
|
||||
expect(lines).toContain("transcript.tool-error");
|
||||
expect(lines).toContain("transcript.timeout-recovery");
|
||||
expect(lines).toContain("evidence.tool timeline: satisfied");
|
||||
expect(lines).toContain("evidence.approval decision: satisfied");
|
||||
expect(lines).toContain("evidence.sandbox policy: satisfied");
|
||||
expect(lines).toContain("evidence.error recovery transcript: satisfied");
|
||||
expect(lines).toContain("failure.approval bypass: covered");
|
||||
expect(lines).toContain("failure.tool result missing: covered");
|
||||
expect(lines).toContain("failure.timeout without recovery: covered");
|
||||
expect(lines).toContain("failure.unsafe tool exposed: covered");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,221 @@
|
||||
const REPORT_SCHEMA_VERSION = "lime.i18n.patchMetricsReport.v1";
|
||||
|
||||
function asFiniteNumber(value, fallback = 0) {
|
||||
return Number.isFinite(Number(value)) ? Number(value) : fallback;
|
||||
}
|
||||
|
||||
function asNonNegativeNumber(value, fallback = 0) {
|
||||
return Math.max(0, asFiniteNumber(value, fallback));
|
||||
}
|
||||
|
||||
function asArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function asString(value, fallback = "") {
|
||||
return typeof value === "string" ? value : fallback;
|
||||
}
|
||||
|
||||
function normalizeRun(run) {
|
||||
return {
|
||||
durationMs: asNonNegativeNumber(run?.durationMs),
|
||||
language: asString(run?.language, "unknown"),
|
||||
matchedSegments: asNonNegativeNumber(run?.matchedSegments),
|
||||
replacedNodes: asNonNegativeNumber(run?.replacedNodes),
|
||||
rootKind: asString(run?.rootKind, "unknown"),
|
||||
timestamp: asNonNegativeNumber(run?.timestamp),
|
||||
};
|
||||
}
|
||||
|
||||
function sumRuns(runs, field) {
|
||||
return runs.reduce((sum, run) => sum + asNonNegativeNumber(run[field]), 0);
|
||||
}
|
||||
|
||||
function normalizeRuntimeMetrics(metrics) {
|
||||
const recentRuns = asArray(metrics?.recentRuns ?? metrics?.runs).map(normalizeRun);
|
||||
const patchTimes = asArray(metrics?.patchTimes).map((value) =>
|
||||
asNonNegativeNumber(value),
|
||||
);
|
||||
const totalPatchTime = patchTimes.reduce((sum, value) => sum + value, 0);
|
||||
|
||||
const totalRuns = asNonNegativeNumber(
|
||||
metrics?.totalRuns,
|
||||
recentRuns.length,
|
||||
);
|
||||
const totalReplacedNodes = asNonNegativeNumber(
|
||||
metrics?.totalReplacedNodes,
|
||||
sumRuns(recentRuns, "replacedNodes"),
|
||||
);
|
||||
const totalMatchedSegments = asNonNegativeNumber(
|
||||
metrics?.totalMatchedSegments,
|
||||
sumRuns(recentRuns, "matchedSegments"),
|
||||
);
|
||||
const averagePatchTimeMs = asNonNegativeNumber(
|
||||
metrics?.averagePatchTimeMs,
|
||||
patchTimes.length > 0 ? totalPatchTime / patchTimes.length : 0,
|
||||
);
|
||||
const slowestPatchTimeMs = asNonNegativeNumber(
|
||||
metrics?.slowestPatchTimeMs,
|
||||
patchTimes.length > 0 ? Math.max(...patchTimes) : 0,
|
||||
);
|
||||
const lastRun = metrics?.lastRun
|
||||
? normalizeRun(metrics.lastRun)
|
||||
: (recentRuns.at(-1) ?? null);
|
||||
|
||||
return {
|
||||
averagePatchTimeMs,
|
||||
languageChanges: asNonNegativeNumber(metrics?.languageChanges),
|
||||
lastRun,
|
||||
recentRuns,
|
||||
slowestPatchTimeMs,
|
||||
totalMatchedSegments,
|
||||
totalReplacedNodes,
|
||||
totalRuns,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveStatus(summary) {
|
||||
if (summary.totalRuns === 0) {
|
||||
return "missing-metrics";
|
||||
}
|
||||
|
||||
if (
|
||||
summary.totalMatchedSegments === 0 &&
|
||||
summary.totalReplacedNodes === 0
|
||||
) {
|
||||
return "no-hit";
|
||||
}
|
||||
|
||||
return "active-patch";
|
||||
}
|
||||
|
||||
function buildRecommendations(status) {
|
||||
if (status === "missing-metrics") {
|
||||
return [
|
||||
"先在 GUI smoke 或 Playwright 验证中导出 window.__I18N_METRICS__,再判断 Patch 退出进度。",
|
||||
];
|
||||
}
|
||||
|
||||
if (status === "no-hit") {
|
||||
return [
|
||||
"当前样本未命中 legacy Patch,可进入 current 主路径依赖审计,但还不能直接删除 DOM replacer。",
|
||||
];
|
||||
}
|
||||
|
||||
return [
|
||||
"仍有 legacy Patch 命中,优先把 recentRuns 覆盖到的页面文案迁入 key-based namespace。",
|
||||
];
|
||||
}
|
||||
|
||||
function normalizeThresholds(thresholds = {}) {
|
||||
return {
|
||||
maxMatchedSegments:
|
||||
thresholds.maxMatchedSegments === undefined
|
||||
? null
|
||||
: asNonNegativeNumber(thresholds.maxMatchedSegments),
|
||||
maxReplacedNodes:
|
||||
thresholds.maxReplacedNodes === undefined
|
||||
? null
|
||||
: asNonNegativeNumber(thresholds.maxReplacedNodes),
|
||||
maxRuns:
|
||||
thresholds.maxRuns === undefined
|
||||
? null
|
||||
: asNonNegativeNumber(thresholds.maxRuns),
|
||||
};
|
||||
}
|
||||
|
||||
function buildThresholdIssues(summary, thresholds) {
|
||||
const issues = [];
|
||||
const checks = [
|
||||
["maxMatchedSegments", "totalMatchedSegments", "命中文本段数"],
|
||||
["maxReplacedNodes", "totalReplacedNodes", "替换节点数"],
|
||||
["maxRuns", "totalRuns", "Patch 运行次数"],
|
||||
];
|
||||
|
||||
for (const [thresholdKey, summaryKey, label] of checks) {
|
||||
const threshold = thresholds[thresholdKey];
|
||||
if (threshold === null || threshold === undefined) {
|
||||
continue;
|
||||
}
|
||||
const actual = summary[summaryKey];
|
||||
if (actual > threshold) {
|
||||
issues.push({
|
||||
actual,
|
||||
field: summaryKey,
|
||||
message: `${label} ${actual} 超过门限 ${threshold}`,
|
||||
threshold,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
export function createI18nPatchMetricsReport({
|
||||
generatedAt = new Date().toISOString(),
|
||||
metrics,
|
||||
sourcePath = "",
|
||||
thresholds,
|
||||
} = {}) {
|
||||
const normalizedMetrics = normalizeRuntimeMetrics(metrics ?? {});
|
||||
const status = resolveStatus(normalizedMetrics);
|
||||
const normalizedThresholds = normalizeThresholds(thresholds);
|
||||
const thresholdIssues = buildThresholdIssues(
|
||||
normalizedMetrics,
|
||||
normalizedThresholds,
|
||||
);
|
||||
|
||||
return {
|
||||
schemaVersion: REPORT_SCHEMA_VERSION,
|
||||
generatedAt,
|
||||
sourcePath,
|
||||
status,
|
||||
retirementCandidate: status === "no-hit",
|
||||
summary: {
|
||||
averagePatchTimeMs: normalizedMetrics.averagePatchTimeMs,
|
||||
languageChanges: normalizedMetrics.languageChanges,
|
||||
lastRunAt: normalizedMetrics.lastRun?.timestamp ?? null,
|
||||
lastRunLanguage: normalizedMetrics.lastRun?.language ?? null,
|
||||
lastRunRootKind: normalizedMetrics.lastRun?.rootKind ?? null,
|
||||
recentRunCount: normalizedMetrics.recentRuns.length,
|
||||
slowestPatchTimeMs: normalizedMetrics.slowestPatchTimeMs,
|
||||
totalMatchedSegments: normalizedMetrics.totalMatchedSegments,
|
||||
totalReplacedNodes: normalizedMetrics.totalReplacedNodes,
|
||||
totalRuns: normalizedMetrics.totalRuns,
|
||||
},
|
||||
thresholds: normalizedThresholds,
|
||||
thresholdIssues,
|
||||
recentRuns: normalizedMetrics.recentRuns,
|
||||
recommendations: buildRecommendations(status),
|
||||
};
|
||||
}
|
||||
|
||||
export function renderI18nPatchMetricsTextReport(report) {
|
||||
const lines = [
|
||||
"Lime i18n Patch Metrics Report",
|
||||
`状态: ${report.status}`,
|
||||
`来源: ${report.sourcePath || "(未指定)"}`,
|
||||
`生成时间: ${report.generatedAt}`,
|
||||
`Patch 运行次数: ${report.summary.totalRuns}`,
|
||||
`替换节点数: ${report.summary.totalReplacedNodes}`,
|
||||
`命中文本段数: ${report.summary.totalMatchedSegments}`,
|
||||
`语言切换次数: ${report.summary.languageChanges}`,
|
||||
`平均耗时: ${report.summary.averagePatchTimeMs.toFixed(2)}ms`,
|
||||
`最慢耗时: ${report.summary.slowestPatchTimeMs.toFixed(2)}ms`,
|
||||
`退出候选: ${report.retirementCandidate ? "yes" : "no"}`,
|
||||
];
|
||||
|
||||
if (report.thresholdIssues.length > 0) {
|
||||
lines.push("", "门限问题:");
|
||||
for (const issue of report.thresholdIssues) {
|
||||
lines.push(`- ${issue.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("", "建议:");
|
||||
for (const recommendation of report.recommendations) {
|
||||
lines.push(`- ${recommendation}`);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
createI18nPatchMetricsReport,
|
||||
renderI18nPatchMetricsTextReport,
|
||||
} from "./i18n-patch-metrics-report-core.mjs";
|
||||
|
||||
describe("i18n-patch-metrics-report-core", () => {
|
||||
it("应把 runtime metrics 转成稳定报告,并保留门限问题", () => {
|
||||
const report = createI18nPatchMetricsReport({
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
sourcePath: ".lime/i18n/patch-metrics.json",
|
||||
thresholds: {
|
||||
maxMatchedSegments: 1,
|
||||
maxReplacedNodes: 1,
|
||||
},
|
||||
metrics: {
|
||||
patchTimes: [10, 20],
|
||||
languageChanges: 2,
|
||||
totalRuns: 2,
|
||||
totalReplacedNodes: 3,
|
||||
totalMatchedSegments: 4,
|
||||
runs: [
|
||||
{
|
||||
durationMs: 10,
|
||||
language: "en",
|
||||
matchedSegments: 4,
|
||||
replacedNodes: 3,
|
||||
rootKind: "document",
|
||||
timestamp: 1778400000000,
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.schemaVersion).toBe("lime.i18n.patchMetricsReport.v1");
|
||||
expect(report.status).toBe("active-patch");
|
||||
expect(report.retirementCandidate).toBe(false);
|
||||
expect(report.summary.totalMatchedSegments).toBe(4);
|
||||
expect(report.summary.averagePatchTimeMs).toBe(15);
|
||||
expect(report.thresholdIssues.map((issue) => issue.field)).toEqual([
|
||||
"totalMatchedSegments",
|
||||
"totalReplacedNodes",
|
||||
]);
|
||||
});
|
||||
|
||||
it("应识别零命中样本为退出候选,但不等价于可直接删除", () => {
|
||||
const report = createI18nPatchMetricsReport({
|
||||
metrics: {
|
||||
languageChanges: 1,
|
||||
recentRuns: [
|
||||
{
|
||||
durationMs: 2,
|
||||
language: "en",
|
||||
matchedSegments: 0,
|
||||
replacedNodes: 0,
|
||||
rootKind: "document",
|
||||
timestamp: 1778400000000,
|
||||
},
|
||||
],
|
||||
totalMatchedSegments: 0,
|
||||
totalReplacedNodes: 0,
|
||||
totalRuns: 1,
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.status).toBe("no-hit");
|
||||
expect(report.retirementCandidate).toBe(true);
|
||||
expect(report.recommendations.join("\n")).toContain("不能直接删除");
|
||||
});
|
||||
|
||||
it("应渲染可读文本报告", () => {
|
||||
const report = createI18nPatchMetricsReport({
|
||||
generatedAt: "2026-05-10T00:00:00.000Z",
|
||||
metrics: {},
|
||||
sourcePath: "",
|
||||
});
|
||||
const text = renderI18nPatchMetricsTextReport(report);
|
||||
|
||||
expect(text).toContain("Lime i18n Patch Metrics Report");
|
||||
expect(text).toContain("状态: missing-metrics");
|
||||
expect(text).toContain("退出候选: no");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
const TauriAliasPatterns = [
|
||||
["^@tauri-apps/api/core$", "core.ts"],
|
||||
["^@tauri-apps/api/event$", "event.ts"],
|
||||
["^@tauri-apps/api/window$", "window.ts"],
|
||||
["^@tauri-apps/api/app$", "window.ts"],
|
||||
["^@tauri-apps/api/path$", "window.ts"],
|
||||
["^@tauri-apps/plugin-dialog$", "plugin-dialog.ts"],
|
||||
["^@tauri-apps/plugin-shell$", "plugin-shell.ts"],
|
||||
["^@tauri-apps/plugin-deep-link$", "plugin-deep-link.ts"],
|
||||
["^@tauri-apps/plugin-global-shortcut$", "plugin-global-shortcut.ts"],
|
||||
];
|
||||
|
||||
function toFileUrl(filePath) {
|
||||
return pathToFileURL(filePath).href;
|
||||
}
|
||||
|
||||
function createVitestSmokeConfig(rootDir) {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "lime-vitest-smoke-"));
|
||||
const configPath = path.join(tempDir, "vitest.config.mjs");
|
||||
const cacheDir = path.join(tempDir, "vite-cache");
|
||||
const tauriMockDir = path.join(rootDir, "src/lib/tauri-mock");
|
||||
const aliasSpecs = TauriAliasPatterns.map(([pattern, target]) => ({
|
||||
pattern,
|
||||
replacement: path.join(tauriMockDir, target),
|
||||
}));
|
||||
|
||||
fs.writeFileSync(
|
||||
configPath,
|
||||
`import path from "node:path";\n` +
|
||||
`import { defineConfig } from ${JSON.stringify(
|
||||
toFileUrl(path.join(rootDir, "node_modules/vite/dist/node/index.js")),
|
||||
)};\n` +
|
||||
`import react from ${JSON.stringify(
|
||||
toFileUrl(
|
||||
path.join(rootDir, "node_modules/@vitejs/plugin-react/dist/index.js"),
|
||||
),
|
||||
)};\n` +
|
||||
`import svgr from ${JSON.stringify(
|
||||
toFileUrl(path.join(rootDir, "node_modules/vite-plugin-svgr/dist/index.js")),
|
||||
)};\n` +
|
||||
`const rootDir = ${JSON.stringify(rootDir)};\n` +
|
||||
`const aliasSpecs = ${JSON.stringify(aliasSpecs)};\n` +
|
||||
`export default defineConfig({\n` +
|
||||
` root: rootDir,\n` +
|
||||
` cacheDir: ${JSON.stringify(cacheDir)},\n` +
|
||||
` define: { "import.meta.env.VITE_APP_VERSION": JSON.stringify(process.env.VITE_APP_VERSION || "test") },\n` +
|
||||
` plugins: [react({ jsxRuntime: "automatic", jsxImportSource: "react", babel: { compact: true } }), svgr()],\n` +
|
||||
` resolve: { alias: [\n` +
|
||||
` { find: "@", replacement: path.resolve(rootDir, "src") },\n` +
|
||||
` ...aliasSpecs.map((entry) => ({ find: new RegExp(entry.pattern), replacement: entry.replacement })),\n` +
|
||||
` ] },\n` +
|
||||
` test: {\n` +
|
||||
` globals: true,\n` +
|
||||
` environment: "jsdom",\n` +
|
||||
` exclude: ["**/node_modules/**", "**/dist/**", "**/src-tauri/target/**"],\n` +
|
||||
` },\n` +
|
||||
`});\n`,
|
||||
);
|
||||
|
||||
return {
|
||||
configPath,
|
||||
cleanup() {
|
||||
fs.rmSync(tempDir, { recursive: true, force: true });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function runVitestSmoke({ rootDir, label, args, logPrefix, env }) {
|
||||
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
|
||||
const config = createVitestSmokeConfig(rootDir);
|
||||
const startedAt = Date.now();
|
||||
|
||||
console.log(`\n[${logPrefix}] > ${label}`);
|
||||
|
||||
try {
|
||||
const result = spawnSync(
|
||||
npmCommand,
|
||||
["exec", "--", "vitest", "run", ...args, "--config", config.configPath],
|
||||
{
|
||||
cwd: rootDir,
|
||||
stdio: "inherit",
|
||||
env: { ...process.env, ...env },
|
||||
},
|
||||
);
|
||||
|
||||
if (result.error) {
|
||||
throw result.error;
|
||||
}
|
||||
|
||||
if (typeof result.status === "number" && result.status !== 0) {
|
||||
const error = new Error(`[${logPrefix}] ${label} 失败`);
|
||||
error.exitCode = result.status;
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
label,
|
||||
status: "pass",
|
||||
durationMs: Date.now() - startedAt,
|
||||
args,
|
||||
};
|
||||
} finally {
|
||||
config.cleanup();
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,7 @@ const rootDir = process.cwd();
|
||||
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
|
||||
const cargoCommand = process.platform === "win32" ? "cargo.exe" : "cargo";
|
||||
const BRIDGE_REASON_LABELS = {
|
||||
agent_qc_contract: "Agent QC contract",
|
||||
bridge_contracts: "bridge/contracts",
|
||||
bridge_runtime: "DevBridge / mock / bridge runtime",
|
||||
fallback_full_suite: "兜底全量",
|
||||
|
||||
@@ -42,11 +42,26 @@ const FRONTEND_TOOLING_FILES = new Set([
|
||||
|
||||
const BRIDGE_FILES = new Set([
|
||||
"vite.config.ts",
|
||||
"scripts/agent-qc-completion-audit.mjs",
|
||||
"docs/test/agent-qc-evidence.schema.json",
|
||||
"docs/test/agent-qc-gui-flows.manifest.json",
|
||||
"docs/test/agent-qc-scenarios.manifest.json",
|
||||
"scripts/agent-qc-export-evidence.mjs",
|
||||
"scripts/agent-qc-gui-flow-report.mjs",
|
||||
"scripts/agent-qc-qcloop-job.mjs",
|
||||
"scripts/agent-qc-release-summary.mjs",
|
||||
"scripts/agent-qc-report.mjs",
|
||||
"scripts/check-command-contracts.mjs",
|
||||
"scripts/check-generated-slop-report.mjs",
|
||||
"scripts/check-dev-bridge-health.mjs",
|
||||
"scripts/harness-eval-history-record.mjs",
|
||||
"scripts/harness-eval-trend-report.mjs",
|
||||
"scripts/lib/agent-qc-completion-audit-core.mjs",
|
||||
"scripts/lib/agent-qc-evidence-core.mjs",
|
||||
"scripts/lib/agent-qc-gui-flow-core.mjs",
|
||||
"scripts/lib/agent-qc-qcloop-job-core.mjs",
|
||||
"scripts/lib/agent-qc-release-summary-core.mjs",
|
||||
"scripts/lib/agent-qc-report-core.mjs",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/social-workbench-e2e-smoke.mjs",
|
||||
"scripts/chrome-bridge-e2e.mjs",
|
||||
@@ -65,6 +80,27 @@ const HARNESS_CLEANUP_CONTRACT_FILES = new Set([
|
||||
"scripts/lib/harness-dashboard-core.mjs",
|
||||
]);
|
||||
|
||||
const AGENT_QC_CONTRACT_FILES = new Set([
|
||||
"docs/tests/agent-ops-qc.md",
|
||||
"docs/tests/agent-qc-p0-scenarios.md",
|
||||
"docs/tests/lime-agent-qc-rollout-plan.md",
|
||||
"docs/test/agent-qc-evidence.schema.json",
|
||||
"scripts/agent-qc-completion-audit.mjs",
|
||||
"docs/test/agent-qc-gui-flows.manifest.json",
|
||||
"docs/test/agent-qc-scenarios.manifest.json",
|
||||
"scripts/agent-qc-export-evidence.mjs",
|
||||
"scripts/agent-qc-gui-flow-report.mjs",
|
||||
"scripts/agent-qc-qcloop-job.mjs",
|
||||
"scripts/agent-qc-release-summary.mjs",
|
||||
"scripts/agent-qc-report.mjs",
|
||||
"scripts/lib/agent-qc-completion-audit-core.mjs",
|
||||
"scripts/lib/agent-qc-evidence-core.mjs",
|
||||
"scripts/lib/agent-qc-gui-flow-core.mjs",
|
||||
"scripts/lib/agent-qc-qcloop-job-core.mjs",
|
||||
"scripts/lib/agent-qc-release-summary-core.mjs",
|
||||
"scripts/lib/agent-qc-report-core.mjs",
|
||||
]);
|
||||
|
||||
const INTEGRITY_FILES = new Set([
|
||||
"package.json",
|
||||
"packages/lime-cli-npm/package.json",
|
||||
@@ -264,7 +300,10 @@ function isDocsChange(file) {
|
||||
}
|
||||
|
||||
function isDocsOnlyChange(files) {
|
||||
return files.length > 0 && files.every((file) => isDocsChange(file));
|
||||
return (
|
||||
files.length > 0 &&
|
||||
files.every((file) => isDocsChange(file) && !isAgentQcContractChange(file))
|
||||
);
|
||||
}
|
||||
|
||||
function isFrontendChange(file) {
|
||||
@@ -291,6 +330,10 @@ function isHarnessCleanupContractChange(file) {
|
||||
return HARNESS_CLEANUP_CONTRACT_FILES.has(file);
|
||||
}
|
||||
|
||||
function isAgentQcContractChange(file) {
|
||||
return AGENT_QC_CONTRACT_FILES.has(file);
|
||||
}
|
||||
|
||||
function collectBridgeReasons(
|
||||
changedFiles,
|
||||
{ full = false, fallback = false, workflow = false } = {},
|
||||
@@ -309,6 +352,10 @@ function collectBridgeReasons(
|
||||
|
||||
const reasons = [];
|
||||
|
||||
if (changedFiles.some(isAgentQcContractChange)) {
|
||||
reasons.push("agent_qc_contract");
|
||||
}
|
||||
|
||||
if (changedFiles.some(isHarnessCleanupContractChange)) {
|
||||
reasons.push("harness_cleanup_contract");
|
||||
}
|
||||
|
||||
@@ -3,6 +3,26 @@ import { describe, expect, it } from "vitest";
|
||||
import { detectTasks } from "./quality-task-planner.mjs";
|
||||
|
||||
describe("quality-task-planner", () => {
|
||||
it("应把 Agent QC manifest/schema 归到 bridge/contracts 风险", () => {
|
||||
const tasks = detectTasks([
|
||||
"docs/tests/agent-ops-qc.md",
|
||||
"docs/tests/agent-qc-p0-scenarios.md",
|
||||
"docs/tests/lime-agent-qc-rollout-plan.md",
|
||||
"docs/test/agent-qc-scenarios.manifest.json",
|
||||
"docs/test/agent-qc-evidence.schema.json",
|
||||
"docs/test/agent-qc-gui-flows.manifest.json",
|
||||
"scripts/lib/agent-qc-completion-audit-core.mjs",
|
||||
"scripts/lib/agent-qc-evidence-core.mjs",
|
||||
"scripts/lib/agent-qc-gui-flow-core.mjs",
|
||||
"scripts/lib/agent-qc-qcloop-job-core.mjs",
|
||||
"scripts/lib/agent-qc-release-summary-core.mjs",
|
||||
]);
|
||||
|
||||
expect(tasks.bridge).toBe(true);
|
||||
expect(tasks.bridgeReasons).toContain("agent_qc_contract");
|
||||
expect(tasks.docsOnly).toBe(false);
|
||||
});
|
||||
|
||||
it("应把 harness cleanup/report 主链文件归到 bridge/contracts 风险", () => {
|
||||
const tasks = detectTasks([
|
||||
"scripts/lib/generated-slop-report-core.mjs",
|
||||
|
||||
@@ -6,6 +6,10 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import {
|
||||
createI18nPatchMetricsReport,
|
||||
renderI18nPatchMetricsTextReport,
|
||||
} from "./lib/i18n-patch-metrics-report-core.mjs";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
@@ -40,6 +44,18 @@ const HEADLESS_TAURI_CONFIG_PATH = path.join(
|
||||
"src-tauri",
|
||||
"tauri.conf.headless.json",
|
||||
);
|
||||
const DEFAULT_I18N_PATCH_METRICS_OUTPUT = path.join(
|
||||
rootDir,
|
||||
".lime",
|
||||
"i18n",
|
||||
"patch-metrics.json",
|
||||
);
|
||||
const DEFAULT_I18N_PATCH_REPORT_OUTPUT = path.join(
|
||||
rootDir,
|
||||
".lime",
|
||||
"i18n",
|
||||
"patch-metrics-report.json",
|
||||
);
|
||||
const tauriCommand =
|
||||
process.platform === "win32"
|
||||
? path.join(rootDir, "node_modules", ".bin", "tauri.cmd")
|
||||
@@ -173,6 +189,9 @@ const DEFAULTS = {
|
||||
intervalMs: 1_000,
|
||||
reuseRunning: false,
|
||||
includeKnowledgeProductE2e: false,
|
||||
i18nPatchMetricsOutput: DEFAULT_I18N_PATCH_METRICS_OUTPUT,
|
||||
i18nPatchReportOutput: DEFAULT_I18N_PATCH_REPORT_OUTPUT,
|
||||
skipI18nPatchMetrics: false,
|
||||
sampleProjectName: "Lime Smoke Workspace",
|
||||
};
|
||||
DEFAULTS.timeoutMs = resolveDefaultTimeoutMs(DEFAULTS.cargoTargetDir);
|
||||
@@ -213,6 +232,11 @@ Lime GUI 冒烟入口
|
||||
--reuse-running 复用已启动的 headless Tauri,不主动拉起
|
||||
--include-knowledge-product-e2e
|
||||
额外执行项目资料产品 E2E 闭环验收
|
||||
--i18n-patch-metrics-output <path>
|
||||
导出 GUI 页面上的 window.__I18N_METRICS__,默认 .lime/i18n/patch-metrics.json
|
||||
--i18n-patch-report-output <path>
|
||||
写入 Patch metrics JSON 报告,默认 .lime/i18n/patch-metrics-report.json
|
||||
--skip-i18n-patch-metrics 跳过 Patch metrics 导出与报告
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
@@ -278,6 +302,27 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--i18n-patch-metrics-output" && argv[index + 1]) {
|
||||
options.i18nPatchMetricsOutput = path.resolve(
|
||||
String(argv[index + 1]).trim(),
|
||||
);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--i18n-patch-report-output" && argv[index + 1]) {
|
||||
options.i18nPatchReportOutput = path.resolve(
|
||||
String(argv[index + 1]).trim(),
|
||||
);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--skip-i18n-patch-metrics") {
|
||||
options.skipI18nPatchMetrics = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
printHelp();
|
||||
process.exit(0);
|
||||
@@ -315,6 +360,15 @@ function parseArgs(argv) {
|
||||
throw new Error("--cargo-target-dir 不能为空");
|
||||
}
|
||||
|
||||
if (!options.skipI18nPatchMetrics) {
|
||||
if (!options.i18nPatchMetricsOutput) {
|
||||
throw new Error("--i18n-patch-metrics-output 不能为空");
|
||||
}
|
||||
if (!options.i18nPatchReportOutput) {
|
||||
throw new Error("--i18n-patch-report-output 不能为空");
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
@@ -855,6 +909,45 @@ async function cleanupStaleGuiSmokeChromeProfiles(
|
||||
}
|
||||
}
|
||||
|
||||
function writeI18nPatchMetricsReport(options) {
|
||||
if (options.skipI18nPatchMetrics) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (!fs.existsSync(options.i18nPatchMetricsOutput)) {
|
||||
console.warn(
|
||||
`[verify:gui-smoke] 未发现 i18n Patch metrics: ${options.i18nPatchMetricsOutput}`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
const metrics = JSON.parse(
|
||||
fs.readFileSync(options.i18nPatchMetricsOutput, "utf8"),
|
||||
);
|
||||
const report = createI18nPatchMetricsReport({
|
||||
metrics,
|
||||
sourcePath: path.relative(rootDir, options.i18nPatchMetricsOutput),
|
||||
});
|
||||
fs.mkdirSync(path.dirname(options.i18nPatchReportOutput), {
|
||||
recursive: true,
|
||||
});
|
||||
fs.writeFileSync(
|
||||
options.i18nPatchReportOutput,
|
||||
JSON.stringify(report, null, 2),
|
||||
"utf8",
|
||||
);
|
||||
console.log(
|
||||
renderI18nPatchMetricsTextReport(report)
|
||||
.trim()
|
||||
.split("\n")
|
||||
.map((line) => `[verify:gui-smoke] ${line}`)
|
||||
.join("\n"),
|
||||
);
|
||||
console.log(
|
||||
`[verify:gui-smoke] i18n Patch metrics report: ${options.i18nPatchReportOutput}`,
|
||||
);
|
||||
}
|
||||
|
||||
function listListeningCommandsForPort(port) {
|
||||
if (!port || process.platform === "win32") {
|
||||
return [];
|
||||
@@ -1481,7 +1574,7 @@ async function main() {
|
||||
npmCommand,
|
||||
[
|
||||
"run",
|
||||
"smoke:knowledge-gui",
|
||||
"smoke:claw-chat-ready-streaming",
|
||||
"--",
|
||||
"--app-url",
|
||||
options.appUrl,
|
||||
@@ -1494,10 +1587,39 @@ async function main() {
|
||||
"--interval-ms",
|
||||
String(options.intervalMs),
|
||||
],
|
||||
"smoke:claw-chat-ready-streaming",
|
||||
options.timeoutMs + 30_000,
|
||||
);
|
||||
|
||||
await runCommand(
|
||||
npmCommand,
|
||||
[
|
||||
"run",
|
||||
"smoke:knowledge-gui",
|
||||
"--",
|
||||
"--app-url",
|
||||
options.appUrl,
|
||||
"--health-url",
|
||||
options.healthUrl,
|
||||
"--invoke-url",
|
||||
options.invokeUrl,
|
||||
"--timeout-ms",
|
||||
String(options.timeoutMs),
|
||||
"--interval-ms",
|
||||
String(options.intervalMs),
|
||||
...(options.skipI18nPatchMetrics
|
||||
? []
|
||||
: [
|
||||
"--i18n-patch-metrics-output",
|
||||
options.i18nPatchMetricsOutput,
|
||||
]),
|
||||
],
|
||||
"smoke:knowledge-gui",
|
||||
options.timeoutMs + 30_000,
|
||||
);
|
||||
|
||||
writeI18nPatchMetricsReport(options);
|
||||
|
||||
if (options.includeKnowledgeProductE2e) {
|
||||
await runCommand(
|
||||
npmCommand,
|
||||
|
||||
Reference in New Issue
Block a user