release: v1.9.0

This commit is contained in:
coso
2026-04-13 02:50:53 +08:00
parent 52624b1488
commit 0f15b8a71e
313 changed files with 19875 additions and 9864 deletions
+18 -1
View File
@@ -32,6 +32,20 @@ function parseWorkspaceField(section, fieldName) {
return pattern.test(section);
}
function readPackageJsonVersion(repoRoot) {
const packageJsonPath = path.join(repoRoot, "package.json");
if (!fs.existsSync(packageJsonPath)) {
return null;
}
try {
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
return typeof packageJson.version === "string" ? packageJson.version : null;
} catch {
return null;
}
}
export function readCargoVersions(cargoTomlPath) {
const content = fs.readFileSync(cargoTomlPath, "utf8");
const workspacePackageSection = extractSection(content, "workspace.package");
@@ -50,8 +64,11 @@ export function readCargoVersions(cargoTomlPath) {
export function readWorkspaceAppVersion(repoRoot = process.cwd()) {
const cargoTomlPath = path.join(repoRoot, "src-tauri", "Cargo.toml");
if (!fs.existsSync(cargoTomlPath)) {
return readPackageJsonVersion(repoRoot);
}
const { workspaceVersion } = readCargoVersions(cargoTomlPath);
return workspaceVersion;
return workspaceVersion ?? readPackageJsonVersion(repoRoot);
}
const currentFilePath = fileURLToPath(import.meta.url);
+13 -6
View File
@@ -13,9 +13,12 @@ const DEFAULTS = {
streamMode: "both",
};
const INVOKE_TIMEOUT_MS = 60_000;
const INVOKE_RETRY_COUNT = 3;
const INVOKE_RETRY_DELAY_MS = 500;
const INVOKE_TIMEOUT_CEILING_MS = 180_000;
const INVOKE_RETRY_COUNT = 10;
const INVOKE_RETRY_DELAY_MS = 1_000;
const POST_HEALTH_SETTLE_MS = 3_000;
const POST_LAUNCH_SETTLE_MS = 1_500;
const READ_PAGE_TIMEOUT_MS = 45_000;
function printHelp() {
console.log(`
@@ -123,13 +126,14 @@ function isTransientInvokeError(error) {
}
async function invoke(options, cmd, args) {
const invokeTimeoutMs = Math.min(options.timeoutMs, INVOKE_TIMEOUT_CEILING_MS);
const requestInit = {
method: "POST",
headers: {
"content-type": "application/json",
},
body: JSON.stringify({ cmd, args }),
signal: AbortSignal.timeout(Math.min(options.timeoutMs, INVOKE_TIMEOUT_MS)),
signal: AbortSignal.timeout(invokeTimeoutMs),
};
for (let attempt = 1; attempt <= INVOKE_RETRY_COUNT; attempt += 1) {
@@ -151,7 +155,7 @@ async function invoke(options, cmd, args) {
if (!isTransientInvokeError(error) || attempt >= INVOKE_RETRY_COUNT) {
if (error?.name === "TimeoutError") {
throw new Error(
`[smoke:browser-runtime] ${cmd} 超时,${Math.min(options.timeoutMs, INVOKE_TIMEOUT_MS)}ms 内未收到 DevBridge 响应`,
`[smoke:browser-runtime] ${cmd} 超时,${invokeTimeoutMs}ms 内未收到 DevBridge 响应`,
);
}
throw new Error(`[smoke:browser-runtime] ${cmd} 请求失败: ${detail}`);
@@ -209,6 +213,7 @@ async function main() {
const options = parseArgs(process.argv.slice(2));
await waitForHealth(options);
await sleep(POST_HEALTH_SETTLE_MS);
const profileKey = `smoke-browser-runtime-${Date.now()}`;
let sessionId = null;
@@ -256,11 +261,13 @@ async function main() {
"get_browser_session_state 未返回 target_id",
);
await sleep(POST_LAUNCH_SETTLE_MS);
const actionResult = await invoke(options, "browser_execute_action", {
request: {
profile_key: profileKey,
action: "read_page",
timeout_ms: 20_000,
timeout_ms: Math.min(options.timeoutMs, READ_PAGE_TIMEOUT_MS),
},
});
assert(actionResult?.success === true, "browser_execute_action(read_page) 未成功");
+158
View File
@@ -0,0 +1,158 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { assertGeneratedSlopReportContract } from "./lib/generated-slop-report-core.mjs";
const GENERATED_SLOP_REPORT_SCRIPT = "scripts/report-generated-slop.mjs";
function parseArgs(argv) {
const result = {
format: "text",
generateCurrent: false,
help: false,
input: ".lime/harness/reports/harness-cleanup-report.json",
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--input" && argv[index + 1]) {
result.input = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--format" && argv[index + 1]) {
result.format = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--generate-current") {
result.generateCurrent = true;
continue;
}
if (arg === "--help" || arg === "-h") {
result.help = true;
}
}
return result;
}
function printHelp() {
console.log(`
Lime Harness Cleanup Report Contract Check
用法:
node scripts/check-generated-slop-report.mjs
node scripts/check-generated-slop-report.mjs --generate-current
node scripts/check-generated-slop-report.mjs --input ".lime/harness/reports/harness-cleanup-report.json"
node scripts/check-generated-slop-report.mjs --input "./tmp/harness-cleanup-report.json" --format json
选项:
--input PATH cleanup report JSON 路径,默认 ".lime/harness/reports/harness-cleanup-report.json"
--generate-current 先生成当前 cleanup report,再校验其契约
--format FMT 输出格式:text | json
-h, --help 显示帮助
`);
}
function resolveInputPath(inputPath) {
return path.resolve(process.cwd(), inputPath);
}
function buildCurrentCleanupReport() {
const tempRoot = fs.mkdtempSync(
path.join(os.tmpdir(), "lime-cleanup-report-contract-"),
);
const outputPath = path.join(tempRoot, "harness-cleanup-report.json");
try {
execFileSync(
process.execPath,
[
path.resolve(process.cwd(), GENERATED_SLOP_REPORT_SCRIPT),
"--format",
"json",
"--output-json",
outputPath,
],
{
cwd: process.cwd(),
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
},
);
return outputPath;
} catch (error) {
fs.rmSync(tempRoot, { recursive: true, force: true });
throw error;
}
}
function buildSummary(report, resolvedPath) {
return {
status: "ok",
inputPath: resolvedPath,
recommendationCount: Array.isArray(report?.recommendations)
? report.recommendations.length
: 0,
verificationFailureFocusCount: Array.isArray(
report?.focus?.observabilityVerificationOutcomes,
)
? report.focus.observabilityVerificationOutcomes.length
: 0,
currentRecoveredBaselineCount: Array.isArray(
report?.focus?.currentRecoveredObservabilityVerificationOutcomes,
)
? report.focus.currentRecoveredObservabilityVerificationOutcomes.length
: 0,
};
}
function renderSummary(summary, format) {
if (format === "json") {
return `${JSON.stringify(summary, null, 2)}\n`;
}
return [
"[harness-cleanup-contract] ok",
`[harness-cleanup-contract] input: ${summary.inputPath}`,
`[harness-cleanup-contract] recommendations: ${summary.recommendationCount}`,
`[harness-cleanup-contract] verification failure focus: ${summary.verificationFailureFocusCount}`,
`[harness-cleanup-contract] current recovered baseline: ${summary.currentRecoveredBaselineCount}`,
].join("\n") + "\n";
}
function main() {
const options = parseArgs(process.argv.slice(2));
if (options.help) {
printHelp();
return;
}
const resolvedPath = options.generateCurrent
? buildCurrentCleanupReport()
: resolveInputPath(options.input);
try {
const report = JSON.parse(fs.readFileSync(resolvedPath, "utf8"));
const validatedReport = assertGeneratedSlopReportContract(report);
const summary = buildSummary(validatedReport, resolvedPath);
process.stdout.write(renderSummary(summary, options.format));
} finally {
if (options.generateCurrent) {
fs.rmSync(path.dirname(resolvedPath), { recursive: true, force: true });
}
}
}
main();
+375 -19
View File
@@ -6,21 +6,34 @@ import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
import { renderHarnessDashboardHtml } from "./lib/harness-dashboard-core.mjs";
const RUNNER_PATH = "scripts/harness-eval-runner.mjs";
const TREND_PATH = "scripts/harness-eval-trend-report.mjs";
const CLEANUP_PATH = "scripts/report-generated-slop.mjs";
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
"repaired",
"success",
"passed",
"clean",
]);
function parseArgs(argv) {
const result = {
cleanupJson: "",
cleanupMarkdown: "",
dashboardHtml: "",
dashboardTitle: "Lime Harness Dashboard",
format: "text",
help: false,
historyDir: "./.lime/harness/history",
manifest: "",
outputJson: "",
retain: 30,
skipCleanup: false,
skipTrend: false,
summaryJson: "",
summaryMarkdown: "",
trendJson: "",
trendMarkdown: "",
workspaceRoot: process.cwd(),
@@ -41,6 +54,12 @@ function parseArgs(argv) {
continue;
}
if (arg === "--manifest" && argv[index + 1]) {
result.manifest = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--retain" && argv[index + 1]) {
result.retain = Number.parseInt(String(argv[index + 1]).trim(), 10);
index += 1;
@@ -59,6 +78,18 @@ function parseArgs(argv) {
continue;
}
if (arg === "--summary-json" && argv[index + 1]) {
result.summaryJson = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--summary-markdown" && argv[index + 1]) {
result.summaryMarkdown = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--trend-json" && argv[index + 1]) {
result.trendJson = String(argv[index + 1]).trim();
index += 1;
@@ -83,6 +114,18 @@ function parseArgs(argv) {
continue;
}
if (arg === "--dashboard-html" && argv[index + 1]) {
result.dashboardHtml = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--dashboard-title" && argv[index + 1]) {
result.dashboardTitle = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--skip-trend") {
result.skipTrend = true;
continue;
@@ -113,11 +156,16 @@ Lime Harness Eval History Record
选项:
--history-dir PATH summary 历史目录,默认 ./.lime/harness/history
--workspace-root PATH 生成当前 summary 时使用的工作区根目录
--manifest PATH 透传给 harness eval runner,覆盖默认 manifest
--retain N 历史窗口保留数量,默认 30
--trend-json PATH trend JSON 输出路径
--trend-markdown PATH trend Markdown 输出路径
--cleanup-json PATH cleanup JSON 输出路径
--cleanup-markdown PATH cleanup Markdown 输出路径
--summary-json PATH summary JSON 输出路径,默认写入 reports/harness-eval-summary.json
--summary-markdown PATH summary Markdown 输出路径,默认写入 reports/harness-eval-summary.md
--trend-json PATH trend JSON 输出路径,默认写入 reports/harness-eval-trend.json
--trend-markdown PATH trend Markdown 输出路径,默认写入 reports/harness-eval-trend.md
--cleanup-json PATH cleanup JSON 输出路径,默认写入 reports/harness-cleanup-report.json
--cleanup-markdown PATH cleanup Markdown 输出路径,默认写入 reports/harness-cleanup-report.md
--dashboard-html PATH dashboard HTML 输出路径,依赖 summary / trend / cleanup;默认写入 reports/harness-dashboard.html
--dashboard-title TEXT dashboard 页面标题
--skip-trend 只记录 summary,不生成 trend
--skip-cleanup 只记录 summary / trend,不生成 cleanup
--format FMT 标准输出格式:text | json
@@ -134,8 +182,9 @@ function ensureParentDirectory(filePath) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
}
function readJsonFile(filePath) {
return JSON.parse(fs.readFileSync(filePath, "utf8"));
function writeTextFile(filePath, contents) {
ensureParentDirectory(filePath);
fs.writeFileSync(filePath, contents, "utf8");
}
function timestampForFilename() {
@@ -154,20 +203,32 @@ function collectHistoryFiles(historyDir) {
.sort((left, right) => left.localeCompare(right));
}
function buildCurrentSummary(repoRoot, workspaceRoot) {
function runRunner(repoRoot, args) {
const runnerPath = resolvePath(repoRoot, RUNNER_PATH);
const output = execFileSync(
process.execPath,
[runnerPath, "--format", "json", "--workspace-root", workspaceRoot],
{
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "inherit"],
},
);
return execFileSync(process.execPath, [runnerPath, ...args], {
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "inherit"],
});
}
function buildCurrentSummary(repoRoot, workspaceRoot, manifestPath) {
const args = ["--format", "json", "--workspace-root", workspaceRoot];
if (manifestPath) {
args.push("--manifest", manifestPath);
}
const output = runRunner(repoRoot, args);
return JSON.parse(output);
}
function buildCurrentSummaryMarkdown(repoRoot, workspaceRoot, manifestPath) {
const args = ["--format", "markdown", "--workspace-root", workspaceRoot];
if (manifestPath) {
args.push("--manifest", manifestPath);
}
return runRunner(repoRoot, args);
}
function writeJsonFile(filePath, payload) {
ensureParentDirectory(filePath);
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
@@ -214,13 +275,157 @@ function buildDefaultArtifactPaths(historyDir) {
? path.join(historyParent, "reports")
: historyParent;
return {
summaryJson: path.join(artifactsRoot, "harness-eval-summary.json"),
summaryMarkdown: path.join(artifactsRoot, "harness-eval-summary.md"),
trendJson: path.join(artifactsRoot, "harness-eval-trend.json"),
trendMarkdown: path.join(artifactsRoot, "harness-eval-trend.md"),
cleanupJson: path.join(artifactsRoot, "harness-cleanup-report.json"),
cleanupMarkdown: path.join(artifactsRoot, "harness-cleanup-report.md"),
dashboardHtml: path.join(artifactsRoot, "harness-dashboard.html"),
};
}
function toVerificationFailureOutcomeFocus(cleanupReport) {
const currentEntries = Array.isArray(
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentObservabilityVerificationOutcomes
: [];
const fallbackEntries = Array.isArray(
cleanupReport?.focus?.observabilityVerificationOutcomes,
)
? cleanupReport.focus.observabilityVerificationOutcomes
: [];
const entries =
currentEntries.length > 0 ? currentEntries : fallbackEntries;
return entries
.map((entry) => {
const signal = typeof entry?.signal === "string" ? entry.signal.trim() : "";
const outcome =
typeof entry?.outcome === "string" ? entry.outcome.trim() : "";
return signal && outcome ? `${signal}:${outcome}` : "";
})
.filter(Boolean);
}
function toCurrentRecoveredBaselineFocus(cleanupReport) {
const explicitRecoveredEntries = Array.isArray(
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes
: [];
const currentEntries = Array.isArray(
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentObservabilityVerificationOutcomes
: [];
const fallbackEntries = Array.isArray(
cleanupReport?.focus?.observabilityVerificationOutcomes,
)
? cleanupReport.focus.observabilityVerificationOutcomes
: [];
const entries =
explicitRecoveredEntries.length > 0
? explicitRecoveredEntries
: currentEntries.length > 0
? currentEntries
: fallbackEntries;
return entries
.filter((entry) =>
RECOVERED_VERIFICATION_OUTCOMES.has(
typeof entry?.outcome === "string" ? entry.outcome.trim() : "",
),
)
.map((entry) => {
const signal = typeof entry?.signal === "string" ? entry.signal.trim() : "";
const outcome =
typeof entry?.outcome === "string" ? entry.outcome.trim() : "";
return signal && outcome ? `${signal}:${outcome}` : "";
})
.filter(Boolean);
}
function toVerificationOutcomeCounts(cleanupReport) {
const summary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.verificationOutcomes &&
typeof cleanupReport.summary.verificationOutcomes === "object"
? cleanupReport.summary.verificationOutcomes
: {};
const currentSummary =
summary &&
typeof summary.current === "object" &&
!Array.isArray(summary.current)
? summary.current
: {};
const degradedSummary =
summary &&
typeof summary.degraded === "object" &&
!Array.isArray(summary.degraded)
? summary.degraded
: {};
return {
failureCaseCount:
typeof summary.failureCaseCount === "number" &&
Number.isFinite(summary.failureCaseCount)
? summary.failureCaseCount
: 0,
blockingFailureCaseCount:
typeof currentSummary.blockingFailureCaseCount === "number" &&
Number.isFinite(currentSummary.blockingFailureCaseCount)
? currentSummary.blockingFailureCaseCount
: 0,
advisoryFailureCaseCount:
typeof currentSummary.advisoryFailureCaseCount === "number" &&
Number.isFinite(currentSummary.advisoryFailureCaseCount)
? currentSummary.advisoryFailureCaseCount
: 0,
recoveredCaseCount:
typeof summary.recoveredCaseCount === "number" &&
Number.isFinite(summary.recoveredCaseCount)
? summary.recoveredCaseCount
: 0,
currentRecoveredCaseCount:
typeof currentSummary.recoveredCaseCount === "number" &&
Number.isFinite(currentSummary.recoveredCaseCount)
? currentSummary.recoveredCaseCount
: 0,
degradedBlockingFailureCaseCount:
typeof degradedSummary.blockingFailureCaseCount === "number" &&
Number.isFinite(degradedSummary.blockingFailureCaseCount)
? degradedSummary.blockingFailureCaseCount
: 0,
};
}
function toTrendCurrentRecoveredBaselineFocus(trendReport) {
const entries = Array.isArray(
trendReport?.classificationDeltas?.currentRecoveredObservabilityVerificationOutcomes,
)
? trendReport.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes
: [];
return entries
.filter((entry) => {
const latestCaseCount =
typeof entry?.latest?.caseCount === "number" &&
Number.isFinite(entry.latest.caseCount)
? entry.latest.caseCount
: 0;
return latestCaseCount > 0;
})
.map((entry) =>
typeof entry?.name === "string" ? entry.name.trim() : "",
)
.filter(Boolean)
.slice(0, 3);
}
function renderOutput(result, format) {
if (format === "json") {
return `${JSON.stringify(result, null, 2)}\n`;
@@ -234,8 +439,31 @@ function renderOutput(result, format) {
`[lime] trimmed files: ${result.trimmedPaths.length}`,
];
if (result.summary) {
if (result.summary.outputJsonPath) {
lines.push(`[lime] summary json: ${result.summary.outputJsonPath}`);
}
if (result.summary.outputMarkdownPath) {
lines.push(`[lime] summary markdown: ${result.summary.outputMarkdownPath}`);
}
}
if (result.trend) {
lines.push(`[lime] trend sample count: ${result.trend.sampleCount}`);
lines.push(
`[lime] trend current observability gap cases: ${result.trend.currentObservabilityGapCaseCount}`,
);
lines.push(
`[lime] trend degraded observability gap cases: ${result.trend.degradedObservabilityGapCaseCount}`,
);
lines.push(
`[lime] trend current recovered baseline cases: ${result.trend.currentRecoveredVerificationCaseCount}`,
);
if (result.trend.currentRecoveredBaselineFocus.length > 0) {
lines.push(
`[lime] trend current recovered baseline: ${result.trend.currentRecoveredBaselineFocus.join(", ")}`,
);
}
if (result.trend.outputJsonPath) {
lines.push(`[lime] trend json: ${result.trend.outputJsonPath}`);
}
@@ -245,11 +473,49 @@ function renderOutput(result, format) {
lines.push(
`[lime] cleanup trend samples: ${result.cleanup.trendSampleCount}`,
);
lines.push(
`[lime] cleanup current observability gap cases: ${result.cleanup.currentObservabilityGapCaseCount}`,
);
lines.push(
`[lime] cleanup degraded observability gap cases: ${result.cleanup.degradedObservabilityGapCaseCount}`,
);
if (result.cleanup.verificationFailureOutcomeFocus.length > 0) {
lines.push(
`[lime] cleanup verification failure outcomes: ${result.cleanup.verificationFailureOutcomeFocus.join(", ")}`,
);
}
lines.push(
`[lime] cleanup verification failure cases: ${result.cleanup.verificationFailureCaseCount}`,
);
lines.push(
`[lime] cleanup verification blocking failure cases: ${result.cleanup.verificationBlockingFailureCaseCount}`,
);
lines.push(
`[lime] cleanup verification advisory failure cases: ${result.cleanup.verificationAdvisoryFailureCaseCount}`,
);
lines.push(
`[lime] cleanup degraded blocking verification failure cases: ${result.cleanup.verificationDegradedBlockingFailureCaseCount}`,
);
lines.push(
`[lime] cleanup verification recovered cases: ${result.cleanup.verificationRecoveredCaseCount}`,
);
lines.push(
`[lime] cleanup current recovered baseline cases: ${result.cleanup.currentVerificationRecoveredCaseCount}`,
);
if (result.cleanup.currentRecoveredBaselineFocus.length > 0) {
lines.push(
`[lime] cleanup current recovered baseline: ${result.cleanup.currentRecoveredBaselineFocus.join(", ")}`,
);
}
if (result.cleanup.outputJsonPath) {
lines.push(`[lime] cleanup json: ${result.cleanup.outputJsonPath}`);
}
}
if (result.dashboard?.outputHtmlPath) {
lines.push(`[lime] dashboard html: ${result.dashboard.outputHtmlPath}`);
}
return `${lines.join("\n")}\n`;
}
@@ -260,11 +526,24 @@ function runHistoryRecordCli() {
return;
}
if (options.dashboardHtml && (options.skipTrend || options.skipCleanup)) {
throw new Error(
"生成 dashboard 需要同时启用 trend 与 cleanup,请移除 --skip-trend / --skip-cleanup。",
);
}
const repoRoot = process.cwd();
const historyDir = resolvePath(repoRoot, options.historyDir);
fs.mkdirSync(historyDir, { recursive: true });
const summary = buildCurrentSummary(repoRoot, options.workspaceRoot);
const effectiveManifest = options.manifest
? resolvePath(repoRoot, options.manifest)
: "";
const summary = buildCurrentSummary(
repoRoot,
options.workspaceRoot,
effectiveManifest,
);
const summaryFilePath = writeUniqueHistorySummary(historyDir, summary);
const trimmedPaths = trimHistoryFiles(historyDir, options.retain);
const historyCount = collectHistoryFiles(historyDir).length;
@@ -276,8 +555,33 @@ function runHistoryRecordCli() {
recordedSummaryPath: summaryFilePath,
historyCount,
trimmedPaths,
summary: null,
trend: null,
cleanup: null,
dashboard: null,
};
let trendReport = null;
let cleanupReport = null;
const summaryJsonPath = resolvePath(
repoRoot,
options.summaryJson || defaults.summaryJson,
);
const summaryMarkdownPath = resolvePath(
repoRoot,
options.summaryMarkdown || defaults.summaryMarkdown,
);
writeJsonFile(summaryJsonPath, summary);
const summaryMarkdown = buildCurrentSummaryMarkdown(
repoRoot,
options.workspaceRoot,
effectiveManifest,
);
writeTextFile(summaryMarkdownPath, summaryMarkdown);
result.summary = {
outputJsonPath: summaryJsonPath,
outputMarkdownPath: summaryMarkdownPath,
};
if (!options.skipTrend) {
@@ -309,9 +613,18 @@ function runHistoryRecordCli() {
stdio: ["ignore", "pipe", "inherit"],
},
);
const trendReport = JSON.parse(trendOutput);
trendReport = JSON.parse(trendOutput);
const trendCurrentRecoveredBaselineFocus =
toTrendCurrentRecoveredBaselineFocus(trendReport);
result.trend = {
sampleCount: trendReport.sampleCount,
currentObservabilityGapCaseCount:
trendReport.latest?.totals?.currentObservabilityGapCaseCount ?? 0,
degradedObservabilityGapCaseCount:
trendReport.latest?.totals?.degradedObservabilityGapCaseCount ?? 0,
currentRecoveredVerificationCaseCount:
trendReport.latest?.totals?.currentRecoveredVerificationCaseCount ?? 0,
currentRecoveredBaselineFocus: trendCurrentRecoveredBaselineFocus,
outputJsonPath: trendJsonPath,
outputMarkdownPath: trendMarkdownPath,
};
@@ -346,14 +659,57 @@ function runHistoryRecordCli() {
stdio: ["ignore", "pipe", "inherit"],
},
);
const cleanupReport = JSON.parse(cleanupOutput);
cleanupReport = JSON.parse(cleanupOutput);
const verificationFailureOutcomeFocus =
toVerificationFailureOutcomeFocus(cleanupReport);
const currentRecoveredBaselineFocus =
toCurrentRecoveredBaselineFocus(cleanupReport);
const verificationOutcomeCounts =
toVerificationOutcomeCounts(cleanupReport);
result.cleanup = {
trendSampleCount: cleanupReport.summary?.trend?.sampleCount ?? 0,
currentObservabilityGapCaseCount:
cleanupReport.summary?.trend?.latestCurrentObservabilityGapCaseCount ?? 0,
degradedObservabilityGapCaseCount:
cleanupReport.summary?.trend?.latestDegradedObservabilityGapCaseCount ?? 0,
verificationFailureOutcomeFocus,
verificationFailureCaseCount:
verificationOutcomeCounts.failureCaseCount,
verificationBlockingFailureCaseCount:
verificationOutcomeCounts.blockingFailureCaseCount,
verificationAdvisoryFailureCaseCount:
verificationOutcomeCounts.advisoryFailureCaseCount,
verificationDegradedBlockingFailureCaseCount:
verificationOutcomeCounts.degradedBlockingFailureCaseCount,
verificationRecoveredCaseCount:
verificationOutcomeCounts.recoveredCaseCount,
currentVerificationRecoveredCaseCount:
verificationOutcomeCounts.currentRecoveredCaseCount,
currentRecoveredBaselineFocus,
outputJsonPath: cleanupJsonPath,
outputMarkdownPath: cleanupMarkdownPath,
};
}
const dashboardHtmlPath =
options.skipTrend || options.skipCleanup
? ""
: resolvePath(repoRoot, options.dashboardHtml || defaults.dashboardHtml);
if (dashboardHtmlPath) {
const dashboardHtml = renderHarnessDashboardHtml({
summaryReport: summary,
trendReport,
cleanupReport,
title: options.dashboardTitle,
});
writeTextFile(dashboardHtmlPath, dashboardHtml);
result.dashboard = {
outputHtmlPath: dashboardHtmlPath,
title: options.dashboardTitle,
};
}
const rendered = renderOutput(result, options.format);
if (options.outputJson) {
const outputPath = resolvePath(repoRoot, options.outputJson);
+251 -87
View File
@@ -22,16 +22,15 @@ const REVIEW_DECISION_RISK_LEVEL_SET = new Set([
"high",
"unknown",
]);
const OBSERVABILITY_GAP_SUITE_TAG = "observability-gap";
function parseArgs(argv) {
const result = {
format: "text",
help: false,
historyRetain: 30,
manifest: DEFAULT_MANIFEST_PATH,
outputJson: "",
outputMarkdown: "",
recordHistoryDir: "",
strict: true,
workspaceRoot: process.cwd(),
};
@@ -69,18 +68,6 @@ function parseArgs(argv) {
continue;
}
if (arg === "--record-history-dir" && argv[index + 1]) {
result.recordHistoryDir = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--history-retain" && argv[index + 1]) {
result.historyRetain = Number.parseInt(String(argv[index + 1]), 10);
index += 1;
continue;
}
if (arg === "--no-strict") {
result.strict = false;
continue;
@@ -108,7 +95,6 @@ Lime Harness Eval Runner
node scripts/harness-eval-runner.mjs --format json
node scripts/harness-eval-runner.mjs --workspace-root "/path/to/workspace"
node scripts/harness-eval-runner.mjs --output-json "./tmp/harness-eval-summary.json" --output-markdown "./tmp/harness-eval-summary.md"
node scripts/harness-eval-runner.mjs --record-history-dir "./artifacts/history"
选项:
--manifest PATH 指定 manifest,默认 docs/test/harness-evals.manifest.json
@@ -116,8 +102,6 @@ Lime Harness Eval Runner
--format FMT 控制标准输出格式:text | json | markdown
--output-json PATH 将 JSON 摘要写入指定路径
--output-markdown PATH 将 Markdown 摘要写入指定路径
--record-history-dir PATH 将当前 summary 追加写入历史目录,供 trend/nightly 复用
--history-retain N 历史目录最多保留多少条 summary,默认 30
--strict 严格模式(默认),发现 invalid case 时返回非 0
--no-strict 非严格模式,只输出摘要,不因 invalid case 退出失败
-h, --help 显示帮助
@@ -136,56 +120,6 @@ function ensureParentDirectory(filePath) {
fs.mkdirSync(path.dirname(filePath), { recursive: true });
}
function ensureDirectory(dirPath) {
fs.mkdirSync(dirPath, { recursive: true });
}
function toHistoryTimestamp(generatedAt) {
const parsed = new Date(generatedAt);
const normalized = Number.isNaN(parsed.getTime())
? new Date().toISOString()
: parsed.toISOString();
return normalized.replace(/[-:]/g, "").replace(/\.(\d{3})Z$/, "$1Z");
}
function trimHistoryDirectory(historyDir, retainCount) {
const normalizedRetainCount =
Number.isInteger(retainCount) && retainCount > 0 ? retainCount : 30;
const historyFiles = fs
.readdirSync(historyDir, { withFileTypes: true })
.filter(
(entry) =>
entry.isFile() &&
entry.name.endsWith("-harness-eval-summary.json"),
)
.map((entry) => path.join(historyDir, entry.name))
.sort((left, right) => right.localeCompare(left));
for (const staleFile of historyFiles.slice(normalizedRetainCount)) {
fs.rmSync(staleFile, { force: true });
}
}
function recordSummaryHistory(repoRoot, summary, options) {
if (!options.recordHistoryDir) {
return "";
}
const historyDir = resolvePath(repoRoot, options.recordHistoryDir);
ensureDirectory(historyDir);
const historyFilePath = path.join(
historyDir,
`${toHistoryTimestamp(summary.generatedAt)}-harness-eval-summary.json`,
);
fs.writeFileSync(
historyFilePath,
`${JSON.stringify(summary, null, 2)}\n`,
"utf8",
);
trimHistoryDirectory(historyDir, options.historyRetain);
return historyFilePath;
}
function normalizeStringList(value) {
if (!Array.isArray(value)) {
return [];
@@ -213,6 +147,10 @@ function normalizeEnumString(value, allowedValues, fallback = "") {
return fallback;
}
function isObject(value) {
return value != null && typeof value === "object" && !Array.isArray(value);
}
function createBreakdownEntry(name) {
return {
name,
@@ -255,6 +193,25 @@ function aggregateCaseBreakdown(cases, selector) {
});
}
function isDegradedObservabilityGapCase(entry) {
return (
entry.observabilityGapCount > 0 &&
normalizeStringList(entry.tags).includes(OBSERVABILITY_GAP_SUITE_TAG)
);
}
function isCurrentObservabilityGapCase(entry) {
return entry.observabilityGapCount > 0 && !isDegradedObservabilityGapCase(entry);
}
function isDegradedObservabilityDiagnosticCase(entry) {
return normalizeStringList(entry?.tags).includes(OBSERVABILITY_GAP_SUITE_TAG);
}
function isCurrentObservabilityDiagnosticCase(entry) {
return !isDegradedObservabilityDiagnosticCase(entry);
}
function getValueByPath(target, dottedPath) {
return dottedPath
.split(".")
@@ -388,19 +345,39 @@ function loadReviewDecisionForCase(caseDir, inlineReviewDecision) {
};
}
function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
const inlineSummary =
inputPayload?.observability &&
typeof inputPayload.observability === "object" &&
!Array.isArray(inputPayload.observability)
? inputPayload.observability
: evidencePayload?.observabilitySummary &&
typeof evidencePayload.observabilitySummary === "object" &&
!Array.isArray(evidencePayload.observabilitySummary)
? evidencePayload.observabilitySummary
: null;
function resolveObservabilitySummary(inputPayload, evidencePayload) {
const inlineSummary = isObject(inputPayload?.observability)
? inputPayload.observability
: null;
const evidenceSummary = isObject(evidencePayload?.observabilitySummary)
? evidencePayload.observabilitySummary
: null;
if (!inlineSummary) {
if (!inlineSummary && !evidenceSummary) {
return null;
}
return {
signalCoverage: Array.isArray(inlineSummary?.signalCoverage)
? inlineSummary.signalCoverage
: Array.isArray(evidenceSummary?.signalCoverage)
? evidenceSummary.signalCoverage
: [],
verificationSummary: isObject(inlineSummary?.verificationSummary)
? inlineSummary.verificationSummary
: isObject(evidenceSummary?.verificationSummary)
? evidenceSummary.verificationSummary
: null,
};
}
function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
const observabilitySummary = resolveObservabilitySummary(
inputPayload,
evidencePayload,
);
if (!observabilitySummary) {
return [
{
signal: "observabilitySummary",
@@ -409,8 +386,8 @@ function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
];
}
const signalCoverage = Array.isArray(inlineSummary.signalCoverage)
? inlineSummary.signalCoverage
const signalCoverage = Array.isArray(observabilitySummary.signalCoverage)
? observabilitySummary.signalCoverage
: [];
if (signalCoverage.length === 0) {
@@ -461,6 +438,129 @@ function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
return coverage;
}
function pushUniqueObservabilityVerificationOutcome(outcomes, seen, signal, outcome) {
const normalizedSignal = normalizeOptionalString(signal);
const normalizedOutcome = normalizeOptionalString(outcome);
if (!normalizedSignal || !normalizedOutcome) {
return;
}
const fingerprint = `${normalizedSignal}:${normalizedOutcome}`;
if (seen.has(fingerprint)) {
return;
}
seen.add(fingerprint);
outcomes.push({
signal: normalizedSignal,
outcome: normalizedOutcome,
});
}
function normalizeObservabilityVerificationSummary(inputPayload, evidencePayload) {
const observabilitySummary = resolveObservabilitySummary(
inputPayload,
evidencePayload,
);
const verificationSummary = isObject(observabilitySummary?.verificationSummary)
? observabilitySummary.verificationSummary
: null;
if (!verificationSummary) {
return [];
}
const outcomes = [];
const seen = new Set();
const artifactValidator = isObject(verificationSummary.artifactValidator)
? verificationSummary.artifactValidator
: null;
if (artifactValidator?.applicable === true) {
const recordCount = Number(artifactValidator.recordCount) || 0;
const issueCount = Number(artifactValidator.issueCount) || 0;
const repairedCount = Number(artifactValidator.repairedCount) || 0;
const fallbackUsedCount = Number(artifactValidator.fallbackUsedCount) || 0;
if (recordCount > 0 && issueCount === 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"artifactValidator",
"clean",
);
}
if (issueCount > 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"artifactValidator",
"issues_present",
);
}
if (repairedCount > 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"artifactValidator",
"repaired",
);
}
if (fallbackUsedCount > 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"artifactValidator",
"fallback_used",
);
}
}
const browserVerification = isObject(verificationSummary.browserVerification)
? verificationSummary.browserVerification
: null;
if (browserVerification) {
if ((Number(browserVerification.successCount) || 0) > 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"browserVerification",
"success",
);
}
if ((Number(browserVerification.failureCount) || 0) > 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"browserVerification",
"failure",
);
}
if ((Number(browserVerification.unknownCount) || 0) > 0) {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"browserVerification",
"unknown",
);
}
}
const guiSmoke = isObject(verificationSummary.guiSmoke)
? verificationSummary.guiSmoke
: null;
if (guiSmoke && typeof guiSmoke.passed === "boolean") {
pushUniqueObservabilityVerificationOutcome(
outcomes,
seen,
"guiSmoke",
guiSmoke.passed ? "passed" : "failed",
);
}
return outcomes;
}
function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
const requiredArtifacts = normalizeStringList(
caseConfig.requiredArtifacts ?? defaults.requiredArtifacts,
@@ -551,6 +651,12 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
const observabilityGapCount = observabilityCoverage.filter(
(entry) => entry.status !== "exported",
).length;
const observabilityVerificationCoverage =
normalizeObservabilityVerificationSummary(inputPayload, evidencePayload);
const observabilityVerificationOutcomes =
observabilityVerificationCoverage.map(
(entry) => `${entry.signal}:${entry.outcome}`,
);
const pendingRequestCount = Array.isArray(
inputPayload?.runtimeContext?.pendingRequests,
@@ -606,6 +712,8 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
observabilityCoverage,
observabilitySignals,
observabilityGapCount,
observabilityVerificationCoverage,
observabilityVerificationOutcomes,
preferredMode,
status: issues.length === 0 ? "ready" : "invalid",
issues,
@@ -675,6 +783,8 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
],
observabilitySignals: ["observabilitySummary:missing"],
observabilityGapCount: 1,
observabilityVerificationCoverage: [],
observabilityVerificationOutcomes: [],
preferredMode: "",
status: "invalid",
issues: [
@@ -727,6 +837,8 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
observabilityCoverage: [{ signal: "observabilitySummary", status: "missing" }],
observabilitySignals: ["observabilitySummary:missing"],
observabilityGapCount: 1,
observabilityVerificationCoverage: [],
observabilityVerificationOutcomes: [],
preferredMode: "",
status: "invalid",
issues: [`不支持的 case source: ${source || "(empty)"}`],
@@ -773,6 +885,12 @@ function buildSummary(manifest, suites, options) {
const observabilityGapCases = allCases.filter(
(entry) => entry.observabilityGapCount > 0,
);
const currentObservabilityGapCases = observabilityGapCases.filter((entry) =>
isCurrentObservabilityGapCase(entry),
);
const degradedObservabilityGapCases = observabilityGapCases.filter((entry) =>
isDegradedObservabilityGapCase(entry),
);
return {
manifestVersion: String(manifest.manifestVersion ?? "unknown"),
@@ -790,6 +908,8 @@ function buildSummary(manifest, suites, options) {
pendingRequestCaseCount: pendingCases.length,
reviewDecisionRecordedCount: recordedReviewDecisionCases.length,
observabilityGapCaseCount: observabilityGapCases.length,
currentObservabilityGapCaseCount: currentObservabilityGapCases.length,
degradedObservabilityGapCaseCount: degradedObservabilityGapCases.length,
},
breakdowns: {
suiteTags: aggregateCaseBreakdown(allCases, (entry) => entry.tags),
@@ -807,6 +927,18 @@ function buildSummary(manifest, suites, options) {
allCases,
(entry) => entry.observabilitySignals,
),
observabilityVerificationOutcomes: aggregateCaseBreakdown(
allCases,
(entry) => entry.observabilityVerificationOutcomes,
),
currentObservabilityVerificationOutcomes: aggregateCaseBreakdown(
allCases.filter((entry) => isCurrentObservabilityDiagnosticCase(entry)),
(entry) => entry.observabilityVerificationOutcomes,
),
degradedObservabilityVerificationOutcomes: aggregateCaseBreakdown(
allCases.filter((entry) => isDegradedObservabilityDiagnosticCase(entry)),
(entry) => entry.observabilityVerificationOutcomes,
),
},
suites,
};
@@ -824,6 +956,8 @@ function renderText(summary) {
`[harness-eval] needs-review cases : ${summary.totals.needsHumanReviewCount}`,
`[harness-eval] recorded review decisions: ${summary.totals.reviewDecisionRecordedCount}`,
`[harness-eval] observability-gap cases: ${summary.totals.observabilityGapCaseCount}`,
`[harness-eval] current observability-gap cases: ${summary.totals.currentObservabilityGapCaseCount}`,
`[harness-eval] degraded observability-gap cases: ${summary.totals.degradedObservabilityGapCaseCount}`,
];
const topFailureModes = summary.breakdowns.failureModes.slice(0, 5);
@@ -869,6 +1003,17 @@ function renderText(summary) {
}
}
const topVerificationOutcomes =
summary.breakdowns.observabilityVerificationOutcomes.slice(0, 5);
if (topVerificationOutcomes.length > 0) {
lines.push("[harness-eval] observability verification outcomes:");
for (const entry of topVerificationOutcomes) {
lines.push(
` - ${entry.name}: case=${entry.caseCount}, ready=${entry.readyCount}, invalid=${entry.invalidCount}`,
);
}
}
for (const suite of summary.suites) {
lines.push(
`[harness-eval] suite ${suite.id}: ready ${suite.stats.readyCount} / ${suite.stats.caseCount}`,
@@ -893,6 +1038,11 @@ function renderText(summary) {
` observability: ${entry.observabilitySignals.join(", ")}`,
);
}
if (entry.observabilityVerificationOutcomes.length > 0) {
lines.push(
` verification: ${entry.observabilityVerificationOutcomes.join(", ")}`,
);
}
for (const issue of entry.issues) {
lines.push(` * ${issue}`);
}
@@ -917,6 +1067,8 @@ function renderMarkdown(summary) {
`- needs review case:${summary.totals.needsHumanReviewCount}`,
`- 已记录人工审核:${summary.totals.reviewDecisionRecordedCount}`,
`- observability gap case:${summary.totals.observabilityGapCaseCount}`,
`- current observability gap case:${summary.totals.currentObservabilityGapCaseCount}`,
`- degraded observability gap case:${summary.totals.degradedObservabilityGapCaseCount}`,
"",
];
@@ -987,6 +1139,19 @@ function renderMarkdown(summary) {
lines.push("");
}
if (summary.breakdowns.observabilityVerificationOutcomes.length > 0) {
lines.push("## Observability Verification Outcome 分布");
lines.push("");
lines.push("| Outcome | case | ready | invalid |");
lines.push("| --- | --- | --- | --- |");
for (const entry of summary.breakdowns.observabilityVerificationOutcomes) {
lines.push(
`| ${entry.name} | ${entry.caseCount} | ${entry.readyCount} | ${entry.invalidCount} |`,
);
}
lines.push("");
}
for (const suite of summary.suites) {
lines.push(`## ${suite.title}`);
lines.push("");
@@ -1025,6 +1190,11 @@ function renderMarkdown(summary) {
`observability: ${entry.observabilitySignals.join(", ")}`,
);
}
if (entry.observabilityVerificationOutcomes.length > 0) {
classificationText.push(
`verification: ${entry.observabilityVerificationOutcomes.join(", ")}`,
);
}
const reviewText = entry.reviewDecisionStatus
? [
entry.reviewDecisionStatus,
@@ -1078,7 +1248,6 @@ function main() {
const jsonOutput = `${JSON.stringify(summary, null, 2)}\n`;
const markdownOutput = renderMarkdown(summary);
const textOutput = renderText(summary);
const historyFilePath = recordSummaryHistory(repoRoot, summary, options);
if (options.outputJson) {
const outputPath = resolvePath(repoRoot, options.outputJson);
@@ -1098,11 +1267,6 @@ function main() {
process.stdout.write(markdownOutput);
} else {
process.stdout.write(textOutput);
if (historyFilePath) {
process.stdout.write(
`[harness-eval] history snapshot: ${historyFilePath}\n`,
);
}
}
const exitCode = determineExitCode(summary, options);
+321 -41
View File
@@ -6,6 +6,20 @@ import path from "node:path";
import process from "node:process";
const RUNNER_PATH = "scripts/harness-eval-runner.mjs";
const OBSERVABILITY_GAP_SUITE_TAG = "observability-gap";
const OBSERVABILITY_FAILURE_OUTCOMES = new Set([
"artifactValidator:issues_present",
"artifactValidator:fallback_used",
"browserVerification:failure",
"browserVerification:unknown",
"guiSmoke:failed",
]);
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
"guiSmoke:clean",
]);
function parseArgs(argv) {
const result = {
@@ -170,12 +184,113 @@ function normalizeNumber(value) {
return typeof value === "number" && Number.isFinite(value) ? value : 0;
}
function listAllCases(summary) {
const suites = Array.isArray(summary?.suites) ? summary.suites : [];
return suites.flatMap((suite) => (Array.isArray(suite?.cases) ? suite.cases : []));
}
function getBreakdownEntries(summary, key) {
return Array.isArray(summary?.breakdowns?.[key]) ? summary.breakdowns[key] : [];
}
function getBreakdownCaseCount(summary, key, namesSet) {
return getBreakdownEntries(summary, key).reduce((total, entry) => {
const name = String(entry?.name ?? "");
if (!namesSet.has(name)) {
return total;
}
return total + normalizeNumber(entry?.caseCount);
}, 0);
}
function buildObservabilityGapTotals(summary) {
const rawTotals =
summary?.totals && typeof summary.totals === "object" ? summary.totals : {};
let total = normalizeNumber(rawTotals.observabilityGapCaseCount);
let current = normalizeNumber(rawTotals.currentObservabilityGapCaseCount);
let degraded = normalizeNumber(rawTotals.degradedObservabilityGapCaseCount);
const allCases = listAllCases(summary);
if (allCases.length === 0) {
if (current === 0 && degraded === 0 && total > 0) {
current = total;
}
return {
total: total || current + degraded,
current,
degraded,
};
}
let derivedCurrent = 0;
let derivedDegraded = 0;
for (const entry of allCases) {
if (normalizeNumber(entry?.observabilityGapCount) <= 0) {
continue;
}
const tags = Array.isArray(entry?.tags) ? entry.tags : [];
if (tags.includes(OBSERVABILITY_GAP_SUITE_TAG)) {
derivedDegraded += 1;
} else {
derivedCurrent += 1;
}
}
const derivedTotal = derivedCurrent + derivedDegraded;
if (derivedTotal === 0) {
return {
total: total || current + degraded,
current,
degraded,
};
}
return {
total: derivedTotal,
current: derivedCurrent,
degraded: derivedDegraded,
};
}
function buildNormalizedTotals(summary) {
const rawTotals =
summary?.totals && typeof summary.totals === "object" ? summary.totals : {};
const gapTotals = buildObservabilityGapTotals(summary);
const currentVerificationOutcomeEntries = getBreakdownEntries(
summary,
"currentObservabilityVerificationOutcomes",
);
return {
suiteCount: normalizeNumber(rawTotals.suiteCount),
caseCount: normalizeNumber(rawTotals.caseCount),
readyCount: normalizeNumber(rawTotals.readyCount),
invalidCount: normalizeNumber(rawTotals.invalidCount),
pendingRequestCaseCount: normalizeNumber(rawTotals.pendingRequestCaseCount),
needsHumanReviewCount: normalizeNumber(rawTotals.needsHumanReviewCount),
reviewDecisionRecordedCount: normalizeNumber(
rawTotals.reviewDecisionRecordedCount,
),
observabilityGapCaseCount: gapTotals.total,
currentObservabilityGapCaseCount: gapTotals.current,
degradedObservabilityGapCaseCount: gapTotals.degraded,
currentRecoveredVerificationCaseCount:
currentVerificationOutcomeEntries.length > 0
? getBreakdownCaseCount(
summary,
"currentObservabilityVerificationOutcomes",
RECOVERED_VERIFICATION_OUTCOMES,
)
: normalizeNumber(rawTotals.currentRecoveredVerificationCaseCount),
};
}
function computeReadyRate(summary) {
const caseCount = normalizeNumber(summary?.totals?.caseCount);
const totals = buildNormalizedTotals(summary);
const caseCount = totals.caseCount;
if (caseCount <= 0) {
return 0;
}
return normalizeNumber(summary?.totals?.readyCount) / caseCount;
return totals.readyCount / caseCount;
}
function getSuiteMap(summary) {
@@ -195,9 +310,7 @@ function getSuiteMap(summary) {
}
function getBreakdownMap(summary, key) {
const entries = Array.isArray(summary?.breakdowns?.[key])
? summary.breakdowns[key]
: [];
const entries = getBreakdownEntries(summary, key);
return new Map(
entries.map((entry) => [
String(entry.name ?? ""),
@@ -309,6 +422,12 @@ function buildBreakdownDeltas(baseline, latest, key) {
});
}
function buildFilteredBreakdownDeltas(baseline, latest, key, predicate) {
return buildBreakdownDeltas(baseline, latest, key).filter((entry) =>
predicate(entry),
);
}
function buildStatusSignals(baseline, latest, sampleCount) {
const signals = [];
@@ -318,15 +437,17 @@ function buildStatusSignals(baseline, latest, sampleCount) {
}
const readyRateDelta = computeReadyRate(latest) - computeReadyRate(baseline);
const invalidDelta =
normalizeNumber(latest?.totals?.invalidCount) -
normalizeNumber(baseline?.totals?.invalidCount);
const baselineTotals = buildNormalizedTotals(baseline);
const latestTotals = buildNormalizedTotals(latest);
const invalidDelta = latestTotals.invalidCount - baselineTotals.invalidCount;
const pendingDelta =
normalizeNumber(latest?.totals?.pendingRequestCaseCount) -
normalizeNumber(baseline?.totals?.pendingRequestCaseCount);
const observabilityGapDelta =
normalizeNumber(latest?.totals?.observabilityGapCaseCount) -
normalizeNumber(baseline?.totals?.observabilityGapCaseCount);
latestTotals.pendingRequestCaseCount - baselineTotals.pendingRequestCaseCount;
const currentObservabilityGapDelta =
latestTotals.currentObservabilityGapCaseCount -
baselineTotals.currentObservabilityGapCaseCount;
const currentRecoveredVerificationDelta =
latestTotals.currentRecoveredVerificationCaseCount -
baselineTotals.currentRecoveredVerificationCaseCount;
if (invalidDelta > 0) {
signals.push(`invalid case 增加 ${invalidDelta},存在回归候选。`);
@@ -344,9 +465,15 @@ function buildStatusSignals(baseline, latest, sampleCount) {
);
}
if (observabilityGapDelta > 0) {
if (currentObservabilityGapDelta > 0) {
signals.push(
`observability gap case 增加 ${observabilityGapDelta},需先补 evidence / analysis / replay 的证据覆盖。`,
`current observability gap case 增加 ${currentObservabilityGapDelta},说明主线样本开始带缺口,需先补 evidence / analysis / replay 的证据覆盖。`,
);
}
if (currentRecoveredVerificationDelta > 0) {
signals.push(
`current recovered verification case 增加 ${currentRecoveredVerificationDelta},说明主线路径正在累积正向守卫。`,
);
}
@@ -364,6 +491,43 @@ function buildStatusSignals(baseline, latest, sampleCount) {
);
}
const verificationOutcomeDeltas = buildBreakdownDeltas(
baseline,
latest,
"observabilityVerificationOutcomes",
);
const increasedVerificationFailure = verificationOutcomeDeltas.find(
(entry) =>
OBSERVABILITY_FAILURE_OUTCOMES.has(entry.name) &&
entry.delta.caseCount > 0,
);
if (increasedVerificationFailure) {
signals.push(
`verification outcome \`${increasedVerificationFailure.name}\` 新增 ${increasedVerificationFailure.delta.caseCount} 个 case。`,
);
}
const currentRecoveredVerificationDeltas = buildFilteredBreakdownDeltas(
baseline,
latest,
"currentObservabilityVerificationOutcomes",
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
);
for (const entry of currentRecoveredVerificationDeltas.filter(
(candidate) => candidate.delta.caseCount < 0,
)) {
signals.push(
`current recovered verification baseline \`${entry.name}\` 减少 ${Math.abs(entry.delta.caseCount)},说明正向守卫可能回退。`,
);
}
for (const entry of currentRecoveredVerificationDeltas.filter(
(candidate) => candidate.delta.caseCount > 0,
)) {
signals.push(
`current recovered verification baseline \`${entry.name}\` 新增 ${entry.delta.caseCount},说明主线路径正在形成正向基线。`,
);
}
if (signals.length === 0) {
signals.push("当前没有检测到明显退化信号。");
}
@@ -389,6 +553,8 @@ function buildTrendReport(samples, repoRoot) {
const latestEntry = sortedSamples[sortedSamples.length - 1];
const baseline = baselineEntry.summary;
const latest = latestEntry.summary;
const baselineTotals = buildNormalizedTotals(baseline);
const latestTotals = buildNormalizedTotals(latest);
const readyRateDelta = computeReadyRate(latest) - computeReadyRate(baseline);
return {
@@ -399,45 +565,46 @@ function buildTrendReport(samples, repoRoot) {
baseline: {
generatedAt: baseline.generatedAt,
sourcePath: baselineEntry.sourcePath,
totals: baseline.totals,
totals: baselineTotals,
},
latest: {
generatedAt: latest.generatedAt,
sourcePath: latestEntry.sourcePath,
totals: latest.totals,
totals: latestTotals,
},
delta: {
suiteCount:
normalizeNumber(latest?.totals?.suiteCount) -
normalizeNumber(baseline?.totals?.suiteCount),
caseCount:
normalizeNumber(latest?.totals?.caseCount) -
normalizeNumber(baseline?.totals?.caseCount),
readyCount:
normalizeNumber(latest?.totals?.readyCount) -
normalizeNumber(baseline?.totals?.readyCount),
invalidCount:
normalizeNumber(latest?.totals?.invalidCount) -
normalizeNumber(baseline?.totals?.invalidCount),
suiteCount: latestTotals.suiteCount - baselineTotals.suiteCount,
caseCount: latestTotals.caseCount - baselineTotals.caseCount,
readyCount: latestTotals.readyCount - baselineTotals.readyCount,
invalidCount: latestTotals.invalidCount - baselineTotals.invalidCount,
pendingRequestCaseCount:
normalizeNumber(latest?.totals?.pendingRequestCaseCount) -
normalizeNumber(baseline?.totals?.pendingRequestCaseCount),
latestTotals.pendingRequestCaseCount -
baselineTotals.pendingRequestCaseCount,
needsHumanReviewCount:
normalizeNumber(latest?.totals?.needsHumanReviewCount) -
normalizeNumber(baseline?.totals?.needsHumanReviewCount),
latestTotals.needsHumanReviewCount -
baselineTotals.needsHumanReviewCount,
reviewDecisionRecordedCount:
normalizeNumber(latest?.totals?.reviewDecisionRecordedCount) -
normalizeNumber(baseline?.totals?.reviewDecisionRecordedCount),
latestTotals.reviewDecisionRecordedCount -
baselineTotals.reviewDecisionRecordedCount,
observabilityGapCaseCount:
normalizeNumber(latest?.totals?.observabilityGapCaseCount) -
normalizeNumber(baseline?.totals?.observabilityGapCaseCount),
latestTotals.observabilityGapCaseCount -
baselineTotals.observabilityGapCaseCount,
currentObservabilityGapCaseCount:
latestTotals.currentObservabilityGapCaseCount -
baselineTotals.currentObservabilityGapCaseCount,
degradedObservabilityGapCaseCount:
latestTotals.degradedObservabilityGapCaseCount -
baselineTotals.degradedObservabilityGapCaseCount,
currentRecoveredVerificationCaseCount:
latestTotals.currentRecoveredVerificationCaseCount -
baselineTotals.currentRecoveredVerificationCaseCount,
readyRate: readyRateDelta,
},
signals: buildStatusSignals(baseline, latest, sortedSamples.length),
samples: sortedSamples.map((entry) => ({
generatedAt: entry.summary.generatedAt,
sourcePath: entry.sourcePath,
totals: entry.summary.totals,
totals: buildNormalizedTotals(entry.summary),
})),
suiteDeltas: buildSuiteDeltas(baseline, latest),
classificationDeltas: {
@@ -458,6 +625,28 @@ function buildTrendReport(samples, repoRoot) {
latest,
"observabilitySignals",
),
observabilityVerificationOutcomes: buildBreakdownDeltas(
baseline,
latest,
"observabilityVerificationOutcomes",
),
currentRecoveredObservabilityVerificationOutcomes:
buildFilteredBreakdownDeltas(
baseline,
latest,
"currentObservabilityVerificationOutcomes",
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
),
currentObservabilityVerificationOutcomes: buildBreakdownDeltas(
baseline,
latest,
"currentObservabilityVerificationOutcomes",
),
degradedObservabilityVerificationOutcomes: buildBreakdownDeltas(
baseline,
latest,
"degradedObservabilityVerificationOutcomes",
),
},
};
}
@@ -473,6 +662,12 @@ function renderText(report) {
`[harness-eval-trend] delta pendingRequestCaseCount: ${report.delta.pendingRequestCaseCount}`,
`[harness-eval-trend] delta reviewDecisionRecordedCount: ${report.delta.reviewDecisionRecordedCount}`,
`[harness-eval-trend] delta observabilityGapCaseCount: ${report.delta.observabilityGapCaseCount}`,
`[harness-eval-trend] delta currentObservabilityGapCaseCount: ${report.delta.currentObservabilityGapCaseCount}`,
`[harness-eval-trend] delta degradedObservabilityGapCaseCount: ${report.delta.degradedObservabilityGapCaseCount}`,
`[harness-eval-trend] delta currentRecoveredVerificationCaseCount: ${report.delta.currentRecoveredVerificationCaseCount}`,
`[harness-eval-trend] latest currentObservabilityGapCaseCount: ${report.latest.totals.currentObservabilityGapCaseCount}`,
`[harness-eval-trend] latest degradedObservabilityGapCaseCount: ${report.latest.totals.degradedObservabilityGapCaseCount}`,
`[harness-eval-trend] latest currentRecoveredVerificationCaseCount: ${report.latest.totals.currentRecoveredVerificationCaseCount}`,
`[harness-eval-trend] delta readyRate: ${(report.delta.readyRate * 100).toFixed(1)}%`,
];
@@ -515,6 +710,42 @@ function renderText(report) {
}
}
const topVerificationOutcomeDeltas =
report.classificationDeltas.observabilityVerificationOutcomes.slice(0, 5);
if (topVerificationOutcomeDeltas.length > 0) {
lines.push("[harness-eval-trend] observability verification outcome deltas:");
for (const entry of topVerificationOutcomeDeltas) {
lines.push(
` - ${entry.name}: delta_case=${entry.delta.caseCount}, latest_case=${entry.latest.caseCount}, latest_invalid=${entry.latest.invalidCount}`,
);
}
}
const topCurrentRecoveredVerificationDeltas =
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes.slice(
0,
5,
);
if (topCurrentRecoveredVerificationDeltas.length > 0) {
lines.push("[harness-eval-trend] current recovered verification baseline deltas:");
for (const entry of topCurrentRecoveredVerificationDeltas) {
lines.push(
` - ${entry.name}: baseline_case=${entry.baseline.caseCount}, latest_case=${entry.latest.caseCount}, delta_case=${entry.delta.caseCount}`,
);
}
}
lines.push("[harness-eval-trend] observability gap roles:");
lines.push(
` - total: baseline=${report.baseline.totals.observabilityGapCaseCount}, latest=${report.latest.totals.observabilityGapCaseCount}, delta=${report.delta.observabilityGapCaseCount}`,
);
lines.push(
` - current: baseline=${report.baseline.totals.currentObservabilityGapCaseCount}, latest=${report.latest.totals.currentObservabilityGapCaseCount}, delta=${report.delta.currentObservabilityGapCaseCount}`,
);
lines.push(
` - degraded: baseline=${report.baseline.totals.degradedObservabilityGapCaseCount}, latest=${report.latest.totals.degradedObservabilityGapCaseCount}, delta=${report.delta.degradedObservabilityGapCaseCount}`,
);
return `${lines.join("\n")}\n`;
}
@@ -537,6 +768,9 @@ function renderMarkdown(report) {
`- needs review case 变化:${report.delta.needsHumanReviewCount}`,
`- 已记录人工审核变化:${report.delta.reviewDecisionRecordedCount}`,
`- observability gap case 变化:${report.delta.observabilityGapCaseCount}`,
`- current observability gap case 变化:${report.delta.currentObservabilityGapCaseCount}`,
`- degraded observability gap case 变化:${report.delta.degradedObservabilityGapCaseCount}`,
`- current recovered verification case 变化:${report.delta.currentRecoveredVerificationCaseCount}`,
`- ready rate 变化:${(report.delta.readyRate * 100).toFixed(1)}%`,
"",
"## 信号",
@@ -616,14 +850,60 @@ function renderMarkdown(report) {
}
}
if (report.classificationDeltas.observabilityVerificationOutcomes.length > 0) {
lines.push("");
lines.push("## Observability Verification Outcome 变化");
lines.push("");
lines.push("| Outcome | baseline case | latest case | delta case | delta invalid |");
lines.push("| --- | --- | --- | --- | --- |");
for (const entry of report.classificationDeltas.observabilityVerificationOutcomes) {
lines.push(
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} | ${entry.delta.invalidCount} |`,
);
}
}
if (
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes
.length > 0
) {
lines.push("");
lines.push("## Current Recovered Baseline 变化");
lines.push("");
lines.push("| Outcome | baseline case | latest case | delta case |");
lines.push("| --- | --- | --- | --- |");
for (const entry of report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes) {
lines.push(
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} |`,
);
}
}
lines.push("");
lines.push("## Observability Gap 角色变化");
lines.push("");
lines.push("| 角色 | baseline case | latest case | delta case |");
lines.push("| --- | --- | --- | --- |");
lines.push(
`| total | ${report.baseline.totals.observabilityGapCaseCount} | ${report.latest.totals.observabilityGapCaseCount} | ${report.delta.observabilityGapCaseCount} |`,
);
lines.push(
`| current | ${report.baseline.totals.currentObservabilityGapCaseCount} | ${report.latest.totals.currentObservabilityGapCaseCount} | ${report.delta.currentObservabilityGapCaseCount} |`,
);
lines.push(
`| degraded | ${report.baseline.totals.degradedObservabilityGapCaseCount} | ${report.latest.totals.degradedObservabilityGapCaseCount} | ${report.delta.degradedObservabilityGapCaseCount} |`,
);
lines.push("");
lines.push("## 时间线样本");
lines.push("");
lines.push("| 时间 | 来源 | case | ready | invalid | pending_request |");
lines.push("| --- | --- | --- | --- | --- | --- |");
lines.push(
"| 时间 | 来源 | case | ready | invalid | pending_request | current_gap | degraded_gap |",
);
lines.push("| --- | --- | --- | --- | --- | --- | --- | --- |");
for (const sample of report.samples) {
lines.push(
`| ${sample.generatedAt} | \`${sample.sourcePath}\` | ${sample.totals.caseCount} | ${sample.totals.readyCount} | ${sample.totals.invalidCount} | ${sample.totals.pendingRequestCaseCount} |`,
`| ${sample.generatedAt} | \`${sample.sourcePath}\` | ${sample.totals.caseCount} | ${sample.totals.readyCount} | ${sample.totals.invalidCount} | ${sample.totals.pendingRequestCaseCount} | ${sample.totals.currentObservabilityGapCaseCount} | ${sample.totals.degradedObservabilityGapCaseCount} |`,
);
}
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+765
View File
@@ -0,0 +1,765 @@
function normalizeNumber(value) {
return typeof value === "number" && Number.isFinite(value) ? value : 0;
}
function normalizeString(value, fallback = "") {
return typeof value === "string" && value.trim().length > 0
? value.trim()
: fallback;
}
function escapeHtml(value) {
return String(value)
.replaceAll("&", "&amp;")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;")
.replaceAll("'", "&#39;");
}
function formatTimestamp(value) {
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime())) {
return normalizeString(value, "-");
}
return parsed.toLocaleString("zh-CN", {
hour12: false,
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
});
}
function renderStatCard(label, value, tone = "neutral") {
return `
<article class="stat-card ${tone}">
<span>${escapeHtml(label)}</span>
<strong>${escapeHtml(String(value))}</strong>
</article>
`;
}
function renderSignalList(signals) {
const items = Array.isArray(signals) ? signals.filter(Boolean) : [];
if (items.length === 0) {
return `<p class="empty">当前没有额外信号。</p>`;
}
return `
<ul class="signal-list">
${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}
</ul>
`;
}
function renderRecommendationList(recommendations) {
const items = Array.isArray(recommendations) ? recommendations : [];
if (items.length === 0) {
return `<p class="empty">当前没有新的治理建议。</p>`;
}
return items
.map(
(entry) => {
const focusVerificationFailureOutcomes = Array.isArray(
entry.focusVerificationFailureOutcomes,
)
? entry.focusVerificationFailureOutcomes
: [];
const focusVerificationRecoveredOutcomes = Array.isArray(
entry.focusVerificationRecoveredOutcomes,
)
? entry.focusVerificationRecoveredOutcomes
: [];
return `
<article class="recommendation-card">
<div class="recommendation-head">
<span class="priority">${escapeHtml(normalizeString(entry.priority, "P?"))}</span>
<h3>${escapeHtml(normalizeString(entry.title, "未命名建议"))}</h3>
</div>
<p>${escapeHtml(
Array.isArray(entry.rationale) ? entry.rationale.join(" ") : "",
)}</p>
${
focusVerificationFailureOutcomes.length > 0
? `
<p class="recommendation-meta">
<strong>关注 failure outcome:</strong>${escapeHtml(
focusVerificationFailureOutcomes.join("、"),
)}
</p>
`
: ""
}
${
focusVerificationRecoveredOutcomes.length > 0
? `
<p class="recommendation-meta">
<strong>关注 recovered outcome:</strong>${escapeHtml(
focusVerificationRecoveredOutcomes.join("、"),
)}
</p>
`
: ""
}
${
Array.isArray(entry.backlogTools) && entry.backlogTools.length > 0
? `
<div class="recommendation-subsection">
<strong>后续动作</strong>
<ul class="backlog-list">
${entry.backlogTools
.map((item) => `<li>${escapeHtml(item)}</li>`)
.join("")}
</ul>
</div>
`
: ""
}
${
Array.isArray(entry.commands) && entry.commands.length > 0
? `
<div class="recommendation-subsection">
<strong>推荐命令</strong>
<div class="command-list">
${entry.commands
.map((command) => `<code>${escapeHtml(command)}</code>`)
.join("")}
</div>
</div>
`
: ""
}
</article>
`;
},
)
.join("");
}
function describeVerificationOutcome(entry) {
const signal = normalizeString(entry?.signal, "unknown");
const outcome = normalizeString(entry?.outcome, "unknown");
if (signal === "artifactValidator" && outcome === "issues_present") {
return "当前 evidence 已记录 artifact 校验问题,优先回看 validator issue 明细。";
}
if (signal === "artifactValidator" && outcome === "fallback_used") {
return "当前 artifact 导出仍触发 fallback,说明产物结构或修复链未完全稳定。";
}
if (signal === "browserVerification" && outcome === "failure") {
return "浏览器验证已有明确失败结果,优先回挂到 replay 或 smoke 断言。";
}
if (signal === "browserVerification" && outcome === "unknown") {
return "浏览器验证结果仍不明确,需要先补 outcome 再继续扩分析。";
}
if (signal === "guiSmoke" && outcome === "failed") {
return "GUI smoke 已明确失败,应优先收敛到受影响主路径。";
}
if (signal === "guiSmoke" && outcome === "passed") {
return "GUI smoke 已通过,可继续把注意力放回 gap 与其它失败面。";
}
if (signal === "artifactValidator" && outcome === "repaired") {
return "artifact validator 已执行修复,可结合 issues/fallback 判断是否还需继续治理。";
}
if (signal === "browserVerification" && outcome === "success") {
return "浏览器验证已有成功样本,可作为 current 主线路径的正向基线。";
}
return "当前 verification outcome 已进入 cleanup 主线,可直接据此定位先修哪层。";
}
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
"repaired",
"success",
"passed",
"clean",
]);
function isRecoveredVerificationOutcome(entry) {
return RECOVERED_VERIFICATION_OUTCOMES.has(
normalizeString(entry?.outcome, "unknown"),
);
}
function renderFocusTable(title, entries, columns) {
const rows = Array.isArray(entries) ? entries : [];
if (rows.length === 0) {
return `
<section class="panel">
<div class="section-header">
<h2>${escapeHtml(title)}</h2>
</div>
<p class="empty">当前没有数据。</p>
</section>
`;
}
return `
<section class="panel">
<div class="section-header">
<h2>${escapeHtml(title)}</h2>
</div>
<div class="table-wrap">
<table>
<thead>
<tr>
${columns.map((column) => `<th>${escapeHtml(column.label)}</th>`).join("")}
</tr>
</thead>
<tbody>
${rows
.map(
(entry) => `
<tr>
${columns
.map((column) => `<td>${escapeHtml(String(column.render(entry)))}</td>`)
.join("")}
</tr>
`,
)
.join("")}
</tbody>
</table>
</div>
</section>
`;
}
export function renderHarnessDashboardHtml({
summaryReport,
trendReport,
cleanupReport,
title = "Lime Harness Dashboard",
}) {
const summaryTotals =
summaryReport && typeof summaryReport === "object" && summaryReport.totals
? summaryReport.totals
: {};
const trendSummary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.trend
? cleanupReport.summary.trend
: {};
const governanceSummary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.governance
? cleanupReport.summary.governance
: {};
const verificationSummary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.verificationOutcomes
? cleanupReport.summary.verificationOutcomes
: {};
const currentVerificationSummary =
verificationSummary &&
typeof verificationSummary.current === "object" &&
!Array.isArray(verificationSummary.current)
? verificationSummary.current
: {};
const degradedVerificationSummary =
verificationSummary &&
typeof verificationSummary.degraded === "object" &&
!Array.isArray(verificationSummary.degraded)
? verificationSummary.degraded
: {};
const trendSignals = Array.isArray(trendReport?.signals) ? trendReport.signals : [];
const cleanupSignals = Array.isArray(cleanupReport?.signals)
? cleanupReport.signals
: [];
const recommendations = Array.isArray(cleanupReport?.recommendations)
? cleanupReport.recommendations
: [];
const sampleRows = Array.isArray(trendReport?.samples) ? trendReport.samples : [];
const currentVerificationFocusRows = Array.isArray(
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentObservabilityVerificationOutcomes.map((entry) => ({
...entry,
role: "current",
}))
: [];
const degradedVerificationFocusRows = Array.isArray(
cleanupReport?.focus?.degradedObservabilityVerificationOutcomes,
)
? cleanupReport.focus.degradedObservabilityVerificationOutcomes.map(
(entry) => ({
...entry,
role: "degraded",
}),
)
: [];
const fallbackVerificationFocusRows = Array.isArray(
cleanupReport?.focus?.observabilityVerificationOutcomes,
)
? cleanupReport.focus.observabilityVerificationOutcomes.map((entry) => ({
...entry,
role: "mixed",
}))
: [];
const explicitCurrentRecoveredVerificationRows = Array.isArray(
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes.map(
(entry) => ({
...entry,
role: "current",
}),
)
: [];
const verificationFocusRows =
currentVerificationFocusRows.length > 0 ||
degradedVerificationFocusRows.length > 0
? [...currentVerificationFocusRows, ...degradedVerificationFocusRows]
: fallbackVerificationFocusRows;
const currentRecoveredVerificationRows =
explicitCurrentRecoveredVerificationRows.length > 0
? explicitCurrentRecoveredVerificationRows
: currentVerificationFocusRows.length > 0
? currentVerificationFocusRows.filter((entry) =>
isRecoveredVerificationOutcome(entry),
)
: fallbackVerificationFocusRows.filter((entry) =>
isRecoveredVerificationOutcome(entry),
);
const currentRecoveredVerificationSummary = currentRecoveredVerificationRows
.slice(0, 3)
.map(
(entry) =>
`${normalizeString(entry?.signal, "-")} (${normalizeString(entry?.outcome, "-")})`,
)
.join("、");
return `<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>${escapeHtml(title)}</title>
<style>
:root {
color-scheme: light;
--bg: #f3f6f4;
--panel: #ffffff;
--panel-muted: #f8faf8;
--border: #d8e1dc;
--text: #112118;
--muted: #5f6d64;
--accent: #0f4c5c;
--success: #177245;
--warning: #b45309;
--danger: #b42318;
--shadow: 0 16px 32px rgba(17, 33, 24, 0.08);
}
* { box-sizing: border-box; }
body {
margin: 0;
font-family:
"SF Pro Display",
"Segoe UI",
"PingFang SC",
"Hiragino Sans GB",
"Microsoft YaHei",
sans-serif;
color: var(--text);
background:
radial-gradient(circle at top left, rgba(15, 76, 92, 0.12), transparent 32%),
radial-gradient(circle at top right, rgba(23, 114, 69, 0.1), transparent 28%),
var(--bg);
}
.page {
max-width: 1440px;
margin: 0 auto;
padding: 28px 22px 42px;
}
.hero,
.panel {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 24px;
box-shadow: var(--shadow);
}
.hero {
padding: 24px 26px;
display: grid;
grid-template-columns: minmax(0, 1.5fr) minmax(280px, 0.9fr);
gap: 18px;
}
.eyebrow {
display: inline-flex;
align-items: center;
padding: 6px 12px;
border-radius: 999px;
background: #e8f7ef;
color: var(--success);
font-size: 13px;
font-weight: 700;
}
h1, h2, h3, p {
margin: 0;
}
h1 {
margin-top: 14px;
font-size: 32px;
line-height: 1.12;
}
.hero p {
margin-top: 12px;
color: var(--muted);
line-height: 1.7;
}
.meta-grid,
.stat-grid,
.panel-grid {
display: grid;
gap: 14px;
}
.meta-grid {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.meta-card,
.stat-card {
border: 1px solid var(--border);
border-radius: 18px;
background: var(--panel-muted);
padding: 14px 16px;
}
.meta-card span,
.stat-card span {
display: block;
font-size: 12px;
color: var(--muted);
margin-bottom: 8px;
}
.meta-card strong,
.stat-card strong {
font-size: 20px;
}
.stat-grid {
margin-top: 18px;
grid-template-columns: repeat(6, minmax(0, 1fr));
}
.stat-card.warning strong { color: var(--warning); }
.stat-card.danger strong { color: var(--danger); }
.stat-card.success strong { color: var(--success); }
.panel-grid {
margin-top: 20px;
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.panel {
padding: 20px 22px;
}
.section-header {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 12px;
margin-bottom: 14px;
}
.section-header p {
color: var(--muted);
line-height: 1.6;
}
.signal-list {
margin: 0;
padding-left: 18px;
display: grid;
gap: 10px;
color: var(--muted);
}
.empty {
color: var(--muted);
line-height: 1.6;
}
.table-wrap {
overflow-x: auto;
}
table {
width: 100%;
border-collapse: collapse;
font-size: 14px;
}
th, td {
padding: 10px 12px;
border-bottom: 1px solid var(--border);
text-align: left;
vertical-align: top;
}
th {
font-size: 12px;
color: var(--muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
tr:last-child td {
border-bottom: none;
}
.recommendation-stack {
display: grid;
gap: 12px;
}
.recommendation-card {
border: 1px solid var(--border);
border-radius: 18px;
padding: 16px;
background: var(--panel-muted);
}
.recommendation-head {
display: flex;
gap: 10px;
align-items: center;
margin-bottom: 10px;
}
.priority {
display: inline-flex;
align-items: center;
justify-content: center;
min-width: 44px;
height: 28px;
padding: 0 10px;
border-radius: 999px;
background: #fff3e8;
color: var(--warning);
font-size: 12px;
font-weight: 700;
}
.recommendation-card p {
color: var(--muted);
line-height: 1.7;
}
.recommendation-meta {
margin-top: 12px;
font-size: 14px;
}
.recommendation-subsection {
margin-top: 14px;
}
.recommendation-subsection strong {
display: block;
margin-bottom: 8px;
font-size: 13px;
}
.backlog-list {
margin: 0;
padding-left: 18px;
display: grid;
gap: 8px;
color: var(--muted);
}
.command-list {
display: flex;
flex-wrap: wrap;
gap: 10px;
}
code {
display: inline-flex;
padding: 8px 10px;
border-radius: 10px;
background: #102a2c;
color: #ecfeff;
font-size: 12px;
white-space: pre-wrap;
}
@media (max-width: 1120px) {
.hero,
.panel-grid {
grid-template-columns: 1fr;
}
.stat-grid {
grid-template-columns: repeat(3, minmax(0, 1fr));
}
}
@media (max-width: 720px) {
.page {
padding: 18px 14px 30px;
}
.meta-grid,
.stat-grid {
grid-template-columns: 1fr 1fr;
}
}
</style>
</head>
<body>
<main class="page">
<section class="hero">
<div>
<span class="eyebrow">Harness Engine Nightly Dashboard</span>
<h1>${escapeHtml(title)}</h1>
<p>同一份 evidence-first 主线下的 summary、trend、cleanup 视图。当前面板会明确区分主线 current 缺口与故意保留的 degraded 诊断基线,避免治理优先级被误导。</p>
<div class="stat-grid">
${renderStatCard("Ready Case", normalizeNumber(summaryTotals.readyCount), "success")}
${renderStatCard("Invalid Case", normalizeNumber(summaryTotals.invalidCount), normalizeNumber(summaryTotals.invalidCount) > 0 ? "danger" : "neutral")}
${renderStatCard("Current Gap", normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount), normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount) > 0 ? "danger" : "success")}
${renderStatCard("Degraded Gap", normalizeNumber(trendSummary.latestDegradedObservabilityGapCaseCount), "warning")}
${renderStatCard("Current Blocking", normalizeNumber(currentVerificationSummary.blockingFailureCaseCount), normalizeNumber(currentVerificationSummary.blockingFailureCaseCount) > 0 ? "danger" : "success")}
${renderStatCard("Current Advisory", normalizeNumber(currentVerificationSummary.advisoryFailureCaseCount), normalizeNumber(currentVerificationSummary.advisoryFailureCaseCount) > 0 ? "warning" : "neutral")}
${renderStatCard("Current Recovered", normalizeNumber(currentVerificationSummary.recoveredCaseCount), normalizeNumber(currentVerificationSummary.recoveredCaseCount) > 0 ? "success" : "neutral")}
${renderStatCard("Degraded Blocking", normalizeNumber(degradedVerificationSummary.blockingFailureCaseCount), normalizeNumber(degradedVerificationSummary.blockingFailureCaseCount) > 0 ? "warning" : "neutral")}
${renderStatCard("Recovered Outcomes", normalizeNumber(verificationSummary.recoveredCaseCount), normalizeNumber(verificationSummary.recoveredCaseCount) > 0 ? "success" : "neutral")}
${renderStatCard("Trend Samples", normalizeNumber(trendSummary.sampleCount))}
${renderStatCard("Governance Violations", normalizeNumber(governanceSummary.violationCount), normalizeNumber(governanceSummary.violationCount) > 0 ? "danger" : "neutral")}
</div>
</div>
<div class="meta-grid">
<div class="meta-card">
<span>Summary 生成时间</span>
<strong>${escapeHtml(formatTimestamp(summaryReport?.generatedAt))}</strong>
</div>
<div class="meta-card">
<span>Trend 生成时间</span>
<strong>${escapeHtml(formatTimestamp(trendReport?.generatedAt))}</strong>
</div>
<div class="meta-card">
<span>Cleanup 生成时间</span>
<strong>${escapeHtml(formatTimestamp(cleanupReport?.generatedAt))}</strong>
</div>
<div class="meta-card">
<span>当前事实源</span>
<strong>summary - history - trend - cleanup</strong>
</div>
<div class="meta-card">
<span>当前 Recovered 基线</span>
<strong>${escapeHtml(
currentRecoveredVerificationSummary ||
"当前没有额外的 recovered baseline",
)}</strong>
</div>
</div>
</section>
<section class="panel-grid">
<section class="panel">
<div class="section-header">
<div>
<h2>Trend 信号</h2>
<p>只把 current gap 当主线风险,degraded gap 仅作为诊断基线保留。</p>
</div>
</div>
${renderSignalList(trendSignals)}
</section>
<section class="panel">
<div class="section-header">
<div>
<h2>Cleanup 信号</h2>
<p>nightly 治理建议与当前风险摘要。</p>
</div>
</div>
${renderSignalList(cleanupSignals)}
</section>
</section>
${renderFocusTable("Observability Gap 角色", [
{
role: "current",
latest: normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount),
delta: normalizeNumber(trendSummary.currentObservabilityGapCaseDelta),
meaning: "主线样本里的证据缺口,必须优先治理。",
},
{
role: "degraded",
latest: normalizeNumber(trendSummary.latestDegradedObservabilityGapCaseCount),
delta: normalizeNumber(trendSummary.degradedObservabilityGapCaseDelta),
meaning: "刻意保留的诊断基线,不应直接抬高主线优先级。",
},
], [
{ label: "角色", render: (entry) => entry.role },
{ label: "latest case", render: (entry) => entry.latest },
{ label: "delta case", render: (entry) => entry.delta },
{ label: "说明", render: (entry) => entry.meaning },
])}
${renderFocusTable("Verification Outcome 焦点", verificationFocusRows, [
{ label: "Role", render: (entry) => normalizeString(entry.role, "-") },
{ label: "Signal", render: (entry) => normalizeString(entry.signal, "-") },
{ label: "Outcome", render: (entry) => normalizeString(entry.outcome, "-") },
{ label: "State", render: (entry) => normalizeString(entry.state, "-") },
{ label: "Latest Case", render: (entry) => normalizeNumber(entry?.latest?.caseCount) },
{ label: "Delta Case", render: (entry) => normalizeNumber(entry?.delta?.caseCount) },
{ label: "说明", render: (entry) => describeVerificationOutcome(entry) },
])}
${renderFocusTable("Current Recovered Baseline", currentRecoveredVerificationRows, [
{ label: "Signal", render: (entry) => normalizeString(entry.signal, "-") },
{ label: "Outcome", render: (entry) => normalizeString(entry.outcome, "-") },
{ label: "Latest Case", render: (entry) => normalizeNumber(entry?.latest?.caseCount) },
{ label: "Delta Case", render: (entry) => normalizeNumber(entry?.delta?.caseCount) },
{ label: "说明", render: (entry) => describeVerificationOutcome(entry) },
])}
${renderFocusTable("历史窗口样本", sampleRows, [
{ label: "时间", render: (entry) => formatTimestamp(entry.generatedAt) },
{ label: "来源", render: (entry) => normalizeString(entry.sourcePath, "-") },
{ label: "case", render: (entry) => normalizeNumber(entry?.totals?.caseCount) },
{ label: "ready", render: (entry) => normalizeNumber(entry?.totals?.readyCount) },
{ label: "invalid", render: (entry) => normalizeNumber(entry?.totals?.invalidCount) },
{ label: "current gap", render: (entry) => normalizeNumber(entry?.totals?.currentObservabilityGapCaseCount) },
{ label: "degraded gap", render: (entry) => normalizeNumber(entry?.totals?.degradedObservabilityGapCaseCount) },
])}
<section class="panel">
<div class="section-header">
<div>
<h2>Cleanup 建议</h2>
<p>当前 nightly 建议按已有优先级排序展示,直接复用 cleanup 报告事实源。</p>
</div>
</div>
<div class="recommendation-stack">
${renderRecommendationList(recommendations)}
</div>
</section>
</main>
</body>
</html>`;
}
+184
View File
@@ -0,0 +1,184 @@
import { describe, expect, it } from "vitest";
import { renderHarnessDashboardHtml } from "./harness-dashboard-core.mjs";
describe("harness-dashboard-core", () => {
it("应把 summary、trend、cleanup 渲染成单一事实源 dashboard", () => {
const html = renderHarnessDashboardHtml({
title: "Harness Engine Dashboard",
summaryReport: {
generatedAt: "2026-04-12T08:00:00.000Z",
totals: {
readyCount: 2,
invalidCount: 1,
},
},
trendReport: {
generatedAt: "2026-04-12T08:01:00.000Z",
signals: ["current gap 保持为 0。"],
samples: [
{
generatedAt: "2026-04-12T08:00:00.000Z",
sourcePath: "/tmp/history/summary.json",
totals: {
caseCount: 2,
readyCount: 2,
invalidCount: 0,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 1,
},
},
],
},
cleanupReport: {
generatedAt: "2026-04-12T08:02:00.000Z",
signals: ["当前没有新的治理风险。"],
recommendations: [
{
priority: "P1",
title: "保持 current gap 为 0",
rationale: ["继续沿用 current/degraded 分层。"],
backlogTools: [
"回看 browser replay / browser verification 失败样本,并把失败断言回挂到受影响主路径。",
],
commands: ["npm run harness:eval:history:record"],
focusVerificationFailureOutcomes: [
"browserVerification (failure)",
],
focusVerificationRecoveredOutcomes: [
"artifactValidator (repaired)",
"browserVerification (success)",
],
},
],
focus: {
currentObservabilityVerificationOutcomes: [
{
signal: "browserVerification",
outcome: "failure",
state: "regressing",
latest: {
caseCount: 1,
},
delta: {
caseCount: 1,
},
},
],
currentRecoveredObservabilityVerificationOutcomes: [
{
signal: "artifactValidator",
outcome: "repaired",
state: "expanding",
latest: {
caseCount: 1,
},
delta: {
caseCount: 1,
},
},
{
signal: "browserVerification",
outcome: "success",
state: "present",
latest: {
caseCount: 1,
},
delta: {
caseCount: 0,
},
},
{
signal: "guiSmoke",
outcome: "passed",
state: "present",
latest: {
caseCount: 1,
},
delta: {
caseCount: 0,
},
},
],
degradedObservabilityVerificationOutcomes: [
{
signal: "guiSmoke",
outcome: "failed",
state: "present",
latest: {
caseCount: 1,
},
delta: {
caseCount: 0,
},
},
],
},
summary: {
trend: {
sampleCount: 1,
latestCurrentObservabilityGapCaseCount: 0,
latestDegradedObservabilityGapCaseCount: 1,
currentObservabilityGapCaseDelta: 0,
degradedObservabilityGapCaseDelta: 0,
},
verificationOutcomes: {
blockingFailureCaseCount: 2,
advisoryFailureCaseCount: 0,
recoveredCaseCount: 3,
current: {
blockingFailureCaseCount: 1,
advisoryFailureCaseCount: 0,
recoveredCaseCount: 3,
},
degraded: {
blockingFailureCaseCount: 1,
advisoryFailureCaseCount: 0,
},
},
governance: {
violationCount: 0,
},
},
},
});
expect(html).toContain("Harness Engine Nightly Dashboard");
expect(html).toContain("Harness Engine Dashboard");
expect(html).toContain("Current Gap");
expect(html).toContain("Degraded Gap");
expect(html).toContain("Current Blocking");
expect(html).toContain("Current Advisory");
expect(html).toContain("Current Recovered");
expect(html).toContain("Degraded Blocking");
expect(html).toContain("Recovered Outcomes");
expect(html).toContain("Cleanup 建议");
expect(html).toContain("Observability Gap 角色");
expect(html).toContain("Verification Outcome 焦点");
expect(html).toContain("Current Recovered Baseline");
expect(html).toContain("current");
expect(html).toContain("degraded");
expect(html).toContain("browserVerification");
expect(html).toContain("failure");
expect(html).toContain("success");
expect(html).toContain("artifactValidator");
expect(html).toContain("repaired");
expect(html).toContain("guiSmoke");
expect(html).toContain("failed");
expect(html).toContain("artifact validator 已执行修复");
expect(html).toContain("浏览器验证已有成功样本");
expect(html).toContain("当前 Recovered 基线");
expect(html).toContain(
"artifactValidator (repaired)、browserVerification (success)、guiSmoke (passed)",
);
expect(html).toContain("关注 failure outcome");
expect(html).toContain("关注 recovered outcome");
expect(html).toContain("artifactValidator (repaired)");
expect(html).toContain("后续动作");
expect(html).toContain(
"回看 browser replay / browser verification 失败样本,并把失败断言回挂到受影响主路径。",
);
expect(html).toContain("推荐命令");
expect(html).toContain("summary - history - trend - cleanup");
});
});
@@ -59,10 +59,125 @@ afterEach(() => {
});
describe("harness-eval-history-record", () => {
it("默认入口应产出完整 harness artifact 套件", () => {
const tempRoot = createTempRoot();
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
const workspaceRoot = path.join(tempRoot, "workspace");
const result = runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--history-dir",
historyDir,
"--workspace-root",
workspaceRoot,
]);
const reportsRoot = path.join(tempRoot, ".lime", "harness", "reports");
const summaryJsonPath = path.join(reportsRoot, "harness-eval-summary.json");
const summaryMarkdownPath = path.join(reportsRoot, "harness-eval-summary.md");
const trendJsonPath = path.join(reportsRoot, "harness-eval-trend.json");
const trendMarkdownPath = path.join(reportsRoot, "harness-eval-trend.md");
const cleanupJsonPath = path.join(reportsRoot, "harness-cleanup-report.json");
const cleanupMarkdownPath = path.join(reportsRoot, "harness-cleanup-report.md");
const dashboardHtmlPath = path.join(reportsRoot, "harness-dashboard.html");
expect(result.summary.outputJsonPath).toBe(summaryJsonPath);
expect(result.summary.outputMarkdownPath).toBe(summaryMarkdownPath);
expect(result.trend.outputJsonPath).toBe(trendJsonPath);
expect(result.trend.outputMarkdownPath).toBe(trendMarkdownPath);
expect(result.trend.currentRecoveredVerificationCaseCount).toBe(3);
expect(result.trend.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(result.cleanup.outputJsonPath).toBe(cleanupJsonPath);
expect(result.cleanup.outputMarkdownPath).toBe(cleanupMarkdownPath);
expect(result.cleanup.verificationFailureOutcomeFocus).toEqual(
expect.arrayContaining([
"artifactValidator:issues_present",
]),
);
expect(result.cleanup.verificationFailureCaseCount).toBe(1);
expect(result.cleanup.verificationBlockingFailureCaseCount).toBe(0);
expect(result.cleanup.verificationAdvisoryFailureCaseCount).toBe(1);
expect(result.cleanup.verificationDegradedBlockingFailureCaseCount).toBe(0);
expect(result.cleanup.verificationRecoveredCaseCount).toBe(3);
expect(result.cleanup.currentVerificationRecoveredCaseCount).toBe(3);
expect(result.cleanup.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(result.dashboard.outputHtmlPath).toBe(dashboardHtmlPath);
expect(fs.existsSync(summaryJsonPath)).toBe(true);
expect(fs.existsSync(summaryMarkdownPath)).toBe(true);
expect(fs.existsSync(trendJsonPath)).toBe(true);
expect(fs.existsSync(trendMarkdownPath)).toBe(true);
expect(fs.existsSync(cleanupJsonPath)).toBe(true);
expect(fs.existsSync(cleanupMarkdownPath)).toBe(true);
expect(fs.existsSync(dashboardHtmlPath)).toBe(true);
const cleanupJson = JSON.parse(fs.readFileSync(cleanupJsonPath, "utf8"));
expect(
cleanupJson.recommendations.every(
(entry: Record<string, unknown>) =>
!Object.prototype.hasOwnProperty.call(
entry,
"focusObservabilityVerificationOutcomes",
),
),
).toBe(true);
expect(() =>
execFileSync(
process.execPath,
[
path.join(repoRoot, "scripts/check-generated-slop-report.mjs"),
"--input",
cleanupJsonPath,
"--format",
"json",
],
{
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
},
),
).not.toThrow();
}, 30_000);
it("应记录本地 summary 历史,并生成非 seed 的 trend / cleanup", () => {
const tempRoot = createTempRoot();
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
const workspaceRoot = path.join(tempRoot, "workspace");
const summaryJsonPath = path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-eval-summary.json",
);
const summaryMarkdownPath = path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-eval-summary.md",
);
const dashboardHtmlPath = path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-dashboard.html",
);
const firstResult = runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
@@ -71,6 +186,12 @@ describe("harness-eval-history-record", () => {
historyDir,
"--workspace-root",
workspaceRoot,
"--summary-json",
summaryJsonPath,
"--summary-markdown",
summaryMarkdownPath,
"--dashboard-html",
dashboardHtmlPath,
"--output-json",
path.join(tempRoot, "first.json"),
]);
@@ -84,6 +205,12 @@ describe("harness-eval-history-record", () => {
historyDir,
"--workspace-root",
workspaceRoot,
"--summary-json",
summaryJsonPath,
"--summary-markdown",
summaryMarkdownPath,
"--dashboard-html",
dashboardHtmlPath,
"--output-json",
path.join(tempRoot, "second.json"),
],
@@ -91,11 +218,72 @@ describe("harness-eval-history-record", () => {
expect(firstResult.historyCount).toBe(1);
expect(firstResult.trend.sampleCount).toBe(1);
expect(firstResult.trend.currentObservabilityGapCaseCount).toBe(0);
expect(firstResult.trend.degradedObservabilityGapCaseCount).toBe(1);
expect(firstResult.trend.currentRecoveredVerificationCaseCount).toBe(3);
expect(firstResult.trend.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(firstResult.summary.outputJsonPath).toBe(summaryJsonPath);
expect(firstResult.summary.outputMarkdownPath).toBe(summaryMarkdownPath);
expect(firstResult.dashboard.outputHtmlPath).toBe(dashboardHtmlPath);
expect(secondResult.historyCount).toBe(2);
expect(secondResult.trend.sampleCount).toBe(2);
expect(secondResult.trend.currentObservabilityGapCaseCount).toBe(0);
expect(secondResult.trend.degradedObservabilityGapCaseCount).toBe(1);
expect(secondResult.trend.currentRecoveredVerificationCaseCount).toBe(3);
expect(secondResult.trend.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(secondResult.cleanup.trendSampleCount).toBe(2);
expect(secondResult.cleanup.currentObservabilityGapCaseCount).toBe(0);
expect(secondResult.cleanup.degradedObservabilityGapCaseCount).toBe(1);
expect(secondResult.cleanup.verificationFailureOutcomeFocus).toEqual(
expect.arrayContaining([
"artifactValidator:issues_present",
]),
);
expect(secondResult.cleanup.verificationFailureCaseCount).toBe(1);
expect(secondResult.cleanup.verificationBlockingFailureCaseCount).toBe(0);
expect(secondResult.cleanup.verificationAdvisoryFailureCaseCount).toBe(1);
expect(secondResult.cleanup.verificationDegradedBlockingFailureCaseCount).toBe(0);
expect(secondResult.cleanup.verificationRecoveredCaseCount).toBe(3);
expect(secondResult.cleanup.currentVerificationRecoveredCaseCount).toBe(3);
expect(secondResult.cleanup.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(fs.existsSync(secondResult.recordedSummaryPath)).toBe(true);
expect(fs.existsSync(summaryJsonPath)).toBe(true);
expect(fs.existsSync(summaryMarkdownPath)).toBe(true);
expect(fs.existsSync(dashboardHtmlPath)).toBe(true);
const dashboardHtml = fs.readFileSync(dashboardHtmlPath, "utf8");
expect(dashboardHtml).toContain("Harness Engine Nightly Dashboard");
expect(dashboardHtml).toContain("Current Gap");
expect(dashboardHtml).toContain("Degraded Gap");
expect(dashboardHtml).toContain("Current Blocking");
expect(dashboardHtml).toContain("Current Advisory");
expect(dashboardHtml).toContain("Current Recovered");
expect(dashboardHtml).toContain("Degraded Blocking");
expect(dashboardHtml).toContain("Recovered Outcomes");
expect(dashboardHtml).toContain("Cleanup 建议");
expect(dashboardHtml).toContain("Observability Gap 角色");
expect(dashboardHtml).toContain("Verification Outcome 焦点");
expect(dashboardHtml).toContain("Current Recovered Baseline");
expect(dashboardHtml).toContain("artifactValidator");
expect(dashboardHtml).toContain("issues_present");
expect(
fs.existsSync(
path.join(
+11 -14
View File
@@ -159,7 +159,7 @@ afterEach(() => {
});
describe("Harness eval history window", () => {
it("runner 应记录并裁剪历史窗口,trend 应复用该目录", () => {
it("history-record 应记录并裁剪历史窗口,trend 应复用该目录", () => {
const tempRoot = createTempRoot();
const caseDir = createReplayFixture(tempRoot);
const manifestPath = path.join(tempRoot, "manifest.json");
@@ -167,40 +167,37 @@ describe("Harness eval history window", () => {
writeJson(manifestPath, createHarnessManifest(caseDir));
runNodeScript("scripts/harness-eval-runner.mjs", [
runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--manifest",
manifestPath,
"--record-history-dir",
"--history-dir",
historyDir,
"--history-retain",
"--retain",
"2",
"--no-strict",
]);
sleepMs(10);
runNodeScript("scripts/harness-eval-runner.mjs", [
runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--manifest",
manifestPath,
"--record-history-dir",
"--history-dir",
historyDir,
"--history-retain",
"--retain",
"2",
"--no-strict",
]);
sleepMs(10);
runNodeScript("scripts/harness-eval-runner.mjs", [
runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--manifest",
manifestPath,
"--record-history-dir",
"--history-dir",
historyDir,
"--history-retain",
"--retain",
"2",
"--no-strict",
]);
const historyFiles = fs
@@ -222,5 +219,5 @@ describe("Harness eval history window", () => {
expect(report.signals).not.toContain(
"样本数不足 2,当前仅形成 trend seed,还不能判断长期退化。",
);
});
}, 30_000);
});
@@ -0,0 +1,155 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { afterEach, describe, expect, it } from "vitest";
const repoRoot = process.cwd();
const tempRoots: string[] = [];
function createTempRoot() {
const tempRoot = fs.mkdtempSync(
path.join(os.tmpdir(), "lime-harness-repo-fixtures-"),
);
tempRoots.push(tempRoot);
return tempRoot;
}
function runHarnessEval(args: string[]) {
const output = execFileSync(
process.execPath,
[path.join(repoRoot, "scripts/harness-eval-runner.mjs"), ...args],
{
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
},
);
return JSON.parse(output);
}
afterEach(() => {
while (tempRoots.length > 0) {
const tempRoot = tempRoots.pop();
if (tempRoot) {
fs.rmSync(tempRoot, { recursive: true, force: true });
}
}
});
describe("Harness repo fixtures", () => {
it("应固定 current 与 degraded observability 样本职责", () => {
const workspaceRoot = createTempRoot();
const summary = runHarnessEval([
"--format",
"json",
"--manifest",
"docs/test/harness-evals.manifest.json",
"--workspace-root",
workspaceRoot,
"--no-strict",
]);
const repoFixtureSuite = summary.suites.find(
(entry: { id: string }) => entry.id === "repo-fixtures",
);
expect(repoFixtureSuite).toBeTruthy();
expect(repoFixtureSuite.stats.caseCount).toBe(2);
expect(repoFixtureSuite.stats.readyCount).toBe(2);
expect(summary.totals.observabilityGapCaseCount).toBe(1);
expect(summary.totals.currentObservabilityGapCaseCount).toBe(0);
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(1);
const currentCase = repoFixtureSuite.cases.find(
(entry: { caseId: string }) =>
entry.caseId === "fixture-minimal-pending-request",
);
expect(currentCase).toBeTruthy();
expect(currentCase.status).toBe("ready");
expect(currentCase.observabilityGapCount).toBe(0);
expect(currentCase.observabilitySignals).toContain(
"requestTelemetry:exported",
);
expect(currentCase.observabilitySignals).toContain(
"artifactValidator:exported",
);
expect(currentCase.observabilitySignals).toContain(
"browserVerification:exported",
);
expect(currentCase.observabilitySignals).toContain("guiSmoke:exported");
expect(currentCase.observabilityVerificationOutcomes).toContain(
"artifactValidator:issues_present",
);
expect(currentCase.observabilityVerificationOutcomes).toContain(
"artifactValidator:repaired",
);
expect(currentCase.observabilityVerificationOutcomes).toContain(
"browserVerification:success",
);
expect(currentCase.observabilityVerificationOutcomes).toContain(
"guiSmoke:passed",
);
const degradedCase = repoFixtureSuite.cases.find(
(entry: { caseId: string }) =>
entry.caseId === "fixture-minimal-observability-gap",
);
expect(degradedCase).toBeTruthy();
expect(degradedCase.status).toBe("ready");
expect(degradedCase.observabilityGapCount).toBe(2);
expect(degradedCase.observabilitySignals).toContain(
"requestTelemetry:known_gap",
);
expect(degradedCase.observabilitySignals).toContain(
"artifactValidator:known_gap",
);
const gapCaseIds = repoFixtureSuite.cases
.filter(
(entry: { observabilityGapCount: number }) =>
entry.observabilityGapCount > 0,
)
.map((entry: { caseId: string }) => entry.caseId);
expect(gapCaseIds).toEqual(["fixture-minimal-observability-gap"]);
const observabilityBreakdownNames = summary.breakdowns.observabilitySignals.map(
(entry: { name: string }) => entry.name,
);
expect(observabilityBreakdownNames).toContain("requestTelemetry:exported");
expect(observabilityBreakdownNames).toContain("requestTelemetry:known_gap");
expect(observabilityBreakdownNames).toContain("artifactValidator:exported");
expect(observabilityBreakdownNames).toContain(
"artifactValidator:known_gap",
);
const verificationOutcomeBreakdownNames =
summary.breakdowns.observabilityVerificationOutcomes.map(
(entry: { name: string }) => entry.name,
);
expect(verificationOutcomeBreakdownNames).toContain(
"artifactValidator:issues_present",
);
expect(verificationOutcomeBreakdownNames).toContain(
"artifactValidator:repaired",
);
expect(verificationOutcomeBreakdownNames).toContain(
"browserVerification:success",
);
expect(verificationOutcomeBreakdownNames).toContain("guiSmoke:passed");
const currentVerificationOutcomeBreakdownNames =
summary.breakdowns.currentObservabilityVerificationOutcomes.map(
(entry: { name: string }) => entry.name,
);
expect(currentVerificationOutcomeBreakdownNames).toContain(
"artifactValidator:issues_present",
);
expect(currentVerificationOutcomeBreakdownNames).toContain(
"browserVerification:success",
);
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
[],
);
});
});
+348 -14
View File
@@ -116,13 +116,36 @@ function createWorkspaceSessionArtifacts(tempRoot: string, sessionId: string) {
},
{
signal: "requestTelemetry",
status: "unlinked",
status: "exported",
},
{
signal: "artifactValidator",
status: "known_gap",
},
],
verificationSummary: {
artifactValidator: {
applicable: true,
recordCount: 1,
issueCount: 1,
repairedCount: 1,
fallbackUsedCount: 0,
},
browserVerification: {
recordCount: 1,
successCount: 1,
failureCount: 0,
unknownCount: 0,
latestUpdatedAt: "2026-03-27T11:22:00Z",
},
guiSmoke: {
status: "completed",
exitCode: 0,
passed: true,
updatedAt: "2026-03-27T11:23:30Z",
hasOutputPreview: true,
},
},
},
linkedArtifacts: {
handoffBundle: {
@@ -325,6 +348,8 @@ describe("Harness review decision / eval integration", () => {
expect(summary.totals.reviewDecisionRecordedCount).toBe(1);
expect(summary.totals.observabilityGapCaseCount).toBe(1);
expect(summary.totals.currentObservabilityGapCaseCount).toBe(1);
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(0);
expect(summary.breakdowns.reviewDecisionStatuses).toEqual(
expect.arrayContaining([
expect.objectContaining({
@@ -343,21 +368,54 @@ describe("Harness review decision / eval integration", () => {
);
expect(summary.breakdowns.observabilitySignals).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "requestTelemetry:unlinked",
caseCount: 1,
}),
expect.objectContaining({
name: "artifactValidator:known_gap",
caseCount: 1,
}),
]),
);
expect(summary.breakdowns.observabilityVerificationOutcomes).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "artifactValidator:issues_present",
caseCount: 1,
}),
expect.objectContaining({
name: "artifactValidator:repaired",
caseCount: 1,
}),
expect.objectContaining({
name: "browserVerification:success",
caseCount: 1,
}),
expect.objectContaining({
name: "guiSmoke:passed",
caseCount: 1,
}),
]),
);
expect(summary.breakdowns.currentObservabilityVerificationOutcomes).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:success",
caseCount: 1,
}),
]),
);
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
[],
);
expect(summary.suites[0].cases[0]).toMatchObject({
reviewDecisionStatus: "accepted",
reviewRiskLevel: "high",
reviewHumanReviewer: "Lime Maintainer",
observabilityGapCount: 2,
observabilityGapCount: 1,
observabilityVerificationOutcomes: expect.arrayContaining([
"artifactValidator:issues_present",
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
});
});
@@ -376,6 +434,8 @@ describe("Harness review decision / eval integration", () => {
pendingRequestCaseCount: 0,
needsHumanReviewCount: 1,
observabilityGapCaseCount: 1,
currentObservabilityGapCaseCount: 1,
degradedObservabilityGapCaseCount: 0,
},
breakdowns: {
suiteTags: [],
@@ -400,9 +460,10 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 1,
},
],
observabilitySignals: [
observabilitySignals: [],
observabilityVerificationOutcomes: [
{
name: "requestTelemetry:unlinked",
name: "browserVerification:unknown",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
@@ -410,6 +471,17 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 1,
},
],
currentObservabilityVerificationOutcomes: [
{
name: "browserVerification:unknown",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 1,
},
],
degradedObservabilityVerificationOutcomes: [],
},
suites: [],
});
@@ -424,6 +496,8 @@ describe("Harness review decision / eval integration", () => {
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 1,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 1,
},
breakdowns: {
suiteTags: [],
@@ -458,6 +532,43 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 0,
},
],
observabilityVerificationOutcomes: [
{
name: "browserVerification:failure",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:failed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
currentObservabilityVerificationOutcomes: [],
degradedObservabilityVerificationOutcomes: [
{
name: "browserVerification:failure",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:failed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
},
suites: [],
});
@@ -505,12 +616,6 @@ describe("Harness review decision / eval integration", () => {
);
expect(report.classificationDeltas.observabilitySignals).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "requestTelemetry:unlinked",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
expect.objectContaining({
name: "artifactValidator:known_gap",
delta: expect.objectContaining({
@@ -519,5 +624,234 @@ describe("Harness review decision / eval integration", () => {
}),
]),
);
expect(
report.classificationDeltas.observabilitySignals.find(
(entry) => entry.name === "requestTelemetry:known_gap",
),
).toBeUndefined();
expect(report.classificationDeltas.observabilityVerificationOutcomes).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:failure",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
expect.objectContaining({
name: "browserVerification:unknown",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
expect.objectContaining({
name: "guiSmoke:failed",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
expect(
report.classificationDeltas.currentObservabilityVerificationOutcomes,
).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:unknown",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
]),
);
expect(
report.classificationDeltas.degradedObservabilityVerificationOutcomes,
).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:failure",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
expect.objectContaining({
name: "guiSmoke:failed",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
expect(report.delta.currentObservabilityGapCaseCount).toBe(-1);
expect(report.delta.degradedObservabilityGapCaseCount).toBe(1);
expect(report.latest.totals.currentObservabilityGapCaseCount).toBe(0);
expect(report.latest.totals.degradedObservabilityGapCaseCount).toBe(1);
});
it("harness-eval-trend-report 应暴露 current recovered baseline 的新增趋势", () => {
const tempRoot = createTempRoot();
const baselinePath = path.join(tempRoot, "baseline-recovered.json");
const latestPath = path.join(tempRoot, "latest-recovered.json");
writeJson(baselinePath, {
generatedAt: "2026-03-28T10:00:00Z",
totals: {
suiteCount: 1,
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 0,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 0,
},
breakdowns: {
suiteTags: [],
failureModes: [],
reviewDecisionStatuses: [],
reviewRiskLevels: [],
observabilitySignals: [],
observabilityVerificationOutcomes: [
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
currentObservabilityVerificationOutcomes: [
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
degradedObservabilityVerificationOutcomes: [],
},
suites: [],
});
writeJson(latestPath, {
generatedAt: "2026-03-28T12:00:00Z",
totals: {
suiteCount: 1,
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 0,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 0,
},
breakdowns: {
suiteTags: [],
failureModes: [],
reviewDecisionStatuses: [],
reviewRiskLevels: [],
observabilitySignals: [],
observabilityVerificationOutcomes: [
{
name: "artifactValidator:repaired",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:passed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
currentObservabilityVerificationOutcomes: [
{
name: "artifactValidator:repaired",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:passed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
degradedObservabilityVerificationOutcomes: [],
},
suites: [],
});
const report = runNodeScript("scripts/harness-eval-trend-report.mjs", [
"--format",
"json",
"--input",
baselinePath,
"--input",
latestPath,
]);
expect(report.latest.totals.currentRecoveredVerificationCaseCount).toBe(3);
expect(report.delta.currentRecoveredVerificationCaseCount).toBe(2);
expect(
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes,
).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "artifactValidator:repaired",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
expect.objectContaining({
name: "browserVerification:success",
delta: expect.objectContaining({
caseCount: 0,
}),
}),
expect.objectContaining({
name: "guiSmoke:passed",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
expect(report.signals).toContain(
"current recovered verification baseline `artifactValidator:repaired` 新增 1,说明主线路径正在形成正向基线。",
);
expect(report.signals).toContain(
"current recovered verification baseline `guiSmoke:passed` 新增 1,说明主线路径正在形成正向基线。",
);
});
});
+18 -1
View File
@@ -10,6 +10,14 @@ const rootDir = process.cwd();
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
const cargoCommand = process.platform === "win32" ? "cargo.exe" : "cargo";
const BRIDGE_REASON_LABELS = {
bridge_contracts: "bridge/contracts",
bridge_runtime: "DevBridge / mock / bridge runtime",
fallback_full_suite: "兜底全量",
full_suite: "full 模式",
harness_cleanup_contract: "harness cleanup contract",
workflow_full_suite: "workflow 全量",
};
function parseArgs(argv) {
const result = {
@@ -107,7 +115,16 @@ function printSummary(changedFiles, tasks) {
console.log("[local-ci] - 前端校验");
}
if (tasks.bridge) {
console.log("[local-ci] - bridge 校验");
const bridgeReasonLabels = Array.isArray(tasks.bridgeReasons)
? tasks.bridgeReasons
.map((reason) => BRIDGE_REASON_LABELS[reason] ?? reason)
.filter(Boolean)
: [];
console.log(
bridgeReasonLabels.length > 0
? `[local-ci] - bridge 校验(${bridgeReasonLabels.join(" / ")})`
: "[local-ci] - bridge 校验",
);
}
if (tasks.guiSmoke) {
console.log("[local-ci] - GUI 冒烟");
+69 -2
View File
@@ -39,13 +39,28 @@ const FRONTEND_TOOLING_FILES = new Set([
const BRIDGE_FILES = new Set([
"vite.config.ts",
"scripts/check-command-contracts.mjs",
"scripts/check-generated-slop-report.mjs",
"scripts/check-dev-bridge-health.mjs",
"scripts/harness-eval-history-record.mjs",
"scripts/harness-eval-trend-report.mjs",
"scripts/report-generated-slop.mjs",
"scripts/social-workbench-e2e-smoke.mjs",
"scripts/chrome-bridge-e2e.mjs",
"scripts/verify-gui-smoke.mjs",
"scripts/lib/generated-slop-report-core.mjs",
"scripts/lib/harness-dashboard-core.mjs",
"docs/aiprompts/playwright-e2e.md",
]);
const HARNESS_CLEANUP_CONTRACT_FILES = new Set([
"scripts/check-generated-slop-report.mjs",
"scripts/harness-eval-history-record.mjs",
"scripts/harness-eval-trend-report.mjs",
"scripts/report-generated-slop.mjs",
"scripts/lib/generated-slop-report-core.mjs",
"scripts/lib/harness-dashboard-core.mjs",
]);
const INTEGRITY_FILES = new Set([
"package.json",
"src-tauri/Cargo.toml",
@@ -249,6 +264,45 @@ function isBridgeChange(file) {
);
}
function isHarnessCleanupContractChange(file) {
return HARNESS_CLEANUP_CONTRACT_FILES.has(file);
}
function collectBridgeReasons(changedFiles, { full = false, fallback = false, workflow = false } = {}) {
if (full) {
return ["full_suite"];
}
if (fallback) {
return ["fallback_full_suite"];
}
if (workflow) {
return ["workflow_full_suite"];
}
const reasons = [];
if (changedFiles.some(isHarnessCleanupContractChange)) {
reasons.push("harness_cleanup_contract");
}
if (
changedFiles.some(
(file) =>
isBridgeChange(file) && !isHarnessCleanupContractChange(file),
)
) {
reasons.push("bridge_runtime");
}
if (reasons.length === 0 && changedFiles.some(isBridgeChange)) {
reasons.push("bridge_contracts");
}
return reasons;
}
function isGuiSmokeChange(file) {
return (
GUI_SMOKE_FILES.has(file) ||
@@ -272,6 +326,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
frontend: true,
rust: true,
bridge: true,
bridgeReasons: collectBridgeReasons([], { full: true }),
guiSmoke: true,
docs: true,
docsOnly: false,
@@ -286,6 +341,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
frontend: true,
rust: true,
bridge: true,
bridgeReasons: collectBridgeReasons([], { fallback: true }),
guiSmoke: true,
docs: true,
docsOnly: false,
@@ -301,6 +357,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
frontend: true,
rust: true,
bridge: true,
bridgeReasons: collectBridgeReasons([], { workflow: true }),
guiSmoke: true,
docs: true,
docsOnly: false,
@@ -315,6 +372,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
frontend: false,
rust: false,
bridge: false,
bridgeReasons: [],
guiSmoke: false,
docs: true,
docsOnly: true,
@@ -323,11 +381,14 @@ function detectTasks(changedFiles, { full = false } = {}) {
};
}
const bridge = changedFiles.some(isBridgeChange);
return {
integrity: changedFiles.some(isIntegrityChange),
frontend: changedFiles.some(isFrontendChange),
rust: changedFiles.some(isRustChange),
bridge: changedFiles.some(isBridgeChange),
bridge,
bridgeReasons: bridge ? collectBridgeReasons(changedFiles) : [],
guiSmoke: changedFiles.some(isGuiSmokeChange),
docs: changedFiles.some(isDocsChange),
docsOnly: false,
@@ -359,4 +420,10 @@ function planQualityTasks({
};
}
export { collectChangedFiles, detectTasks, planQualityTasks, resolveDiffBase };
export {
collectBridgeReasons,
collectChangedFiles,
detectTasks,
planQualityTasks,
resolveDiffBase,
};
+33
View File
@@ -0,0 +1,33 @@
import { describe, expect, it } from "vitest";
import { detectTasks } from "./quality-task-planner.mjs";
describe("quality-task-planner", () => {
it("应把 harness cleanup/report 主链文件归到 bridge/contracts 风险", () => {
const tasks = detectTasks([
"scripts/lib/generated-slop-report-core.mjs",
"scripts/check-generated-slop-report.mjs",
"scripts/harness-eval-history-record.mjs",
]);
expect(tasks.bridge).toBe(true);
expect(tasks.bridgeReasons).toContain("harness_cleanup_contract");
expect(tasks.docsOnly).toBe(false);
});
it("应把 harness dashboard 渲染文件归到 bridge/contracts 风险", () => {
const tasks = detectTasks(["scripts/lib/harness-dashboard-core.mjs"]);
expect(tasks.bridge).toBe(true);
expect(tasks.bridgeReasons).toContain("harness_cleanup_contract");
expect(tasks.docsOnly).toBe(false);
});
it("应把 DevBridge 主链改动标记为 bridge runtime 风险", () => {
const tasks = detectTasks(["src/lib/dev-bridge/safeInvoke.ts"]);
expect(tasks.bridge).toBe(true);
expect(tasks.bridgeReasons).toContain("bridge_runtime");
expect(tasks.docsOnly).toBe(false);
});
});
+1
View File
@@ -68,6 +68,7 @@ function printGithubFormat(result) {
`frontend=${tasks.frontend}`,
`rust=${tasks.rust}`,
`bridge=${tasks.bridge}`,
`bridge_reasons=${Array.isArray(tasks.bridgeReasons) ? tasks.bridgeReasons.join(",") : ""}`,
`gui_smoke=${tasks.guiSmoke}`,
`docs=${tasks.docs}`,
`docs_only=${tasks.docsOnly}`,
+37
View File
@@ -5,7 +5,11 @@ import process from 'node:process';
const DEV_URL = process.env.LIME_WEB_BRIDGE_URL?.trim() || 'http://127.0.0.1:1420/';
const DEV_URL_TIMEOUT_MS = 1_500;
const REUSE_ONLY_TIMEOUT_MS = 10_000;
const REUSE_ONLY_INTERVAL_MS = 500;
const ROOT_MARKERS = ['<title>Lime</title>', '<div id="root"></div>'];
const REUSE_EXISTING_ONLY =
process.env.LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY?.trim() === '1';
const env = { ...process.env };
delete env.TAURI_ENV_PLATFORM;
@@ -39,6 +43,32 @@ async function probeExistingDevServer(url) {
}
}
async function waitForExistingDevServer(url) {
const startedAt = Date.now();
let lastProbe = { reachable: false };
while (Date.now() - startedAt < REUSE_ONLY_TIMEOUT_MS) {
lastProbe = await probeExistingDevServer(url);
if (lastProbe.reachable && lastProbe.isLimeDevShell) {
return lastProbe;
}
if (lastProbe.reachable && !lastProbe.isLimeDevShell) {
const statusLabel = `${lastProbe.status} ${lastProbe.statusText}`.trim();
throw new Error(
`[dev:web-bridge] ${DEV_URL} 已被其他服务占用,且返回内容不是 Lime dev shell(${statusLabel})。请先关闭占用进程后重试。`,
);
}
await new Promise((resolve) => setTimeout(resolve, REUSE_ONLY_INTERVAL_MS));
}
throw new Error(
`[dev:web-bridge] 要求复用已有 Lime dev server,但 ${url} 在 ${REUSE_ONLY_TIMEOUT_MS}ms 内未就绪。`,
);
}
async function waitForExitSignal() {
await new Promise((resolve) => {
const handleExit = () => resolve();
@@ -64,6 +94,13 @@ function startVite() {
}
async function main() {
if (REUSE_EXISTING_ONLY) {
await waitForExistingDevServer(DEV_URL);
console.log(`[dev:web-bridge] 复用已存在的 Lime dev server: ${DEV_URL}`);
await waitForExitSignal();
return;
}
const existingServer = await probeExistingDevServer(DEV_URL);
if (existingServer.reachable) {
+741 -69
View File
@@ -1,31 +1,108 @@
#!/usr/bin/env node
import { spawn, spawnSync } from "node:child_process";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import process from "node:process";
import { fileURLToPath } from "node:url";
const DEFAULTS = {
appUrl: "http://127.0.0.1:1420/",
healthUrl: "http://127.0.0.1:3030/health",
timeoutMs: 600_000,
intervalMs: 1_000,
reuseRunning: false,
sampleProjectName: "Lime Smoke Workspace",
};
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
const rootDir = path.resolve(__dirname, "..");
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
const LIME_SKIP_STARTUP_WINDOW_REVEAL = "LIME_SKIP_STARTUP_WINDOW_REVEAL";
const LIME_DISABLE_SINGLE_INSTANCE = "LIME_DISABLE_SINGLE_INSTANCE";
const LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY =
"LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY";
const LIME_WEB_BRIDGE_URL = "LIME_WEB_BRIDGE_URL";
const ROOT_MARKERS = ['<title>Lime</title>', '<div id="root"></div>'];
const SHARED_TAURI_TARGET_DIR = path.join(rootDir, "src-tauri", "target");
const ISOLATED_GUI_SMOKE_TARGET_DIR = path.join(
os.tmpdir(),
"lime-gui-smoke-target",
);
const GUI_SMOKE_TEMP_CONFIG_BASENAME_PREFIX = "lime-gui-smoke-tauri-";
const GUI_SMOKE_COLD_TIMEOUT_MS = 1_800_000;
const GUI_SMOKE_WARM_TIMEOUT_MS = 600_000;
const GUI_SMOKE_BRIDGE_HEARTBEAT_MS = 30_000;
const GUI_SMOKE_COMPILE_GRACE_MS = 900_000;
const GUI_SMOKE_MAX_COMPILE_GRACE_EXTENSIONS = 2;
const GUI_SMOKE_BOOT_GRACE_MS = 60_000;
const HEADLESS_TAURI_CONFIG_PATH = path.join(
rootDir,
"src-tauri",
"tauri.conf.headless.json",
);
const tauriCommand =
process.platform === "win32"
? path.join(rootDir, "node_modules", ".bin", "tauri.cmd")
: path.join(rootDir, "node_modules", ".bin", "tauri");
const state = {
child: null,
cleanedUp: false,
tempConfigPath: null,
};
function resolveTargetBinaryPath(targetDir) {
const appBinaryName = process.platform === "win32" ? "lime.exe" : "lime";
return path.join(targetDir, "debug", appBinaryName);
}
function listTargetLockHolderCommands(targetDir) {
if (process.platform === "win32") {
return [];
}
const lockPath = path.join(targetDir, "debug", ".cargo-lock");
const pidOutput = runQuietCommand("lsof", ["-t", "--", lockPath]);
if (!pidOutput) {
return [];
}
const pids = [
...new Set(
pidOutput
.split("\n")
.map((item) => item.trim())
.filter(Boolean),
),
];
return pids
.map((pid) => runQuietCommand("ps", ["-p", pid, "-o", "command="]))
.filter(Boolean);
}
function resolvePreferredCargoTargetDir() {
const sharedTargetLockHolders = listTargetLockHolderCommands(
SHARED_TAURI_TARGET_DIR,
);
if (sharedTargetLockHolders.length === 0) {
return SHARED_TAURI_TARGET_DIR;
}
return ISOLATED_GUI_SMOKE_TARGET_DIR;
}
function resolveDefaultTimeoutMs(cargoTargetDir) {
const binaryPath = resolveTargetBinaryPath(cargoTargetDir);
return fs.existsSync(binaryPath)
? GUI_SMOKE_WARM_TIMEOUT_MS
: GUI_SMOKE_COLD_TIMEOUT_MS;
}
const DEFAULTS = {
appUrl: "http://127.0.0.1:1420/",
healthUrl: "http://127.0.0.1:3030/health",
cargoTargetDir: resolvePreferredCargoTargetDir(),
intervalMs: 1_000,
reuseRunning: false,
sampleProjectName: "Lime Smoke Workspace",
};
DEFAULTS.timeoutMs = resolveDefaultTimeoutMs(DEFAULTS.cargoTargetDir);
function printHelp() {
console.log(`
Lime GUI 冒烟入口
@@ -41,9 +118,10 @@ Lime GUI 冒烟入口
选项:
--app-url <url> 前端地址,默认 http://127.0.0.1:1420/
--health-url <url> DevBridge 健康检查地址,默认 http://127.0.0.1:3030/health
--timeout-ms <ms> 等待 headless / bridge / smoke 的超时,默认 600000
--timeout-ms <ms> 等待 headless / bridge / smoke 的超时,默认冷启动 1800000 / 热启动 600000
--interval-ms <ms> 轮询间隔,默认 1000
--sample-project-name <s> workspace 路径校验使用的示例项目名
--cargo-target-dir <dir> 指定 Cargo target 目录;默认优先复用 src-tauri/target,无锁时共享,否则回退独立 GUI smoke target
--reuse-running 复用已启动的 headless Tauri,不主动拉起
-h, --help 显示帮助
`);
@@ -85,6 +163,12 @@ function parseArgs(argv) {
continue;
}
if (arg === "--cargo-target-dir" && argv[index + 1]) {
options.cargoTargetDir = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--reuse-running") {
options.reuseRunning = true;
continue;
@@ -116,6 +200,10 @@ function parseArgs(argv) {
throw new Error("--sample-project-name 不能为空");
}
if (!options.cargoTargetDir) {
throw new Error("--cargo-target-dir 不能为空");
}
return options;
}
@@ -129,10 +217,22 @@ function assert(condition, message) {
}
}
function isLimeDevShell(html) {
return ROOT_MARKERS.some((marker) => html.includes(marker));
}
function formatCommand(command, args) {
return [command, ...args].join(" ");
}
function trimTrailingSlash(value) {
return value.endsWith("/") ? value.slice(0, -1) : value;
}
function buildNoopBeforeDevCommand() {
return `"${process.execPath}" -e "process.exit(0)"`;
}
function runCommand(command, args, label, timeoutMs) {
console.log(`\n[verify:gui-smoke] > ${formatCommand(command, args)}`);
const result = spawnSync(command, args, {
@@ -160,66 +260,435 @@ function runCommand(command, args, label, timeoutMs) {
}
}
function startHeadlessTauri() {
console.log("[verify:gui-smoke] 启动 headless Tauri 环境...");
state.child = spawn(npmCommand, ["run", "tauri:dev:headless"], {
cwd: rootDir,
stdio: "inherit",
env: {
...process.env,
[LIME_SKIP_STARTUP_WINDOW_REVEAL]: "1",
[LIME_DISABLE_SINGLE_INSTANCE]: "1",
},
detached: process.platform !== "win32",
});
}
function createHeadlessTauriConfig(options, startupMode) {
const rawConfig = fs.readFileSync(HEADLESS_TAURI_CONFIG_PATH, "utf8");
const config = JSON.parse(rawConfig);
async function stopHeadlessTauri() {
const child = state.child;
if (!child || state.cleanedUp) {
return;
config.build = {
...(config.build || {}),
devUrl: trimTrailingSlash(options.appUrl),
};
if (startupMode.reuseExistingAppShell) {
config.build.beforeDevCommand = buildNoopBeforeDevCommand();
}
state.cleanedUp = true;
console.log("[verify:gui-smoke] 停止 headless Tauri 环境...");
const tempConfigPath = path.join(
os.tmpdir(),
`lime-gui-smoke-tauri-${process.pid}.json`,
);
fs.writeFileSync(tempConfigPath, JSON.stringify(config, null, 2));
state.tempConfigPath = tempConfigPath;
return tempConfigPath;
}
if (typeof child.pid !== "number") {
function resolveUrlPort(url) {
try {
return new URL(url).port || (url.startsWith("https:") ? "443" : "80");
} catch {
return "";
}
}
function runQuietCommand(command, args) {
try {
return spawnSync(command, args, {
cwd: rootDir,
stdio: ["ignore", "pipe", "pipe"],
env: process.env,
encoding: "utf8",
}).stdout.trim();
} catch {
return "";
}
}
function listProcessTable() {
if (process.platform === "win32") {
return [];
}
const output = runQuietCommand("ps", ["-axo", "pid=,ppid=,pgid=,command="]);
if (!output) {
return [];
}
return output
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/);
if (!match) {
return null;
}
return {
pid: Number(match[1]),
ppid: Number(match[2]),
pgid: Number(match[3]),
command: match[4],
};
})
.filter(Boolean);
}
function listProcessStats() {
if (process.platform === "win32") {
return [];
}
const output = runQuietCommand("ps", [
"-axo",
"pid=,ppid=,pgid=,%cpu=,%mem=,etime=,stat=,command=",
]);
if (!output) {
return [];
}
return output
.split("\n")
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const match = line.match(
/^(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\S+)\s+(\S+)\s+(.*)$/,
);
if (!match) {
return null;
}
return {
pid: Number(match[1]),
ppid: Number(match[2]),
pgid: Number(match[3]),
cpu: Number(match[4]),
mem: Number(match[5]),
etime: match[6],
stat: match[7],
command: match[8],
};
})
.filter(Boolean);
}
function isGuiSmokeVerifierCommand(command) {
return (
command.includes("scripts/verify-gui-smoke.mjs") ||
command.includes("npm run verify:gui-smoke")
);
}
function isGuiSmokeTauriCommand(command) {
return (
command.includes("tauri dev") &&
command.includes(GUI_SMOKE_TEMP_CONFIG_BASENAME_PREFIX)
);
}
function summarizeGuiSmokeProcessCommand(command) {
if (command.includes("/bin/rustc")) {
return "rustc";
}
if (command.includes("cargo run --no-default-features")) {
return "cargo";
}
if (command.includes("tauri dev")) {
return "tauri";
}
if (command.includes("start-web-bridge-dev.mjs")) {
return "web-bridge";
}
if (command.includes("npm run dev:web-bridge")) {
return "npm:dev:web-bridge";
}
if (command.includes("npm exec vite") || command.includes("/bin/vite")) {
return "vite";
}
return command.split(/\s+/)[0]?.split("/").pop() || "process";
}
function resolveGuiSmokeProcessGroupId(startedByScript) {
if (startedByScript && typeof state.child?.pid === "number") {
return state.child.pid;
}
const snapshot = inspectGuiSmokeTauriProcesses();
const firstActive = snapshot.active[0];
return firstActive ? firstActive.pgid || firstActive.pid : null;
}
function listGuiSmokeGroupProcesses(startedByScript) {
const targetGroupId = resolveGuiSmokeProcessGroupId(startedByScript);
if (!Number.isInteger(targetGroupId) || targetGroupId < 1) {
return [];
}
return listProcessStats().filter((item) => item.pgid === targetGroupId);
}
function hasActiveGuiSmokeCompile(startedByScript) {
return listGuiSmokeGroupProcesses(startedByScript).some(
(item) =>
item.command.includes("/bin/rustc") ||
item.command.includes("cargo run --no-default-features"),
);
}
function describeGuiSmokeHeartbeat(startedByScript) {
const interestingProcesses = listGuiSmokeGroupProcesses(startedByScript)
.filter(
(item) =>
item.command.includes("tauri dev") ||
item.command.includes("cargo run --no-default-features") ||
item.command.includes("/bin/rustc") ||
item.command.includes("start-web-bridge-dev.mjs") ||
item.command.includes("npm run dev:web-bridge") ||
item.command.includes("npm exec vite") ||
item.command.includes("/bin/vite"),
)
.sort((left, right) => right.cpu - left.cpu || left.pid - right.pid)
.slice(0, 5);
if (interestingProcesses.length === 0) {
return "";
}
return interestingProcesses
.map((item) => {
const cpu = Number.isFinite(item.cpu) ? item.cpu.toFixed(1) : "0.0";
return `${summarizeGuiSmokeProcessCommand(item.command)} pid=${item.pid} etime=${item.etime} cpu=${cpu}% stat=${item.stat}`;
})
.join(" | ");
}
function inspectGuiSmokeTauriProcesses() {
const processTable = listProcessTable();
const processByPid = new Map(processTable.map((item) => [item.pid, item]));
const active = [];
const stale = [];
for (const item of processTable) {
if (!isGuiSmokeTauriCommand(item.command)) {
continue;
}
if (item.pid === process.pid) {
continue;
}
const parent = processByPid.get(item.ppid);
if (!parent || !isGuiSmokeVerifierCommand(parent.command)) {
stale.push(item);
continue;
}
active.push(item);
}
return { active, stale };
}
function isProcessAlive(pid) {
if (!Number.isInteger(pid) || pid < 1) {
return false;
}
try {
process.kill(pid, 0);
return true;
} catch {
return false;
}
}
function signalProcessTree(pid, signal) {
if (!Number.isInteger(pid) || pid < 1) {
return;
}
if (process.platform === "win32") {
spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
if (signal === "SIGKILL") {
spawnSync("taskkill", ["/pid", String(pid), "/T", "/F"], {
stdio: "ignore",
});
return;
}
spawnSync("taskkill", ["/pid", String(pid), "/T"], {
stdio: "ignore",
});
return;
}
try {
process.kill(-child.pid, "SIGTERM");
} catch {
try {
child.kill("SIGTERM");
} catch {
return;
}
const target = `-${pid}`;
spawnSync("kill", [`-${signal}`, target], {
stdio: "ignore",
});
}
async function stopProcessTree(pid, label) {
if (!Number.isInteger(pid) || pid < 1) {
return;
}
console.log(`[verify:gui-smoke] 清理残留 ${label} 进程组: ${pid}`);
signalProcessTree(pid, "SIGTERM");
for (let attempt = 0; attempt < 25; attempt += 1) {
if (child.exitCode !== null || child.signalCode) {
if (!isProcessAlive(pid)) {
return;
}
await sleep(200);
}
try {
process.kill(-child.pid, "SIGKILL");
} catch {
try {
child.kill("SIGKILL");
} catch {
// ignore
signalProcessTree(pid, "SIGKILL");
}
async function cleanupStaleGuiSmokeProcesses() {
const snapshot = inspectGuiSmokeTauriProcesses();
if (snapshot.stale.length === 0) {
return 0;
}
for (const item of snapshot.stale) {
const groupPid = item.pgid || item.pid;
await stopProcessTree(groupPid, `GUI smoke Tauri(PID=${item.pid})`);
}
return snapshot.stale.length;
}
function listListeningCommandsForPort(port) {
if (!port || process.platform === "win32") {
return [];
}
const pidOutput = runQuietCommand("lsof", [
"-nP",
`-iTCP:${port}`,
"-sTCP:LISTEN",
"-t",
]);
if (!pidOutput) {
return [];
}
const pids = [...new Set(pidOutput.split("\n").map((item) => item.trim()).filter(Boolean))];
return pids
.map((pid) => runQuietCommand("ps", ["-p", pid, "-o", "command="]))
.filter(Boolean);
}
function isLikelyLimeFrontendListener(command) {
return (
command.includes(rootDir) &&
(command.includes("vite") ||
command.includes("npm run dev") ||
command.includes("tauri dev"))
);
}
function startHeadlessTauri(options, startupMode) {
console.log("[verify:gui-smoke] 启动 headless Tauri 环境...");
runCommand(
npmCommand,
["run", "generate:agent-runtime-clients"],
"generate:agent-runtime-clients",
options.timeoutMs,
);
runCommand(
npmCommand,
["run", "generate:extension-site-adapters"],
"generate:extension-site-adapters",
options.timeoutMs,
);
const tauriConfigPath = createHeadlessTauriConfig(options, startupMode);
state.child = spawn(
tauriCommand,
["dev", "--no-watch", "--config", tauriConfigPath],
{
cwd: rootDir,
stdio: "inherit",
env: {
...process.env,
CARGO_TARGET_DIR: options.cargoTargetDir,
[LIME_SKIP_STARTUP_WINDOW_REVEAL]: "1",
[LIME_DISABLE_SINGLE_INSTANCE]: "1",
[LIME_WEB_BRIDGE_URL]: options.appUrl,
...(startupMode.reuseExistingAppShell
? {
[LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY]: "1",
}
: {}),
},
detached: process.platform !== "win32",
},
);
}
async function stopHeadlessTauri() {
const child = state.child;
if (state.cleanedUp) {
return;
}
state.cleanedUp = true;
if (child) {
console.log("[verify:gui-smoke] 停止 headless Tauri 环境...");
}
if (typeof child?.pid === "number") {
if (process.platform === "win32") {
spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
stdio: "ignore",
});
} else {
try {
process.kill(-child.pid, "SIGTERM");
} catch {
try {
child.kill("SIGTERM");
} catch {
// ignore
}
}
for (let attempt = 0; attempt < 25; attempt += 1) {
if (child.exitCode !== null || child.signalCode) {
break;
}
await sleep(200);
}
try {
process.kill(-child.pid, "SIGKILL");
} catch {
try {
child.kill("SIGKILL");
} catch {
// ignore
}
}
}
}
try {
if (state.tempConfigPath) {
fs.unlinkSync(state.tempConfigPath);
}
} catch {
// ignore
}
state.tempConfigPath = null;
}
async function waitForAppShell(options) {
@@ -236,9 +705,7 @@ async function waitForAppShell(options) {
}
assert(
html.includes("<title>Lime</title>") ||
html.includes('<div id="root"></div>') ||
html.includes('<div id="root"></div'),
isLimeDevShell(html) || html.includes('<div id="root"></div'),
"前端首页返回成功,但未检测到 Lime 根页面标记",
);
@@ -273,19 +740,221 @@ async function isUrlReady(url, timeoutMs) {
}
}
function describeChildExit(child) {
if (!child) {
return "unknown";
}
const parts = [];
if (typeof child.exitCode === "number") {
parts.push(`exitCode=${child.exitCode}`);
}
if (child.signalCode) {
parts.push(`signal=${child.signalCode}`);
}
return parts.length > 0 ? parts.join(" ") : "unknown";
}
async function waitForBridgeHealth(options, startedByScript) {
const startedAt = Date.now();
let deadlineAt = startedAt + options.timeoutMs;
let compileGraceCount = 0;
let bootGraceUsed = false;
let lastError = null;
let lastHeartbeatAt = startedAt;
console.log(`[bridge:health] 开始检查: ${options.healthUrl}`);
while (true) {
if (
startedByScript &&
state.child &&
(typeof state.child.exitCode === "number" || state.child.signalCode)
) {
const exitDetail = describeChildExit(state.child);
const lastDetail =
lastError instanceof Error
? `;最近一次健康检查错误: ${lastError.message}`
: "";
throw new Error(
`[verify:gui-smoke] headless Tauri 在 DevBridge 就绪前提前退出(${exitDetail})${lastDetail}`,
);
}
try {
const response = await fetch(options.healthUrl, {
method: "GET",
signal: AbortSignal.timeout(Math.min(options.intervalMs, 1_500)),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
let payload = null;
try {
payload = text ? JSON.parse(text) : null;
} catch {
payload = null;
}
const elapsed = Date.now() - startedAt;
const status =
payload && typeof payload === "object" ? payload.status : undefined;
console.log(
`[bridge:health] 就绪: ${options.healthUrl} (${elapsed}ms)${status ? ` status=${status}` : ""}`,
);
return;
} catch (error) {
lastError = error;
const heartbeatAt = Date.now();
if (heartbeatAt >= deadlineAt) {
const heartbeat = describeGuiSmokeHeartbeat(startedByScript);
if (
compileGraceCount < GUI_SMOKE_MAX_COMPILE_GRACE_EXTENSIONS &&
hasActiveGuiSmokeCompile(startedByScript)
) {
compileGraceCount += 1;
deadlineAt = heartbeatAt + GUI_SMOKE_COMPILE_GRACE_MS;
console.log(
`[bridge:health] 检测到 GUI smoke 仍在编译,第 ${compileGraceCount} 次额外延长 ${GUI_SMOKE_COMPILE_GRACE_MS}ms 等待 DevBridge${heartbeat ? `;进程组: ${heartbeat}` : ""}`,
);
continue;
}
if (
!bootGraceUsed &&
startedByScript &&
state.child &&
state.child.exitCode === null &&
!state.child.signalCode
) {
bootGraceUsed = true;
deadlineAt = heartbeatAt + GUI_SMOKE_BOOT_GRACE_MS;
console.log(
`[bridge:health] 编译链已结束,额外延长 ${GUI_SMOKE_BOOT_GRACE_MS}ms 等待 headless Tauri 拉起 DevBridge${heartbeat ? `;进程组: ${heartbeat}` : ""}`,
);
continue;
}
break;
}
if (heartbeatAt - lastHeartbeatAt >= GUI_SMOKE_BRIDGE_HEARTBEAT_MS) {
lastHeartbeatAt = heartbeatAt;
const detail =
error instanceof Error
? error.message
: String(error || "unknown error");
const heartbeat = describeGuiSmokeHeartbeat(startedByScript);
console.log(
`[bridge:health] 等待中: ${options.healthUrl} (${heartbeatAt - startedAt}ms);最近错误: ${detail}${heartbeat ? `;进程组: ${heartbeat}` : ""}`,
);
}
await sleep(options.intervalMs);
}
}
const detail =
lastError instanceof Error
? lastError.message
: String(lastError || "unknown error");
throw new Error(
`[bridge:health] 超时未就绪: ${options.healthUrl}。最后错误: ${detail}`,
);
}
async function probeAppShell(url, timeoutMs) {
try {
const response = await fetch(url, {
method: "GET",
signal: AbortSignal.timeout(timeoutMs),
});
const html = await response.text();
return {
reachable: response.ok,
status: response.status,
statusText: response.statusText,
isLimeDevShell: response.ok && isLimeDevShell(html),
};
} catch {
return {
reachable: false,
status: null,
statusText: "",
isLimeDevShell: false,
};
}
}
async function resolveStartupMode(options) {
if (options.reuseRunning) {
return {
shouldStart: false,
reusedExisting: true,
reuseExistingAppShell: false,
};
}
const existingAppShell = await isUrlReady(options.appUrl, 1_500);
if (!existingAppShell) {
const staleProcessCount = await cleanupStaleGuiSmokeProcesses();
if (staleProcessCount > 0) {
console.log(
`[verify:gui-smoke] 已清理 ${staleProcessCount} 条残留 GUI smoke headless 链路。`,
);
}
const guiSmokeProcesses = inspectGuiSmokeTauriProcesses();
const existingAppShell = await probeAppShell(options.appUrl, 1_500);
if (existingAppShell.reachable && !existingAppShell.isLimeDevShell) {
const statusLabel = `${existingAppShell.status || "unknown"} ${existingAppShell.statusText || ""}`.trim();
throw new Error(
`[verify:gui-smoke] ${options.appUrl} 已被其他服务占用,且返回内容不是 Lime 前端壳(${statusLabel})。请先关闭占用进程后重试。`,
);
}
if (!existingAppShell.isLimeDevShell) {
const existingListeners = listListeningCommandsForPort(
resolveUrlPort(options.appUrl),
);
const hasLimeFrontendListener = existingListeners.some(
isLikelyLimeFrontendListener,
);
if (guiSmokeProcesses.active.length > 0) {
const pidList = guiSmokeProcesses.active.map((item) => item.pid).join(", ");
console.log(
`[verify:gui-smoke] 检测到已有 GUI smoke headless 进程正在启动(PID: ${pidList});本次将直接复用现有链路并等待 DevBridge。`,
);
return {
shouldStart: false,
reusedExisting: true,
reuseExistingAppShell: hasLimeFrontendListener,
};
}
if (hasLimeFrontendListener) {
console.log(
`[verify:gui-smoke] 检测到 ${options.appUrl} 已由当前仓库的前端启动链监听,但页面尚未完全就绪;将复用现有前端启动链,只拉起 headless Tauri 与 DevBridge。`,
);
return {
shouldStart: true,
reusedExisting: false,
reuseExistingAppShell: true,
};
}
if (existingListeners.length > 0) {
throw new Error(
`[verify:gui-smoke] ${options.appUrl} 已被其他进程占用,且当前无法确认为 Lime 前端壳。请先关闭占用进程后重试。`,
);
}
return {
shouldStart: true,
reusedExisting: false,
reuseExistingAppShell: false,
};
}
@@ -297,6 +966,19 @@ async function resolveStartupMode(options) {
return {
shouldStart: false,
reusedExisting: true,
reuseExistingAppShell: true,
};
}
if (guiSmokeProcesses.active.length > 0) {
const pidList = guiSmokeProcesses.active.map((item) => item.pid).join(", ");
console.log(
`[verify:gui-smoke] 检测到已有 GUI smoke headless 进程正在启动(PID: ${pidList});本次将直接复用现有前端与 headless 链路。`,
);
return {
shouldStart: false,
reusedExisting: true,
reuseExistingAppShell: true,
};
}
@@ -306,6 +988,7 @@ async function resolveStartupMode(options) {
return {
shouldStart: true,
reusedExisting: false,
reuseExistingAppShell: true,
};
}
@@ -318,6 +1001,10 @@ async function main() {
const startupMode = await resolveStartupMode(options);
const startedByScript = startupMode.shouldStart;
console.log(
`[verify:gui-smoke] Cargo target: ${options.cargoTargetDir}`,
);
const handleSignal = async (signal) => {
try {
await stopHeadlessTauri();
@@ -335,28 +1022,13 @@ async function main() {
try {
if (startedByScript) {
startHeadlessTauri();
startHeadlessTauri(options, startupMode);
await sleep(1_500);
} else if (startupMode.reusedExisting) {
console.log("[verify:gui-smoke] 复用已运行的 headless Tauri 环境。");
}
runCommand(
npmCommand,
[
"run",
"bridge:health",
"--",
"--url",
options.healthUrl,
"--timeout-ms",
String(options.timeoutMs),
"--interval-ms",
String(options.intervalMs),
],
"bridge:health",
options.timeoutMs + 5_000,
);
await waitForBridgeHealth(options, startedByScript);
await waitForAppShell(options);