mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-09-24 23:10:56 +08:00
release: v1.9.0
This commit is contained in:
+18
-1
@@ -32,6 +32,20 @@ function parseWorkspaceField(section, fieldName) {
|
||||
return pattern.test(section);
|
||||
}
|
||||
|
||||
function readPackageJsonVersion(repoRoot) {
|
||||
const packageJsonPath = path.join(repoRoot, "package.json");
|
||||
if (!fs.existsSync(packageJsonPath)) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const packageJson = JSON.parse(fs.readFileSync(packageJsonPath, "utf8"));
|
||||
return typeof packageJson.version === "string" ? packageJson.version : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function readCargoVersions(cargoTomlPath) {
|
||||
const content = fs.readFileSync(cargoTomlPath, "utf8");
|
||||
const workspacePackageSection = extractSection(content, "workspace.package");
|
||||
@@ -50,8 +64,11 @@ export function readCargoVersions(cargoTomlPath) {
|
||||
|
||||
export function readWorkspaceAppVersion(repoRoot = process.cwd()) {
|
||||
const cargoTomlPath = path.join(repoRoot, "src-tauri", "Cargo.toml");
|
||||
if (!fs.existsSync(cargoTomlPath)) {
|
||||
return readPackageJsonVersion(repoRoot);
|
||||
}
|
||||
const { workspaceVersion } = readCargoVersions(cargoTomlPath);
|
||||
return workspaceVersion;
|
||||
return workspaceVersion ?? readPackageJsonVersion(repoRoot);
|
||||
}
|
||||
|
||||
const currentFilePath = fileURLToPath(import.meta.url);
|
||||
|
||||
@@ -13,9 +13,12 @@ const DEFAULTS = {
|
||||
streamMode: "both",
|
||||
};
|
||||
|
||||
const INVOKE_TIMEOUT_MS = 60_000;
|
||||
const INVOKE_RETRY_COUNT = 3;
|
||||
const INVOKE_RETRY_DELAY_MS = 500;
|
||||
const INVOKE_TIMEOUT_CEILING_MS = 180_000;
|
||||
const INVOKE_RETRY_COUNT = 10;
|
||||
const INVOKE_RETRY_DELAY_MS = 1_000;
|
||||
const POST_HEALTH_SETTLE_MS = 3_000;
|
||||
const POST_LAUNCH_SETTLE_MS = 1_500;
|
||||
const READ_PAGE_TIMEOUT_MS = 45_000;
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
@@ -123,13 +126,14 @@ function isTransientInvokeError(error) {
|
||||
}
|
||||
|
||||
async function invoke(options, cmd, args) {
|
||||
const invokeTimeoutMs = Math.min(options.timeoutMs, INVOKE_TIMEOUT_CEILING_MS);
|
||||
const requestInit = {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify({ cmd, args }),
|
||||
signal: AbortSignal.timeout(Math.min(options.timeoutMs, INVOKE_TIMEOUT_MS)),
|
||||
signal: AbortSignal.timeout(invokeTimeoutMs),
|
||||
};
|
||||
|
||||
for (let attempt = 1; attempt <= INVOKE_RETRY_COUNT; attempt += 1) {
|
||||
@@ -151,7 +155,7 @@ async function invoke(options, cmd, args) {
|
||||
if (!isTransientInvokeError(error) || attempt >= INVOKE_RETRY_COUNT) {
|
||||
if (error?.name === "TimeoutError") {
|
||||
throw new Error(
|
||||
`[smoke:browser-runtime] ${cmd} 超时,${Math.min(options.timeoutMs, INVOKE_TIMEOUT_MS)}ms 内未收到 DevBridge 响应`,
|
||||
`[smoke:browser-runtime] ${cmd} 超时,${invokeTimeoutMs}ms 内未收到 DevBridge 响应`,
|
||||
);
|
||||
}
|
||||
throw new Error(`[smoke:browser-runtime] ${cmd} 请求失败: ${detail}`);
|
||||
@@ -209,6 +213,7 @@ async function main() {
|
||||
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
await waitForHealth(options);
|
||||
await sleep(POST_HEALTH_SETTLE_MS);
|
||||
|
||||
const profileKey = `smoke-browser-runtime-${Date.now()}`;
|
||||
let sessionId = null;
|
||||
@@ -256,11 +261,13 @@ async function main() {
|
||||
"get_browser_session_state 未返回 target_id",
|
||||
);
|
||||
|
||||
await sleep(POST_LAUNCH_SETTLE_MS);
|
||||
|
||||
const actionResult = await invoke(options, "browser_execute_action", {
|
||||
request: {
|
||||
profile_key: profileKey,
|
||||
action: "read_page",
|
||||
timeout_ms: 20_000,
|
||||
timeout_ms: Math.min(options.timeoutMs, READ_PAGE_TIMEOUT_MS),
|
||||
},
|
||||
});
|
||||
assert(actionResult?.success === true, "browser_execute_action(read_page) 未成功");
|
||||
|
||||
@@ -0,0 +1,158 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
import { assertGeneratedSlopReportContract } from "./lib/generated-slop-report-core.mjs";
|
||||
|
||||
const GENERATED_SLOP_REPORT_SCRIPT = "scripts/report-generated-slop.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
format: "text",
|
||||
generateCurrent: false,
|
||||
help: false,
|
||||
input: ".lime/harness/reports/harness-cleanup-report.json",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
|
||||
if (arg === "--input" && argv[index + 1]) {
|
||||
result.input = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--generate-current") {
|
||||
result.generateCurrent = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Harness Cleanup Report Contract Check
|
||||
|
||||
用法:
|
||||
node scripts/check-generated-slop-report.mjs
|
||||
node scripts/check-generated-slop-report.mjs --generate-current
|
||||
node scripts/check-generated-slop-report.mjs --input ".lime/harness/reports/harness-cleanup-report.json"
|
||||
node scripts/check-generated-slop-report.mjs --input "./tmp/harness-cleanup-report.json" --format json
|
||||
|
||||
选项:
|
||||
--input PATH cleanup report JSON 路径,默认 ".lime/harness/reports/harness-cleanup-report.json"
|
||||
--generate-current 先生成当前 cleanup report,再校验其契约
|
||||
--format FMT 输出格式:text | json
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function resolveInputPath(inputPath) {
|
||||
return path.resolve(process.cwd(), inputPath);
|
||||
}
|
||||
|
||||
function buildCurrentCleanupReport() {
|
||||
const tempRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "lime-cleanup-report-contract-"),
|
||||
);
|
||||
const outputPath = path.join(tempRoot, "harness-cleanup-report.json");
|
||||
|
||||
try {
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
path.resolve(process.cwd(), GENERATED_SLOP_REPORT_SCRIPT),
|
||||
"--format",
|
||||
"json",
|
||||
"--output-json",
|
||||
outputPath,
|
||||
],
|
||||
{
|
||||
cwd: process.cwd(),
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return outputPath;
|
||||
} catch (error) {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
function buildSummary(report, resolvedPath) {
|
||||
return {
|
||||
status: "ok",
|
||||
inputPath: resolvedPath,
|
||||
recommendationCount: Array.isArray(report?.recommendations)
|
||||
? report.recommendations.length
|
||||
: 0,
|
||||
verificationFailureFocusCount: Array.isArray(
|
||||
report?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? report.focus.observabilityVerificationOutcomes.length
|
||||
: 0,
|
||||
currentRecoveredBaselineCount: Array.isArray(
|
||||
report?.focus?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? report.focus.currentRecoveredObservabilityVerificationOutcomes.length
|
||||
: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function renderSummary(summary, format) {
|
||||
if (format === "json") {
|
||||
return `${JSON.stringify(summary, null, 2)}\n`;
|
||||
}
|
||||
|
||||
return [
|
||||
"[harness-cleanup-contract] ok",
|
||||
`[harness-cleanup-contract] input: ${summary.inputPath}`,
|
||||
`[harness-cleanup-contract] recommendations: ${summary.recommendationCount}`,
|
||||
`[harness-cleanup-contract] verification failure focus: ${summary.verificationFailureFocusCount}`,
|
||||
`[harness-cleanup-contract] current recovered baseline: ${summary.currentRecoveredBaselineCount}`,
|
||||
].join("\n") + "\n";
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const resolvedPath = options.generateCurrent
|
||||
? buildCurrentCleanupReport()
|
||||
: resolveInputPath(options.input);
|
||||
|
||||
try {
|
||||
const report = JSON.parse(fs.readFileSync(resolvedPath, "utf8"));
|
||||
const validatedReport = assertGeneratedSlopReportContract(report);
|
||||
const summary = buildSummary(validatedReport, resolvedPath);
|
||||
process.stdout.write(renderSummary(summary, options.format));
|
||||
} finally {
|
||||
if (options.generateCurrent) {
|
||||
fs.rmSync(path.dirname(resolvedPath), { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -6,21 +6,34 @@ import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { renderHarnessDashboardHtml } from "./lib/harness-dashboard-core.mjs";
|
||||
|
||||
const RUNNER_PATH = "scripts/harness-eval-runner.mjs";
|
||||
const TREND_PATH = "scripts/harness-eval-trend-report.mjs";
|
||||
const CLEANUP_PATH = "scripts/report-generated-slop.mjs";
|
||||
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
|
||||
"repaired",
|
||||
"success",
|
||||
"passed",
|
||||
"clean",
|
||||
]);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
cleanupJson: "",
|
||||
cleanupMarkdown: "",
|
||||
dashboardHtml: "",
|
||||
dashboardTitle: "Lime Harness Dashboard",
|
||||
format: "text",
|
||||
help: false,
|
||||
historyDir: "./.lime/harness/history",
|
||||
manifest: "",
|
||||
outputJson: "",
|
||||
retain: 30,
|
||||
skipCleanup: false,
|
||||
skipTrend: false,
|
||||
summaryJson: "",
|
||||
summaryMarkdown: "",
|
||||
trendJson: "",
|
||||
trendMarkdown: "",
|
||||
workspaceRoot: process.cwd(),
|
||||
@@ -41,6 +54,12 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--manifest" && argv[index + 1]) {
|
||||
result.manifest = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--retain" && argv[index + 1]) {
|
||||
result.retain = Number.parseInt(String(argv[index + 1]).trim(), 10);
|
||||
index += 1;
|
||||
@@ -59,6 +78,18 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--summary-json" && argv[index + 1]) {
|
||||
result.summaryJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--summary-markdown" && argv[index + 1]) {
|
||||
result.summaryMarkdown = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--trend-json" && argv[index + 1]) {
|
||||
result.trendJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
@@ -83,6 +114,18 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--dashboard-html" && argv[index + 1]) {
|
||||
result.dashboardHtml = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--dashboard-title" && argv[index + 1]) {
|
||||
result.dashboardTitle = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--skip-trend") {
|
||||
result.skipTrend = true;
|
||||
continue;
|
||||
@@ -113,11 +156,16 @@ Lime Harness Eval History Record
|
||||
选项:
|
||||
--history-dir PATH summary 历史目录,默认 ./.lime/harness/history
|
||||
--workspace-root PATH 生成当前 summary 时使用的工作区根目录
|
||||
--manifest PATH 透传给 harness eval runner,覆盖默认 manifest
|
||||
--retain N 历史窗口保留数量,默认 30
|
||||
--trend-json PATH trend JSON 输出路径
|
||||
--trend-markdown PATH trend Markdown 输出路径
|
||||
--cleanup-json PATH cleanup JSON 输出路径
|
||||
--cleanup-markdown PATH cleanup Markdown 输出路径
|
||||
--summary-json PATH summary JSON 输出路径,默认写入 reports/harness-eval-summary.json
|
||||
--summary-markdown PATH summary Markdown 输出路径,默认写入 reports/harness-eval-summary.md
|
||||
--trend-json PATH trend JSON 输出路径,默认写入 reports/harness-eval-trend.json
|
||||
--trend-markdown PATH trend Markdown 输出路径,默认写入 reports/harness-eval-trend.md
|
||||
--cleanup-json PATH cleanup JSON 输出路径,默认写入 reports/harness-cleanup-report.json
|
||||
--cleanup-markdown PATH cleanup Markdown 输出路径,默认写入 reports/harness-cleanup-report.md
|
||||
--dashboard-html PATH dashboard HTML 输出路径,依赖 summary / trend / cleanup;默认写入 reports/harness-dashboard.html
|
||||
--dashboard-title TEXT dashboard 页面标题
|
||||
--skip-trend 只记录 summary,不生成 trend
|
||||
--skip-cleanup 只记录 summary / trend,不生成 cleanup
|
||||
--format FMT 标准输出格式:text | json
|
||||
@@ -134,8 +182,9 @@ function ensureParentDirectory(filePath) {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
function writeTextFile(filePath, contents) {
|
||||
ensureParentDirectory(filePath);
|
||||
fs.writeFileSync(filePath, contents, "utf8");
|
||||
}
|
||||
|
||||
function timestampForFilename() {
|
||||
@@ -154,20 +203,32 @@ function collectHistoryFiles(historyDir) {
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function buildCurrentSummary(repoRoot, workspaceRoot) {
|
||||
function runRunner(repoRoot, args) {
|
||||
const runnerPath = resolvePath(repoRoot, RUNNER_PATH);
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[runnerPath, "--format", "json", "--workspace-root", workspaceRoot],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
},
|
||||
);
|
||||
return execFileSync(process.execPath, [runnerPath, ...args], {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
}
|
||||
|
||||
function buildCurrentSummary(repoRoot, workspaceRoot, manifestPath) {
|
||||
const args = ["--format", "json", "--workspace-root", workspaceRoot];
|
||||
if (manifestPath) {
|
||||
args.push("--manifest", manifestPath);
|
||||
}
|
||||
const output = runRunner(repoRoot, args);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
function buildCurrentSummaryMarkdown(repoRoot, workspaceRoot, manifestPath) {
|
||||
const args = ["--format", "markdown", "--workspace-root", workspaceRoot];
|
||||
if (manifestPath) {
|
||||
args.push("--manifest", manifestPath);
|
||||
}
|
||||
return runRunner(repoRoot, args);
|
||||
}
|
||||
|
||||
function writeJsonFile(filePath, payload) {
|
||||
ensureParentDirectory(filePath);
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
@@ -214,13 +275,157 @@ function buildDefaultArtifactPaths(historyDir) {
|
||||
? path.join(historyParent, "reports")
|
||||
: historyParent;
|
||||
return {
|
||||
summaryJson: path.join(artifactsRoot, "harness-eval-summary.json"),
|
||||
summaryMarkdown: path.join(artifactsRoot, "harness-eval-summary.md"),
|
||||
trendJson: path.join(artifactsRoot, "harness-eval-trend.json"),
|
||||
trendMarkdown: path.join(artifactsRoot, "harness-eval-trend.md"),
|
||||
cleanupJson: path.join(artifactsRoot, "harness-cleanup-report.json"),
|
||||
cleanupMarkdown: path.join(artifactsRoot, "harness-cleanup-report.md"),
|
||||
dashboardHtml: path.join(artifactsRoot, "harness-dashboard.html"),
|
||||
};
|
||||
}
|
||||
|
||||
function toVerificationFailureOutcomeFocus(cleanupReport) {
|
||||
const currentEntries = Array.isArray(
|
||||
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentObservabilityVerificationOutcomes
|
||||
: [];
|
||||
const fallbackEntries = Array.isArray(
|
||||
cleanupReport?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.observabilityVerificationOutcomes
|
||||
: [];
|
||||
const entries =
|
||||
currentEntries.length > 0 ? currentEntries : fallbackEntries;
|
||||
|
||||
return entries
|
||||
.map((entry) => {
|
||||
const signal = typeof entry?.signal === "string" ? entry.signal.trim() : "";
|
||||
const outcome =
|
||||
typeof entry?.outcome === "string" ? entry.outcome.trim() : "";
|
||||
return signal && outcome ? `${signal}:${outcome}` : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function toCurrentRecoveredBaselineFocus(cleanupReport) {
|
||||
const explicitRecoveredEntries = Array.isArray(
|
||||
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes
|
||||
: [];
|
||||
const currentEntries = Array.isArray(
|
||||
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentObservabilityVerificationOutcomes
|
||||
: [];
|
||||
const fallbackEntries = Array.isArray(
|
||||
cleanupReport?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.observabilityVerificationOutcomes
|
||||
: [];
|
||||
const entries =
|
||||
explicitRecoveredEntries.length > 0
|
||||
? explicitRecoveredEntries
|
||||
: currentEntries.length > 0
|
||||
? currentEntries
|
||||
: fallbackEntries;
|
||||
|
||||
return entries
|
||||
.filter((entry) =>
|
||||
RECOVERED_VERIFICATION_OUTCOMES.has(
|
||||
typeof entry?.outcome === "string" ? entry.outcome.trim() : "",
|
||||
),
|
||||
)
|
||||
.map((entry) => {
|
||||
const signal = typeof entry?.signal === "string" ? entry.signal.trim() : "";
|
||||
const outcome =
|
||||
typeof entry?.outcome === "string" ? entry.outcome.trim() : "";
|
||||
return signal && outcome ? `${signal}:${outcome}` : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function toVerificationOutcomeCounts(cleanupReport) {
|
||||
const summary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.verificationOutcomes &&
|
||||
typeof cleanupReport.summary.verificationOutcomes === "object"
|
||||
? cleanupReport.summary.verificationOutcomes
|
||||
: {};
|
||||
const currentSummary =
|
||||
summary &&
|
||||
typeof summary.current === "object" &&
|
||||
!Array.isArray(summary.current)
|
||||
? summary.current
|
||||
: {};
|
||||
const degradedSummary =
|
||||
summary &&
|
||||
typeof summary.degraded === "object" &&
|
||||
!Array.isArray(summary.degraded)
|
||||
? summary.degraded
|
||||
: {};
|
||||
|
||||
return {
|
||||
failureCaseCount:
|
||||
typeof summary.failureCaseCount === "number" &&
|
||||
Number.isFinite(summary.failureCaseCount)
|
||||
? summary.failureCaseCount
|
||||
: 0,
|
||||
blockingFailureCaseCount:
|
||||
typeof currentSummary.blockingFailureCaseCount === "number" &&
|
||||
Number.isFinite(currentSummary.blockingFailureCaseCount)
|
||||
? currentSummary.blockingFailureCaseCount
|
||||
: 0,
|
||||
advisoryFailureCaseCount:
|
||||
typeof currentSummary.advisoryFailureCaseCount === "number" &&
|
||||
Number.isFinite(currentSummary.advisoryFailureCaseCount)
|
||||
? currentSummary.advisoryFailureCaseCount
|
||||
: 0,
|
||||
recoveredCaseCount:
|
||||
typeof summary.recoveredCaseCount === "number" &&
|
||||
Number.isFinite(summary.recoveredCaseCount)
|
||||
? summary.recoveredCaseCount
|
||||
: 0,
|
||||
currentRecoveredCaseCount:
|
||||
typeof currentSummary.recoveredCaseCount === "number" &&
|
||||
Number.isFinite(currentSummary.recoveredCaseCount)
|
||||
? currentSummary.recoveredCaseCount
|
||||
: 0,
|
||||
degradedBlockingFailureCaseCount:
|
||||
typeof degradedSummary.blockingFailureCaseCount === "number" &&
|
||||
Number.isFinite(degradedSummary.blockingFailureCaseCount)
|
||||
? degradedSummary.blockingFailureCaseCount
|
||||
: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function toTrendCurrentRecoveredBaselineFocus(trendReport) {
|
||||
const entries = Array.isArray(
|
||||
trendReport?.classificationDeltas?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? trendReport.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes
|
||||
: [];
|
||||
|
||||
return entries
|
||||
.filter((entry) => {
|
||||
const latestCaseCount =
|
||||
typeof entry?.latest?.caseCount === "number" &&
|
||||
Number.isFinite(entry.latest.caseCount)
|
||||
? entry.latest.caseCount
|
||||
: 0;
|
||||
return latestCaseCount > 0;
|
||||
})
|
||||
.map((entry) =>
|
||||
typeof entry?.name === "string" ? entry.name.trim() : "",
|
||||
)
|
||||
.filter(Boolean)
|
||||
.slice(0, 3);
|
||||
}
|
||||
|
||||
function renderOutput(result, format) {
|
||||
if (format === "json") {
|
||||
return `${JSON.stringify(result, null, 2)}\n`;
|
||||
@@ -234,8 +439,31 @@ function renderOutput(result, format) {
|
||||
`[lime] trimmed files: ${result.trimmedPaths.length}`,
|
||||
];
|
||||
|
||||
if (result.summary) {
|
||||
if (result.summary.outputJsonPath) {
|
||||
lines.push(`[lime] summary json: ${result.summary.outputJsonPath}`);
|
||||
}
|
||||
if (result.summary.outputMarkdownPath) {
|
||||
lines.push(`[lime] summary markdown: ${result.summary.outputMarkdownPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.trend) {
|
||||
lines.push(`[lime] trend sample count: ${result.trend.sampleCount}`);
|
||||
lines.push(
|
||||
`[lime] trend current observability gap cases: ${result.trend.currentObservabilityGapCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] trend degraded observability gap cases: ${result.trend.degradedObservabilityGapCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] trend current recovered baseline cases: ${result.trend.currentRecoveredVerificationCaseCount}`,
|
||||
);
|
||||
if (result.trend.currentRecoveredBaselineFocus.length > 0) {
|
||||
lines.push(
|
||||
`[lime] trend current recovered baseline: ${result.trend.currentRecoveredBaselineFocus.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (result.trend.outputJsonPath) {
|
||||
lines.push(`[lime] trend json: ${result.trend.outputJsonPath}`);
|
||||
}
|
||||
@@ -245,11 +473,49 @@ function renderOutput(result, format) {
|
||||
lines.push(
|
||||
`[lime] cleanup trend samples: ${result.cleanup.trendSampleCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup current observability gap cases: ${result.cleanup.currentObservabilityGapCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup degraded observability gap cases: ${result.cleanup.degradedObservabilityGapCaseCount}`,
|
||||
);
|
||||
if (result.cleanup.verificationFailureOutcomeFocus.length > 0) {
|
||||
lines.push(
|
||||
`[lime] cleanup verification failure outcomes: ${result.cleanup.verificationFailureOutcomeFocus.join(", ")}`,
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
`[lime] cleanup verification failure cases: ${result.cleanup.verificationFailureCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup verification blocking failure cases: ${result.cleanup.verificationBlockingFailureCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup verification advisory failure cases: ${result.cleanup.verificationAdvisoryFailureCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup degraded blocking verification failure cases: ${result.cleanup.verificationDegradedBlockingFailureCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup verification recovered cases: ${result.cleanup.verificationRecoveredCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
`[lime] cleanup current recovered baseline cases: ${result.cleanup.currentVerificationRecoveredCaseCount}`,
|
||||
);
|
||||
if (result.cleanup.currentRecoveredBaselineFocus.length > 0) {
|
||||
lines.push(
|
||||
`[lime] cleanup current recovered baseline: ${result.cleanup.currentRecoveredBaselineFocus.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (result.cleanup.outputJsonPath) {
|
||||
lines.push(`[lime] cleanup json: ${result.cleanup.outputJsonPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.dashboard?.outputHtmlPath) {
|
||||
lines.push(`[lime] dashboard html: ${result.dashboard.outputHtmlPath}`);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
@@ -260,11 +526,24 @@ function runHistoryRecordCli() {
|
||||
return;
|
||||
}
|
||||
|
||||
if (options.dashboardHtml && (options.skipTrend || options.skipCleanup)) {
|
||||
throw new Error(
|
||||
"生成 dashboard 需要同时启用 trend 与 cleanup,请移除 --skip-trend / --skip-cleanup。",
|
||||
);
|
||||
}
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const historyDir = resolvePath(repoRoot, options.historyDir);
|
||||
fs.mkdirSync(historyDir, { recursive: true });
|
||||
|
||||
const summary = buildCurrentSummary(repoRoot, options.workspaceRoot);
|
||||
const effectiveManifest = options.manifest
|
||||
? resolvePath(repoRoot, options.manifest)
|
||||
: "";
|
||||
const summary = buildCurrentSummary(
|
||||
repoRoot,
|
||||
options.workspaceRoot,
|
||||
effectiveManifest,
|
||||
);
|
||||
const summaryFilePath = writeUniqueHistorySummary(historyDir, summary);
|
||||
const trimmedPaths = trimHistoryFiles(historyDir, options.retain);
|
||||
const historyCount = collectHistoryFiles(historyDir).length;
|
||||
@@ -276,8 +555,33 @@ function runHistoryRecordCli() {
|
||||
recordedSummaryPath: summaryFilePath,
|
||||
historyCount,
|
||||
trimmedPaths,
|
||||
summary: null,
|
||||
trend: null,
|
||||
cleanup: null,
|
||||
dashboard: null,
|
||||
};
|
||||
let trendReport = null;
|
||||
let cleanupReport = null;
|
||||
|
||||
const summaryJsonPath = resolvePath(
|
||||
repoRoot,
|
||||
options.summaryJson || defaults.summaryJson,
|
||||
);
|
||||
const summaryMarkdownPath = resolvePath(
|
||||
repoRoot,
|
||||
options.summaryMarkdown || defaults.summaryMarkdown,
|
||||
);
|
||||
|
||||
writeJsonFile(summaryJsonPath, summary);
|
||||
const summaryMarkdown = buildCurrentSummaryMarkdown(
|
||||
repoRoot,
|
||||
options.workspaceRoot,
|
||||
effectiveManifest,
|
||||
);
|
||||
writeTextFile(summaryMarkdownPath, summaryMarkdown);
|
||||
result.summary = {
|
||||
outputJsonPath: summaryJsonPath,
|
||||
outputMarkdownPath: summaryMarkdownPath,
|
||||
};
|
||||
|
||||
if (!options.skipTrend) {
|
||||
@@ -309,9 +613,18 @@ function runHistoryRecordCli() {
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
},
|
||||
);
|
||||
const trendReport = JSON.parse(trendOutput);
|
||||
trendReport = JSON.parse(trendOutput);
|
||||
const trendCurrentRecoveredBaselineFocus =
|
||||
toTrendCurrentRecoveredBaselineFocus(trendReport);
|
||||
result.trend = {
|
||||
sampleCount: trendReport.sampleCount,
|
||||
currentObservabilityGapCaseCount:
|
||||
trendReport.latest?.totals?.currentObservabilityGapCaseCount ?? 0,
|
||||
degradedObservabilityGapCaseCount:
|
||||
trendReport.latest?.totals?.degradedObservabilityGapCaseCount ?? 0,
|
||||
currentRecoveredVerificationCaseCount:
|
||||
trendReport.latest?.totals?.currentRecoveredVerificationCaseCount ?? 0,
|
||||
currentRecoveredBaselineFocus: trendCurrentRecoveredBaselineFocus,
|
||||
outputJsonPath: trendJsonPath,
|
||||
outputMarkdownPath: trendMarkdownPath,
|
||||
};
|
||||
@@ -346,14 +659,57 @@ function runHistoryRecordCli() {
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
},
|
||||
);
|
||||
const cleanupReport = JSON.parse(cleanupOutput);
|
||||
cleanupReport = JSON.parse(cleanupOutput);
|
||||
const verificationFailureOutcomeFocus =
|
||||
toVerificationFailureOutcomeFocus(cleanupReport);
|
||||
const currentRecoveredBaselineFocus =
|
||||
toCurrentRecoveredBaselineFocus(cleanupReport);
|
||||
const verificationOutcomeCounts =
|
||||
toVerificationOutcomeCounts(cleanupReport);
|
||||
result.cleanup = {
|
||||
trendSampleCount: cleanupReport.summary?.trend?.sampleCount ?? 0,
|
||||
currentObservabilityGapCaseCount:
|
||||
cleanupReport.summary?.trend?.latestCurrentObservabilityGapCaseCount ?? 0,
|
||||
degradedObservabilityGapCaseCount:
|
||||
cleanupReport.summary?.trend?.latestDegradedObservabilityGapCaseCount ?? 0,
|
||||
verificationFailureOutcomeFocus,
|
||||
verificationFailureCaseCount:
|
||||
verificationOutcomeCounts.failureCaseCount,
|
||||
verificationBlockingFailureCaseCount:
|
||||
verificationOutcomeCounts.blockingFailureCaseCount,
|
||||
verificationAdvisoryFailureCaseCount:
|
||||
verificationOutcomeCounts.advisoryFailureCaseCount,
|
||||
verificationDegradedBlockingFailureCaseCount:
|
||||
verificationOutcomeCounts.degradedBlockingFailureCaseCount,
|
||||
verificationRecoveredCaseCount:
|
||||
verificationOutcomeCounts.recoveredCaseCount,
|
||||
currentVerificationRecoveredCaseCount:
|
||||
verificationOutcomeCounts.currentRecoveredCaseCount,
|
||||
currentRecoveredBaselineFocus,
|
||||
outputJsonPath: cleanupJsonPath,
|
||||
outputMarkdownPath: cleanupMarkdownPath,
|
||||
};
|
||||
}
|
||||
|
||||
const dashboardHtmlPath =
|
||||
options.skipTrend || options.skipCleanup
|
||||
? ""
|
||||
: resolvePath(repoRoot, options.dashboardHtml || defaults.dashboardHtml);
|
||||
|
||||
if (dashboardHtmlPath) {
|
||||
const dashboardHtml = renderHarnessDashboardHtml({
|
||||
summaryReport: summary,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
title: options.dashboardTitle,
|
||||
});
|
||||
writeTextFile(dashboardHtmlPath, dashboardHtml);
|
||||
result.dashboard = {
|
||||
outputHtmlPath: dashboardHtmlPath,
|
||||
title: options.dashboardTitle,
|
||||
};
|
||||
}
|
||||
|
||||
const rendered = renderOutput(result, options.format);
|
||||
if (options.outputJson) {
|
||||
const outputPath = resolvePath(repoRoot, options.outputJson);
|
||||
|
||||
+251
-87
@@ -22,16 +22,15 @@ const REVIEW_DECISION_RISK_LEVEL_SET = new Set([
|
||||
"high",
|
||||
"unknown",
|
||||
]);
|
||||
const OBSERVABILITY_GAP_SUITE_TAG = "observability-gap";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
format: "text",
|
||||
help: false,
|
||||
historyRetain: 30,
|
||||
manifest: DEFAULT_MANIFEST_PATH,
|
||||
outputJson: "",
|
||||
outputMarkdown: "",
|
||||
recordHistoryDir: "",
|
||||
strict: true,
|
||||
workspaceRoot: process.cwd(),
|
||||
};
|
||||
@@ -69,18 +68,6 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--record-history-dir" && argv[index + 1]) {
|
||||
result.recordHistoryDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--history-retain" && argv[index + 1]) {
|
||||
result.historyRetain = Number.parseInt(String(argv[index + 1]), 10);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--no-strict") {
|
||||
result.strict = false;
|
||||
continue;
|
||||
@@ -108,7 +95,6 @@ Lime Harness Eval Runner
|
||||
node scripts/harness-eval-runner.mjs --format json
|
||||
node scripts/harness-eval-runner.mjs --workspace-root "/path/to/workspace"
|
||||
node scripts/harness-eval-runner.mjs --output-json "./tmp/harness-eval-summary.json" --output-markdown "./tmp/harness-eval-summary.md"
|
||||
node scripts/harness-eval-runner.mjs --record-history-dir "./artifacts/history"
|
||||
|
||||
选项:
|
||||
--manifest PATH 指定 manifest,默认 docs/test/harness-evals.manifest.json
|
||||
@@ -116,8 +102,6 @@ Lime Harness Eval Runner
|
||||
--format FMT 控制标准输出格式:text | json | markdown
|
||||
--output-json PATH 将 JSON 摘要写入指定路径
|
||||
--output-markdown PATH 将 Markdown 摘要写入指定路径
|
||||
--record-history-dir PATH 将当前 summary 追加写入历史目录,供 trend/nightly 复用
|
||||
--history-retain N 历史目录最多保留多少条 summary,默认 30
|
||||
--strict 严格模式(默认),发现 invalid case 时返回非 0
|
||||
--no-strict 非严格模式,只输出摘要,不因 invalid case 退出失败
|
||||
-h, --help 显示帮助
|
||||
@@ -136,56 +120,6 @@ function ensureParentDirectory(filePath) {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
}
|
||||
|
||||
function ensureDirectory(dirPath) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function toHistoryTimestamp(generatedAt) {
|
||||
const parsed = new Date(generatedAt);
|
||||
const normalized = Number.isNaN(parsed.getTime())
|
||||
? new Date().toISOString()
|
||||
: parsed.toISOString();
|
||||
return normalized.replace(/[-:]/g, "").replace(/\.(\d{3})Z$/, "$1Z");
|
||||
}
|
||||
|
||||
function trimHistoryDirectory(historyDir, retainCount) {
|
||||
const normalizedRetainCount =
|
||||
Number.isInteger(retainCount) && retainCount > 0 ? retainCount : 30;
|
||||
const historyFiles = fs
|
||||
.readdirSync(historyDir, { withFileTypes: true })
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.isFile() &&
|
||||
entry.name.endsWith("-harness-eval-summary.json"),
|
||||
)
|
||||
.map((entry) => path.join(historyDir, entry.name))
|
||||
.sort((left, right) => right.localeCompare(left));
|
||||
|
||||
for (const staleFile of historyFiles.slice(normalizedRetainCount)) {
|
||||
fs.rmSync(staleFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function recordSummaryHistory(repoRoot, summary, options) {
|
||||
if (!options.recordHistoryDir) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const historyDir = resolvePath(repoRoot, options.recordHistoryDir);
|
||||
ensureDirectory(historyDir);
|
||||
const historyFilePath = path.join(
|
||||
historyDir,
|
||||
`${toHistoryTimestamp(summary.generatedAt)}-harness-eval-summary.json`,
|
||||
);
|
||||
fs.writeFileSync(
|
||||
historyFilePath,
|
||||
`${JSON.stringify(summary, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
trimHistoryDirectory(historyDir, options.historyRetain);
|
||||
return historyFilePath;
|
||||
}
|
||||
|
||||
function normalizeStringList(value) {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
@@ -213,6 +147,10 @@ function normalizeEnumString(value, allowedValues, fallback = "") {
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function isObject(value) {
|
||||
return value != null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function createBreakdownEntry(name) {
|
||||
return {
|
||||
name,
|
||||
@@ -255,6 +193,25 @@ function aggregateCaseBreakdown(cases, selector) {
|
||||
});
|
||||
}
|
||||
|
||||
function isDegradedObservabilityGapCase(entry) {
|
||||
return (
|
||||
entry.observabilityGapCount > 0 &&
|
||||
normalizeStringList(entry.tags).includes(OBSERVABILITY_GAP_SUITE_TAG)
|
||||
);
|
||||
}
|
||||
|
||||
function isCurrentObservabilityGapCase(entry) {
|
||||
return entry.observabilityGapCount > 0 && !isDegradedObservabilityGapCase(entry);
|
||||
}
|
||||
|
||||
function isDegradedObservabilityDiagnosticCase(entry) {
|
||||
return normalizeStringList(entry?.tags).includes(OBSERVABILITY_GAP_SUITE_TAG);
|
||||
}
|
||||
|
||||
function isCurrentObservabilityDiagnosticCase(entry) {
|
||||
return !isDegradedObservabilityDiagnosticCase(entry);
|
||||
}
|
||||
|
||||
function getValueByPath(target, dottedPath) {
|
||||
return dottedPath
|
||||
.split(".")
|
||||
@@ -388,19 +345,39 @@ function loadReviewDecisionForCase(caseDir, inlineReviewDecision) {
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
|
||||
const inlineSummary =
|
||||
inputPayload?.observability &&
|
||||
typeof inputPayload.observability === "object" &&
|
||||
!Array.isArray(inputPayload.observability)
|
||||
? inputPayload.observability
|
||||
: evidencePayload?.observabilitySummary &&
|
||||
typeof evidencePayload.observabilitySummary === "object" &&
|
||||
!Array.isArray(evidencePayload.observabilitySummary)
|
||||
? evidencePayload.observabilitySummary
|
||||
: null;
|
||||
function resolveObservabilitySummary(inputPayload, evidencePayload) {
|
||||
const inlineSummary = isObject(inputPayload?.observability)
|
||||
? inputPayload.observability
|
||||
: null;
|
||||
const evidenceSummary = isObject(evidencePayload?.observabilitySummary)
|
||||
? evidencePayload.observabilitySummary
|
||||
: null;
|
||||
|
||||
if (!inlineSummary) {
|
||||
if (!inlineSummary && !evidenceSummary) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
signalCoverage: Array.isArray(inlineSummary?.signalCoverage)
|
||||
? inlineSummary.signalCoverage
|
||||
: Array.isArray(evidenceSummary?.signalCoverage)
|
||||
? evidenceSummary.signalCoverage
|
||||
: [],
|
||||
verificationSummary: isObject(inlineSummary?.verificationSummary)
|
||||
? inlineSummary.verificationSummary
|
||||
: isObject(evidenceSummary?.verificationSummary)
|
||||
? evidenceSummary.verificationSummary
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
|
||||
const observabilitySummary = resolveObservabilitySummary(
|
||||
inputPayload,
|
||||
evidencePayload,
|
||||
);
|
||||
|
||||
if (!observabilitySummary) {
|
||||
return [
|
||||
{
|
||||
signal: "observabilitySummary",
|
||||
@@ -409,8 +386,8 @@ function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
|
||||
];
|
||||
}
|
||||
|
||||
const signalCoverage = Array.isArray(inlineSummary.signalCoverage)
|
||||
? inlineSummary.signalCoverage
|
||||
const signalCoverage = Array.isArray(observabilitySummary.signalCoverage)
|
||||
? observabilitySummary.signalCoverage
|
||||
: [];
|
||||
|
||||
if (signalCoverage.length === 0) {
|
||||
@@ -461,6 +438,129 @@ function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
|
||||
return coverage;
|
||||
}
|
||||
|
||||
function pushUniqueObservabilityVerificationOutcome(outcomes, seen, signal, outcome) {
|
||||
const normalizedSignal = normalizeOptionalString(signal);
|
||||
const normalizedOutcome = normalizeOptionalString(outcome);
|
||||
if (!normalizedSignal || !normalizedOutcome) {
|
||||
return;
|
||||
}
|
||||
|
||||
const fingerprint = `${normalizedSignal}:${normalizedOutcome}`;
|
||||
if (seen.has(fingerprint)) {
|
||||
return;
|
||||
}
|
||||
|
||||
seen.add(fingerprint);
|
||||
outcomes.push({
|
||||
signal: normalizedSignal,
|
||||
outcome: normalizedOutcome,
|
||||
});
|
||||
}
|
||||
|
||||
function normalizeObservabilityVerificationSummary(inputPayload, evidencePayload) {
|
||||
const observabilitySummary = resolveObservabilitySummary(
|
||||
inputPayload,
|
||||
evidencePayload,
|
||||
);
|
||||
const verificationSummary = isObject(observabilitySummary?.verificationSummary)
|
||||
? observabilitySummary.verificationSummary
|
||||
: null;
|
||||
|
||||
if (!verificationSummary) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const outcomes = [];
|
||||
const seen = new Set();
|
||||
|
||||
const artifactValidator = isObject(verificationSummary.artifactValidator)
|
||||
? verificationSummary.artifactValidator
|
||||
: null;
|
||||
if (artifactValidator?.applicable === true) {
|
||||
const recordCount = Number(artifactValidator.recordCount) || 0;
|
||||
const issueCount = Number(artifactValidator.issueCount) || 0;
|
||||
const repairedCount = Number(artifactValidator.repairedCount) || 0;
|
||||
const fallbackUsedCount = Number(artifactValidator.fallbackUsedCount) || 0;
|
||||
|
||||
if (recordCount > 0 && issueCount === 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"artifactValidator",
|
||||
"clean",
|
||||
);
|
||||
}
|
||||
if (issueCount > 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"artifactValidator",
|
||||
"issues_present",
|
||||
);
|
||||
}
|
||||
if (repairedCount > 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"artifactValidator",
|
||||
"repaired",
|
||||
);
|
||||
}
|
||||
if (fallbackUsedCount > 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"artifactValidator",
|
||||
"fallback_used",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const browserVerification = isObject(verificationSummary.browserVerification)
|
||||
? verificationSummary.browserVerification
|
||||
: null;
|
||||
if (browserVerification) {
|
||||
if ((Number(browserVerification.successCount) || 0) > 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"browserVerification",
|
||||
"success",
|
||||
);
|
||||
}
|
||||
if ((Number(browserVerification.failureCount) || 0) > 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"browserVerification",
|
||||
"failure",
|
||||
);
|
||||
}
|
||||
if ((Number(browserVerification.unknownCount) || 0) > 0) {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"browserVerification",
|
||||
"unknown",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const guiSmoke = isObject(verificationSummary.guiSmoke)
|
||||
? verificationSummary.guiSmoke
|
||||
: null;
|
||||
if (guiSmoke && typeof guiSmoke.passed === "boolean") {
|
||||
pushUniqueObservabilityVerificationOutcome(
|
||||
outcomes,
|
||||
seen,
|
||||
"guiSmoke",
|
||||
guiSmoke.passed ? "passed" : "failed",
|
||||
);
|
||||
}
|
||||
|
||||
return outcomes;
|
||||
}
|
||||
|
||||
function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
const requiredArtifacts = normalizeStringList(
|
||||
caseConfig.requiredArtifacts ?? defaults.requiredArtifacts,
|
||||
@@ -551,6 +651,12 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
const observabilityGapCount = observabilityCoverage.filter(
|
||||
(entry) => entry.status !== "exported",
|
||||
).length;
|
||||
const observabilityVerificationCoverage =
|
||||
normalizeObservabilityVerificationSummary(inputPayload, evidencePayload);
|
||||
const observabilityVerificationOutcomes =
|
||||
observabilityVerificationCoverage.map(
|
||||
(entry) => `${entry.signal}:${entry.outcome}`,
|
||||
);
|
||||
|
||||
const pendingRequestCount = Array.isArray(
|
||||
inputPayload?.runtimeContext?.pendingRequests,
|
||||
@@ -606,6 +712,8 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
observabilityCoverage,
|
||||
observabilitySignals,
|
||||
observabilityGapCount,
|
||||
observabilityVerificationCoverage,
|
||||
observabilityVerificationOutcomes,
|
||||
preferredMode,
|
||||
status: issues.length === 0 ? "ready" : "invalid",
|
||||
issues,
|
||||
@@ -675,6 +783,8 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
|
||||
],
|
||||
observabilitySignals: ["observabilitySummary:missing"],
|
||||
observabilityGapCount: 1,
|
||||
observabilityVerificationCoverage: [],
|
||||
observabilityVerificationOutcomes: [],
|
||||
preferredMode: "",
|
||||
status: "invalid",
|
||||
issues: [
|
||||
@@ -727,6 +837,8 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
|
||||
observabilityCoverage: [{ signal: "observabilitySummary", status: "missing" }],
|
||||
observabilitySignals: ["observabilitySummary:missing"],
|
||||
observabilityGapCount: 1,
|
||||
observabilityVerificationCoverage: [],
|
||||
observabilityVerificationOutcomes: [],
|
||||
preferredMode: "",
|
||||
status: "invalid",
|
||||
issues: [`不支持的 case source: ${source || "(empty)"}`],
|
||||
@@ -773,6 +885,12 @@ function buildSummary(manifest, suites, options) {
|
||||
const observabilityGapCases = allCases.filter(
|
||||
(entry) => entry.observabilityGapCount > 0,
|
||||
);
|
||||
const currentObservabilityGapCases = observabilityGapCases.filter((entry) =>
|
||||
isCurrentObservabilityGapCase(entry),
|
||||
);
|
||||
const degradedObservabilityGapCases = observabilityGapCases.filter((entry) =>
|
||||
isDegradedObservabilityGapCase(entry),
|
||||
);
|
||||
|
||||
return {
|
||||
manifestVersion: String(manifest.manifestVersion ?? "unknown"),
|
||||
@@ -790,6 +908,8 @@ function buildSummary(manifest, suites, options) {
|
||||
pendingRequestCaseCount: pendingCases.length,
|
||||
reviewDecisionRecordedCount: recordedReviewDecisionCases.length,
|
||||
observabilityGapCaseCount: observabilityGapCases.length,
|
||||
currentObservabilityGapCaseCount: currentObservabilityGapCases.length,
|
||||
degradedObservabilityGapCaseCount: degradedObservabilityGapCases.length,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: aggregateCaseBreakdown(allCases, (entry) => entry.tags),
|
||||
@@ -807,6 +927,18 @@ function buildSummary(manifest, suites, options) {
|
||||
allCases,
|
||||
(entry) => entry.observabilitySignals,
|
||||
),
|
||||
observabilityVerificationOutcomes: aggregateCaseBreakdown(
|
||||
allCases,
|
||||
(entry) => entry.observabilityVerificationOutcomes,
|
||||
),
|
||||
currentObservabilityVerificationOutcomes: aggregateCaseBreakdown(
|
||||
allCases.filter((entry) => isCurrentObservabilityDiagnosticCase(entry)),
|
||||
(entry) => entry.observabilityVerificationOutcomes,
|
||||
),
|
||||
degradedObservabilityVerificationOutcomes: aggregateCaseBreakdown(
|
||||
allCases.filter((entry) => isDegradedObservabilityDiagnosticCase(entry)),
|
||||
(entry) => entry.observabilityVerificationOutcomes,
|
||||
),
|
||||
},
|
||||
suites,
|
||||
};
|
||||
@@ -824,6 +956,8 @@ function renderText(summary) {
|
||||
`[harness-eval] needs-review cases : ${summary.totals.needsHumanReviewCount}`,
|
||||
`[harness-eval] recorded review decisions: ${summary.totals.reviewDecisionRecordedCount}`,
|
||||
`[harness-eval] observability-gap cases: ${summary.totals.observabilityGapCaseCount}`,
|
||||
`[harness-eval] current observability-gap cases: ${summary.totals.currentObservabilityGapCaseCount}`,
|
||||
`[harness-eval] degraded observability-gap cases: ${summary.totals.degradedObservabilityGapCaseCount}`,
|
||||
];
|
||||
|
||||
const topFailureModes = summary.breakdowns.failureModes.slice(0, 5);
|
||||
@@ -869,6 +1003,17 @@ function renderText(summary) {
|
||||
}
|
||||
}
|
||||
|
||||
const topVerificationOutcomes =
|
||||
summary.breakdowns.observabilityVerificationOutcomes.slice(0, 5);
|
||||
if (topVerificationOutcomes.length > 0) {
|
||||
lines.push("[harness-eval] observability verification outcomes:");
|
||||
for (const entry of topVerificationOutcomes) {
|
||||
lines.push(
|
||||
` - ${entry.name}: case=${entry.caseCount}, ready=${entry.readyCount}, invalid=${entry.invalidCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const suite of summary.suites) {
|
||||
lines.push(
|
||||
`[harness-eval] suite ${suite.id}: ready ${suite.stats.readyCount} / ${suite.stats.caseCount}`,
|
||||
@@ -893,6 +1038,11 @@ function renderText(summary) {
|
||||
` observability: ${entry.observabilitySignals.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (entry.observabilityVerificationOutcomes.length > 0) {
|
||||
lines.push(
|
||||
` verification: ${entry.observabilityVerificationOutcomes.join(", ")}`,
|
||||
);
|
||||
}
|
||||
for (const issue of entry.issues) {
|
||||
lines.push(` * ${issue}`);
|
||||
}
|
||||
@@ -917,6 +1067,8 @@ function renderMarkdown(summary) {
|
||||
`- needs review case:${summary.totals.needsHumanReviewCount}`,
|
||||
`- 已记录人工审核:${summary.totals.reviewDecisionRecordedCount}`,
|
||||
`- observability gap case:${summary.totals.observabilityGapCaseCount}`,
|
||||
`- current observability gap case:${summary.totals.currentObservabilityGapCaseCount}`,
|
||||
`- degraded observability gap case:${summary.totals.degradedObservabilityGapCaseCount}`,
|
||||
"",
|
||||
];
|
||||
|
||||
@@ -987,6 +1139,19 @@ function renderMarkdown(summary) {
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (summary.breakdowns.observabilityVerificationOutcomes.length > 0) {
|
||||
lines.push("## Observability Verification Outcome 分布");
|
||||
lines.push("");
|
||||
lines.push("| Outcome | case | ready | invalid |");
|
||||
lines.push("| --- | --- | --- | --- |");
|
||||
for (const entry of summary.breakdowns.observabilityVerificationOutcomes) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.caseCount} | ${entry.readyCount} | ${entry.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
for (const suite of summary.suites) {
|
||||
lines.push(`## ${suite.title}`);
|
||||
lines.push("");
|
||||
@@ -1025,6 +1190,11 @@ function renderMarkdown(summary) {
|
||||
`observability: ${entry.observabilitySignals.join(", ")}`,
|
||||
);
|
||||
}
|
||||
if (entry.observabilityVerificationOutcomes.length > 0) {
|
||||
classificationText.push(
|
||||
`verification: ${entry.observabilityVerificationOutcomes.join(", ")}`,
|
||||
);
|
||||
}
|
||||
const reviewText = entry.reviewDecisionStatus
|
||||
? [
|
||||
entry.reviewDecisionStatus,
|
||||
@@ -1078,7 +1248,6 @@ function main() {
|
||||
const jsonOutput = `${JSON.stringify(summary, null, 2)}\n`;
|
||||
const markdownOutput = renderMarkdown(summary);
|
||||
const textOutput = renderText(summary);
|
||||
const historyFilePath = recordSummaryHistory(repoRoot, summary, options);
|
||||
|
||||
if (options.outputJson) {
|
||||
const outputPath = resolvePath(repoRoot, options.outputJson);
|
||||
@@ -1098,11 +1267,6 @@ function main() {
|
||||
process.stdout.write(markdownOutput);
|
||||
} else {
|
||||
process.stdout.write(textOutput);
|
||||
if (historyFilePath) {
|
||||
process.stdout.write(
|
||||
`[harness-eval] history snapshot: ${historyFilePath}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const exitCode = determineExitCode(summary, options);
|
||||
|
||||
@@ -6,6 +6,20 @@ import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
const RUNNER_PATH = "scripts/harness-eval-runner.mjs";
|
||||
const OBSERVABILITY_GAP_SUITE_TAG = "observability-gap";
|
||||
const OBSERVABILITY_FAILURE_OUTCOMES = new Set([
|
||||
"artifactValidator:issues_present",
|
||||
"artifactValidator:fallback_used",
|
||||
"browserVerification:failure",
|
||||
"browserVerification:unknown",
|
||||
"guiSmoke:failed",
|
||||
]);
|
||||
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
"guiSmoke:clean",
|
||||
]);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
@@ -170,12 +184,113 @@ function normalizeNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function listAllCases(summary) {
|
||||
const suites = Array.isArray(summary?.suites) ? summary.suites : [];
|
||||
return suites.flatMap((suite) => (Array.isArray(suite?.cases) ? suite.cases : []));
|
||||
}
|
||||
|
||||
function getBreakdownEntries(summary, key) {
|
||||
return Array.isArray(summary?.breakdowns?.[key]) ? summary.breakdowns[key] : [];
|
||||
}
|
||||
|
||||
function getBreakdownCaseCount(summary, key, namesSet) {
|
||||
return getBreakdownEntries(summary, key).reduce((total, entry) => {
|
||||
const name = String(entry?.name ?? "");
|
||||
if (!namesSet.has(name)) {
|
||||
return total;
|
||||
}
|
||||
return total + normalizeNumber(entry?.caseCount);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
function buildObservabilityGapTotals(summary) {
|
||||
const rawTotals =
|
||||
summary?.totals && typeof summary.totals === "object" ? summary.totals : {};
|
||||
let total = normalizeNumber(rawTotals.observabilityGapCaseCount);
|
||||
let current = normalizeNumber(rawTotals.currentObservabilityGapCaseCount);
|
||||
let degraded = normalizeNumber(rawTotals.degradedObservabilityGapCaseCount);
|
||||
const allCases = listAllCases(summary);
|
||||
|
||||
if (allCases.length === 0) {
|
||||
if (current === 0 && degraded === 0 && total > 0) {
|
||||
current = total;
|
||||
}
|
||||
return {
|
||||
total: total || current + degraded,
|
||||
current,
|
||||
degraded,
|
||||
};
|
||||
}
|
||||
|
||||
let derivedCurrent = 0;
|
||||
let derivedDegraded = 0;
|
||||
for (const entry of allCases) {
|
||||
if (normalizeNumber(entry?.observabilityGapCount) <= 0) {
|
||||
continue;
|
||||
}
|
||||
const tags = Array.isArray(entry?.tags) ? entry.tags : [];
|
||||
if (tags.includes(OBSERVABILITY_GAP_SUITE_TAG)) {
|
||||
derivedDegraded += 1;
|
||||
} else {
|
||||
derivedCurrent += 1;
|
||||
}
|
||||
}
|
||||
|
||||
const derivedTotal = derivedCurrent + derivedDegraded;
|
||||
if (derivedTotal === 0) {
|
||||
return {
|
||||
total: total || current + degraded,
|
||||
current,
|
||||
degraded,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
total: derivedTotal,
|
||||
current: derivedCurrent,
|
||||
degraded: derivedDegraded,
|
||||
};
|
||||
}
|
||||
|
||||
function buildNormalizedTotals(summary) {
|
||||
const rawTotals =
|
||||
summary?.totals && typeof summary.totals === "object" ? summary.totals : {};
|
||||
const gapTotals = buildObservabilityGapTotals(summary);
|
||||
const currentVerificationOutcomeEntries = getBreakdownEntries(
|
||||
summary,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
);
|
||||
return {
|
||||
suiteCount: normalizeNumber(rawTotals.suiteCount),
|
||||
caseCount: normalizeNumber(rawTotals.caseCount),
|
||||
readyCount: normalizeNumber(rawTotals.readyCount),
|
||||
invalidCount: normalizeNumber(rawTotals.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(rawTotals.pendingRequestCaseCount),
|
||||
needsHumanReviewCount: normalizeNumber(rawTotals.needsHumanReviewCount),
|
||||
reviewDecisionRecordedCount: normalizeNumber(
|
||||
rawTotals.reviewDecisionRecordedCount,
|
||||
),
|
||||
observabilityGapCaseCount: gapTotals.total,
|
||||
currentObservabilityGapCaseCount: gapTotals.current,
|
||||
degradedObservabilityGapCaseCount: gapTotals.degraded,
|
||||
currentRecoveredVerificationCaseCount:
|
||||
currentVerificationOutcomeEntries.length > 0
|
||||
? getBreakdownCaseCount(
|
||||
summary,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
RECOVERED_VERIFICATION_OUTCOMES,
|
||||
)
|
||||
: normalizeNumber(rawTotals.currentRecoveredVerificationCaseCount),
|
||||
};
|
||||
}
|
||||
|
||||
function computeReadyRate(summary) {
|
||||
const caseCount = normalizeNumber(summary?.totals?.caseCount);
|
||||
const totals = buildNormalizedTotals(summary);
|
||||
const caseCount = totals.caseCount;
|
||||
if (caseCount <= 0) {
|
||||
return 0;
|
||||
}
|
||||
return normalizeNumber(summary?.totals?.readyCount) / caseCount;
|
||||
return totals.readyCount / caseCount;
|
||||
}
|
||||
|
||||
function getSuiteMap(summary) {
|
||||
@@ -195,9 +310,7 @@ function getSuiteMap(summary) {
|
||||
}
|
||||
|
||||
function getBreakdownMap(summary, key) {
|
||||
const entries = Array.isArray(summary?.breakdowns?.[key])
|
||||
? summary.breakdowns[key]
|
||||
: [];
|
||||
const entries = getBreakdownEntries(summary, key);
|
||||
return new Map(
|
||||
entries.map((entry) => [
|
||||
String(entry.name ?? ""),
|
||||
@@ -309,6 +422,12 @@ function buildBreakdownDeltas(baseline, latest, key) {
|
||||
});
|
||||
}
|
||||
|
||||
function buildFilteredBreakdownDeltas(baseline, latest, key, predicate) {
|
||||
return buildBreakdownDeltas(baseline, latest, key).filter((entry) =>
|
||||
predicate(entry),
|
||||
);
|
||||
}
|
||||
|
||||
function buildStatusSignals(baseline, latest, sampleCount) {
|
||||
const signals = [];
|
||||
|
||||
@@ -318,15 +437,17 @@ function buildStatusSignals(baseline, latest, sampleCount) {
|
||||
}
|
||||
|
||||
const readyRateDelta = computeReadyRate(latest) - computeReadyRate(baseline);
|
||||
const invalidDelta =
|
||||
normalizeNumber(latest?.totals?.invalidCount) -
|
||||
normalizeNumber(baseline?.totals?.invalidCount);
|
||||
const baselineTotals = buildNormalizedTotals(baseline);
|
||||
const latestTotals = buildNormalizedTotals(latest);
|
||||
const invalidDelta = latestTotals.invalidCount - baselineTotals.invalidCount;
|
||||
const pendingDelta =
|
||||
normalizeNumber(latest?.totals?.pendingRequestCaseCount) -
|
||||
normalizeNumber(baseline?.totals?.pendingRequestCaseCount);
|
||||
const observabilityGapDelta =
|
||||
normalizeNumber(latest?.totals?.observabilityGapCaseCount) -
|
||||
normalizeNumber(baseline?.totals?.observabilityGapCaseCount);
|
||||
latestTotals.pendingRequestCaseCount - baselineTotals.pendingRequestCaseCount;
|
||||
const currentObservabilityGapDelta =
|
||||
latestTotals.currentObservabilityGapCaseCount -
|
||||
baselineTotals.currentObservabilityGapCaseCount;
|
||||
const currentRecoveredVerificationDelta =
|
||||
latestTotals.currentRecoveredVerificationCaseCount -
|
||||
baselineTotals.currentRecoveredVerificationCaseCount;
|
||||
|
||||
if (invalidDelta > 0) {
|
||||
signals.push(`invalid case 增加 ${invalidDelta},存在回归候选。`);
|
||||
@@ -344,9 +465,15 @@ function buildStatusSignals(baseline, latest, sampleCount) {
|
||||
);
|
||||
}
|
||||
|
||||
if (observabilityGapDelta > 0) {
|
||||
if (currentObservabilityGapDelta > 0) {
|
||||
signals.push(
|
||||
`observability gap case 增加 ${observabilityGapDelta},需先补 evidence / analysis / replay 的证据覆盖。`,
|
||||
`current observability gap case 增加 ${currentObservabilityGapDelta},说明主线样本开始带缺口,需先补 evidence / analysis / replay 的证据覆盖。`,
|
||||
);
|
||||
}
|
||||
|
||||
if (currentRecoveredVerificationDelta > 0) {
|
||||
signals.push(
|
||||
`current recovered verification case 增加 ${currentRecoveredVerificationDelta},说明主线路径正在累积正向守卫。`,
|
||||
);
|
||||
}
|
||||
|
||||
@@ -364,6 +491,43 @@ function buildStatusSignals(baseline, latest, sampleCount) {
|
||||
);
|
||||
}
|
||||
|
||||
const verificationOutcomeDeltas = buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"observabilityVerificationOutcomes",
|
||||
);
|
||||
const increasedVerificationFailure = verificationOutcomeDeltas.find(
|
||||
(entry) =>
|
||||
OBSERVABILITY_FAILURE_OUTCOMES.has(entry.name) &&
|
||||
entry.delta.caseCount > 0,
|
||||
);
|
||||
if (increasedVerificationFailure) {
|
||||
signals.push(
|
||||
`verification outcome \`${increasedVerificationFailure.name}\` 新增 ${increasedVerificationFailure.delta.caseCount} 个 case。`,
|
||||
);
|
||||
}
|
||||
|
||||
const currentRecoveredVerificationDeltas = buildFilteredBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
|
||||
);
|
||||
for (const entry of currentRecoveredVerificationDeltas.filter(
|
||||
(candidate) => candidate.delta.caseCount < 0,
|
||||
)) {
|
||||
signals.push(
|
||||
`current recovered verification baseline \`${entry.name}\` 减少 ${Math.abs(entry.delta.caseCount)},说明正向守卫可能回退。`,
|
||||
);
|
||||
}
|
||||
for (const entry of currentRecoveredVerificationDeltas.filter(
|
||||
(candidate) => candidate.delta.caseCount > 0,
|
||||
)) {
|
||||
signals.push(
|
||||
`current recovered verification baseline \`${entry.name}\` 新增 ${entry.delta.caseCount},说明主线路径正在形成正向基线。`,
|
||||
);
|
||||
}
|
||||
|
||||
if (signals.length === 0) {
|
||||
signals.push("当前没有检测到明显退化信号。");
|
||||
}
|
||||
@@ -389,6 +553,8 @@ function buildTrendReport(samples, repoRoot) {
|
||||
const latestEntry = sortedSamples[sortedSamples.length - 1];
|
||||
const baseline = baselineEntry.summary;
|
||||
const latest = latestEntry.summary;
|
||||
const baselineTotals = buildNormalizedTotals(baseline);
|
||||
const latestTotals = buildNormalizedTotals(latest);
|
||||
const readyRateDelta = computeReadyRate(latest) - computeReadyRate(baseline);
|
||||
|
||||
return {
|
||||
@@ -399,45 +565,46 @@ function buildTrendReport(samples, repoRoot) {
|
||||
baseline: {
|
||||
generatedAt: baseline.generatedAt,
|
||||
sourcePath: baselineEntry.sourcePath,
|
||||
totals: baseline.totals,
|
||||
totals: baselineTotals,
|
||||
},
|
||||
latest: {
|
||||
generatedAt: latest.generatedAt,
|
||||
sourcePath: latestEntry.sourcePath,
|
||||
totals: latest.totals,
|
||||
totals: latestTotals,
|
||||
},
|
||||
delta: {
|
||||
suiteCount:
|
||||
normalizeNumber(latest?.totals?.suiteCount) -
|
||||
normalizeNumber(baseline?.totals?.suiteCount),
|
||||
caseCount:
|
||||
normalizeNumber(latest?.totals?.caseCount) -
|
||||
normalizeNumber(baseline?.totals?.caseCount),
|
||||
readyCount:
|
||||
normalizeNumber(latest?.totals?.readyCount) -
|
||||
normalizeNumber(baseline?.totals?.readyCount),
|
||||
invalidCount:
|
||||
normalizeNumber(latest?.totals?.invalidCount) -
|
||||
normalizeNumber(baseline?.totals?.invalidCount),
|
||||
suiteCount: latestTotals.suiteCount - baselineTotals.suiteCount,
|
||||
caseCount: latestTotals.caseCount - baselineTotals.caseCount,
|
||||
readyCount: latestTotals.readyCount - baselineTotals.readyCount,
|
||||
invalidCount: latestTotals.invalidCount - baselineTotals.invalidCount,
|
||||
pendingRequestCaseCount:
|
||||
normalizeNumber(latest?.totals?.pendingRequestCaseCount) -
|
||||
normalizeNumber(baseline?.totals?.pendingRequestCaseCount),
|
||||
latestTotals.pendingRequestCaseCount -
|
||||
baselineTotals.pendingRequestCaseCount,
|
||||
needsHumanReviewCount:
|
||||
normalizeNumber(latest?.totals?.needsHumanReviewCount) -
|
||||
normalizeNumber(baseline?.totals?.needsHumanReviewCount),
|
||||
latestTotals.needsHumanReviewCount -
|
||||
baselineTotals.needsHumanReviewCount,
|
||||
reviewDecisionRecordedCount:
|
||||
normalizeNumber(latest?.totals?.reviewDecisionRecordedCount) -
|
||||
normalizeNumber(baseline?.totals?.reviewDecisionRecordedCount),
|
||||
latestTotals.reviewDecisionRecordedCount -
|
||||
baselineTotals.reviewDecisionRecordedCount,
|
||||
observabilityGapCaseCount:
|
||||
normalizeNumber(latest?.totals?.observabilityGapCaseCount) -
|
||||
normalizeNumber(baseline?.totals?.observabilityGapCaseCount),
|
||||
latestTotals.observabilityGapCaseCount -
|
||||
baselineTotals.observabilityGapCaseCount,
|
||||
currentObservabilityGapCaseCount:
|
||||
latestTotals.currentObservabilityGapCaseCount -
|
||||
baselineTotals.currentObservabilityGapCaseCount,
|
||||
degradedObservabilityGapCaseCount:
|
||||
latestTotals.degradedObservabilityGapCaseCount -
|
||||
baselineTotals.degradedObservabilityGapCaseCount,
|
||||
currentRecoveredVerificationCaseCount:
|
||||
latestTotals.currentRecoveredVerificationCaseCount -
|
||||
baselineTotals.currentRecoveredVerificationCaseCount,
|
||||
readyRate: readyRateDelta,
|
||||
},
|
||||
signals: buildStatusSignals(baseline, latest, sortedSamples.length),
|
||||
samples: sortedSamples.map((entry) => ({
|
||||
generatedAt: entry.summary.generatedAt,
|
||||
sourcePath: entry.sourcePath,
|
||||
totals: entry.summary.totals,
|
||||
totals: buildNormalizedTotals(entry.summary),
|
||||
})),
|
||||
suiteDeltas: buildSuiteDeltas(baseline, latest),
|
||||
classificationDeltas: {
|
||||
@@ -458,6 +625,28 @@ function buildTrendReport(samples, repoRoot) {
|
||||
latest,
|
||||
"observabilitySignals",
|
||||
),
|
||||
observabilityVerificationOutcomes: buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"observabilityVerificationOutcomes",
|
||||
),
|
||||
currentRecoveredObservabilityVerificationOutcomes:
|
||||
buildFilteredBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
|
||||
),
|
||||
currentObservabilityVerificationOutcomes: buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
),
|
||||
degradedObservabilityVerificationOutcomes: buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"degradedObservabilityVerificationOutcomes",
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -473,6 +662,12 @@ function renderText(report) {
|
||||
`[harness-eval-trend] delta pendingRequestCaseCount: ${report.delta.pendingRequestCaseCount}`,
|
||||
`[harness-eval-trend] delta reviewDecisionRecordedCount: ${report.delta.reviewDecisionRecordedCount}`,
|
||||
`[harness-eval-trend] delta observabilityGapCaseCount: ${report.delta.observabilityGapCaseCount}`,
|
||||
`[harness-eval-trend] delta currentObservabilityGapCaseCount: ${report.delta.currentObservabilityGapCaseCount}`,
|
||||
`[harness-eval-trend] delta degradedObservabilityGapCaseCount: ${report.delta.degradedObservabilityGapCaseCount}`,
|
||||
`[harness-eval-trend] delta currentRecoveredVerificationCaseCount: ${report.delta.currentRecoveredVerificationCaseCount}`,
|
||||
`[harness-eval-trend] latest currentObservabilityGapCaseCount: ${report.latest.totals.currentObservabilityGapCaseCount}`,
|
||||
`[harness-eval-trend] latest degradedObservabilityGapCaseCount: ${report.latest.totals.degradedObservabilityGapCaseCount}`,
|
||||
`[harness-eval-trend] latest currentRecoveredVerificationCaseCount: ${report.latest.totals.currentRecoveredVerificationCaseCount}`,
|
||||
`[harness-eval-trend] delta readyRate: ${(report.delta.readyRate * 100).toFixed(1)}%`,
|
||||
];
|
||||
|
||||
@@ -515,6 +710,42 @@ function renderText(report) {
|
||||
}
|
||||
}
|
||||
|
||||
const topVerificationOutcomeDeltas =
|
||||
report.classificationDeltas.observabilityVerificationOutcomes.slice(0, 5);
|
||||
if (topVerificationOutcomeDeltas.length > 0) {
|
||||
lines.push("[harness-eval-trend] observability verification outcome deltas:");
|
||||
for (const entry of topVerificationOutcomeDeltas) {
|
||||
lines.push(
|
||||
` - ${entry.name}: delta_case=${entry.delta.caseCount}, latest_case=${entry.latest.caseCount}, latest_invalid=${entry.latest.invalidCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const topCurrentRecoveredVerificationDeltas =
|
||||
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes.slice(
|
||||
0,
|
||||
5,
|
||||
);
|
||||
if (topCurrentRecoveredVerificationDeltas.length > 0) {
|
||||
lines.push("[harness-eval-trend] current recovered verification baseline deltas:");
|
||||
for (const entry of topCurrentRecoveredVerificationDeltas) {
|
||||
lines.push(
|
||||
` - ${entry.name}: baseline_case=${entry.baseline.caseCount}, latest_case=${entry.latest.caseCount}, delta_case=${entry.delta.caseCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("[harness-eval-trend] observability gap roles:");
|
||||
lines.push(
|
||||
` - total: baseline=${report.baseline.totals.observabilityGapCaseCount}, latest=${report.latest.totals.observabilityGapCaseCount}, delta=${report.delta.observabilityGapCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
` - current: baseline=${report.baseline.totals.currentObservabilityGapCaseCount}, latest=${report.latest.totals.currentObservabilityGapCaseCount}, delta=${report.delta.currentObservabilityGapCaseCount}`,
|
||||
);
|
||||
lines.push(
|
||||
` - degraded: baseline=${report.baseline.totals.degradedObservabilityGapCaseCount}, latest=${report.latest.totals.degradedObservabilityGapCaseCount}, delta=${report.delta.degradedObservabilityGapCaseCount}`,
|
||||
);
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
@@ -537,6 +768,9 @@ function renderMarkdown(report) {
|
||||
`- needs review case 变化:${report.delta.needsHumanReviewCount}`,
|
||||
`- 已记录人工审核变化:${report.delta.reviewDecisionRecordedCount}`,
|
||||
`- observability gap case 变化:${report.delta.observabilityGapCaseCount}`,
|
||||
`- current observability gap case 变化:${report.delta.currentObservabilityGapCaseCount}`,
|
||||
`- degraded observability gap case 变化:${report.delta.degradedObservabilityGapCaseCount}`,
|
||||
`- current recovered verification case 变化:${report.delta.currentRecoveredVerificationCaseCount}`,
|
||||
`- ready rate 变化:${(report.delta.readyRate * 100).toFixed(1)}%`,
|
||||
"",
|
||||
"## 信号",
|
||||
@@ -616,14 +850,60 @@ function renderMarkdown(report) {
|
||||
}
|
||||
}
|
||||
|
||||
if (report.classificationDeltas.observabilityVerificationOutcomes.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## Observability Verification Outcome 变化");
|
||||
lines.push("");
|
||||
lines.push("| Outcome | baseline case | latest case | delta case | delta invalid |");
|
||||
lines.push("| --- | --- | --- | --- | --- |");
|
||||
for (const entry of report.classificationDeltas.observabilityVerificationOutcomes) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} | ${entry.delta.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes
|
||||
.length > 0
|
||||
) {
|
||||
lines.push("");
|
||||
lines.push("## Current Recovered Baseline 变化");
|
||||
lines.push("");
|
||||
lines.push("| Outcome | baseline case | latest case | delta case |");
|
||||
lines.push("| --- | --- | --- | --- |");
|
||||
for (const entry of report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("");
|
||||
lines.push("## Observability Gap 角色变化");
|
||||
lines.push("");
|
||||
lines.push("| 角色 | baseline case | latest case | delta case |");
|
||||
lines.push("| --- | --- | --- | --- |");
|
||||
lines.push(
|
||||
`| total | ${report.baseline.totals.observabilityGapCaseCount} | ${report.latest.totals.observabilityGapCaseCount} | ${report.delta.observabilityGapCaseCount} |`,
|
||||
);
|
||||
lines.push(
|
||||
`| current | ${report.baseline.totals.currentObservabilityGapCaseCount} | ${report.latest.totals.currentObservabilityGapCaseCount} | ${report.delta.currentObservabilityGapCaseCount} |`,
|
||||
);
|
||||
lines.push(
|
||||
`| degraded | ${report.baseline.totals.degradedObservabilityGapCaseCount} | ${report.latest.totals.degradedObservabilityGapCaseCount} | ${report.delta.degradedObservabilityGapCaseCount} |`,
|
||||
);
|
||||
|
||||
lines.push("");
|
||||
lines.push("## 时间线样本");
|
||||
lines.push("");
|
||||
lines.push("| 时间 | 来源 | case | ready | invalid | pending_request |");
|
||||
lines.push("| --- | --- | --- | --- | --- | --- |");
|
||||
lines.push(
|
||||
"| 时间 | 来源 | case | ready | invalid | pending_request | current_gap | degraded_gap |",
|
||||
);
|
||||
lines.push("| --- | --- | --- | --- | --- | --- | --- | --- |");
|
||||
for (const sample of report.samples) {
|
||||
lines.push(
|
||||
`| ${sample.generatedAt} | \`${sample.sourcePath}\` | ${sample.totals.caseCount} | ${sample.totals.readyCount} | ${sample.totals.invalidCount} | ${sample.totals.pendingRequestCaseCount} |`,
|
||||
`| ${sample.generatedAt} | \`${sample.sourcePath}\` | ${sample.totals.caseCount} | ${sample.totals.readyCount} | ${sample.totals.invalidCount} | ${sample.totals.pendingRequestCaseCount} | ${sample.totals.currentObservabilityGapCaseCount} | ${sample.totals.degradedObservabilityGapCaseCount} |`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,765 @@
|
||||
function normalizeNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function normalizeString(value, fallback = "") {
|
||||
return typeof value === "string" && value.trim().length > 0
|
||||
? value.trim()
|
||||
: fallback;
|
||||
}
|
||||
|
||||
function escapeHtml(value) {
|
||||
return String(value)
|
||||
.replaceAll("&", "&")
|
||||
.replaceAll("<", "<")
|
||||
.replaceAll(">", ">")
|
||||
.replaceAll('"', """)
|
||||
.replaceAll("'", "'");
|
||||
}
|
||||
|
||||
function formatTimestamp(value) {
|
||||
const parsed = new Date(value);
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
return normalizeString(value, "-");
|
||||
}
|
||||
return parsed.toLocaleString("zh-CN", {
|
||||
hour12: false,
|
||||
year: "numeric",
|
||||
month: "2-digit",
|
||||
day: "2-digit",
|
||||
hour: "2-digit",
|
||||
minute: "2-digit",
|
||||
second: "2-digit",
|
||||
});
|
||||
}
|
||||
|
||||
function renderStatCard(label, value, tone = "neutral") {
|
||||
return `
|
||||
<article class="stat-card ${tone}">
|
||||
<span>${escapeHtml(label)}</span>
|
||||
<strong>${escapeHtml(String(value))}</strong>
|
||||
</article>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderSignalList(signals) {
|
||||
const items = Array.isArray(signals) ? signals.filter(Boolean) : [];
|
||||
if (items.length === 0) {
|
||||
return `<p class="empty">当前没有额外信号。</p>`;
|
||||
}
|
||||
|
||||
return `
|
||||
<ul class="signal-list">
|
||||
${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}
|
||||
</ul>
|
||||
`;
|
||||
}
|
||||
|
||||
function renderRecommendationList(recommendations) {
|
||||
const items = Array.isArray(recommendations) ? recommendations : [];
|
||||
if (items.length === 0) {
|
||||
return `<p class="empty">当前没有新的治理建议。</p>`;
|
||||
}
|
||||
|
||||
return items
|
||||
.map(
|
||||
(entry) => {
|
||||
const focusVerificationFailureOutcomes = Array.isArray(
|
||||
entry.focusVerificationFailureOutcomes,
|
||||
)
|
||||
? entry.focusVerificationFailureOutcomes
|
||||
: [];
|
||||
const focusVerificationRecoveredOutcomes = Array.isArray(
|
||||
entry.focusVerificationRecoveredOutcomes,
|
||||
)
|
||||
? entry.focusVerificationRecoveredOutcomes
|
||||
: [];
|
||||
|
||||
return `
|
||||
<article class="recommendation-card">
|
||||
<div class="recommendation-head">
|
||||
<span class="priority">${escapeHtml(normalizeString(entry.priority, "P?"))}</span>
|
||||
<h3>${escapeHtml(normalizeString(entry.title, "未命名建议"))}</h3>
|
||||
</div>
|
||||
<p>${escapeHtml(
|
||||
Array.isArray(entry.rationale) ? entry.rationale.join(" ") : "",
|
||||
)}</p>
|
||||
${
|
||||
focusVerificationFailureOutcomes.length > 0
|
||||
? `
|
||||
<p class="recommendation-meta">
|
||||
<strong>关注 failure outcome:</strong>${escapeHtml(
|
||||
focusVerificationFailureOutcomes.join("、"),
|
||||
)}
|
||||
</p>
|
||||
`
|
||||
: ""
|
||||
}
|
||||
${
|
||||
focusVerificationRecoveredOutcomes.length > 0
|
||||
? `
|
||||
<p class="recommendation-meta">
|
||||
<strong>关注 recovered outcome:</strong>${escapeHtml(
|
||||
focusVerificationRecoveredOutcomes.join("、"),
|
||||
)}
|
||||
</p>
|
||||
`
|
||||
: ""
|
||||
}
|
||||
${
|
||||
Array.isArray(entry.backlogTools) && entry.backlogTools.length > 0
|
||||
? `
|
||||
<div class="recommendation-subsection">
|
||||
<strong>后续动作</strong>
|
||||
<ul class="backlog-list">
|
||||
${entry.backlogTools
|
||||
.map((item) => `<li>${escapeHtml(item)}</li>`)
|
||||
.join("")}
|
||||
</ul>
|
||||
</div>
|
||||
`
|
||||
: ""
|
||||
}
|
||||
${
|
||||
Array.isArray(entry.commands) && entry.commands.length > 0
|
||||
? `
|
||||
<div class="recommendation-subsection">
|
||||
<strong>推荐命令</strong>
|
||||
<div class="command-list">
|
||||
${entry.commands
|
||||
.map((command) => `<code>${escapeHtml(command)}</code>`)
|
||||
.join("")}
|
||||
</div>
|
||||
</div>
|
||||
`
|
||||
: ""
|
||||
}
|
||||
</article>
|
||||
`;
|
||||
},
|
||||
)
|
||||
.join("");
|
||||
}
|
||||
|
||||
function describeVerificationOutcome(entry) {
|
||||
const signal = normalizeString(entry?.signal, "unknown");
|
||||
const outcome = normalizeString(entry?.outcome, "unknown");
|
||||
|
||||
if (signal === "artifactValidator" && outcome === "issues_present") {
|
||||
return "当前 evidence 已记录 artifact 校验问题,优先回看 validator issue 明细。";
|
||||
}
|
||||
if (signal === "artifactValidator" && outcome === "fallback_used") {
|
||||
return "当前 artifact 导出仍触发 fallback,说明产物结构或修复链未完全稳定。";
|
||||
}
|
||||
if (signal === "browserVerification" && outcome === "failure") {
|
||||
return "浏览器验证已有明确失败结果,优先回挂到 replay 或 smoke 断言。";
|
||||
}
|
||||
if (signal === "browserVerification" && outcome === "unknown") {
|
||||
return "浏览器验证结果仍不明确,需要先补 outcome 再继续扩分析。";
|
||||
}
|
||||
if (signal === "guiSmoke" && outcome === "failed") {
|
||||
return "GUI smoke 已明确失败,应优先收敛到受影响主路径。";
|
||||
}
|
||||
if (signal === "guiSmoke" && outcome === "passed") {
|
||||
return "GUI smoke 已通过,可继续把注意力放回 gap 与其它失败面。";
|
||||
}
|
||||
if (signal === "artifactValidator" && outcome === "repaired") {
|
||||
return "artifact validator 已执行修复,可结合 issues/fallback 判断是否还需继续治理。";
|
||||
}
|
||||
if (signal === "browserVerification" && outcome === "success") {
|
||||
return "浏览器验证已有成功样本,可作为 current 主线路径的正向基线。";
|
||||
}
|
||||
|
||||
return "当前 verification outcome 已进入 cleanup 主线,可直接据此定位先修哪层。";
|
||||
}
|
||||
|
||||
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
|
||||
"repaired",
|
||||
"success",
|
||||
"passed",
|
||||
"clean",
|
||||
]);
|
||||
|
||||
function isRecoveredVerificationOutcome(entry) {
|
||||
return RECOVERED_VERIFICATION_OUTCOMES.has(
|
||||
normalizeString(entry?.outcome, "unknown"),
|
||||
);
|
||||
}
|
||||
|
||||
function renderFocusTable(title, entries, columns) {
|
||||
const rows = Array.isArray(entries) ? entries : [];
|
||||
if (rows.length === 0) {
|
||||
return `
|
||||
<section class="panel">
|
||||
<div class="section-header">
|
||||
<h2>${escapeHtml(title)}</h2>
|
||||
</div>
|
||||
<p class="empty">当前没有数据。</p>
|
||||
</section>
|
||||
`;
|
||||
}
|
||||
|
||||
return `
|
||||
<section class="panel">
|
||||
<div class="section-header">
|
||||
<h2>${escapeHtml(title)}</h2>
|
||||
</div>
|
||||
<div class="table-wrap">
|
||||
<table>
|
||||
<thead>
|
||||
<tr>
|
||||
${columns.map((column) => `<th>${escapeHtml(column.label)}</th>`).join("")}
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
${rows
|
||||
.map(
|
||||
(entry) => `
|
||||
<tr>
|
||||
${columns
|
||||
.map((column) => `<td>${escapeHtml(String(column.render(entry)))}</td>`)
|
||||
.join("")}
|
||||
</tr>
|
||||
`,
|
||||
)
|
||||
.join("")}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</section>
|
||||
`;
|
||||
}
|
||||
|
||||
export function renderHarnessDashboardHtml({
|
||||
summaryReport,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
title = "Lime Harness Dashboard",
|
||||
}) {
|
||||
const summaryTotals =
|
||||
summaryReport && typeof summaryReport === "object" && summaryReport.totals
|
||||
? summaryReport.totals
|
||||
: {};
|
||||
const trendSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.trend
|
||||
? cleanupReport.summary.trend
|
||||
: {};
|
||||
const governanceSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.governance
|
||||
? cleanupReport.summary.governance
|
||||
: {};
|
||||
const verificationSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.verificationOutcomes
|
||||
? cleanupReport.summary.verificationOutcomes
|
||||
: {};
|
||||
const currentVerificationSummary =
|
||||
verificationSummary &&
|
||||
typeof verificationSummary.current === "object" &&
|
||||
!Array.isArray(verificationSummary.current)
|
||||
? verificationSummary.current
|
||||
: {};
|
||||
const degradedVerificationSummary =
|
||||
verificationSummary &&
|
||||
typeof verificationSummary.degraded === "object" &&
|
||||
!Array.isArray(verificationSummary.degraded)
|
||||
? verificationSummary.degraded
|
||||
: {};
|
||||
const trendSignals = Array.isArray(trendReport?.signals) ? trendReport.signals : [];
|
||||
const cleanupSignals = Array.isArray(cleanupReport?.signals)
|
||||
? cleanupReport.signals
|
||||
: [];
|
||||
const recommendations = Array.isArray(cleanupReport?.recommendations)
|
||||
? cleanupReport.recommendations
|
||||
: [];
|
||||
const sampleRows = Array.isArray(trendReport?.samples) ? trendReport.samples : [];
|
||||
const currentVerificationFocusRows = Array.isArray(
|
||||
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentObservabilityVerificationOutcomes.map((entry) => ({
|
||||
...entry,
|
||||
role: "current",
|
||||
}))
|
||||
: [];
|
||||
const degradedVerificationFocusRows = Array.isArray(
|
||||
cleanupReport?.focus?.degradedObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.degradedObservabilityVerificationOutcomes.map(
|
||||
(entry) => ({
|
||||
...entry,
|
||||
role: "degraded",
|
||||
}),
|
||||
)
|
||||
: [];
|
||||
const fallbackVerificationFocusRows = Array.isArray(
|
||||
cleanupReport?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.observabilityVerificationOutcomes.map((entry) => ({
|
||||
...entry,
|
||||
role: "mixed",
|
||||
}))
|
||||
: [];
|
||||
const explicitCurrentRecoveredVerificationRows = Array.isArray(
|
||||
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes.map(
|
||||
(entry) => ({
|
||||
...entry,
|
||||
role: "current",
|
||||
}),
|
||||
)
|
||||
: [];
|
||||
const verificationFocusRows =
|
||||
currentVerificationFocusRows.length > 0 ||
|
||||
degradedVerificationFocusRows.length > 0
|
||||
? [...currentVerificationFocusRows, ...degradedVerificationFocusRows]
|
||||
: fallbackVerificationFocusRows;
|
||||
const currentRecoveredVerificationRows =
|
||||
explicitCurrentRecoveredVerificationRows.length > 0
|
||||
? explicitCurrentRecoveredVerificationRows
|
||||
: currentVerificationFocusRows.length > 0
|
||||
? currentVerificationFocusRows.filter((entry) =>
|
||||
isRecoveredVerificationOutcome(entry),
|
||||
)
|
||||
: fallbackVerificationFocusRows.filter((entry) =>
|
||||
isRecoveredVerificationOutcome(entry),
|
||||
);
|
||||
const currentRecoveredVerificationSummary = currentRecoveredVerificationRows
|
||||
.slice(0, 3)
|
||||
.map(
|
||||
(entry) =>
|
||||
`${normalizeString(entry?.signal, "-")} (${normalizeString(entry?.outcome, "-")})`,
|
||||
)
|
||||
.join("、");
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>${escapeHtml(title)}</title>
|
||||
<style>
|
||||
:root {
|
||||
color-scheme: light;
|
||||
--bg: #f3f6f4;
|
||||
--panel: #ffffff;
|
||||
--panel-muted: #f8faf8;
|
||||
--border: #d8e1dc;
|
||||
--text: #112118;
|
||||
--muted: #5f6d64;
|
||||
--accent: #0f4c5c;
|
||||
--success: #177245;
|
||||
--warning: #b45309;
|
||||
--danger: #b42318;
|
||||
--shadow: 0 16px 32px rgba(17, 33, 24, 0.08);
|
||||
}
|
||||
|
||||
* { box-sizing: border-box; }
|
||||
body {
|
||||
margin: 0;
|
||||
font-family:
|
||||
"SF Pro Display",
|
||||
"Segoe UI",
|
||||
"PingFang SC",
|
||||
"Hiragino Sans GB",
|
||||
"Microsoft YaHei",
|
||||
sans-serif;
|
||||
color: var(--text);
|
||||
background:
|
||||
radial-gradient(circle at top left, rgba(15, 76, 92, 0.12), transparent 32%),
|
||||
radial-gradient(circle at top right, rgba(23, 114, 69, 0.1), transparent 28%),
|
||||
var(--bg);
|
||||
}
|
||||
|
||||
.page {
|
||||
max-width: 1440px;
|
||||
margin: 0 auto;
|
||||
padding: 28px 22px 42px;
|
||||
}
|
||||
|
||||
.hero,
|
||||
.panel {
|
||||
background: var(--panel);
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 24px;
|
||||
box-shadow: var(--shadow);
|
||||
}
|
||||
|
||||
.hero {
|
||||
padding: 24px 26px;
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1.5fr) minmax(280px, 0.9fr);
|
||||
gap: 18px;
|
||||
}
|
||||
|
||||
.eyebrow {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
padding: 6px 12px;
|
||||
border-radius: 999px;
|
||||
background: #e8f7ef;
|
||||
color: var(--success);
|
||||
font-size: 13px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
h1, h2, h3, p {
|
||||
margin: 0;
|
||||
}
|
||||
|
||||
h1 {
|
||||
margin-top: 14px;
|
||||
font-size: 32px;
|
||||
line-height: 1.12;
|
||||
}
|
||||
|
||||
.hero p {
|
||||
margin-top: 12px;
|
||||
color: var(--muted);
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.meta-grid,
|
||||
.stat-grid,
|
||||
.panel-grid {
|
||||
display: grid;
|
||||
gap: 14px;
|
||||
}
|
||||
|
||||
.meta-grid {
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.meta-card,
|
||||
.stat-card {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 18px;
|
||||
background: var(--panel-muted);
|
||||
padding: 14px 16px;
|
||||
}
|
||||
|
||||
.meta-card span,
|
||||
.stat-card span {
|
||||
display: block;
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
margin-bottom: 8px;
|
||||
}
|
||||
|
||||
.meta-card strong,
|
||||
.stat-card strong {
|
||||
font-size: 20px;
|
||||
}
|
||||
|
||||
.stat-grid {
|
||||
margin-top: 18px;
|
||||
grid-template-columns: repeat(6, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.stat-card.warning strong { color: var(--warning); }
|
||||
.stat-card.danger strong { color: var(--danger); }
|
||||
.stat-card.success strong { color: var(--success); }
|
||||
|
||||
.panel-grid {
|
||||
margin-top: 20px;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
}
|
||||
|
||||
.panel {
|
||||
padding: 20px 22px;
|
||||
}
|
||||
|
||||
.section-header {
|
||||
display: flex;
|
||||
align-items: flex-start;
|
||||
justify-content: space-between;
|
||||
gap: 12px;
|
||||
margin-bottom: 14px;
|
||||
}
|
||||
|
||||
.section-header p {
|
||||
color: var(--muted);
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.signal-list {
|
||||
margin: 0;
|
||||
padding-left: 18px;
|
||||
display: grid;
|
||||
gap: 10px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.empty {
|
||||
color: var(--muted);
|
||||
line-height: 1.6;
|
||||
}
|
||||
|
||||
.table-wrap {
|
||||
overflow-x: auto;
|
||||
}
|
||||
|
||||
table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
th, td {
|
||||
padding: 10px 12px;
|
||||
border-bottom: 1px solid var(--border);
|
||||
text-align: left;
|
||||
vertical-align: top;
|
||||
}
|
||||
|
||||
th {
|
||||
font-size: 12px;
|
||||
color: var(--muted);
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.04em;
|
||||
}
|
||||
|
||||
tr:last-child td {
|
||||
border-bottom: none;
|
||||
}
|
||||
|
||||
.recommendation-stack {
|
||||
display: grid;
|
||||
gap: 12px;
|
||||
}
|
||||
|
||||
.recommendation-card {
|
||||
border: 1px solid var(--border);
|
||||
border-radius: 18px;
|
||||
padding: 16px;
|
||||
background: var(--panel-muted);
|
||||
}
|
||||
|
||||
.recommendation-head {
|
||||
display: flex;
|
||||
gap: 10px;
|
||||
align-items: center;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
|
||||
.priority {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
min-width: 44px;
|
||||
height: 28px;
|
||||
padding: 0 10px;
|
||||
border-radius: 999px;
|
||||
background: #fff3e8;
|
||||
color: var(--warning);
|
||||
font-size: 12px;
|
||||
font-weight: 700;
|
||||
}
|
||||
|
||||
.recommendation-card p {
|
||||
color: var(--muted);
|
||||
line-height: 1.7;
|
||||
}
|
||||
|
||||
.recommendation-meta {
|
||||
margin-top: 12px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.recommendation-subsection {
|
||||
margin-top: 14px;
|
||||
}
|
||||
|
||||
.recommendation-subsection strong {
|
||||
display: block;
|
||||
margin-bottom: 8px;
|
||||
font-size: 13px;
|
||||
}
|
||||
|
||||
.backlog-list {
|
||||
margin: 0;
|
||||
padding-left: 18px;
|
||||
display: grid;
|
||||
gap: 8px;
|
||||
color: var(--muted);
|
||||
}
|
||||
|
||||
.command-list {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
code {
|
||||
display: inline-flex;
|
||||
padding: 8px 10px;
|
||||
border-radius: 10px;
|
||||
background: #102a2c;
|
||||
color: #ecfeff;
|
||||
font-size: 12px;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
@media (max-width: 1120px) {
|
||||
.hero,
|
||||
.panel-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.stat-grid {
|
||||
grid-template-columns: repeat(3, minmax(0, 1fr));
|
||||
}
|
||||
}
|
||||
|
||||
@media (max-width: 720px) {
|
||||
.page {
|
||||
padding: 18px 14px 30px;
|
||||
}
|
||||
|
||||
.meta-grid,
|
||||
.stat-grid {
|
||||
grid-template-columns: 1fr 1fr;
|
||||
}
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<main class="page">
|
||||
<section class="hero">
|
||||
<div>
|
||||
<span class="eyebrow">Harness Engine Nightly Dashboard</span>
|
||||
<h1>${escapeHtml(title)}</h1>
|
||||
<p>同一份 evidence-first 主线下的 summary、trend、cleanup 视图。当前面板会明确区分主线 current 缺口与故意保留的 degraded 诊断基线,避免治理优先级被误导。</p>
|
||||
<div class="stat-grid">
|
||||
${renderStatCard("Ready Case", normalizeNumber(summaryTotals.readyCount), "success")}
|
||||
${renderStatCard("Invalid Case", normalizeNumber(summaryTotals.invalidCount), normalizeNumber(summaryTotals.invalidCount) > 0 ? "danger" : "neutral")}
|
||||
${renderStatCard("Current Gap", normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount), normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount) > 0 ? "danger" : "success")}
|
||||
${renderStatCard("Degraded Gap", normalizeNumber(trendSummary.latestDegradedObservabilityGapCaseCount), "warning")}
|
||||
${renderStatCard("Current Blocking", normalizeNumber(currentVerificationSummary.blockingFailureCaseCount), normalizeNumber(currentVerificationSummary.blockingFailureCaseCount) > 0 ? "danger" : "success")}
|
||||
${renderStatCard("Current Advisory", normalizeNumber(currentVerificationSummary.advisoryFailureCaseCount), normalizeNumber(currentVerificationSummary.advisoryFailureCaseCount) > 0 ? "warning" : "neutral")}
|
||||
${renderStatCard("Current Recovered", normalizeNumber(currentVerificationSummary.recoveredCaseCount), normalizeNumber(currentVerificationSummary.recoveredCaseCount) > 0 ? "success" : "neutral")}
|
||||
${renderStatCard("Degraded Blocking", normalizeNumber(degradedVerificationSummary.blockingFailureCaseCount), normalizeNumber(degradedVerificationSummary.blockingFailureCaseCount) > 0 ? "warning" : "neutral")}
|
||||
${renderStatCard("Recovered Outcomes", normalizeNumber(verificationSummary.recoveredCaseCount), normalizeNumber(verificationSummary.recoveredCaseCount) > 0 ? "success" : "neutral")}
|
||||
${renderStatCard("Trend Samples", normalizeNumber(trendSummary.sampleCount))}
|
||||
${renderStatCard("Governance Violations", normalizeNumber(governanceSummary.violationCount), normalizeNumber(governanceSummary.violationCount) > 0 ? "danger" : "neutral")}
|
||||
</div>
|
||||
</div>
|
||||
<div class="meta-grid">
|
||||
<div class="meta-card">
|
||||
<span>Summary 生成时间</span>
|
||||
<strong>${escapeHtml(formatTimestamp(summaryReport?.generatedAt))}</strong>
|
||||
</div>
|
||||
<div class="meta-card">
|
||||
<span>Trend 生成时间</span>
|
||||
<strong>${escapeHtml(formatTimestamp(trendReport?.generatedAt))}</strong>
|
||||
</div>
|
||||
<div class="meta-card">
|
||||
<span>Cleanup 生成时间</span>
|
||||
<strong>${escapeHtml(formatTimestamp(cleanupReport?.generatedAt))}</strong>
|
||||
</div>
|
||||
<div class="meta-card">
|
||||
<span>当前事实源</span>
|
||||
<strong>summary - history - trend - cleanup</strong>
|
||||
</div>
|
||||
<div class="meta-card">
|
||||
<span>当前 Recovered 基线</span>
|
||||
<strong>${escapeHtml(
|
||||
currentRecoveredVerificationSummary ||
|
||||
"当前没有额外的 recovered baseline",
|
||||
)}</strong>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section class="panel-grid">
|
||||
<section class="panel">
|
||||
<div class="section-header">
|
||||
<div>
|
||||
<h2>Trend 信号</h2>
|
||||
<p>只把 current gap 当主线风险,degraded gap 仅作为诊断基线保留。</p>
|
||||
</div>
|
||||
</div>
|
||||
${renderSignalList(trendSignals)}
|
||||
</section>
|
||||
<section class="panel">
|
||||
<div class="section-header">
|
||||
<div>
|
||||
<h2>Cleanup 信号</h2>
|
||||
<p>nightly 治理建议与当前风险摘要。</p>
|
||||
</div>
|
||||
</div>
|
||||
${renderSignalList(cleanupSignals)}
|
||||
</section>
|
||||
</section>
|
||||
|
||||
${renderFocusTable("Observability Gap 角色", [
|
||||
{
|
||||
role: "current",
|
||||
latest: normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount),
|
||||
delta: normalizeNumber(trendSummary.currentObservabilityGapCaseDelta),
|
||||
meaning: "主线样本里的证据缺口,必须优先治理。",
|
||||
},
|
||||
{
|
||||
role: "degraded",
|
||||
latest: normalizeNumber(trendSummary.latestDegradedObservabilityGapCaseCount),
|
||||
delta: normalizeNumber(trendSummary.degradedObservabilityGapCaseDelta),
|
||||
meaning: "刻意保留的诊断基线,不应直接抬高主线优先级。",
|
||||
},
|
||||
], [
|
||||
{ label: "角色", render: (entry) => entry.role },
|
||||
{ label: "latest case", render: (entry) => entry.latest },
|
||||
{ label: "delta case", render: (entry) => entry.delta },
|
||||
{ label: "说明", render: (entry) => entry.meaning },
|
||||
])}
|
||||
|
||||
${renderFocusTable("Verification Outcome 焦点", verificationFocusRows, [
|
||||
{ label: "Role", render: (entry) => normalizeString(entry.role, "-") },
|
||||
{ label: "Signal", render: (entry) => normalizeString(entry.signal, "-") },
|
||||
{ label: "Outcome", render: (entry) => normalizeString(entry.outcome, "-") },
|
||||
{ label: "State", render: (entry) => normalizeString(entry.state, "-") },
|
||||
{ label: "Latest Case", render: (entry) => normalizeNumber(entry?.latest?.caseCount) },
|
||||
{ label: "Delta Case", render: (entry) => normalizeNumber(entry?.delta?.caseCount) },
|
||||
{ label: "说明", render: (entry) => describeVerificationOutcome(entry) },
|
||||
])}
|
||||
|
||||
${renderFocusTable("Current Recovered Baseline", currentRecoveredVerificationRows, [
|
||||
{ label: "Signal", render: (entry) => normalizeString(entry.signal, "-") },
|
||||
{ label: "Outcome", render: (entry) => normalizeString(entry.outcome, "-") },
|
||||
{ label: "Latest Case", render: (entry) => normalizeNumber(entry?.latest?.caseCount) },
|
||||
{ label: "Delta Case", render: (entry) => normalizeNumber(entry?.delta?.caseCount) },
|
||||
{ label: "说明", render: (entry) => describeVerificationOutcome(entry) },
|
||||
])}
|
||||
|
||||
${renderFocusTable("历史窗口样本", sampleRows, [
|
||||
{ label: "时间", render: (entry) => formatTimestamp(entry.generatedAt) },
|
||||
{ label: "来源", render: (entry) => normalizeString(entry.sourcePath, "-") },
|
||||
{ label: "case", render: (entry) => normalizeNumber(entry?.totals?.caseCount) },
|
||||
{ label: "ready", render: (entry) => normalizeNumber(entry?.totals?.readyCount) },
|
||||
{ label: "invalid", render: (entry) => normalizeNumber(entry?.totals?.invalidCount) },
|
||||
{ label: "current gap", render: (entry) => normalizeNumber(entry?.totals?.currentObservabilityGapCaseCount) },
|
||||
{ label: "degraded gap", render: (entry) => normalizeNumber(entry?.totals?.degradedObservabilityGapCaseCount) },
|
||||
])}
|
||||
|
||||
<section class="panel">
|
||||
<div class="section-header">
|
||||
<div>
|
||||
<h2>Cleanup 建议</h2>
|
||||
<p>当前 nightly 建议按已有优先级排序展示,直接复用 cleanup 报告事实源。</p>
|
||||
</div>
|
||||
</div>
|
||||
<div class="recommendation-stack">
|
||||
${renderRecommendationList(recommendations)}
|
||||
</div>
|
||||
</section>
|
||||
</main>
|
||||
</body>
|
||||
</html>`;
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { renderHarnessDashboardHtml } from "./harness-dashboard-core.mjs";
|
||||
|
||||
describe("harness-dashboard-core", () => {
|
||||
it("应把 summary、trend、cleanup 渲染成单一事实源 dashboard", () => {
|
||||
const html = renderHarnessDashboardHtml({
|
||||
title: "Harness Engine Dashboard",
|
||||
summaryReport: {
|
||||
generatedAt: "2026-04-12T08:00:00.000Z",
|
||||
totals: {
|
||||
readyCount: 2,
|
||||
invalidCount: 1,
|
||||
},
|
||||
},
|
||||
trendReport: {
|
||||
generatedAt: "2026-04-12T08:01:00.000Z",
|
||||
signals: ["current gap 保持为 0。"],
|
||||
samples: [
|
||||
{
|
||||
generatedAt: "2026-04-12T08:00:00.000Z",
|
||||
sourcePath: "/tmp/history/summary.json",
|
||||
totals: {
|
||||
caseCount: 2,
|
||||
readyCount: 2,
|
||||
invalidCount: 0,
|
||||
currentObservabilityGapCaseCount: 0,
|
||||
degradedObservabilityGapCaseCount: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
cleanupReport: {
|
||||
generatedAt: "2026-04-12T08:02:00.000Z",
|
||||
signals: ["当前没有新的治理风险。"],
|
||||
recommendations: [
|
||||
{
|
||||
priority: "P1",
|
||||
title: "保持 current gap 为 0",
|
||||
rationale: ["继续沿用 current/degraded 分层。"],
|
||||
backlogTools: [
|
||||
"回看 browser replay / browser verification 失败样本,并把失败断言回挂到受影响主路径。",
|
||||
],
|
||||
commands: ["npm run harness:eval:history:record"],
|
||||
focusVerificationFailureOutcomes: [
|
||||
"browserVerification (failure)",
|
||||
],
|
||||
focusVerificationRecoveredOutcomes: [
|
||||
"artifactValidator (repaired)",
|
||||
"browserVerification (success)",
|
||||
],
|
||||
},
|
||||
],
|
||||
focus: {
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
signal: "browserVerification",
|
||||
outcome: "failure",
|
||||
state: "regressing",
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
outcome: "repaired",
|
||||
state: "expanding",
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 1,
|
||||
},
|
||||
},
|
||||
{
|
||||
signal: "browserVerification",
|
||||
outcome: "success",
|
||||
state: "present",
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 0,
|
||||
},
|
||||
},
|
||||
{
|
||||
signal: "guiSmoke",
|
||||
outcome: "passed",
|
||||
state: "present",
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [
|
||||
{
|
||||
signal: "guiSmoke",
|
||||
outcome: "failed",
|
||||
state: "present",
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
summary: {
|
||||
trend: {
|
||||
sampleCount: 1,
|
||||
latestCurrentObservabilityGapCaseCount: 0,
|
||||
latestDegradedObservabilityGapCaseCount: 1,
|
||||
currentObservabilityGapCaseDelta: 0,
|
||||
degradedObservabilityGapCaseDelta: 0,
|
||||
},
|
||||
verificationOutcomes: {
|
||||
blockingFailureCaseCount: 2,
|
||||
advisoryFailureCaseCount: 0,
|
||||
recoveredCaseCount: 3,
|
||||
current: {
|
||||
blockingFailureCaseCount: 1,
|
||||
advisoryFailureCaseCount: 0,
|
||||
recoveredCaseCount: 3,
|
||||
},
|
||||
degraded: {
|
||||
blockingFailureCaseCount: 1,
|
||||
advisoryFailureCaseCount: 0,
|
||||
},
|
||||
},
|
||||
governance: {
|
||||
violationCount: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(html).toContain("Harness Engine Nightly Dashboard");
|
||||
expect(html).toContain("Harness Engine Dashboard");
|
||||
expect(html).toContain("Current Gap");
|
||||
expect(html).toContain("Degraded Gap");
|
||||
expect(html).toContain("Current Blocking");
|
||||
expect(html).toContain("Current Advisory");
|
||||
expect(html).toContain("Current Recovered");
|
||||
expect(html).toContain("Degraded Blocking");
|
||||
expect(html).toContain("Recovered Outcomes");
|
||||
expect(html).toContain("Cleanup 建议");
|
||||
expect(html).toContain("Observability Gap 角色");
|
||||
expect(html).toContain("Verification Outcome 焦点");
|
||||
expect(html).toContain("Current Recovered Baseline");
|
||||
expect(html).toContain("current");
|
||||
expect(html).toContain("degraded");
|
||||
expect(html).toContain("browserVerification");
|
||||
expect(html).toContain("failure");
|
||||
expect(html).toContain("success");
|
||||
expect(html).toContain("artifactValidator");
|
||||
expect(html).toContain("repaired");
|
||||
expect(html).toContain("guiSmoke");
|
||||
expect(html).toContain("failed");
|
||||
expect(html).toContain("artifact validator 已执行修复");
|
||||
expect(html).toContain("浏览器验证已有成功样本");
|
||||
expect(html).toContain("当前 Recovered 基线");
|
||||
expect(html).toContain(
|
||||
"artifactValidator (repaired)、browserVerification (success)、guiSmoke (passed)",
|
||||
);
|
||||
expect(html).toContain("关注 failure outcome");
|
||||
expect(html).toContain("关注 recovered outcome");
|
||||
expect(html).toContain("artifactValidator (repaired)");
|
||||
expect(html).toContain("后续动作");
|
||||
expect(html).toContain(
|
||||
"回看 browser replay / browser verification 失败样本,并把失败断言回挂到受影响主路径。",
|
||||
);
|
||||
expect(html).toContain("推荐命令");
|
||||
expect(html).toContain("summary - history - trend - cleanup");
|
||||
});
|
||||
});
|
||||
@@ -59,10 +59,125 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("harness-eval-history-record", () => {
|
||||
it("默认入口应产出完整 harness artifact 套件", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
|
||||
const workspaceRoot = path.join(tempRoot, "workspace");
|
||||
|
||||
const result = runNodeScript("scripts/harness-eval-history-record.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
]);
|
||||
|
||||
const reportsRoot = path.join(tempRoot, ".lime", "harness", "reports");
|
||||
const summaryJsonPath = path.join(reportsRoot, "harness-eval-summary.json");
|
||||
const summaryMarkdownPath = path.join(reportsRoot, "harness-eval-summary.md");
|
||||
const trendJsonPath = path.join(reportsRoot, "harness-eval-trend.json");
|
||||
const trendMarkdownPath = path.join(reportsRoot, "harness-eval-trend.md");
|
||||
const cleanupJsonPath = path.join(reportsRoot, "harness-cleanup-report.json");
|
||||
const cleanupMarkdownPath = path.join(reportsRoot, "harness-cleanup-report.md");
|
||||
const dashboardHtmlPath = path.join(reportsRoot, "harness-dashboard.html");
|
||||
|
||||
expect(result.summary.outputJsonPath).toBe(summaryJsonPath);
|
||||
expect(result.summary.outputMarkdownPath).toBe(summaryMarkdownPath);
|
||||
expect(result.trend.outputJsonPath).toBe(trendJsonPath);
|
||||
expect(result.trend.outputMarkdownPath).toBe(trendMarkdownPath);
|
||||
expect(result.trend.currentRecoveredVerificationCaseCount).toBe(3);
|
||||
expect(result.trend.currentRecoveredBaselineFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]),
|
||||
);
|
||||
expect(result.cleanup.outputJsonPath).toBe(cleanupJsonPath);
|
||||
expect(result.cleanup.outputMarkdownPath).toBe(cleanupMarkdownPath);
|
||||
expect(result.cleanup.verificationFailureOutcomeFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:issues_present",
|
||||
]),
|
||||
);
|
||||
expect(result.cleanup.verificationFailureCaseCount).toBe(1);
|
||||
expect(result.cleanup.verificationBlockingFailureCaseCount).toBe(0);
|
||||
expect(result.cleanup.verificationAdvisoryFailureCaseCount).toBe(1);
|
||||
expect(result.cleanup.verificationDegradedBlockingFailureCaseCount).toBe(0);
|
||||
expect(result.cleanup.verificationRecoveredCaseCount).toBe(3);
|
||||
expect(result.cleanup.currentVerificationRecoveredCaseCount).toBe(3);
|
||||
expect(result.cleanup.currentRecoveredBaselineFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]),
|
||||
);
|
||||
expect(result.dashboard.outputHtmlPath).toBe(dashboardHtmlPath);
|
||||
|
||||
expect(fs.existsSync(summaryJsonPath)).toBe(true);
|
||||
expect(fs.existsSync(summaryMarkdownPath)).toBe(true);
|
||||
expect(fs.existsSync(trendJsonPath)).toBe(true);
|
||||
expect(fs.existsSync(trendMarkdownPath)).toBe(true);
|
||||
expect(fs.existsSync(cleanupJsonPath)).toBe(true);
|
||||
expect(fs.existsSync(cleanupMarkdownPath)).toBe(true);
|
||||
expect(fs.existsSync(dashboardHtmlPath)).toBe(true);
|
||||
|
||||
const cleanupJson = JSON.parse(fs.readFileSync(cleanupJsonPath, "utf8"));
|
||||
expect(
|
||||
cleanupJson.recommendations.every(
|
||||
(entry: Record<string, unknown>) =>
|
||||
!Object.prototype.hasOwnProperty.call(
|
||||
entry,
|
||||
"focusObservabilityVerificationOutcomes",
|
||||
),
|
||||
),
|
||||
).toBe(true);
|
||||
expect(() =>
|
||||
execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
path.join(repoRoot, "scripts/check-generated-slop-report.mjs"),
|
||||
"--input",
|
||||
cleanupJsonPath,
|
||||
"--format",
|
||||
"json",
|
||||
],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
),
|
||||
).not.toThrow();
|
||||
}, 30_000);
|
||||
|
||||
it("应记录本地 summary 历史,并生成非 seed 的 trend / cleanup", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
|
||||
const workspaceRoot = path.join(tempRoot, "workspace");
|
||||
const summaryJsonPath = path.join(
|
||||
tempRoot,
|
||||
".lime",
|
||||
"harness",
|
||||
"reports",
|
||||
"harness-eval-summary.json",
|
||||
);
|
||||
const summaryMarkdownPath = path.join(
|
||||
tempRoot,
|
||||
".lime",
|
||||
"harness",
|
||||
"reports",
|
||||
"harness-eval-summary.md",
|
||||
);
|
||||
const dashboardHtmlPath = path.join(
|
||||
tempRoot,
|
||||
".lime",
|
||||
"harness",
|
||||
"reports",
|
||||
"harness-dashboard.html",
|
||||
);
|
||||
|
||||
const firstResult = runNodeScript("scripts/harness-eval-history-record.mjs", [
|
||||
"--format",
|
||||
@@ -71,6 +186,12 @@ describe("harness-eval-history-record", () => {
|
||||
historyDir,
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--summary-json",
|
||||
summaryJsonPath,
|
||||
"--summary-markdown",
|
||||
summaryMarkdownPath,
|
||||
"--dashboard-html",
|
||||
dashboardHtmlPath,
|
||||
"--output-json",
|
||||
path.join(tempRoot, "first.json"),
|
||||
]);
|
||||
@@ -84,6 +205,12 @@ describe("harness-eval-history-record", () => {
|
||||
historyDir,
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--summary-json",
|
||||
summaryJsonPath,
|
||||
"--summary-markdown",
|
||||
summaryMarkdownPath,
|
||||
"--dashboard-html",
|
||||
dashboardHtmlPath,
|
||||
"--output-json",
|
||||
path.join(tempRoot, "second.json"),
|
||||
],
|
||||
@@ -91,11 +218,72 @@ describe("harness-eval-history-record", () => {
|
||||
|
||||
expect(firstResult.historyCount).toBe(1);
|
||||
expect(firstResult.trend.sampleCount).toBe(1);
|
||||
expect(firstResult.trend.currentObservabilityGapCaseCount).toBe(0);
|
||||
expect(firstResult.trend.degradedObservabilityGapCaseCount).toBe(1);
|
||||
expect(firstResult.trend.currentRecoveredVerificationCaseCount).toBe(3);
|
||||
expect(firstResult.trend.currentRecoveredBaselineFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]),
|
||||
);
|
||||
expect(firstResult.summary.outputJsonPath).toBe(summaryJsonPath);
|
||||
expect(firstResult.summary.outputMarkdownPath).toBe(summaryMarkdownPath);
|
||||
expect(firstResult.dashboard.outputHtmlPath).toBe(dashboardHtmlPath);
|
||||
|
||||
expect(secondResult.historyCount).toBe(2);
|
||||
expect(secondResult.trend.sampleCount).toBe(2);
|
||||
expect(secondResult.trend.currentObservabilityGapCaseCount).toBe(0);
|
||||
expect(secondResult.trend.degradedObservabilityGapCaseCount).toBe(1);
|
||||
expect(secondResult.trend.currentRecoveredVerificationCaseCount).toBe(3);
|
||||
expect(secondResult.trend.currentRecoveredBaselineFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]),
|
||||
);
|
||||
expect(secondResult.cleanup.trendSampleCount).toBe(2);
|
||||
expect(secondResult.cleanup.currentObservabilityGapCaseCount).toBe(0);
|
||||
expect(secondResult.cleanup.degradedObservabilityGapCaseCount).toBe(1);
|
||||
expect(secondResult.cleanup.verificationFailureOutcomeFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:issues_present",
|
||||
]),
|
||||
);
|
||||
expect(secondResult.cleanup.verificationFailureCaseCount).toBe(1);
|
||||
expect(secondResult.cleanup.verificationBlockingFailureCaseCount).toBe(0);
|
||||
expect(secondResult.cleanup.verificationAdvisoryFailureCaseCount).toBe(1);
|
||||
expect(secondResult.cleanup.verificationDegradedBlockingFailureCaseCount).toBe(0);
|
||||
expect(secondResult.cleanup.verificationRecoveredCaseCount).toBe(3);
|
||||
expect(secondResult.cleanup.currentVerificationRecoveredCaseCount).toBe(3);
|
||||
expect(secondResult.cleanup.currentRecoveredBaselineFocus).toEqual(
|
||||
expect.arrayContaining([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]),
|
||||
);
|
||||
expect(fs.existsSync(secondResult.recordedSummaryPath)).toBe(true);
|
||||
expect(fs.existsSync(summaryJsonPath)).toBe(true);
|
||||
expect(fs.existsSync(summaryMarkdownPath)).toBe(true);
|
||||
expect(fs.existsSync(dashboardHtmlPath)).toBe(true);
|
||||
const dashboardHtml = fs.readFileSync(dashboardHtmlPath, "utf8");
|
||||
expect(dashboardHtml).toContain("Harness Engine Nightly Dashboard");
|
||||
expect(dashboardHtml).toContain("Current Gap");
|
||||
expect(dashboardHtml).toContain("Degraded Gap");
|
||||
expect(dashboardHtml).toContain("Current Blocking");
|
||||
expect(dashboardHtml).toContain("Current Advisory");
|
||||
expect(dashboardHtml).toContain("Current Recovered");
|
||||
expect(dashboardHtml).toContain("Degraded Blocking");
|
||||
expect(dashboardHtml).toContain("Recovered Outcomes");
|
||||
expect(dashboardHtml).toContain("Cleanup 建议");
|
||||
expect(dashboardHtml).toContain("Observability Gap 角色");
|
||||
expect(dashboardHtml).toContain("Verification Outcome 焦点");
|
||||
expect(dashboardHtml).toContain("Current Recovered Baseline");
|
||||
expect(dashboardHtml).toContain("artifactValidator");
|
||||
expect(dashboardHtml).toContain("issues_present");
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(
|
||||
|
||||
@@ -159,7 +159,7 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("Harness eval history window", () => {
|
||||
it("runner 应记录并裁剪历史窗口,trend 应复用该目录", () => {
|
||||
it("history-record 应记录并裁剪历史窗口,trend 应复用该目录", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const caseDir = createReplayFixture(tempRoot);
|
||||
const manifestPath = path.join(tempRoot, "manifest.json");
|
||||
@@ -167,40 +167,37 @@ describe("Harness eval history window", () => {
|
||||
|
||||
writeJson(manifestPath, createHarnessManifest(caseDir));
|
||||
|
||||
runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
runNodeScript("scripts/harness-eval-history-record.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--record-history-dir",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--history-retain",
|
||||
"--retain",
|
||||
"2",
|
||||
"--no-strict",
|
||||
]);
|
||||
sleepMs(10);
|
||||
runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
runNodeScript("scripts/harness-eval-history-record.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--record-history-dir",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--history-retain",
|
||||
"--retain",
|
||||
"2",
|
||||
"--no-strict",
|
||||
]);
|
||||
sleepMs(10);
|
||||
runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
runNodeScript("scripts/harness-eval-history-record.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--record-history-dir",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--history-retain",
|
||||
"--retain",
|
||||
"2",
|
||||
"--no-strict",
|
||||
]);
|
||||
|
||||
const historyFiles = fs
|
||||
@@ -222,5 +219,5 @@ describe("Harness eval history window", () => {
|
||||
expect(report.signals).not.toContain(
|
||||
"样本数不足 2,当前仅形成 trend seed,还不能判断长期退化。",
|
||||
);
|
||||
});
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
function createTempRoot() {
|
||||
const tempRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "lime-harness-repo-fixtures-"),
|
||||
);
|
||||
tempRoots.push(tempRoot);
|
||||
return tempRoot;
|
||||
}
|
||||
|
||||
function runHarnessEval(args: string[]) {
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[path.join(repoRoot, "scripts/harness-eval-runner.mjs"), ...args],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (tempRoots.length > 0) {
|
||||
const tempRoot = tempRoots.pop();
|
||||
if (tempRoot) {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("Harness repo fixtures", () => {
|
||||
it("应固定 current 与 degraded observability 样本职责", () => {
|
||||
const workspaceRoot = createTempRoot();
|
||||
const summary = runHarnessEval([
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
"docs/test/harness-evals.manifest.json",
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--no-strict",
|
||||
]);
|
||||
|
||||
const repoFixtureSuite = summary.suites.find(
|
||||
(entry: { id: string }) => entry.id === "repo-fixtures",
|
||||
);
|
||||
expect(repoFixtureSuite).toBeTruthy();
|
||||
expect(repoFixtureSuite.stats.caseCount).toBe(2);
|
||||
expect(repoFixtureSuite.stats.readyCount).toBe(2);
|
||||
expect(summary.totals.observabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.currentObservabilityGapCaseCount).toBe(0);
|
||||
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(1);
|
||||
|
||||
const currentCase = repoFixtureSuite.cases.find(
|
||||
(entry: { caseId: string }) =>
|
||||
entry.caseId === "fixture-minimal-pending-request",
|
||||
);
|
||||
expect(currentCase).toBeTruthy();
|
||||
expect(currentCase.status).toBe("ready");
|
||||
expect(currentCase.observabilityGapCount).toBe(0);
|
||||
expect(currentCase.observabilitySignals).toContain(
|
||||
"requestTelemetry:exported",
|
||||
);
|
||||
expect(currentCase.observabilitySignals).toContain(
|
||||
"artifactValidator:exported",
|
||||
);
|
||||
expect(currentCase.observabilitySignals).toContain(
|
||||
"browserVerification:exported",
|
||||
);
|
||||
expect(currentCase.observabilitySignals).toContain("guiSmoke:exported");
|
||||
expect(currentCase.observabilityVerificationOutcomes).toContain(
|
||||
"artifactValidator:issues_present",
|
||||
);
|
||||
expect(currentCase.observabilityVerificationOutcomes).toContain(
|
||||
"artifactValidator:repaired",
|
||||
);
|
||||
expect(currentCase.observabilityVerificationOutcomes).toContain(
|
||||
"browserVerification:success",
|
||||
);
|
||||
expect(currentCase.observabilityVerificationOutcomes).toContain(
|
||||
"guiSmoke:passed",
|
||||
);
|
||||
|
||||
const degradedCase = repoFixtureSuite.cases.find(
|
||||
(entry: { caseId: string }) =>
|
||||
entry.caseId === "fixture-minimal-observability-gap",
|
||||
);
|
||||
expect(degradedCase).toBeTruthy();
|
||||
expect(degradedCase.status).toBe("ready");
|
||||
expect(degradedCase.observabilityGapCount).toBe(2);
|
||||
expect(degradedCase.observabilitySignals).toContain(
|
||||
"requestTelemetry:known_gap",
|
||||
);
|
||||
expect(degradedCase.observabilitySignals).toContain(
|
||||
"artifactValidator:known_gap",
|
||||
);
|
||||
|
||||
const gapCaseIds = repoFixtureSuite.cases
|
||||
.filter(
|
||||
(entry: { observabilityGapCount: number }) =>
|
||||
entry.observabilityGapCount > 0,
|
||||
)
|
||||
.map((entry: { caseId: string }) => entry.caseId);
|
||||
expect(gapCaseIds).toEqual(["fixture-minimal-observability-gap"]);
|
||||
|
||||
const observabilityBreakdownNames = summary.breakdowns.observabilitySignals.map(
|
||||
(entry: { name: string }) => entry.name,
|
||||
);
|
||||
expect(observabilityBreakdownNames).toContain("requestTelemetry:exported");
|
||||
expect(observabilityBreakdownNames).toContain("requestTelemetry:known_gap");
|
||||
expect(observabilityBreakdownNames).toContain("artifactValidator:exported");
|
||||
expect(observabilityBreakdownNames).toContain(
|
||||
"artifactValidator:known_gap",
|
||||
);
|
||||
|
||||
const verificationOutcomeBreakdownNames =
|
||||
summary.breakdowns.observabilityVerificationOutcomes.map(
|
||||
(entry: { name: string }) => entry.name,
|
||||
);
|
||||
expect(verificationOutcomeBreakdownNames).toContain(
|
||||
"artifactValidator:issues_present",
|
||||
);
|
||||
expect(verificationOutcomeBreakdownNames).toContain(
|
||||
"artifactValidator:repaired",
|
||||
);
|
||||
expect(verificationOutcomeBreakdownNames).toContain(
|
||||
"browserVerification:success",
|
||||
);
|
||||
expect(verificationOutcomeBreakdownNames).toContain("guiSmoke:passed");
|
||||
|
||||
const currentVerificationOutcomeBreakdownNames =
|
||||
summary.breakdowns.currentObservabilityVerificationOutcomes.map(
|
||||
(entry: { name: string }) => entry.name,
|
||||
);
|
||||
expect(currentVerificationOutcomeBreakdownNames).toContain(
|
||||
"artifactValidator:issues_present",
|
||||
);
|
||||
expect(currentVerificationOutcomeBreakdownNames).toContain(
|
||||
"browserVerification:success",
|
||||
);
|
||||
|
||||
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
|
||||
[],
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -116,13 +116,36 @@ function createWorkspaceSessionArtifacts(tempRoot: string, sessionId: string) {
|
||||
},
|
||||
{
|
||||
signal: "requestTelemetry",
|
||||
status: "unlinked",
|
||||
status: "exported",
|
||||
},
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
status: "known_gap",
|
||||
},
|
||||
],
|
||||
verificationSummary: {
|
||||
artifactValidator: {
|
||||
applicable: true,
|
||||
recordCount: 1,
|
||||
issueCount: 1,
|
||||
repairedCount: 1,
|
||||
fallbackUsedCount: 0,
|
||||
},
|
||||
browserVerification: {
|
||||
recordCount: 1,
|
||||
successCount: 1,
|
||||
failureCount: 0,
|
||||
unknownCount: 0,
|
||||
latestUpdatedAt: "2026-03-27T11:22:00Z",
|
||||
},
|
||||
guiSmoke: {
|
||||
status: "completed",
|
||||
exitCode: 0,
|
||||
passed: true,
|
||||
updatedAt: "2026-03-27T11:23:30Z",
|
||||
hasOutputPreview: true,
|
||||
},
|
||||
},
|
||||
},
|
||||
linkedArtifacts: {
|
||||
handoffBundle: {
|
||||
@@ -325,6 +348,8 @@ describe("Harness review decision / eval integration", () => {
|
||||
|
||||
expect(summary.totals.reviewDecisionRecordedCount).toBe(1);
|
||||
expect(summary.totals.observabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.currentObservabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(0);
|
||||
expect(summary.breakdowns.reviewDecisionStatuses).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
@@ -343,21 +368,54 @@ describe("Harness review decision / eval integration", () => {
|
||||
);
|
||||
expect(summary.breakdowns.observabilitySignals).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "requestTelemetry:unlinked",
|
||||
caseCount: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "artifactValidator:known_gap",
|
||||
caseCount: 1,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.breakdowns.observabilityVerificationOutcomes).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "artifactValidator:issues_present",
|
||||
caseCount: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "artifactValidator:repaired",
|
||||
caseCount: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "guiSmoke:passed",
|
||||
caseCount: 1,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.breakdowns.currentObservabilityVerificationOutcomes).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
|
||||
[],
|
||||
);
|
||||
expect(summary.suites[0].cases[0]).toMatchObject({
|
||||
reviewDecisionStatus: "accepted",
|
||||
reviewRiskLevel: "high",
|
||||
reviewHumanReviewer: "Lime Maintainer",
|
||||
observabilityGapCount: 2,
|
||||
observabilityGapCount: 1,
|
||||
observabilityVerificationOutcomes: expect.arrayContaining([
|
||||
"artifactValidator:issues_present",
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
@@ -376,6 +434,8 @@ describe("Harness review decision / eval integration", () => {
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
observabilityGapCaseCount: 1,
|
||||
currentObservabilityGapCaseCount: 1,
|
||||
degradedObservabilityGapCaseCount: 0,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: [],
|
||||
@@ -400,9 +460,10 @@ describe("Harness review decision / eval integration", () => {
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
],
|
||||
observabilitySignals: [
|
||||
observabilitySignals: [],
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "requestTelemetry:unlinked",
|
||||
name: "browserVerification:unknown",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
@@ -410,6 +471,17 @@ describe("Harness review decision / eval integration", () => {
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:unknown",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
suites: [],
|
||||
});
|
||||
@@ -424,6 +496,8 @@ describe("Harness review decision / eval integration", () => {
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
observabilityGapCaseCount: 1,
|
||||
currentObservabilityGapCaseCount: 0,
|
||||
degradedObservabilityGapCaseCount: 1,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: [],
|
||||
@@ -458,6 +532,43 @@ describe("Harness review decision / eval integration", () => {
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:failure",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [],
|
||||
degradedObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:failure",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
suites: [],
|
||||
});
|
||||
@@ -505,12 +616,6 @@ describe("Harness review decision / eval integration", () => {
|
||||
);
|
||||
expect(report.classificationDeltas.observabilitySignals).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "requestTelemetry:unlinked",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: -1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "artifactValidator:known_gap",
|
||||
delta: expect.objectContaining({
|
||||
@@ -519,5 +624,234 @@ describe("Harness review decision / eval integration", () => {
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
report.classificationDeltas.observabilitySignals.find(
|
||||
(entry) => entry.name === "requestTelemetry:known_gap",
|
||||
),
|
||||
).toBeUndefined();
|
||||
expect(report.classificationDeltas.observabilityVerificationOutcomes).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:failure",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:unknown",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: -1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "guiSmoke:failed",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
report.classificationDeltas.currentObservabilityVerificationOutcomes,
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:unknown",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: -1,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
report.classificationDeltas.degradedObservabilityVerificationOutcomes,
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:failure",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "guiSmoke:failed",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(report.delta.currentObservabilityGapCaseCount).toBe(-1);
|
||||
expect(report.delta.degradedObservabilityGapCaseCount).toBe(1);
|
||||
expect(report.latest.totals.currentObservabilityGapCaseCount).toBe(0);
|
||||
expect(report.latest.totals.degradedObservabilityGapCaseCount).toBe(1);
|
||||
});
|
||||
|
||||
it("harness-eval-trend-report 应暴露 current recovered baseline 的新增趋势", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const baselinePath = path.join(tempRoot, "baseline-recovered.json");
|
||||
const latestPath = path.join(tempRoot, "latest-recovered.json");
|
||||
|
||||
writeJson(baselinePath, {
|
||||
generatedAt: "2026-03-28T10:00:00Z",
|
||||
totals: {
|
||||
suiteCount: 1,
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
observabilityGapCaseCount: 0,
|
||||
currentObservabilityGapCaseCount: 0,
|
||||
degradedObservabilityGapCaseCount: 0,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: [],
|
||||
failureModes: [],
|
||||
reviewDecisionStatuses: [],
|
||||
reviewRiskLevels: [],
|
||||
observabilitySignals: [],
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
suites: [],
|
||||
});
|
||||
|
||||
writeJson(latestPath, {
|
||||
generatedAt: "2026-03-28T12:00:00Z",
|
||||
totals: {
|
||||
suiteCount: 1,
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
observabilityGapCaseCount: 0,
|
||||
currentObservabilityGapCaseCount: 0,
|
||||
degradedObservabilityGapCaseCount: 0,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: [],
|
||||
failureModes: [],
|
||||
reviewDecisionStatuses: [],
|
||||
reviewRiskLevels: [],
|
||||
observabilitySignals: [],
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "artifactValidator:repaired",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
{
|
||||
name: "guiSmoke:passed",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "artifactValidator:repaired",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
{
|
||||
name: "guiSmoke:passed",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
suites: [],
|
||||
});
|
||||
|
||||
const report = runNodeScript("scripts/harness-eval-trend-report.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--input",
|
||||
baselinePath,
|
||||
"--input",
|
||||
latestPath,
|
||||
]);
|
||||
|
||||
expect(report.latest.totals.currentRecoveredVerificationCaseCount).toBe(3);
|
||||
expect(report.delta.currentRecoveredVerificationCaseCount).toBe(2);
|
||||
expect(
|
||||
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes,
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "artifactValidator:repaired",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:success",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 0,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "guiSmoke:passed",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(report.signals).toContain(
|
||||
"current recovered verification baseline `artifactValidator:repaired` 新增 1,说明主线路径正在形成正向基线。",
|
||||
);
|
||||
expect(report.signals).toContain(
|
||||
"current recovered verification baseline `guiSmoke:passed` 新增 1,说明主线路径正在形成正向基线。",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
+18
-1
@@ -10,6 +10,14 @@ const rootDir = process.cwd();
|
||||
|
||||
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
|
||||
const cargoCommand = process.platform === "win32" ? "cargo.exe" : "cargo";
|
||||
const BRIDGE_REASON_LABELS = {
|
||||
bridge_contracts: "bridge/contracts",
|
||||
bridge_runtime: "DevBridge / mock / bridge runtime",
|
||||
fallback_full_suite: "兜底全量",
|
||||
full_suite: "full 模式",
|
||||
harness_cleanup_contract: "harness cleanup contract",
|
||||
workflow_full_suite: "workflow 全量",
|
||||
};
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
@@ -107,7 +115,16 @@ function printSummary(changedFiles, tasks) {
|
||||
console.log("[local-ci] - 前端校验");
|
||||
}
|
||||
if (tasks.bridge) {
|
||||
console.log("[local-ci] - bridge 校验");
|
||||
const bridgeReasonLabels = Array.isArray(tasks.bridgeReasons)
|
||||
? tasks.bridgeReasons
|
||||
.map((reason) => BRIDGE_REASON_LABELS[reason] ?? reason)
|
||||
.filter(Boolean)
|
||||
: [];
|
||||
console.log(
|
||||
bridgeReasonLabels.length > 0
|
||||
? `[local-ci] - bridge 校验(${bridgeReasonLabels.join(" / ")})`
|
||||
: "[local-ci] - bridge 校验",
|
||||
);
|
||||
}
|
||||
if (tasks.guiSmoke) {
|
||||
console.log("[local-ci] - GUI 冒烟");
|
||||
|
||||
@@ -39,13 +39,28 @@ const FRONTEND_TOOLING_FILES = new Set([
|
||||
const BRIDGE_FILES = new Set([
|
||||
"vite.config.ts",
|
||||
"scripts/check-command-contracts.mjs",
|
||||
"scripts/check-generated-slop-report.mjs",
|
||||
"scripts/check-dev-bridge-health.mjs",
|
||||
"scripts/harness-eval-history-record.mjs",
|
||||
"scripts/harness-eval-trend-report.mjs",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/social-workbench-e2e-smoke.mjs",
|
||||
"scripts/chrome-bridge-e2e.mjs",
|
||||
"scripts/verify-gui-smoke.mjs",
|
||||
"scripts/lib/generated-slop-report-core.mjs",
|
||||
"scripts/lib/harness-dashboard-core.mjs",
|
||||
"docs/aiprompts/playwright-e2e.md",
|
||||
]);
|
||||
|
||||
const HARNESS_CLEANUP_CONTRACT_FILES = new Set([
|
||||
"scripts/check-generated-slop-report.mjs",
|
||||
"scripts/harness-eval-history-record.mjs",
|
||||
"scripts/harness-eval-trend-report.mjs",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/lib/generated-slop-report-core.mjs",
|
||||
"scripts/lib/harness-dashboard-core.mjs",
|
||||
]);
|
||||
|
||||
const INTEGRITY_FILES = new Set([
|
||||
"package.json",
|
||||
"src-tauri/Cargo.toml",
|
||||
@@ -249,6 +264,45 @@ function isBridgeChange(file) {
|
||||
);
|
||||
}
|
||||
|
||||
function isHarnessCleanupContractChange(file) {
|
||||
return HARNESS_CLEANUP_CONTRACT_FILES.has(file);
|
||||
}
|
||||
|
||||
function collectBridgeReasons(changedFiles, { full = false, fallback = false, workflow = false } = {}) {
|
||||
if (full) {
|
||||
return ["full_suite"];
|
||||
}
|
||||
|
||||
if (fallback) {
|
||||
return ["fallback_full_suite"];
|
||||
}
|
||||
|
||||
if (workflow) {
|
||||
return ["workflow_full_suite"];
|
||||
}
|
||||
|
||||
const reasons = [];
|
||||
|
||||
if (changedFiles.some(isHarnessCleanupContractChange)) {
|
||||
reasons.push("harness_cleanup_contract");
|
||||
}
|
||||
|
||||
if (
|
||||
changedFiles.some(
|
||||
(file) =>
|
||||
isBridgeChange(file) && !isHarnessCleanupContractChange(file),
|
||||
)
|
||||
) {
|
||||
reasons.push("bridge_runtime");
|
||||
}
|
||||
|
||||
if (reasons.length === 0 && changedFiles.some(isBridgeChange)) {
|
||||
reasons.push("bridge_contracts");
|
||||
}
|
||||
|
||||
return reasons;
|
||||
}
|
||||
|
||||
function isGuiSmokeChange(file) {
|
||||
return (
|
||||
GUI_SMOKE_FILES.has(file) ||
|
||||
@@ -272,6 +326,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
|
||||
frontend: true,
|
||||
rust: true,
|
||||
bridge: true,
|
||||
bridgeReasons: collectBridgeReasons([], { full: true }),
|
||||
guiSmoke: true,
|
||||
docs: true,
|
||||
docsOnly: false,
|
||||
@@ -286,6 +341,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
|
||||
frontend: true,
|
||||
rust: true,
|
||||
bridge: true,
|
||||
bridgeReasons: collectBridgeReasons([], { fallback: true }),
|
||||
guiSmoke: true,
|
||||
docs: true,
|
||||
docsOnly: false,
|
||||
@@ -301,6 +357,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
|
||||
frontend: true,
|
||||
rust: true,
|
||||
bridge: true,
|
||||
bridgeReasons: collectBridgeReasons([], { workflow: true }),
|
||||
guiSmoke: true,
|
||||
docs: true,
|
||||
docsOnly: false,
|
||||
@@ -315,6 +372,7 @@ function detectTasks(changedFiles, { full = false } = {}) {
|
||||
frontend: false,
|
||||
rust: false,
|
||||
bridge: false,
|
||||
bridgeReasons: [],
|
||||
guiSmoke: false,
|
||||
docs: true,
|
||||
docsOnly: true,
|
||||
@@ -323,11 +381,14 @@ function detectTasks(changedFiles, { full = false } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
const bridge = changedFiles.some(isBridgeChange);
|
||||
|
||||
return {
|
||||
integrity: changedFiles.some(isIntegrityChange),
|
||||
frontend: changedFiles.some(isFrontendChange),
|
||||
rust: changedFiles.some(isRustChange),
|
||||
bridge: changedFiles.some(isBridgeChange),
|
||||
bridge,
|
||||
bridgeReasons: bridge ? collectBridgeReasons(changedFiles) : [],
|
||||
guiSmoke: changedFiles.some(isGuiSmokeChange),
|
||||
docs: changedFiles.some(isDocsChange),
|
||||
docsOnly: false,
|
||||
@@ -359,4 +420,10 @@ function planQualityTasks({
|
||||
};
|
||||
}
|
||||
|
||||
export { collectChangedFiles, detectTasks, planQualityTasks, resolveDiffBase };
|
||||
export {
|
||||
collectBridgeReasons,
|
||||
collectChangedFiles,
|
||||
detectTasks,
|
||||
planQualityTasks,
|
||||
resolveDiffBase,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { detectTasks } from "./quality-task-planner.mjs";
|
||||
|
||||
describe("quality-task-planner", () => {
|
||||
it("应把 harness cleanup/report 主链文件归到 bridge/contracts 风险", () => {
|
||||
const tasks = detectTasks([
|
||||
"scripts/lib/generated-slop-report-core.mjs",
|
||||
"scripts/check-generated-slop-report.mjs",
|
||||
"scripts/harness-eval-history-record.mjs",
|
||||
]);
|
||||
|
||||
expect(tasks.bridge).toBe(true);
|
||||
expect(tasks.bridgeReasons).toContain("harness_cleanup_contract");
|
||||
expect(tasks.docsOnly).toBe(false);
|
||||
});
|
||||
|
||||
it("应把 harness dashboard 渲染文件归到 bridge/contracts 风险", () => {
|
||||
const tasks = detectTasks(["scripts/lib/harness-dashboard-core.mjs"]);
|
||||
|
||||
expect(tasks.bridge).toBe(true);
|
||||
expect(tasks.bridgeReasons).toContain("harness_cleanup_contract");
|
||||
expect(tasks.docsOnly).toBe(false);
|
||||
});
|
||||
|
||||
it("应把 DevBridge 主链改动标记为 bridge runtime 风险", () => {
|
||||
const tasks = detectTasks(["src/lib/dev-bridge/safeInvoke.ts"]);
|
||||
|
||||
expect(tasks.bridge).toBe(true);
|
||||
expect(tasks.bridgeReasons).toContain("bridge_runtime");
|
||||
expect(tasks.docsOnly).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -68,6 +68,7 @@ function printGithubFormat(result) {
|
||||
`frontend=${tasks.frontend}`,
|
||||
`rust=${tasks.rust}`,
|
||||
`bridge=${tasks.bridge}`,
|
||||
`bridge_reasons=${Array.isArray(tasks.bridgeReasons) ? tasks.bridgeReasons.join(",") : ""}`,
|
||||
`gui_smoke=${tasks.guiSmoke}`,
|
||||
`docs=${tasks.docs}`,
|
||||
`docs_only=${tasks.docsOnly}`,
|
||||
|
||||
@@ -5,7 +5,11 @@ import process from 'node:process';
|
||||
|
||||
const DEV_URL = process.env.LIME_WEB_BRIDGE_URL?.trim() || 'http://127.0.0.1:1420/';
|
||||
const DEV_URL_TIMEOUT_MS = 1_500;
|
||||
const REUSE_ONLY_TIMEOUT_MS = 10_000;
|
||||
const REUSE_ONLY_INTERVAL_MS = 500;
|
||||
const ROOT_MARKERS = ['<title>Lime</title>', '<div id="root"></div>'];
|
||||
const REUSE_EXISTING_ONLY =
|
||||
process.env.LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY?.trim() === '1';
|
||||
|
||||
const env = { ...process.env };
|
||||
delete env.TAURI_ENV_PLATFORM;
|
||||
@@ -39,6 +43,32 @@ async function probeExistingDevServer(url) {
|
||||
}
|
||||
}
|
||||
|
||||
async function waitForExistingDevServer(url) {
|
||||
const startedAt = Date.now();
|
||||
let lastProbe = { reachable: false };
|
||||
|
||||
while (Date.now() - startedAt < REUSE_ONLY_TIMEOUT_MS) {
|
||||
lastProbe = await probeExistingDevServer(url);
|
||||
|
||||
if (lastProbe.reachable && lastProbe.isLimeDevShell) {
|
||||
return lastProbe;
|
||||
}
|
||||
|
||||
if (lastProbe.reachable && !lastProbe.isLimeDevShell) {
|
||||
const statusLabel = `${lastProbe.status} ${lastProbe.statusText}`.trim();
|
||||
throw new Error(
|
||||
`[dev:web-bridge] ${DEV_URL} 已被其他服务占用,且返回内容不是 Lime dev shell(${statusLabel})。请先关闭占用进程后重试。`,
|
||||
);
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, REUSE_ONLY_INTERVAL_MS));
|
||||
}
|
||||
|
||||
throw new Error(
|
||||
`[dev:web-bridge] 要求复用已有 Lime dev server,但 ${url} 在 ${REUSE_ONLY_TIMEOUT_MS}ms 内未就绪。`,
|
||||
);
|
||||
}
|
||||
|
||||
async function waitForExitSignal() {
|
||||
await new Promise((resolve) => {
|
||||
const handleExit = () => resolve();
|
||||
@@ -64,6 +94,13 @@ function startVite() {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
if (REUSE_EXISTING_ONLY) {
|
||||
await waitForExistingDevServer(DEV_URL);
|
||||
console.log(`[dev:web-bridge] 复用已存在的 Lime dev server: ${DEV_URL}`);
|
||||
await waitForExitSignal();
|
||||
return;
|
||||
}
|
||||
|
||||
const existingServer = await probeExistingDevServer(DEV_URL);
|
||||
|
||||
if (existingServer.reachable) {
|
||||
|
||||
+741
-69
@@ -1,31 +1,108 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn, spawnSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const DEFAULTS = {
|
||||
appUrl: "http://127.0.0.1:1420/",
|
||||
healthUrl: "http://127.0.0.1:3030/health",
|
||||
timeoutMs: 600_000,
|
||||
intervalMs: 1_000,
|
||||
reuseRunning: false,
|
||||
sampleProjectName: "Lime Smoke Workspace",
|
||||
};
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const rootDir = path.resolve(__dirname, "..");
|
||||
const npmCommand = process.platform === "win32" ? "npm.cmd" : "npm";
|
||||
const LIME_SKIP_STARTUP_WINDOW_REVEAL = "LIME_SKIP_STARTUP_WINDOW_REVEAL";
|
||||
const LIME_DISABLE_SINGLE_INSTANCE = "LIME_DISABLE_SINGLE_INSTANCE";
|
||||
const LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY =
|
||||
"LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY";
|
||||
const LIME_WEB_BRIDGE_URL = "LIME_WEB_BRIDGE_URL";
|
||||
const ROOT_MARKERS = ['<title>Lime</title>', '<div id="root"></div>'];
|
||||
const SHARED_TAURI_TARGET_DIR = path.join(rootDir, "src-tauri", "target");
|
||||
const ISOLATED_GUI_SMOKE_TARGET_DIR = path.join(
|
||||
os.tmpdir(),
|
||||
"lime-gui-smoke-target",
|
||||
);
|
||||
const GUI_SMOKE_TEMP_CONFIG_BASENAME_PREFIX = "lime-gui-smoke-tauri-";
|
||||
const GUI_SMOKE_COLD_TIMEOUT_MS = 1_800_000;
|
||||
const GUI_SMOKE_WARM_TIMEOUT_MS = 600_000;
|
||||
const GUI_SMOKE_BRIDGE_HEARTBEAT_MS = 30_000;
|
||||
const GUI_SMOKE_COMPILE_GRACE_MS = 900_000;
|
||||
const GUI_SMOKE_MAX_COMPILE_GRACE_EXTENSIONS = 2;
|
||||
const GUI_SMOKE_BOOT_GRACE_MS = 60_000;
|
||||
const HEADLESS_TAURI_CONFIG_PATH = path.join(
|
||||
rootDir,
|
||||
"src-tauri",
|
||||
"tauri.conf.headless.json",
|
||||
);
|
||||
const tauriCommand =
|
||||
process.platform === "win32"
|
||||
? path.join(rootDir, "node_modules", ".bin", "tauri.cmd")
|
||||
: path.join(rootDir, "node_modules", ".bin", "tauri");
|
||||
|
||||
const state = {
|
||||
child: null,
|
||||
cleanedUp: false,
|
||||
tempConfigPath: null,
|
||||
};
|
||||
|
||||
function resolveTargetBinaryPath(targetDir) {
|
||||
const appBinaryName = process.platform === "win32" ? "lime.exe" : "lime";
|
||||
return path.join(targetDir, "debug", appBinaryName);
|
||||
}
|
||||
|
||||
function listTargetLockHolderCommands(targetDir) {
|
||||
if (process.platform === "win32") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const lockPath = path.join(targetDir, "debug", ".cargo-lock");
|
||||
const pidOutput = runQuietCommand("lsof", ["-t", "--", lockPath]);
|
||||
if (!pidOutput) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const pids = [
|
||||
...new Set(
|
||||
pidOutput
|
||||
.split("\n")
|
||||
.map((item) => item.trim())
|
||||
.filter(Boolean),
|
||||
),
|
||||
];
|
||||
|
||||
return pids
|
||||
.map((pid) => runQuietCommand("ps", ["-p", pid, "-o", "command="]))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function resolvePreferredCargoTargetDir() {
|
||||
const sharedTargetLockHolders = listTargetLockHolderCommands(
|
||||
SHARED_TAURI_TARGET_DIR,
|
||||
);
|
||||
if (sharedTargetLockHolders.length === 0) {
|
||||
return SHARED_TAURI_TARGET_DIR;
|
||||
}
|
||||
|
||||
return ISOLATED_GUI_SMOKE_TARGET_DIR;
|
||||
}
|
||||
|
||||
function resolveDefaultTimeoutMs(cargoTargetDir) {
|
||||
const binaryPath = resolveTargetBinaryPath(cargoTargetDir);
|
||||
return fs.existsSync(binaryPath)
|
||||
? GUI_SMOKE_WARM_TIMEOUT_MS
|
||||
: GUI_SMOKE_COLD_TIMEOUT_MS;
|
||||
}
|
||||
|
||||
const DEFAULTS = {
|
||||
appUrl: "http://127.0.0.1:1420/",
|
||||
healthUrl: "http://127.0.0.1:3030/health",
|
||||
cargoTargetDir: resolvePreferredCargoTargetDir(),
|
||||
intervalMs: 1_000,
|
||||
reuseRunning: false,
|
||||
sampleProjectName: "Lime Smoke Workspace",
|
||||
};
|
||||
DEFAULTS.timeoutMs = resolveDefaultTimeoutMs(DEFAULTS.cargoTargetDir);
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime GUI 冒烟入口
|
||||
@@ -41,9 +118,10 @@ Lime GUI 冒烟入口
|
||||
选项:
|
||||
--app-url <url> 前端地址,默认 http://127.0.0.1:1420/
|
||||
--health-url <url> DevBridge 健康检查地址,默认 http://127.0.0.1:3030/health
|
||||
--timeout-ms <ms> 等待 headless / bridge / smoke 的超时,默认 600000
|
||||
--timeout-ms <ms> 等待 headless / bridge / smoke 的超时,默认冷启动 1800000 / 热启动 600000
|
||||
--interval-ms <ms> 轮询间隔,默认 1000
|
||||
--sample-project-name <s> workspace 路径校验使用的示例项目名
|
||||
--cargo-target-dir <dir> 指定 Cargo target 目录;默认优先复用 src-tauri/target,无锁时共享,否则回退独立 GUI smoke target
|
||||
--reuse-running 复用已启动的 headless Tauri,不主动拉起
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
@@ -85,6 +163,12 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--cargo-target-dir" && argv[index + 1]) {
|
||||
options.cargoTargetDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--reuse-running") {
|
||||
options.reuseRunning = true;
|
||||
continue;
|
||||
@@ -116,6 +200,10 @@ function parseArgs(argv) {
|
||||
throw new Error("--sample-project-name 不能为空");
|
||||
}
|
||||
|
||||
if (!options.cargoTargetDir) {
|
||||
throw new Error("--cargo-target-dir 不能为空");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
@@ -129,10 +217,22 @@ function assert(condition, message) {
|
||||
}
|
||||
}
|
||||
|
||||
function isLimeDevShell(html) {
|
||||
return ROOT_MARKERS.some((marker) => html.includes(marker));
|
||||
}
|
||||
|
||||
function formatCommand(command, args) {
|
||||
return [command, ...args].join(" ");
|
||||
}
|
||||
|
||||
function trimTrailingSlash(value) {
|
||||
return value.endsWith("/") ? value.slice(0, -1) : value;
|
||||
}
|
||||
|
||||
function buildNoopBeforeDevCommand() {
|
||||
return `"${process.execPath}" -e "process.exit(0)"`;
|
||||
}
|
||||
|
||||
function runCommand(command, args, label, timeoutMs) {
|
||||
console.log(`\n[verify:gui-smoke] > ${formatCommand(command, args)}`);
|
||||
const result = spawnSync(command, args, {
|
||||
@@ -160,66 +260,435 @@ function runCommand(command, args, label, timeoutMs) {
|
||||
}
|
||||
}
|
||||
|
||||
function startHeadlessTauri() {
|
||||
console.log("[verify:gui-smoke] 启动 headless Tauri 环境...");
|
||||
state.child = spawn(npmCommand, ["run", "tauri:dev:headless"], {
|
||||
cwd: rootDir,
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
[LIME_SKIP_STARTUP_WINDOW_REVEAL]: "1",
|
||||
[LIME_DISABLE_SINGLE_INSTANCE]: "1",
|
||||
},
|
||||
detached: process.platform !== "win32",
|
||||
});
|
||||
}
|
||||
function createHeadlessTauriConfig(options, startupMode) {
|
||||
const rawConfig = fs.readFileSync(HEADLESS_TAURI_CONFIG_PATH, "utf8");
|
||||
const config = JSON.parse(rawConfig);
|
||||
|
||||
async function stopHeadlessTauri() {
|
||||
const child = state.child;
|
||||
if (!child || state.cleanedUp) {
|
||||
return;
|
||||
config.build = {
|
||||
...(config.build || {}),
|
||||
devUrl: trimTrailingSlash(options.appUrl),
|
||||
};
|
||||
|
||||
if (startupMode.reuseExistingAppShell) {
|
||||
config.build.beforeDevCommand = buildNoopBeforeDevCommand();
|
||||
}
|
||||
|
||||
state.cleanedUp = true;
|
||||
console.log("[verify:gui-smoke] 停止 headless Tauri 环境...");
|
||||
const tempConfigPath = path.join(
|
||||
os.tmpdir(),
|
||||
`lime-gui-smoke-tauri-${process.pid}.json`,
|
||||
);
|
||||
fs.writeFileSync(tempConfigPath, JSON.stringify(config, null, 2));
|
||||
state.tempConfigPath = tempConfigPath;
|
||||
return tempConfigPath;
|
||||
}
|
||||
|
||||
if (typeof child.pid !== "number") {
|
||||
function resolveUrlPort(url) {
|
||||
try {
|
||||
return new URL(url).port || (url.startsWith("https:") ? "443" : "80");
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function runQuietCommand(command, args) {
|
||||
try {
|
||||
return spawnSync(command, args, {
|
||||
cwd: rootDir,
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
env: process.env,
|
||||
encoding: "utf8",
|
||||
}).stdout.trim();
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function listProcessTable() {
|
||||
if (process.platform === "win32") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const output = runQuietCommand("ps", ["-axo", "pid=,ppid=,pgid=,command="]);
|
||||
if (!output) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return output
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const match = line.match(/^(\d+)\s+(\d+)\s+(\d+)\s+(.*)$/);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
pid: Number(match[1]),
|
||||
ppid: Number(match[2]),
|
||||
pgid: Number(match[3]),
|
||||
command: match[4],
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function listProcessStats() {
|
||||
if (process.platform === "win32") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const output = runQuietCommand("ps", [
|
||||
"-axo",
|
||||
"pid=,ppid=,pgid=,%cpu=,%mem=,etime=,stat=,command=",
|
||||
]);
|
||||
if (!output) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return output
|
||||
.split("\n")
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const match = line.match(
|
||||
/^(\d+)\s+(\d+)\s+(\d+)\s+([\d.]+)\s+([\d.]+)\s+(\S+)\s+(\S+)\s+(.*)$/,
|
||||
);
|
||||
if (!match) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
pid: Number(match[1]),
|
||||
ppid: Number(match[2]),
|
||||
pgid: Number(match[3]),
|
||||
cpu: Number(match[4]),
|
||||
mem: Number(match[5]),
|
||||
etime: match[6],
|
||||
stat: match[7],
|
||||
command: match[8],
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function isGuiSmokeVerifierCommand(command) {
|
||||
return (
|
||||
command.includes("scripts/verify-gui-smoke.mjs") ||
|
||||
command.includes("npm run verify:gui-smoke")
|
||||
);
|
||||
}
|
||||
|
||||
function isGuiSmokeTauriCommand(command) {
|
||||
return (
|
||||
command.includes("tauri dev") &&
|
||||
command.includes(GUI_SMOKE_TEMP_CONFIG_BASENAME_PREFIX)
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeGuiSmokeProcessCommand(command) {
|
||||
if (command.includes("/bin/rustc")) {
|
||||
return "rustc";
|
||||
}
|
||||
|
||||
if (command.includes("cargo run --no-default-features")) {
|
||||
return "cargo";
|
||||
}
|
||||
|
||||
if (command.includes("tauri dev")) {
|
||||
return "tauri";
|
||||
}
|
||||
|
||||
if (command.includes("start-web-bridge-dev.mjs")) {
|
||||
return "web-bridge";
|
||||
}
|
||||
|
||||
if (command.includes("npm run dev:web-bridge")) {
|
||||
return "npm:dev:web-bridge";
|
||||
}
|
||||
|
||||
if (command.includes("npm exec vite") || command.includes("/bin/vite")) {
|
||||
return "vite";
|
||||
}
|
||||
|
||||
return command.split(/\s+/)[0]?.split("/").pop() || "process";
|
||||
}
|
||||
|
||||
function resolveGuiSmokeProcessGroupId(startedByScript) {
|
||||
if (startedByScript && typeof state.child?.pid === "number") {
|
||||
return state.child.pid;
|
||||
}
|
||||
|
||||
const snapshot = inspectGuiSmokeTauriProcesses();
|
||||
const firstActive = snapshot.active[0];
|
||||
return firstActive ? firstActive.pgid || firstActive.pid : null;
|
||||
}
|
||||
|
||||
function listGuiSmokeGroupProcesses(startedByScript) {
|
||||
const targetGroupId = resolveGuiSmokeProcessGroupId(startedByScript);
|
||||
if (!Number.isInteger(targetGroupId) || targetGroupId < 1) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return listProcessStats().filter((item) => item.pgid === targetGroupId);
|
||||
}
|
||||
|
||||
function hasActiveGuiSmokeCompile(startedByScript) {
|
||||
return listGuiSmokeGroupProcesses(startedByScript).some(
|
||||
(item) =>
|
||||
item.command.includes("/bin/rustc") ||
|
||||
item.command.includes("cargo run --no-default-features"),
|
||||
);
|
||||
}
|
||||
|
||||
function describeGuiSmokeHeartbeat(startedByScript) {
|
||||
const interestingProcesses = listGuiSmokeGroupProcesses(startedByScript)
|
||||
.filter(
|
||||
(item) =>
|
||||
item.command.includes("tauri dev") ||
|
||||
item.command.includes("cargo run --no-default-features") ||
|
||||
item.command.includes("/bin/rustc") ||
|
||||
item.command.includes("start-web-bridge-dev.mjs") ||
|
||||
item.command.includes("npm run dev:web-bridge") ||
|
||||
item.command.includes("npm exec vite") ||
|
||||
item.command.includes("/bin/vite"),
|
||||
)
|
||||
.sort((left, right) => right.cpu - left.cpu || left.pid - right.pid)
|
||||
.slice(0, 5);
|
||||
|
||||
if (interestingProcesses.length === 0) {
|
||||
return "";
|
||||
}
|
||||
|
||||
return interestingProcesses
|
||||
.map((item) => {
|
||||
const cpu = Number.isFinite(item.cpu) ? item.cpu.toFixed(1) : "0.0";
|
||||
return `${summarizeGuiSmokeProcessCommand(item.command)} pid=${item.pid} etime=${item.etime} cpu=${cpu}% stat=${item.stat}`;
|
||||
})
|
||||
.join(" | ");
|
||||
}
|
||||
|
||||
function inspectGuiSmokeTauriProcesses() {
|
||||
const processTable = listProcessTable();
|
||||
const processByPid = new Map(processTable.map((item) => [item.pid, item]));
|
||||
const active = [];
|
||||
const stale = [];
|
||||
|
||||
for (const item of processTable) {
|
||||
if (!isGuiSmokeTauriCommand(item.command)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (item.pid === process.pid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const parent = processByPid.get(item.ppid);
|
||||
if (!parent || !isGuiSmokeVerifierCommand(parent.command)) {
|
||||
stale.push(item);
|
||||
continue;
|
||||
}
|
||||
|
||||
active.push(item);
|
||||
}
|
||||
|
||||
return { active, stale };
|
||||
}
|
||||
|
||||
function isProcessAlive(pid) {
|
||||
if (!Number.isInteger(pid) || pid < 1) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
process.kill(pid, 0);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function signalProcessTree(pid, signal) {
|
||||
if (!Number.isInteger(pid) || pid < 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (process.platform === "win32") {
|
||||
spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
|
||||
if (signal === "SIGKILL") {
|
||||
spawnSync("taskkill", ["/pid", String(pid), "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
spawnSync("taskkill", ["/pid", String(pid), "/T"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
process.kill(-child.pid, "SIGTERM");
|
||||
} catch {
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
const target = `-${pid}`;
|
||||
spawnSync("kill", [`-${signal}`, target], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
}
|
||||
|
||||
async function stopProcessTree(pid, label) {
|
||||
if (!Number.isInteger(pid) || pid < 1) {
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`[verify:gui-smoke] 清理残留 ${label} 进程组: ${pid}`);
|
||||
signalProcessTree(pid, "SIGTERM");
|
||||
|
||||
for (let attempt = 0; attempt < 25; attempt += 1) {
|
||||
if (child.exitCode !== null || child.signalCode) {
|
||||
if (!isProcessAlive(pid)) {
|
||||
return;
|
||||
}
|
||||
await sleep(200);
|
||||
}
|
||||
|
||||
try {
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
} catch {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// ignore
|
||||
signalProcessTree(pid, "SIGKILL");
|
||||
}
|
||||
|
||||
async function cleanupStaleGuiSmokeProcesses() {
|
||||
const snapshot = inspectGuiSmokeTauriProcesses();
|
||||
if (snapshot.stale.length === 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
for (const item of snapshot.stale) {
|
||||
const groupPid = item.pgid || item.pid;
|
||||
await stopProcessTree(groupPid, `GUI smoke Tauri(PID=${item.pid})`);
|
||||
}
|
||||
|
||||
return snapshot.stale.length;
|
||||
}
|
||||
|
||||
function listListeningCommandsForPort(port) {
|
||||
if (!port || process.platform === "win32") {
|
||||
return [];
|
||||
}
|
||||
|
||||
const pidOutput = runQuietCommand("lsof", [
|
||||
"-nP",
|
||||
`-iTCP:${port}`,
|
||||
"-sTCP:LISTEN",
|
||||
"-t",
|
||||
]);
|
||||
if (!pidOutput) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const pids = [...new Set(pidOutput.split("\n").map((item) => item.trim()).filter(Boolean))];
|
||||
return pids
|
||||
.map((pid) => runQuietCommand("ps", ["-p", pid, "-o", "command="]))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function isLikelyLimeFrontendListener(command) {
|
||||
return (
|
||||
command.includes(rootDir) &&
|
||||
(command.includes("vite") ||
|
||||
command.includes("npm run dev") ||
|
||||
command.includes("tauri dev"))
|
||||
);
|
||||
}
|
||||
|
||||
function startHeadlessTauri(options, startupMode) {
|
||||
console.log("[verify:gui-smoke] 启动 headless Tauri 环境...");
|
||||
runCommand(
|
||||
npmCommand,
|
||||
["run", "generate:agent-runtime-clients"],
|
||||
"generate:agent-runtime-clients",
|
||||
options.timeoutMs,
|
||||
);
|
||||
runCommand(
|
||||
npmCommand,
|
||||
["run", "generate:extension-site-adapters"],
|
||||
"generate:extension-site-adapters",
|
||||
options.timeoutMs,
|
||||
);
|
||||
const tauriConfigPath = createHeadlessTauriConfig(options, startupMode);
|
||||
|
||||
state.child = spawn(
|
||||
tauriCommand,
|
||||
["dev", "--no-watch", "--config", tauriConfigPath],
|
||||
{
|
||||
cwd: rootDir,
|
||||
stdio: "inherit",
|
||||
env: {
|
||||
...process.env,
|
||||
CARGO_TARGET_DIR: options.cargoTargetDir,
|
||||
[LIME_SKIP_STARTUP_WINDOW_REVEAL]: "1",
|
||||
[LIME_DISABLE_SINGLE_INSTANCE]: "1",
|
||||
[LIME_WEB_BRIDGE_URL]: options.appUrl,
|
||||
...(startupMode.reuseExistingAppShell
|
||||
? {
|
||||
[LIME_WEB_BRIDGE_REUSE_EXISTING_ONLY]: "1",
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
detached: process.platform !== "win32",
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
async function stopHeadlessTauri() {
|
||||
const child = state.child;
|
||||
if (state.cleanedUp) {
|
||||
return;
|
||||
}
|
||||
|
||||
state.cleanedUp = true;
|
||||
if (child) {
|
||||
console.log("[verify:gui-smoke] 停止 headless Tauri 环境...");
|
||||
}
|
||||
|
||||
if (typeof child?.pid === "number") {
|
||||
if (process.platform === "win32") {
|
||||
spawnSync("taskkill", ["/pid", String(child.pid), "/T", "/F"], {
|
||||
stdio: "ignore",
|
||||
});
|
||||
} else {
|
||||
try {
|
||||
process.kill(-child.pid, "SIGTERM");
|
||||
} catch {
|
||||
try {
|
||||
child.kill("SIGTERM");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
for (let attempt = 0; attempt < 25; attempt += 1) {
|
||||
if (child.exitCode !== null || child.signalCode) {
|
||||
break;
|
||||
}
|
||||
await sleep(200);
|
||||
}
|
||||
|
||||
try {
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
} catch {
|
||||
try {
|
||||
child.kill("SIGKILL");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (state.tempConfigPath) {
|
||||
fs.unlinkSync(state.tempConfigPath);
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
|
||||
state.tempConfigPath = null;
|
||||
}
|
||||
|
||||
async function waitForAppShell(options) {
|
||||
@@ -236,9 +705,7 @@ async function waitForAppShell(options) {
|
||||
}
|
||||
|
||||
assert(
|
||||
html.includes("<title>Lime</title>") ||
|
||||
html.includes('<div id="root"></div>') ||
|
||||
html.includes('<div id="root"></div'),
|
||||
isLimeDevShell(html) || html.includes('<div id="root"></div'),
|
||||
"前端首页返回成功,但未检测到 Lime 根页面标记",
|
||||
);
|
||||
|
||||
@@ -273,19 +740,221 @@ async function isUrlReady(url, timeoutMs) {
|
||||
}
|
||||
}
|
||||
|
||||
function describeChildExit(child) {
|
||||
if (!child) {
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
const parts = [];
|
||||
if (typeof child.exitCode === "number") {
|
||||
parts.push(`exitCode=${child.exitCode}`);
|
||||
}
|
||||
if (child.signalCode) {
|
||||
parts.push(`signal=${child.signalCode}`);
|
||||
}
|
||||
return parts.length > 0 ? parts.join(" ") : "unknown";
|
||||
}
|
||||
|
||||
async function waitForBridgeHealth(options, startedByScript) {
|
||||
const startedAt = Date.now();
|
||||
let deadlineAt = startedAt + options.timeoutMs;
|
||||
let compileGraceCount = 0;
|
||||
let bootGraceUsed = false;
|
||||
let lastError = null;
|
||||
let lastHeartbeatAt = startedAt;
|
||||
|
||||
console.log(`[bridge:health] 开始检查: ${options.healthUrl}`);
|
||||
|
||||
while (true) {
|
||||
if (
|
||||
startedByScript &&
|
||||
state.child &&
|
||||
(typeof state.child.exitCode === "number" || state.child.signalCode)
|
||||
) {
|
||||
const exitDetail = describeChildExit(state.child);
|
||||
const lastDetail =
|
||||
lastError instanceof Error
|
||||
? `;最近一次健康检查错误: ${lastError.message}`
|
||||
: "";
|
||||
throw new Error(
|
||||
`[verify:gui-smoke] headless Tauri 在 DevBridge 就绪前提前退出(${exitDetail})${lastDetail}`,
|
||||
);
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(options.healthUrl, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(Math.min(options.intervalMs, 1_500)),
|
||||
});
|
||||
const text = await response.text();
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
let payload = null;
|
||||
try {
|
||||
payload = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
payload = null;
|
||||
}
|
||||
|
||||
const elapsed = Date.now() - startedAt;
|
||||
const status =
|
||||
payload && typeof payload === "object" ? payload.status : undefined;
|
||||
console.log(
|
||||
`[bridge:health] 就绪: ${options.healthUrl} (${elapsed}ms)${status ? ` status=${status}` : ""}`,
|
||||
);
|
||||
return;
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
const heartbeatAt = Date.now();
|
||||
if (heartbeatAt >= deadlineAt) {
|
||||
const heartbeat = describeGuiSmokeHeartbeat(startedByScript);
|
||||
if (
|
||||
compileGraceCount < GUI_SMOKE_MAX_COMPILE_GRACE_EXTENSIONS &&
|
||||
hasActiveGuiSmokeCompile(startedByScript)
|
||||
) {
|
||||
compileGraceCount += 1;
|
||||
deadlineAt = heartbeatAt + GUI_SMOKE_COMPILE_GRACE_MS;
|
||||
console.log(
|
||||
`[bridge:health] 检测到 GUI smoke 仍在编译,第 ${compileGraceCount} 次额外延长 ${GUI_SMOKE_COMPILE_GRACE_MS}ms 等待 DevBridge${heartbeat ? `;进程组: ${heartbeat}` : ""}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (
|
||||
!bootGraceUsed &&
|
||||
startedByScript &&
|
||||
state.child &&
|
||||
state.child.exitCode === null &&
|
||||
!state.child.signalCode
|
||||
) {
|
||||
bootGraceUsed = true;
|
||||
deadlineAt = heartbeatAt + GUI_SMOKE_BOOT_GRACE_MS;
|
||||
console.log(
|
||||
`[bridge:health] 编译链已结束,额外延长 ${GUI_SMOKE_BOOT_GRACE_MS}ms 等待 headless Tauri 拉起 DevBridge${heartbeat ? `;进程组: ${heartbeat}` : ""}`,
|
||||
);
|
||||
continue;
|
||||
}
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if (heartbeatAt - lastHeartbeatAt >= GUI_SMOKE_BRIDGE_HEARTBEAT_MS) {
|
||||
lastHeartbeatAt = heartbeatAt;
|
||||
const detail =
|
||||
error instanceof Error
|
||||
? error.message
|
||||
: String(error || "unknown error");
|
||||
const heartbeat = describeGuiSmokeHeartbeat(startedByScript);
|
||||
console.log(
|
||||
`[bridge:health] 等待中: ${options.healthUrl} (${heartbeatAt - startedAt}ms);最近错误: ${detail}${heartbeat ? `;进程组: ${heartbeat}` : ""}`,
|
||||
);
|
||||
}
|
||||
await sleep(options.intervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
const detail =
|
||||
lastError instanceof Error
|
||||
? lastError.message
|
||||
: String(lastError || "unknown error");
|
||||
throw new Error(
|
||||
`[bridge:health] 超时未就绪: ${options.healthUrl}。最后错误: ${detail}`,
|
||||
);
|
||||
}
|
||||
|
||||
async function probeAppShell(url, timeoutMs) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
signal: AbortSignal.timeout(timeoutMs),
|
||||
});
|
||||
const html = await response.text();
|
||||
|
||||
return {
|
||||
reachable: response.ok,
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
isLimeDevShell: response.ok && isLimeDevShell(html),
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
reachable: false,
|
||||
status: null,
|
||||
statusText: "",
|
||||
isLimeDevShell: false,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function resolveStartupMode(options) {
|
||||
if (options.reuseRunning) {
|
||||
return {
|
||||
shouldStart: false,
|
||||
reusedExisting: true,
|
||||
reuseExistingAppShell: false,
|
||||
};
|
||||
}
|
||||
|
||||
const existingAppShell = await isUrlReady(options.appUrl, 1_500);
|
||||
if (!existingAppShell) {
|
||||
const staleProcessCount = await cleanupStaleGuiSmokeProcesses();
|
||||
if (staleProcessCount > 0) {
|
||||
console.log(
|
||||
`[verify:gui-smoke] 已清理 ${staleProcessCount} 条残留 GUI smoke headless 链路。`,
|
||||
);
|
||||
}
|
||||
|
||||
const guiSmokeProcesses = inspectGuiSmokeTauriProcesses();
|
||||
const existingAppShell = await probeAppShell(options.appUrl, 1_500);
|
||||
if (existingAppShell.reachable && !existingAppShell.isLimeDevShell) {
|
||||
const statusLabel = `${existingAppShell.status || "unknown"} ${existingAppShell.statusText || ""}`.trim();
|
||||
throw new Error(
|
||||
`[verify:gui-smoke] ${options.appUrl} 已被其他服务占用,且返回内容不是 Lime 前端壳(${statusLabel})。请先关闭占用进程后重试。`,
|
||||
);
|
||||
}
|
||||
|
||||
if (!existingAppShell.isLimeDevShell) {
|
||||
const existingListeners = listListeningCommandsForPort(
|
||||
resolveUrlPort(options.appUrl),
|
||||
);
|
||||
const hasLimeFrontendListener = existingListeners.some(
|
||||
isLikelyLimeFrontendListener,
|
||||
);
|
||||
|
||||
if (guiSmokeProcesses.active.length > 0) {
|
||||
const pidList = guiSmokeProcesses.active.map((item) => item.pid).join(", ");
|
||||
console.log(
|
||||
`[verify:gui-smoke] 检测到已有 GUI smoke headless 进程正在启动(PID: ${pidList});本次将直接复用现有链路并等待 DevBridge。`,
|
||||
);
|
||||
return {
|
||||
shouldStart: false,
|
||||
reusedExisting: true,
|
||||
reuseExistingAppShell: hasLimeFrontendListener,
|
||||
};
|
||||
}
|
||||
|
||||
if (hasLimeFrontendListener) {
|
||||
console.log(
|
||||
`[verify:gui-smoke] 检测到 ${options.appUrl} 已由当前仓库的前端启动链监听,但页面尚未完全就绪;将复用现有前端启动链,只拉起 headless Tauri 与 DevBridge。`,
|
||||
);
|
||||
return {
|
||||
shouldStart: true,
|
||||
reusedExisting: false,
|
||||
reuseExistingAppShell: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (existingListeners.length > 0) {
|
||||
throw new Error(
|
||||
`[verify:gui-smoke] ${options.appUrl} 已被其他进程占用,且当前无法确认为 Lime 前端壳。请先关闭占用进程后重试。`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
shouldStart: true,
|
||||
reusedExisting: false,
|
||||
reuseExistingAppShell: false,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -297,6 +966,19 @@ async function resolveStartupMode(options) {
|
||||
return {
|
||||
shouldStart: false,
|
||||
reusedExisting: true,
|
||||
reuseExistingAppShell: true,
|
||||
};
|
||||
}
|
||||
|
||||
if (guiSmokeProcesses.active.length > 0) {
|
||||
const pidList = guiSmokeProcesses.active.map((item) => item.pid).join(", ");
|
||||
console.log(
|
||||
`[verify:gui-smoke] 检测到已有 GUI smoke headless 进程正在启动(PID: ${pidList});本次将直接复用现有前端与 headless 链路。`,
|
||||
);
|
||||
return {
|
||||
shouldStart: false,
|
||||
reusedExisting: true,
|
||||
reuseExistingAppShell: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -306,6 +988,7 @@ async function resolveStartupMode(options) {
|
||||
return {
|
||||
shouldStart: true,
|
||||
reusedExisting: false,
|
||||
reuseExistingAppShell: true,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -318,6 +1001,10 @@ async function main() {
|
||||
const startupMode = await resolveStartupMode(options);
|
||||
const startedByScript = startupMode.shouldStart;
|
||||
|
||||
console.log(
|
||||
`[verify:gui-smoke] Cargo target: ${options.cargoTargetDir}`,
|
||||
);
|
||||
|
||||
const handleSignal = async (signal) => {
|
||||
try {
|
||||
await stopHeadlessTauri();
|
||||
@@ -335,28 +1022,13 @@ async function main() {
|
||||
|
||||
try {
|
||||
if (startedByScript) {
|
||||
startHeadlessTauri();
|
||||
startHeadlessTauri(options, startupMode);
|
||||
await sleep(1_500);
|
||||
} else if (startupMode.reusedExisting) {
|
||||
console.log("[verify:gui-smoke] 复用已运行的 headless Tauri 环境。");
|
||||
}
|
||||
|
||||
runCommand(
|
||||
npmCommand,
|
||||
[
|
||||
"run",
|
||||
"bridge:health",
|
||||
"--",
|
||||
"--url",
|
||||
options.healthUrl,
|
||||
"--timeout-ms",
|
||||
String(options.timeoutMs),
|
||||
"--interval-ms",
|
||||
String(options.intervalMs),
|
||||
],
|
||||
"bridge:health",
|
||||
options.timeoutMs + 5_000,
|
||||
);
|
||||
await waitForBridgeHealth(options, startedByScript);
|
||||
|
||||
await waitForAppShell(options);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user