release: v1.9.0

This commit is contained in:
coso
2026-04-13 02:50:53 +08:00
parent 52624b1488
commit 0f15b8a71e
313 changed files with 19875 additions and 9864 deletions
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+765
View File
@@ -0,0 +1,765 @@
function normalizeNumber(value) {
return typeof value === "number" && Number.isFinite(value) ? value : 0;
}
function normalizeString(value, fallback = "") {
return typeof value === "string" && value.trim().length > 0
? value.trim()
: fallback;
}
function escapeHtml(value) {
return String(value)
.replaceAll("&", "&")
.replaceAll("<", "&lt;")
.replaceAll(">", "&gt;")
.replaceAll('"', "&quot;")
.replaceAll("'", "&#39;");
}
function formatTimestamp(value) {
const parsed = new Date(value);
if (Number.isNaN(parsed.getTime())) {
return normalizeString(value, "-");
}
return parsed.toLocaleString("zh-CN", {
hour12: false,
year: "numeric",
month: "2-digit",
day: "2-digit",
hour: "2-digit",
minute: "2-digit",
second: "2-digit",
});
}
function renderStatCard(label, value, tone = "neutral") {
return `
<article class="stat-card ${tone}">
<span>${escapeHtml(label)}</span>
<strong>${escapeHtml(String(value))}</strong>
</article>
`;
}
function renderSignalList(signals) {
const items = Array.isArray(signals) ? signals.filter(Boolean) : [];
if (items.length === 0) {
return `<p class="empty">当前没有额外信号。</p>`;
}
return `
<ul class="signal-list">
${items.map((item) => `<li>${escapeHtml(item)}</li>`).join("")}
</ul>
`;
}
function renderRecommendationList(recommendations) {
const items = Array.isArray(recommendations) ? recommendations : [];
if (items.length === 0) {
return `<p class="empty">当前没有新的治理建议。</p>`;
}
return items
.map(
(entry) => {
const focusVerificationFailureOutcomes = Array.isArray(
entry.focusVerificationFailureOutcomes,
)
? entry.focusVerificationFailureOutcomes
: [];
const focusVerificationRecoveredOutcomes = Array.isArray(
entry.focusVerificationRecoveredOutcomes,
)
? entry.focusVerificationRecoveredOutcomes
: [];
return `
<article class="recommendation-card">
<div class="recommendation-head">
<span class="priority">${escapeHtml(normalizeString(entry.priority, "P?"))}</span>
<h3>${escapeHtml(normalizeString(entry.title, "未命名建议"))}</h3>
</div>
<p>${escapeHtml(
Array.isArray(entry.rationale) ? entry.rationale.join(" ") : "",
)}</p>
${
focusVerificationFailureOutcomes.length > 0
? `
<p class="recommendation-meta">
<strong>关注 failure outcome:</strong>${escapeHtml(
focusVerificationFailureOutcomes.join("、"),
)}
</p>
`
: ""
}
${
focusVerificationRecoveredOutcomes.length > 0
? `
<p class="recommendation-meta">
<strong>关注 recovered outcome:</strong>${escapeHtml(
focusVerificationRecoveredOutcomes.join("、"),
)}
</p>
`
: ""
}
${
Array.isArray(entry.backlogTools) && entry.backlogTools.length > 0
? `
<div class="recommendation-subsection">
<strong>后续动作</strong>
<ul class="backlog-list">
${entry.backlogTools
.map((item) => `<li>${escapeHtml(item)}</li>`)
.join("")}
</ul>
</div>
`
: ""
}
${
Array.isArray(entry.commands) && entry.commands.length > 0
? `
<div class="recommendation-subsection">
<strong>推荐命令</strong>
<div class="command-list">
${entry.commands
.map((command) => `<code>${escapeHtml(command)}</code>`)
.join("")}
</div>
</div>
`
: ""
}
</article>
`;
},
)
.join("");
}
function describeVerificationOutcome(entry) {
const signal = normalizeString(entry?.signal, "unknown");
const outcome = normalizeString(entry?.outcome, "unknown");
if (signal === "artifactValidator" && outcome === "issues_present") {
return "当前 evidence 已记录 artifact 校验问题,优先回看 validator issue 明细。";
}
if (signal === "artifactValidator" && outcome === "fallback_used") {
return "当前 artifact 导出仍触发 fallback,说明产物结构或修复链未完全稳定。";
}
if (signal === "browserVerification" && outcome === "failure") {
return "浏览器验证已有明确失败结果,优先回挂到 replay 或 smoke 断言。";
}
if (signal === "browserVerification" && outcome === "unknown") {
return "浏览器验证结果仍不明确,需要先补 outcome 再继续扩分析。";
}
if (signal === "guiSmoke" && outcome === "failed") {
return "GUI smoke 已明确失败,应优先收敛到受影响主路径。";
}
if (signal === "guiSmoke" && outcome === "passed") {
return "GUI smoke 已通过,可继续把注意力放回 gap 与其它失败面。";
}
if (signal === "artifactValidator" && outcome === "repaired") {
return "artifact validator 已执行修复,可结合 issues/fallback 判断是否还需继续治理。";
}
if (signal === "browserVerification" && outcome === "success") {
return "浏览器验证已有成功样本,可作为 current 主线路径的正向基线。";
}
return "当前 verification outcome 已进入 cleanup 主线,可直接据此定位先修哪层。";
}
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
"repaired",
"success",
"passed",
"clean",
]);
function isRecoveredVerificationOutcome(entry) {
return RECOVERED_VERIFICATION_OUTCOMES.has(
normalizeString(entry?.outcome, "unknown"),
);
}
function renderFocusTable(title, entries, columns) {
const rows = Array.isArray(entries) ? entries : [];
if (rows.length === 0) {
return `
<section class="panel">
<div class="section-header">
<h2>${escapeHtml(title)}</h2>
</div>
<p class="empty">当前没有数据。</p>
</section>
`;
}
return `
<section class="panel">
<div class="section-header">
<h2>${escapeHtml(title)}</h2>
</div>
<div class="table-wrap">
<table>
<thead>
<tr>
${columns.map((column) => `<th>${escapeHtml(column.label)}</th>`).join("")}
</tr>
</thead>
<tbody>
${rows
.map(
(entry) => `
<tr>
${columns
.map((column) => `<td>${escapeHtml(String(column.render(entry)))}</td>`)
.join("")}
</tr>
`,
)
.join("")}
</tbody>
</table>
</div>
</section>
`;
}
export function renderHarnessDashboardHtml({
summaryReport,
trendReport,
cleanupReport,
title = "Lime Harness Dashboard",
}) {
const summaryTotals =
summaryReport && typeof summaryReport === "object" && summaryReport.totals
? summaryReport.totals
: {};
const trendSummary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.trend
? cleanupReport.summary.trend
: {};
const governanceSummary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.governance
? cleanupReport.summary.governance
: {};
const verificationSummary =
cleanupReport &&
typeof cleanupReport === "object" &&
cleanupReport.summary &&
cleanupReport.summary.verificationOutcomes
? cleanupReport.summary.verificationOutcomes
: {};
const currentVerificationSummary =
verificationSummary &&
typeof verificationSummary.current === "object" &&
!Array.isArray(verificationSummary.current)
? verificationSummary.current
: {};
const degradedVerificationSummary =
verificationSummary &&
typeof verificationSummary.degraded === "object" &&
!Array.isArray(verificationSummary.degraded)
? verificationSummary.degraded
: {};
const trendSignals = Array.isArray(trendReport?.signals) ? trendReport.signals : [];
const cleanupSignals = Array.isArray(cleanupReport?.signals)
? cleanupReport.signals
: [];
const recommendations = Array.isArray(cleanupReport?.recommendations)
? cleanupReport.recommendations
: [];
const sampleRows = Array.isArray(trendReport?.samples) ? trendReport.samples : [];
const currentVerificationFocusRows = Array.isArray(
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentObservabilityVerificationOutcomes.map((entry) => ({
...entry,
role: "current",
}))
: [];
const degradedVerificationFocusRows = Array.isArray(
cleanupReport?.focus?.degradedObservabilityVerificationOutcomes,
)
? cleanupReport.focus.degradedObservabilityVerificationOutcomes.map(
(entry) => ({
...entry,
role: "degraded",
}),
)
: [];
const fallbackVerificationFocusRows = Array.isArray(
cleanupReport?.focus?.observabilityVerificationOutcomes,
)
? cleanupReport.focus.observabilityVerificationOutcomes.map((entry) => ({
...entry,
role: "mixed",
}))
: [];
const explicitCurrentRecoveredVerificationRows = Array.isArray(
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
)
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes.map(
(entry) => ({
...entry,
role: "current",
}),
)
: [];
const verificationFocusRows =
currentVerificationFocusRows.length > 0 ||
degradedVerificationFocusRows.length > 0
? [...currentVerificationFocusRows, ...degradedVerificationFocusRows]
: fallbackVerificationFocusRows;
const currentRecoveredVerificationRows =
explicitCurrentRecoveredVerificationRows.length > 0
? explicitCurrentRecoveredVerificationRows
: currentVerificationFocusRows.length > 0
? currentVerificationFocusRows.filter((entry) =>
isRecoveredVerificationOutcome(entry),
)
: fallbackVerificationFocusRows.filter((entry) =>
isRecoveredVerificationOutcome(entry),
);
const currentRecoveredVerificationSummary = currentRecoveredVerificationRows
.slice(0, 3)
.map(
(entry) =>
`${normalizeString(entry?.signal, "-")} (${normalizeString(entry?.outcome, "-")})`,
)
.join("、");
return `<!doctype html>
<html lang="zh-CN">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />
<title>${escapeHtml(title)}</title>
<style>
:root {
color-scheme: light;
--bg: #f3f6f4;
--panel: #ffffff;
--panel-muted: #f8faf8;
--border: #d8e1dc;
--text: #112118;
--muted: #5f6d64;
--accent: #0f4c5c;
--success: #177245;
--warning: #b45309;
--danger: #b42318;
--shadow: 0 16px 32px rgba(17, 33, 24, 0.08);
}
* { box-sizing: border-box; }
body {
margin: 0;
font-family:
"SF Pro Display",
"Segoe UI",
"PingFang SC",
"Hiragino Sans GB",
"Microsoft YaHei",
sans-serif;
color: var(--text);
background:
radial-gradient(circle at top left, rgba(15, 76, 92, 0.12), transparent 32%),
radial-gradient(circle at top right, rgba(23, 114, 69, 0.1), transparent 28%),
var(--bg);
}
.page {
max-width: 1440px;
margin: 0 auto;
padding: 28px 22px 42px;
}
.hero,
.panel {
background: var(--panel);
border: 1px solid var(--border);
border-radius: 24px;
box-shadow: var(--shadow);
}
.hero {
padding: 24px 26px;
display: grid;
grid-template-columns: minmax(0, 1.5fr) minmax(280px, 0.9fr);
gap: 18px;
}
.eyebrow {
display: inline-flex;
align-items: center;
padding: 6px 12px;
border-radius: 999px;
background: #e8f7ef;
color: var(--success);
font-size: 13px;
font-weight: 700;
}
h1, h2, h3, p {
margin: 0;
}
h1 {
margin-top: 14px;
font-size: 32px;
line-height: 1.12;
}
.hero p {
margin-top: 12px;
color: var(--muted);
line-height: 1.7;
}
.meta-grid,
.stat-grid,
.panel-grid {
display: grid;
gap: 14px;
}
.meta-grid {
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.meta-card,
.stat-card {
border: 1px solid var(--border);
border-radius: 18px;
background: var(--panel-muted);
padding: 14px 16px;
}
.meta-card span,
.stat-card span {
display: block;
font-size: 12px;
color: var(--muted);
margin-bottom: 8px;
}
.meta-card strong,
.stat-card strong {
font-size: 20px;
}
.stat-grid {
margin-top: 18px;
grid-template-columns: repeat(6, minmax(0, 1fr));
}
.stat-card.warning strong { color: var(--warning); }
.stat-card.danger strong { color: var(--danger); }
.stat-card.success strong { color: var(--success); }
.panel-grid {
margin-top: 20px;
grid-template-columns: repeat(2, minmax(0, 1fr));
}
.panel {
padding: 20px 22px;
}
.section-header {
display: flex;
align-items: flex-start;
justify-content: space-between;
gap: 12px;
margin-bottom: 14px;
}
.section-header p {
color: var(--muted);
line-height: 1.6;
}
.signal-list {
margin: 0;
padding-left: 18px;
display: grid;
gap: 10px;
color: var(--muted);
}
.empty {
color: var(--muted);
line-height: 1.6;
}
.table-wrap {
overflow-x: auto;
}
table {
width: 100%;
border-collapse: collapse;
font-size: 14px;
}
th, td {
padding: 10px 12px;
border-bottom: 1px solid var(--border);
text-align: left;
vertical-align: top;
}
th {
font-size: 12px;
color: var(--muted);
text-transform: uppercase;
letter-spacing: 0.04em;
}
tr:last-child td {
border-bottom: none;
}
.recommendation-stack {
display: grid;
gap: 12px;
}
.recommendation-card {
border: 1px solid var(--border);
border-radius: 18px;
padding: 16px;
background: var(--panel-muted);
}
.recommendation-head {
display: flex;
gap: 10px;
align-items: center;
margin-bottom: 10px;
}
.priority {
display: inline-flex;
align-items: center;
justify-content: center;
min-width: 44px;
height: 28px;
padding: 0 10px;
border-radius: 999px;
background: #fff3e8;
color: var(--warning);
font-size: 12px;
font-weight: 700;
}
.recommendation-card p {
color: var(--muted);
line-height: 1.7;
}
.recommendation-meta {
margin-top: 12px;
font-size: 14px;
}
.recommendation-subsection {
margin-top: 14px;
}
.recommendation-subsection strong {
display: block;
margin-bottom: 8px;
font-size: 13px;
}
.backlog-list {
margin: 0;
padding-left: 18px;
display: grid;
gap: 8px;
color: var(--muted);
}
.command-list {
display: flex;
flex-wrap: wrap;
gap: 10px;
}
code {
display: inline-flex;
padding: 8px 10px;
border-radius: 10px;
background: #102a2c;
color: #ecfeff;
font-size: 12px;
white-space: pre-wrap;
}
@media (max-width: 1120px) {
.hero,
.panel-grid {
grid-template-columns: 1fr;
}
.stat-grid {
grid-template-columns: repeat(3, minmax(0, 1fr));
}
}
@media (max-width: 720px) {
.page {
padding: 18px 14px 30px;
}
.meta-grid,
.stat-grid {
grid-template-columns: 1fr 1fr;
}
}
</style>
</head>
<body>
<main class="page">
<section class="hero">
<div>
<span class="eyebrow">Harness Engine Nightly Dashboard</span>
<h1>${escapeHtml(title)}</h1>
<p>同一份 evidence-first 主线下的 summary、trend、cleanup 视图。当前面板会明确区分主线 current 缺口与故意保留的 degraded 诊断基线,避免治理优先级被误导。</p>
<div class="stat-grid">
${renderStatCard("Ready Case", normalizeNumber(summaryTotals.readyCount), "success")}
${renderStatCard("Invalid Case", normalizeNumber(summaryTotals.invalidCount), normalizeNumber(summaryTotals.invalidCount) > 0 ? "danger" : "neutral")}
${renderStatCard("Current Gap", normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount), normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount) > 0 ? "danger" : "success")}
${renderStatCard("Degraded Gap", normalizeNumber(trendSummary.latestDegradedObservabilityGapCaseCount), "warning")}
${renderStatCard("Current Blocking", normalizeNumber(currentVerificationSummary.blockingFailureCaseCount), normalizeNumber(currentVerificationSummary.blockingFailureCaseCount) > 0 ? "danger" : "success")}
${renderStatCard("Current Advisory", normalizeNumber(currentVerificationSummary.advisoryFailureCaseCount), normalizeNumber(currentVerificationSummary.advisoryFailureCaseCount) > 0 ? "warning" : "neutral")}
${renderStatCard("Current Recovered", normalizeNumber(currentVerificationSummary.recoveredCaseCount), normalizeNumber(currentVerificationSummary.recoveredCaseCount) > 0 ? "success" : "neutral")}
${renderStatCard("Degraded Blocking", normalizeNumber(degradedVerificationSummary.blockingFailureCaseCount), normalizeNumber(degradedVerificationSummary.blockingFailureCaseCount) > 0 ? "warning" : "neutral")}
${renderStatCard("Recovered Outcomes", normalizeNumber(verificationSummary.recoveredCaseCount), normalizeNumber(verificationSummary.recoveredCaseCount) > 0 ? "success" : "neutral")}
${renderStatCard("Trend Samples", normalizeNumber(trendSummary.sampleCount))}
${renderStatCard("Governance Violations", normalizeNumber(governanceSummary.violationCount), normalizeNumber(governanceSummary.violationCount) > 0 ? "danger" : "neutral")}
</div>
</div>
<div class="meta-grid">
<div class="meta-card">
<span>Summary 生成时间</span>
<strong>${escapeHtml(formatTimestamp(summaryReport?.generatedAt))}</strong>
</div>
<div class="meta-card">
<span>Trend 生成时间</span>
<strong>${escapeHtml(formatTimestamp(trendReport?.generatedAt))}</strong>
</div>
<div class="meta-card">
<span>Cleanup 生成时间</span>
<strong>${escapeHtml(formatTimestamp(cleanupReport?.generatedAt))}</strong>
</div>
<div class="meta-card">
<span>当前事实源</span>
<strong>summary - history - trend - cleanup</strong>
</div>
<div class="meta-card">
<span>当前 Recovered 基线</span>
<strong>${escapeHtml(
currentRecoveredVerificationSummary ||
"当前没有额外的 recovered baseline",
)}</strong>
</div>
</div>
</section>
<section class="panel-grid">
<section class="panel">
<div class="section-header">
<div>
<h2>Trend 信号</h2>
<p>只把 current gap 当主线风险,degraded gap 仅作为诊断基线保留。</p>
</div>
</div>
${renderSignalList(trendSignals)}
</section>
<section class="panel">
<div class="section-header">
<div>
<h2>Cleanup 信号</h2>
<p>nightly 治理建议与当前风险摘要。</p>
</div>
</div>
${renderSignalList(cleanupSignals)}
</section>
</section>
${renderFocusTable("Observability Gap 角色", [
{
role: "current",
latest: normalizeNumber(trendSummary.latestCurrentObservabilityGapCaseCount),
delta: normalizeNumber(trendSummary.currentObservabilityGapCaseDelta),
meaning: "主线样本里的证据缺口,必须优先治理。",
},
{
role: "degraded",
latest: normalizeNumber(trendSummary.latestDegradedObservabilityGapCaseCount),
delta: normalizeNumber(trendSummary.degradedObservabilityGapCaseDelta),
meaning: "刻意保留的诊断基线,不应直接抬高主线优先级。",
},
], [
{ label: "角色", render: (entry) => entry.role },
{ label: "latest case", render: (entry) => entry.latest },
{ label: "delta case", render: (entry) => entry.delta },
{ label: "说明", render: (entry) => entry.meaning },
])}
${renderFocusTable("Verification Outcome 焦点", verificationFocusRows, [
{ label: "Role", render: (entry) => normalizeString(entry.role, "-") },
{ label: "Signal", render: (entry) => normalizeString(entry.signal, "-") },
{ label: "Outcome", render: (entry) => normalizeString(entry.outcome, "-") },
{ label: "State", render: (entry) => normalizeString(entry.state, "-") },
{ label: "Latest Case", render: (entry) => normalizeNumber(entry?.latest?.caseCount) },
{ label: "Delta Case", render: (entry) => normalizeNumber(entry?.delta?.caseCount) },
{ label: "说明", render: (entry) => describeVerificationOutcome(entry) },
])}
${renderFocusTable("Current Recovered Baseline", currentRecoveredVerificationRows, [
{ label: "Signal", render: (entry) => normalizeString(entry.signal, "-") },
{ label: "Outcome", render: (entry) => normalizeString(entry.outcome, "-") },
{ label: "Latest Case", render: (entry) => normalizeNumber(entry?.latest?.caseCount) },
{ label: "Delta Case", render: (entry) => normalizeNumber(entry?.delta?.caseCount) },
{ label: "说明", render: (entry) => describeVerificationOutcome(entry) },
])}
${renderFocusTable("历史窗口样本", sampleRows, [
{ label: "时间", render: (entry) => formatTimestamp(entry.generatedAt) },
{ label: "来源", render: (entry) => normalizeString(entry.sourcePath, "-") },
{ label: "case", render: (entry) => normalizeNumber(entry?.totals?.caseCount) },
{ label: "ready", render: (entry) => normalizeNumber(entry?.totals?.readyCount) },
{ label: "invalid", render: (entry) => normalizeNumber(entry?.totals?.invalidCount) },
{ label: "current gap", render: (entry) => normalizeNumber(entry?.totals?.currentObservabilityGapCaseCount) },
{ label: "degraded gap", render: (entry) => normalizeNumber(entry?.totals?.degradedObservabilityGapCaseCount) },
])}
<section class="panel">
<div class="section-header">
<div>
<h2>Cleanup 建议</h2>
<p>当前 nightly 建议按已有优先级排序展示,直接复用 cleanup 报告事实源。</p>
</div>
</div>
<div class="recommendation-stack">
${renderRecommendationList(recommendations)}
</div>
</section>
</main>
</body>
</html>`;
}
+184
View File
@@ -0,0 +1,184 @@
import { describe, expect, it } from "vitest";
import { renderHarnessDashboardHtml } from "./harness-dashboard-core.mjs";
describe("harness-dashboard-core", () => {
it("应把 summary、trend、cleanup 渲染成单一事实源 dashboard", () => {
const html = renderHarnessDashboardHtml({
title: "Harness Engine Dashboard",
summaryReport: {
generatedAt: "2026-04-12T08:00:00.000Z",
totals: {
readyCount: 2,
invalidCount: 1,
},
},
trendReport: {
generatedAt: "2026-04-12T08:01:00.000Z",
signals: ["current gap 保持为 0。"],
samples: [
{
generatedAt: "2026-04-12T08:00:00.000Z",
sourcePath: "/tmp/history/summary.json",
totals: {
caseCount: 2,
readyCount: 2,
invalidCount: 0,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 1,
},
},
],
},
cleanupReport: {
generatedAt: "2026-04-12T08:02:00.000Z",
signals: ["当前没有新的治理风险。"],
recommendations: [
{
priority: "P1",
title: "保持 current gap 为 0",
rationale: ["继续沿用 current/degraded 分层。"],
backlogTools: [
"回看 browser replay / browser verification 失败样本,并把失败断言回挂到受影响主路径。",
],
commands: ["npm run harness:eval:history:record"],
focusVerificationFailureOutcomes: [
"browserVerification (failure)",
],
focusVerificationRecoveredOutcomes: [
"artifactValidator (repaired)",
"browserVerification (success)",
],
},
],
focus: {
currentObservabilityVerificationOutcomes: [
{
signal: "browserVerification",
outcome: "failure",
state: "regressing",
latest: {
caseCount: 1,
},
delta: {
caseCount: 1,
},
},
],
currentRecoveredObservabilityVerificationOutcomes: [
{
signal: "artifactValidator",
outcome: "repaired",
state: "expanding",
latest: {
caseCount: 1,
},
delta: {
caseCount: 1,
},
},
{
signal: "browserVerification",
outcome: "success",
state: "present",
latest: {
caseCount: 1,
},
delta: {
caseCount: 0,
},
},
{
signal: "guiSmoke",
outcome: "passed",
state: "present",
latest: {
caseCount: 1,
},
delta: {
caseCount: 0,
},
},
],
degradedObservabilityVerificationOutcomes: [
{
signal: "guiSmoke",
outcome: "failed",
state: "present",
latest: {
caseCount: 1,
},
delta: {
caseCount: 0,
},
},
],
},
summary: {
trend: {
sampleCount: 1,
latestCurrentObservabilityGapCaseCount: 0,
latestDegradedObservabilityGapCaseCount: 1,
currentObservabilityGapCaseDelta: 0,
degradedObservabilityGapCaseDelta: 0,
},
verificationOutcomes: {
blockingFailureCaseCount: 2,
advisoryFailureCaseCount: 0,
recoveredCaseCount: 3,
current: {
blockingFailureCaseCount: 1,
advisoryFailureCaseCount: 0,
recoveredCaseCount: 3,
},
degraded: {
blockingFailureCaseCount: 1,
advisoryFailureCaseCount: 0,
},
},
governance: {
violationCount: 0,
},
},
},
});
expect(html).toContain("Harness Engine Nightly Dashboard");
expect(html).toContain("Harness Engine Dashboard");
expect(html).toContain("Current Gap");
expect(html).toContain("Degraded Gap");
expect(html).toContain("Current Blocking");
expect(html).toContain("Current Advisory");
expect(html).toContain("Current Recovered");
expect(html).toContain("Degraded Blocking");
expect(html).toContain("Recovered Outcomes");
expect(html).toContain("Cleanup 建议");
expect(html).toContain("Observability Gap 角色");
expect(html).toContain("Verification Outcome 焦点");
expect(html).toContain("Current Recovered Baseline");
expect(html).toContain("current");
expect(html).toContain("degraded");
expect(html).toContain("browserVerification");
expect(html).toContain("failure");
expect(html).toContain("success");
expect(html).toContain("artifactValidator");
expect(html).toContain("repaired");
expect(html).toContain("guiSmoke");
expect(html).toContain("failed");
expect(html).toContain("artifact validator 已执行修复");
expect(html).toContain("浏览器验证已有成功样本");
expect(html).toContain("当前 Recovered 基线");
expect(html).toContain(
"artifactValidator (repaired)、browserVerification (success)、guiSmoke (passed)",
);
expect(html).toContain("关注 failure outcome");
expect(html).toContain("关注 recovered outcome");
expect(html).toContain("artifactValidator (repaired)");
expect(html).toContain("后续动作");
expect(html).toContain(
"回看 browser replay / browser verification 失败样本,并把失败断言回挂到受影响主路径。",
);
expect(html).toContain("推荐命令");
expect(html).toContain("summary - history - trend - cleanup");
});
});
@@ -59,10 +59,125 @@ afterEach(() => {
});
describe("harness-eval-history-record", () => {
it("默认入口应产出完整 harness artifact 套件", () => {
const tempRoot = createTempRoot();
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
const workspaceRoot = path.join(tempRoot, "workspace");
const result = runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--history-dir",
historyDir,
"--workspace-root",
workspaceRoot,
]);
const reportsRoot = path.join(tempRoot, ".lime", "harness", "reports");
const summaryJsonPath = path.join(reportsRoot, "harness-eval-summary.json");
const summaryMarkdownPath = path.join(reportsRoot, "harness-eval-summary.md");
const trendJsonPath = path.join(reportsRoot, "harness-eval-trend.json");
const trendMarkdownPath = path.join(reportsRoot, "harness-eval-trend.md");
const cleanupJsonPath = path.join(reportsRoot, "harness-cleanup-report.json");
const cleanupMarkdownPath = path.join(reportsRoot, "harness-cleanup-report.md");
const dashboardHtmlPath = path.join(reportsRoot, "harness-dashboard.html");
expect(result.summary.outputJsonPath).toBe(summaryJsonPath);
expect(result.summary.outputMarkdownPath).toBe(summaryMarkdownPath);
expect(result.trend.outputJsonPath).toBe(trendJsonPath);
expect(result.trend.outputMarkdownPath).toBe(trendMarkdownPath);
expect(result.trend.currentRecoveredVerificationCaseCount).toBe(3);
expect(result.trend.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(result.cleanup.outputJsonPath).toBe(cleanupJsonPath);
expect(result.cleanup.outputMarkdownPath).toBe(cleanupMarkdownPath);
expect(result.cleanup.verificationFailureOutcomeFocus).toEqual(
expect.arrayContaining([
"artifactValidator:issues_present",
]),
);
expect(result.cleanup.verificationFailureCaseCount).toBe(1);
expect(result.cleanup.verificationBlockingFailureCaseCount).toBe(0);
expect(result.cleanup.verificationAdvisoryFailureCaseCount).toBe(1);
expect(result.cleanup.verificationDegradedBlockingFailureCaseCount).toBe(0);
expect(result.cleanup.verificationRecoveredCaseCount).toBe(3);
expect(result.cleanup.currentVerificationRecoveredCaseCount).toBe(3);
expect(result.cleanup.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(result.dashboard.outputHtmlPath).toBe(dashboardHtmlPath);
expect(fs.existsSync(summaryJsonPath)).toBe(true);
expect(fs.existsSync(summaryMarkdownPath)).toBe(true);
expect(fs.existsSync(trendJsonPath)).toBe(true);
expect(fs.existsSync(trendMarkdownPath)).toBe(true);
expect(fs.existsSync(cleanupJsonPath)).toBe(true);
expect(fs.existsSync(cleanupMarkdownPath)).toBe(true);
expect(fs.existsSync(dashboardHtmlPath)).toBe(true);
const cleanupJson = JSON.parse(fs.readFileSync(cleanupJsonPath, "utf8"));
expect(
cleanupJson.recommendations.every(
(entry: Record<string, unknown>) =>
!Object.prototype.hasOwnProperty.call(
entry,
"focusObservabilityVerificationOutcomes",
),
),
).toBe(true);
expect(() =>
execFileSync(
process.execPath,
[
path.join(repoRoot, "scripts/check-generated-slop-report.mjs"),
"--input",
cleanupJsonPath,
"--format",
"json",
],
{
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
},
),
).not.toThrow();
}, 30_000);
it("应记录本地 summary 历史,并生成非 seed 的 trend / cleanup", () => {
const tempRoot = createTempRoot();
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
const workspaceRoot = path.join(tempRoot, "workspace");
const summaryJsonPath = path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-eval-summary.json",
);
const summaryMarkdownPath = path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-eval-summary.md",
);
const dashboardHtmlPath = path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-dashboard.html",
);
const firstResult = runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
@@ -71,6 +186,12 @@ describe("harness-eval-history-record", () => {
historyDir,
"--workspace-root",
workspaceRoot,
"--summary-json",
summaryJsonPath,
"--summary-markdown",
summaryMarkdownPath,
"--dashboard-html",
dashboardHtmlPath,
"--output-json",
path.join(tempRoot, "first.json"),
]);
@@ -84,6 +205,12 @@ describe("harness-eval-history-record", () => {
historyDir,
"--workspace-root",
workspaceRoot,
"--summary-json",
summaryJsonPath,
"--summary-markdown",
summaryMarkdownPath,
"--dashboard-html",
dashboardHtmlPath,
"--output-json",
path.join(tempRoot, "second.json"),
],
@@ -91,11 +218,72 @@ describe("harness-eval-history-record", () => {
expect(firstResult.historyCount).toBe(1);
expect(firstResult.trend.sampleCount).toBe(1);
expect(firstResult.trend.currentObservabilityGapCaseCount).toBe(0);
expect(firstResult.trend.degradedObservabilityGapCaseCount).toBe(1);
expect(firstResult.trend.currentRecoveredVerificationCaseCount).toBe(3);
expect(firstResult.trend.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(firstResult.summary.outputJsonPath).toBe(summaryJsonPath);
expect(firstResult.summary.outputMarkdownPath).toBe(summaryMarkdownPath);
expect(firstResult.dashboard.outputHtmlPath).toBe(dashboardHtmlPath);
expect(secondResult.historyCount).toBe(2);
expect(secondResult.trend.sampleCount).toBe(2);
expect(secondResult.trend.currentObservabilityGapCaseCount).toBe(0);
expect(secondResult.trend.degradedObservabilityGapCaseCount).toBe(1);
expect(secondResult.trend.currentRecoveredVerificationCaseCount).toBe(3);
expect(secondResult.trend.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(secondResult.cleanup.trendSampleCount).toBe(2);
expect(secondResult.cleanup.currentObservabilityGapCaseCount).toBe(0);
expect(secondResult.cleanup.degradedObservabilityGapCaseCount).toBe(1);
expect(secondResult.cleanup.verificationFailureOutcomeFocus).toEqual(
expect.arrayContaining([
"artifactValidator:issues_present",
]),
);
expect(secondResult.cleanup.verificationFailureCaseCount).toBe(1);
expect(secondResult.cleanup.verificationBlockingFailureCaseCount).toBe(0);
expect(secondResult.cleanup.verificationAdvisoryFailureCaseCount).toBe(1);
expect(secondResult.cleanup.verificationDegradedBlockingFailureCaseCount).toBe(0);
expect(secondResult.cleanup.verificationRecoveredCaseCount).toBe(3);
expect(secondResult.cleanup.currentVerificationRecoveredCaseCount).toBe(3);
expect(secondResult.cleanup.currentRecoveredBaselineFocus).toEqual(
expect.arrayContaining([
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
);
expect(fs.existsSync(secondResult.recordedSummaryPath)).toBe(true);
expect(fs.existsSync(summaryJsonPath)).toBe(true);
expect(fs.existsSync(summaryMarkdownPath)).toBe(true);
expect(fs.existsSync(dashboardHtmlPath)).toBe(true);
const dashboardHtml = fs.readFileSync(dashboardHtmlPath, "utf8");
expect(dashboardHtml).toContain("Harness Engine Nightly Dashboard");
expect(dashboardHtml).toContain("Current Gap");
expect(dashboardHtml).toContain("Degraded Gap");
expect(dashboardHtml).toContain("Current Blocking");
expect(dashboardHtml).toContain("Current Advisory");
expect(dashboardHtml).toContain("Current Recovered");
expect(dashboardHtml).toContain("Degraded Blocking");
expect(dashboardHtml).toContain("Recovered Outcomes");
expect(dashboardHtml).toContain("Cleanup 建议");
expect(dashboardHtml).toContain("Observability Gap 角色");
expect(dashboardHtml).toContain("Verification Outcome 焦点");
expect(dashboardHtml).toContain("Current Recovered Baseline");
expect(dashboardHtml).toContain("artifactValidator");
expect(dashboardHtml).toContain("issues_present");
expect(
fs.existsSync(
path.join(
+11 -14
View File
@@ -159,7 +159,7 @@ afterEach(() => {
});
describe("Harness eval history window", () => {
it("runner 应记录并裁剪历史窗口,trend 应复用该目录", () => {
it("history-record 应记录并裁剪历史窗口,trend 应复用该目录", () => {
const tempRoot = createTempRoot();
const caseDir = createReplayFixture(tempRoot);
const manifestPath = path.join(tempRoot, "manifest.json");
@@ -167,40 +167,37 @@ describe("Harness eval history window", () => {
writeJson(manifestPath, createHarnessManifest(caseDir));
runNodeScript("scripts/harness-eval-runner.mjs", [
runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--manifest",
manifestPath,
"--record-history-dir",
"--history-dir",
historyDir,
"--history-retain",
"--retain",
"2",
"--no-strict",
]);
sleepMs(10);
runNodeScript("scripts/harness-eval-runner.mjs", [
runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--manifest",
manifestPath,
"--record-history-dir",
"--history-dir",
historyDir,
"--history-retain",
"--retain",
"2",
"--no-strict",
]);
sleepMs(10);
runNodeScript("scripts/harness-eval-runner.mjs", [
runNodeScript("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--manifest",
manifestPath,
"--record-history-dir",
"--history-dir",
historyDir,
"--history-retain",
"--retain",
"2",
"--no-strict",
]);
const historyFiles = fs
@@ -222,5 +219,5 @@ describe("Harness eval history window", () => {
expect(report.signals).not.toContain(
"样本数不足 2,当前仅形成 trend seed,还不能判断长期退化。",
);
});
}, 30_000);
});
@@ -0,0 +1,155 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { afterEach, describe, expect, it } from "vitest";
const repoRoot = process.cwd();
const tempRoots: string[] = [];
function createTempRoot() {
const tempRoot = fs.mkdtempSync(
path.join(os.tmpdir(), "lime-harness-repo-fixtures-"),
);
tempRoots.push(tempRoot);
return tempRoot;
}
function runHarnessEval(args: string[]) {
const output = execFileSync(
process.execPath,
[path.join(repoRoot, "scripts/harness-eval-runner.mjs"), ...args],
{
cwd: repoRoot,
encoding: "utf8",
stdio: ["ignore", "pipe", "pipe"],
},
);
return JSON.parse(output);
}
afterEach(() => {
while (tempRoots.length > 0) {
const tempRoot = tempRoots.pop();
if (tempRoot) {
fs.rmSync(tempRoot, { recursive: true, force: true });
}
}
});
describe("Harness repo fixtures", () => {
it("应固定 current 与 degraded observability 样本职责", () => {
const workspaceRoot = createTempRoot();
const summary = runHarnessEval([
"--format",
"json",
"--manifest",
"docs/test/harness-evals.manifest.json",
"--workspace-root",
workspaceRoot,
"--no-strict",
]);
const repoFixtureSuite = summary.suites.find(
(entry: { id: string }) => entry.id === "repo-fixtures",
);
expect(repoFixtureSuite).toBeTruthy();
expect(repoFixtureSuite.stats.caseCount).toBe(2);
expect(repoFixtureSuite.stats.readyCount).toBe(2);
expect(summary.totals.observabilityGapCaseCount).toBe(1);
expect(summary.totals.currentObservabilityGapCaseCount).toBe(0);
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(1);
const currentCase = repoFixtureSuite.cases.find(
(entry: { caseId: string }) =>
entry.caseId === "fixture-minimal-pending-request",
);
expect(currentCase).toBeTruthy();
expect(currentCase.status).toBe("ready");
expect(currentCase.observabilityGapCount).toBe(0);
expect(currentCase.observabilitySignals).toContain(
"requestTelemetry:exported",
);
expect(currentCase.observabilitySignals).toContain(
"artifactValidator:exported",
);
expect(currentCase.observabilitySignals).toContain(
"browserVerification:exported",
);
expect(currentCase.observabilitySignals).toContain("guiSmoke:exported");
expect(currentCase.observabilityVerificationOutcomes).toContain(
"artifactValidator:issues_present",
);
expect(currentCase.observabilityVerificationOutcomes).toContain(
"artifactValidator:repaired",
);
expect(currentCase.observabilityVerificationOutcomes).toContain(
"browserVerification:success",
);
expect(currentCase.observabilityVerificationOutcomes).toContain(
"guiSmoke:passed",
);
const degradedCase = repoFixtureSuite.cases.find(
(entry: { caseId: string }) =>
entry.caseId === "fixture-minimal-observability-gap",
);
expect(degradedCase).toBeTruthy();
expect(degradedCase.status).toBe("ready");
expect(degradedCase.observabilityGapCount).toBe(2);
expect(degradedCase.observabilitySignals).toContain(
"requestTelemetry:known_gap",
);
expect(degradedCase.observabilitySignals).toContain(
"artifactValidator:known_gap",
);
const gapCaseIds = repoFixtureSuite.cases
.filter(
(entry: { observabilityGapCount: number }) =>
entry.observabilityGapCount > 0,
)
.map((entry: { caseId: string }) => entry.caseId);
expect(gapCaseIds).toEqual(["fixture-minimal-observability-gap"]);
const observabilityBreakdownNames = summary.breakdowns.observabilitySignals.map(
(entry: { name: string }) => entry.name,
);
expect(observabilityBreakdownNames).toContain("requestTelemetry:exported");
expect(observabilityBreakdownNames).toContain("requestTelemetry:known_gap");
expect(observabilityBreakdownNames).toContain("artifactValidator:exported");
expect(observabilityBreakdownNames).toContain(
"artifactValidator:known_gap",
);
const verificationOutcomeBreakdownNames =
summary.breakdowns.observabilityVerificationOutcomes.map(
(entry: { name: string }) => entry.name,
);
expect(verificationOutcomeBreakdownNames).toContain(
"artifactValidator:issues_present",
);
expect(verificationOutcomeBreakdownNames).toContain(
"artifactValidator:repaired",
);
expect(verificationOutcomeBreakdownNames).toContain(
"browserVerification:success",
);
expect(verificationOutcomeBreakdownNames).toContain("guiSmoke:passed");
const currentVerificationOutcomeBreakdownNames =
summary.breakdowns.currentObservabilityVerificationOutcomes.map(
(entry: { name: string }) => entry.name,
);
expect(currentVerificationOutcomeBreakdownNames).toContain(
"artifactValidator:issues_present",
);
expect(currentVerificationOutcomeBreakdownNames).toContain(
"browserVerification:success",
);
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
[],
);
});
});
+348 -14
View File
@@ -116,13 +116,36 @@ function createWorkspaceSessionArtifacts(tempRoot: string, sessionId: string) {
},
{
signal: "requestTelemetry",
status: "unlinked",
status: "exported",
},
{
signal: "artifactValidator",
status: "known_gap",
},
],
verificationSummary: {
artifactValidator: {
applicable: true,
recordCount: 1,
issueCount: 1,
repairedCount: 1,
fallbackUsedCount: 0,
},
browserVerification: {
recordCount: 1,
successCount: 1,
failureCount: 0,
unknownCount: 0,
latestUpdatedAt: "2026-03-27T11:22:00Z",
},
guiSmoke: {
status: "completed",
exitCode: 0,
passed: true,
updatedAt: "2026-03-27T11:23:30Z",
hasOutputPreview: true,
},
},
},
linkedArtifacts: {
handoffBundle: {
@@ -325,6 +348,8 @@ describe("Harness review decision / eval integration", () => {
expect(summary.totals.reviewDecisionRecordedCount).toBe(1);
expect(summary.totals.observabilityGapCaseCount).toBe(1);
expect(summary.totals.currentObservabilityGapCaseCount).toBe(1);
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(0);
expect(summary.breakdowns.reviewDecisionStatuses).toEqual(
expect.arrayContaining([
expect.objectContaining({
@@ -343,21 +368,54 @@ describe("Harness review decision / eval integration", () => {
);
expect(summary.breakdowns.observabilitySignals).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "requestTelemetry:unlinked",
caseCount: 1,
}),
expect.objectContaining({
name: "artifactValidator:known_gap",
caseCount: 1,
}),
]),
);
expect(summary.breakdowns.observabilityVerificationOutcomes).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "artifactValidator:issues_present",
caseCount: 1,
}),
expect.objectContaining({
name: "artifactValidator:repaired",
caseCount: 1,
}),
expect.objectContaining({
name: "browserVerification:success",
caseCount: 1,
}),
expect.objectContaining({
name: "guiSmoke:passed",
caseCount: 1,
}),
]),
);
expect(summary.breakdowns.currentObservabilityVerificationOutcomes).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:success",
caseCount: 1,
}),
]),
);
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
[],
);
expect(summary.suites[0].cases[0]).toMatchObject({
reviewDecisionStatus: "accepted",
reviewRiskLevel: "high",
reviewHumanReviewer: "Lime Maintainer",
observabilityGapCount: 2,
observabilityGapCount: 1,
observabilityVerificationOutcomes: expect.arrayContaining([
"artifactValidator:issues_present",
"artifactValidator:repaired",
"browserVerification:success",
"guiSmoke:passed",
]),
});
});
@@ -376,6 +434,8 @@ describe("Harness review decision / eval integration", () => {
pendingRequestCaseCount: 0,
needsHumanReviewCount: 1,
observabilityGapCaseCount: 1,
currentObservabilityGapCaseCount: 1,
degradedObservabilityGapCaseCount: 0,
},
breakdowns: {
suiteTags: [],
@@ -400,9 +460,10 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 1,
},
],
observabilitySignals: [
observabilitySignals: [],
observabilityVerificationOutcomes: [
{
name: "requestTelemetry:unlinked",
name: "browserVerification:unknown",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
@@ -410,6 +471,17 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 1,
},
],
currentObservabilityVerificationOutcomes: [
{
name: "browserVerification:unknown",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 1,
},
],
degradedObservabilityVerificationOutcomes: [],
},
suites: [],
});
@@ -424,6 +496,8 @@ describe("Harness review decision / eval integration", () => {
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 1,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 1,
},
breakdowns: {
suiteTags: [],
@@ -458,6 +532,43 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 0,
},
],
observabilityVerificationOutcomes: [
{
name: "browserVerification:failure",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:failed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
currentObservabilityVerificationOutcomes: [],
degradedObservabilityVerificationOutcomes: [
{
name: "browserVerification:failure",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:failed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
},
suites: [],
});
@@ -505,12 +616,6 @@ describe("Harness review decision / eval integration", () => {
);
expect(report.classificationDeltas.observabilitySignals).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "requestTelemetry:unlinked",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
expect.objectContaining({
name: "artifactValidator:known_gap",
delta: expect.objectContaining({
@@ -519,5 +624,234 @@ describe("Harness review decision / eval integration", () => {
}),
]),
);
expect(
report.classificationDeltas.observabilitySignals.find(
(entry) => entry.name === "requestTelemetry:known_gap",
),
).toBeUndefined();
expect(report.classificationDeltas.observabilityVerificationOutcomes).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:failure",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
expect.objectContaining({
name: "browserVerification:unknown",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
expect.objectContaining({
name: "guiSmoke:failed",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
expect(
report.classificationDeltas.currentObservabilityVerificationOutcomes,
).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:unknown",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
]),
);
expect(
report.classificationDeltas.degradedObservabilityVerificationOutcomes,
).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "browserVerification:failure",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
expect.objectContaining({
name: "guiSmoke:failed",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
expect(report.delta.currentObservabilityGapCaseCount).toBe(-1);
expect(report.delta.degradedObservabilityGapCaseCount).toBe(1);
expect(report.latest.totals.currentObservabilityGapCaseCount).toBe(0);
expect(report.latest.totals.degradedObservabilityGapCaseCount).toBe(1);
});
it("harness-eval-trend-report 应暴露 current recovered baseline 的新增趋势", () => {
const tempRoot = createTempRoot();
const baselinePath = path.join(tempRoot, "baseline-recovered.json");
const latestPath = path.join(tempRoot, "latest-recovered.json");
writeJson(baselinePath, {
generatedAt: "2026-03-28T10:00:00Z",
totals: {
suiteCount: 1,
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 0,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 0,
},
breakdowns: {
suiteTags: [],
failureModes: [],
reviewDecisionStatuses: [],
reviewRiskLevels: [],
observabilitySignals: [],
observabilityVerificationOutcomes: [
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
currentObservabilityVerificationOutcomes: [
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
degradedObservabilityVerificationOutcomes: [],
},
suites: [],
});
writeJson(latestPath, {
generatedAt: "2026-03-28T12:00:00Z",
totals: {
suiteCount: 1,
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 0,
currentObservabilityGapCaseCount: 0,
degradedObservabilityGapCaseCount: 0,
},
breakdowns: {
suiteTags: [],
failureModes: [],
reviewDecisionStatuses: [],
reviewRiskLevels: [],
observabilitySignals: [],
observabilityVerificationOutcomes: [
{
name: "artifactValidator:repaired",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:passed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
currentObservabilityVerificationOutcomes: [
{
name: "artifactValidator:repaired",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "browserVerification:success",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
{
name: "guiSmoke:passed",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
degradedObservabilityVerificationOutcomes: [],
},
suites: [],
});
const report = runNodeScript("scripts/harness-eval-trend-report.mjs", [
"--format",
"json",
"--input",
baselinePath,
"--input",
latestPath,
]);
expect(report.latest.totals.currentRecoveredVerificationCaseCount).toBe(3);
expect(report.delta.currentRecoveredVerificationCaseCount).toBe(2);
expect(
report.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes,
).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "artifactValidator:repaired",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
expect.objectContaining({
name: "browserVerification:success",
delta: expect.objectContaining({
caseCount: 0,
}),
}),
expect.objectContaining({
name: "guiSmoke:passed",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
expect(report.signals).toContain(
"current recovered verification baseline `artifactValidator:repaired` 新增 1,说明主线路径正在形成正向基线。",
);
expect(report.signals).toContain(
"current recovered verification baseline `guiSmoke:passed` 新增 1,说明主线路径正在形成正向基线。",
);
});
});