mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-09-24 23:10:56 +08:00
release: v1.10.0
This commit is contained in:
coso
@@ -7,16 +7,13 @@ import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { renderHarnessDashboardHtml } from "./lib/harness-dashboard-core.mjs";
|
||||
import {
|
||||
deriveVerificationHistoryRecordFacts as deriveSharedVerificationHistoryRecordFacts,
|
||||
} from "./lib/harness-verification-facts.mjs";
|
||||
|
||||
const RUNNER_PATH = "scripts/harness-eval-runner.mjs";
|
||||
const TREND_PATH = "scripts/harness-eval-trend-report.mjs";
|
||||
const CLEANUP_PATH = "scripts/report-generated-slop.mjs";
|
||||
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
|
||||
"repaired",
|
||||
"success",
|
||||
"passed",
|
||||
"clean",
|
||||
]);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
@@ -257,6 +254,14 @@ function writeUniqueHistorySummary(historyDir, payload) {
|
||||
throw new Error(`无法在历史目录中创建唯一 summary 文件: ${historyDir}`);
|
||||
}
|
||||
|
||||
function normalizeString(value) {
|
||||
return typeof value === "string" ? value.trim() : "";
|
||||
}
|
||||
|
||||
function normalizeNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
|
||||
function trimHistoryFiles(historyDir, retain) {
|
||||
const files = collectHistoryFiles(historyDir).sort((left, right) =>
|
||||
right.localeCompare(left),
|
||||
@@ -285,145 +290,24 @@ function buildDefaultArtifactPaths(historyDir) {
|
||||
};
|
||||
}
|
||||
|
||||
function toVerificationFailureOutcomeFocus(cleanupReport) {
|
||||
const currentEntries = Array.isArray(
|
||||
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentObservabilityVerificationOutcomes
|
||||
: [];
|
||||
const fallbackEntries = Array.isArray(
|
||||
cleanupReport?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.observabilityVerificationOutcomes
|
||||
: [];
|
||||
const entries =
|
||||
currentEntries.length > 0 ? currentEntries : fallbackEntries;
|
||||
|
||||
return entries
|
||||
.map((entry) => {
|
||||
const signal = typeof entry?.signal === "string" ? entry.signal.trim() : "";
|
||||
const outcome =
|
||||
typeof entry?.outcome === "string" ? entry.outcome.trim() : "";
|
||||
return signal && outcome ? `${signal}:${outcome}` : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function toCurrentRecoveredBaselineFocus(cleanupReport) {
|
||||
const explicitRecoveredEntries = Array.isArray(
|
||||
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes
|
||||
: [];
|
||||
const currentEntries = Array.isArray(
|
||||
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentObservabilityVerificationOutcomes
|
||||
: [];
|
||||
const fallbackEntries = Array.isArray(
|
||||
cleanupReport?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.observabilityVerificationOutcomes
|
||||
: [];
|
||||
const entries =
|
||||
explicitRecoveredEntries.length > 0
|
||||
? explicitRecoveredEntries
|
||||
: currentEntries.length > 0
|
||||
? currentEntries
|
||||
: fallbackEntries;
|
||||
|
||||
return entries
|
||||
.filter((entry) =>
|
||||
RECOVERED_VERIFICATION_OUTCOMES.has(
|
||||
typeof entry?.outcome === "string" ? entry.outcome.trim() : "",
|
||||
),
|
||||
)
|
||||
.map((entry) => {
|
||||
const signal = typeof entry?.signal === "string" ? entry.signal.trim() : "";
|
||||
const outcome =
|
||||
typeof entry?.outcome === "string" ? entry.outcome.trim() : "";
|
||||
return signal && outcome ? `${signal}:${outcome}` : "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function toVerificationOutcomeCounts(cleanupReport) {
|
||||
const summary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.verificationOutcomes &&
|
||||
typeof cleanupReport.summary.verificationOutcomes === "object"
|
||||
? cleanupReport.summary.verificationOutcomes
|
||||
: {};
|
||||
const currentSummary =
|
||||
summary &&
|
||||
typeof summary.current === "object" &&
|
||||
!Array.isArray(summary.current)
|
||||
? summary.current
|
||||
: {};
|
||||
const degradedSummary =
|
||||
summary &&
|
||||
typeof summary.degraded === "object" &&
|
||||
!Array.isArray(summary.degraded)
|
||||
? summary.degraded
|
||||
: {};
|
||||
|
||||
return {
|
||||
failureCaseCount:
|
||||
typeof summary.failureCaseCount === "number" &&
|
||||
Number.isFinite(summary.failureCaseCount)
|
||||
? summary.failureCaseCount
|
||||
: 0,
|
||||
blockingFailureCaseCount:
|
||||
typeof currentSummary.blockingFailureCaseCount === "number" &&
|
||||
Number.isFinite(currentSummary.blockingFailureCaseCount)
|
||||
? currentSummary.blockingFailureCaseCount
|
||||
: 0,
|
||||
advisoryFailureCaseCount:
|
||||
typeof currentSummary.advisoryFailureCaseCount === "number" &&
|
||||
Number.isFinite(currentSummary.advisoryFailureCaseCount)
|
||||
? currentSummary.advisoryFailureCaseCount
|
||||
: 0,
|
||||
recoveredCaseCount:
|
||||
typeof summary.recoveredCaseCount === "number" &&
|
||||
Number.isFinite(summary.recoveredCaseCount)
|
||||
? summary.recoveredCaseCount
|
||||
: 0,
|
||||
currentRecoveredCaseCount:
|
||||
typeof currentSummary.recoveredCaseCount === "number" &&
|
||||
Number.isFinite(currentSummary.recoveredCaseCount)
|
||||
? currentSummary.recoveredCaseCount
|
||||
: 0,
|
||||
degradedBlockingFailureCaseCount:
|
||||
typeof degradedSummary.blockingFailureCaseCount === "number" &&
|
||||
Number.isFinite(degradedSummary.blockingFailureCaseCount)
|
||||
? degradedSummary.blockingFailureCaseCount
|
||||
: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function toTrendCurrentRecoveredBaselineFocus(trendReport) {
|
||||
const entries = Array.isArray(
|
||||
trendReport?.classificationDeltas?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? trendReport.classificationDeltas.currentRecoveredObservabilityVerificationOutcomes
|
||||
: [];
|
||||
return deriveSharedVerificationHistoryRecordFacts({
|
||||
summary: null,
|
||||
trendReport,
|
||||
cleanupReport: null,
|
||||
}).currentRecoveredBaselineFocus.slice(0, 3);
|
||||
}
|
||||
|
||||
return entries
|
||||
.filter((entry) => {
|
||||
const latestCaseCount =
|
||||
typeof entry?.latest?.caseCount === "number" &&
|
||||
Number.isFinite(entry.latest.caseCount)
|
||||
? entry.latest.caseCount
|
||||
: 0;
|
||||
return latestCaseCount > 0;
|
||||
})
|
||||
.map((entry) =>
|
||||
typeof entry?.name === "string" ? entry.name.trim() : "",
|
||||
)
|
||||
.filter(Boolean)
|
||||
.slice(0, 3);
|
||||
export function deriveHistoryRecordVerificationFacts({
|
||||
summary,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
}) {
|
||||
return deriveSharedVerificationHistoryRecordFacts({
|
||||
summary,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
});
|
||||
}
|
||||
|
||||
function renderOutput(result, format) {
|
||||
@@ -660,32 +544,33 @@ function runHistoryRecordCli() {
|
||||
},
|
||||
);
|
||||
cleanupReport = JSON.parse(cleanupOutput);
|
||||
const verificationFailureOutcomeFocus =
|
||||
toVerificationFailureOutcomeFocus(cleanupReport);
|
||||
const currentRecoveredBaselineFocus =
|
||||
toCurrentRecoveredBaselineFocus(cleanupReport);
|
||||
const verificationOutcomeCounts =
|
||||
toVerificationOutcomeCounts(cleanupReport);
|
||||
const verificationFacts = deriveHistoryRecordVerificationFacts({
|
||||
summary,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
});
|
||||
result.cleanup = {
|
||||
trendSampleCount: cleanupReport.summary?.trend?.sampleCount ?? 0,
|
||||
currentObservabilityGapCaseCount:
|
||||
cleanupReport.summary?.trend?.latestCurrentObservabilityGapCaseCount ?? 0,
|
||||
degradedObservabilityGapCaseCount:
|
||||
cleanupReport.summary?.trend?.latestDegradedObservabilityGapCaseCount ?? 0,
|
||||
verificationFailureOutcomeFocus,
|
||||
verificationFailureOutcomeFocus:
|
||||
verificationFacts.verificationFailureOutcomeFocus,
|
||||
verificationFailureCaseCount:
|
||||
verificationOutcomeCounts.failureCaseCount,
|
||||
verificationFacts.verificationOutcomeCounts.failureCaseCount,
|
||||
verificationBlockingFailureCaseCount:
|
||||
verificationOutcomeCounts.blockingFailureCaseCount,
|
||||
verificationFacts.verificationOutcomeCounts.blockingFailureCaseCount,
|
||||
verificationAdvisoryFailureCaseCount:
|
||||
verificationOutcomeCounts.advisoryFailureCaseCount,
|
||||
verificationFacts.verificationOutcomeCounts.advisoryFailureCaseCount,
|
||||
verificationDegradedBlockingFailureCaseCount:
|
||||
verificationOutcomeCounts.degradedBlockingFailureCaseCount,
|
||||
verificationFacts.verificationOutcomeCounts.degradedBlockingFailureCaseCount,
|
||||
verificationRecoveredCaseCount:
|
||||
verificationOutcomeCounts.recoveredCaseCount,
|
||||
verificationFacts.verificationOutcomeCounts.recoveredCaseCount,
|
||||
currentVerificationRecoveredCaseCount:
|
||||
verificationOutcomeCounts.currentRecoveredCaseCount,
|
||||
currentRecoveredBaselineFocus,
|
||||
verificationFacts.verificationOutcomeCounts.currentRecoveredCaseCount,
|
||||
currentRecoveredBaselineFocus:
|
||||
verificationFacts.currentRecoveredBaselineFocus,
|
||||
outputJsonPath: cleanupJsonPath,
|
||||
outputMarkdownPath: cleanupMarkdownPath,
|
||||
};
|
||||
|
||||
@@ -23,6 +23,12 @@ const REVIEW_DECISION_RISK_LEVEL_SET = new Set([
|
||||
"unknown",
|
||||
]);
|
||||
const OBSERVABILITY_GAP_SUITE_TAG = "observability-gap";
|
||||
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
|
||||
"artifactValidator:repaired",
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
"guiSmoke:clean",
|
||||
]);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
@@ -193,6 +199,12 @@ function aggregateCaseBreakdown(cases, selector) {
|
||||
});
|
||||
}
|
||||
|
||||
function collectRecoveredVerificationOutcomes(outcomes) {
|
||||
return mergeUniqueStrings(outcomes).filter((entry) =>
|
||||
RECOVERED_VERIFICATION_OUTCOMES.has(entry),
|
||||
);
|
||||
}
|
||||
|
||||
function isDegradedObservabilityGapCase(entry) {
|
||||
return (
|
||||
entry.observabilityGapCount > 0 &&
|
||||
@@ -891,6 +903,22 @@ function buildSummary(manifest, suites, options) {
|
||||
const degradedObservabilityGapCases = observabilityGapCases.filter((entry) =>
|
||||
isDegradedObservabilityGapCase(entry),
|
||||
);
|
||||
const currentObservabilityDiagnosticCases = allCases.filter((entry) =>
|
||||
isCurrentObservabilityDiagnosticCase(entry),
|
||||
);
|
||||
const currentRecoveredObservabilityVerificationOutcomes =
|
||||
aggregateCaseBreakdown(
|
||||
currentObservabilityDiagnosticCases,
|
||||
(entry) =>
|
||||
collectRecoveredVerificationOutcomes(
|
||||
entry.observabilityVerificationOutcomes,
|
||||
),
|
||||
);
|
||||
const currentRecoveredVerificationCaseCount =
|
||||
currentRecoveredObservabilityVerificationOutcomes.reduce(
|
||||
(total, entry) => total + entry.caseCount,
|
||||
0,
|
||||
);
|
||||
|
||||
return {
|
||||
manifestVersion: String(manifest.manifestVersion ?? "unknown"),
|
||||
@@ -910,6 +938,7 @@ function buildSummary(manifest, suites, options) {
|
||||
observabilityGapCaseCount: observabilityGapCases.length,
|
||||
currentObservabilityGapCaseCount: currentObservabilityGapCases.length,
|
||||
degradedObservabilityGapCaseCount: degradedObservabilityGapCases.length,
|
||||
currentRecoveredVerificationCaseCount,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: aggregateCaseBreakdown(allCases, (entry) => entry.tags),
|
||||
@@ -932,9 +961,10 @@ function buildSummary(manifest, suites, options) {
|
||||
(entry) => entry.observabilityVerificationOutcomes,
|
||||
),
|
||||
currentObservabilityVerificationOutcomes: aggregateCaseBreakdown(
|
||||
allCases.filter((entry) => isCurrentObservabilityDiagnosticCase(entry)),
|
||||
currentObservabilityDiagnosticCases,
|
||||
(entry) => entry.observabilityVerificationOutcomes,
|
||||
),
|
||||
currentRecoveredObservabilityVerificationOutcomes,
|
||||
degradedObservabilityVerificationOutcomes: aggregateCaseBreakdown(
|
||||
allCases.filter((entry) => isDegradedObservabilityDiagnosticCase(entry)),
|
||||
(entry) => entry.observabilityVerificationOutcomes,
|
||||
@@ -958,6 +988,7 @@ function renderText(summary) {
|
||||
`[harness-eval] observability-gap cases: ${summary.totals.observabilityGapCaseCount}`,
|
||||
`[harness-eval] current observability-gap cases: ${summary.totals.currentObservabilityGapCaseCount}`,
|
||||
`[harness-eval] degraded observability-gap cases: ${summary.totals.degradedObservabilityGapCaseCount}`,
|
||||
`[harness-eval] current recovered verification cases: ${summary.totals.currentRecoveredVerificationCaseCount}`,
|
||||
];
|
||||
|
||||
const topFailureModes = summary.breakdowns.failureModes.slice(0, 5);
|
||||
@@ -1014,6 +1045,20 @@ function renderText(summary) {
|
||||
}
|
||||
}
|
||||
|
||||
const topCurrentRecoveredVerificationOutcomes =
|
||||
summary.breakdowns.currentRecoveredObservabilityVerificationOutcomes.slice(
|
||||
0,
|
||||
5,
|
||||
);
|
||||
if (topCurrentRecoveredVerificationOutcomes.length > 0) {
|
||||
lines.push("[harness-eval] current recovered verification outcomes:");
|
||||
for (const entry of topCurrentRecoveredVerificationOutcomes) {
|
||||
lines.push(
|
||||
` - ${entry.name}: case=${entry.caseCount}, ready=${entry.readyCount}, invalid=${entry.invalidCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const suite of summary.suites) {
|
||||
lines.push(
|
||||
`[harness-eval] suite ${suite.id}: ready ${suite.stats.readyCount} / ${suite.stats.caseCount}`,
|
||||
@@ -1069,6 +1114,7 @@ function renderMarkdown(summary) {
|
||||
`- observability gap case：${summary.totals.observabilityGapCaseCount}`,
|
||||
`- current observability gap case：${summary.totals.currentObservabilityGapCaseCount}`,
|
||||
`- degraded observability gap case：${summary.totals.degradedObservabilityGapCaseCount}`,
|
||||
`- current recovered verification case：${summary.totals.currentRecoveredVerificationCaseCount}`,
|
||||
"",
|
||||
];
|
||||
|
||||
@@ -1152,6 +1198,19 @@ function renderMarkdown(summary) {
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (summary.breakdowns.currentRecoveredObservabilityVerificationOutcomes.length > 0) {
|
||||
lines.push("## Current Recovered Verification Outcome 分布");
|
||||
lines.push("");
|
||||
lines.push("| Outcome | case | ready | invalid |");
|
||||
lines.push("| --- | --- | --- | --- |");
|
||||
for (const entry of summary.breakdowns.currentRecoveredObservabilityVerificationOutcomes) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.caseCount} | ${entry.readyCount} | ${entry.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
for (const suite of summary.suites) {
|
||||
lines.push(`## ${suite.title}`);
|
||||
lines.push("");
|
||||
|
||||
@@ -256,10 +256,8 @@ function buildNormalizedTotals(summary) {
|
||||
const rawTotals =
|
||||
summary?.totals && typeof summary.totals === "object" ? summary.totals : {};
|
||||
const gapTotals = buildObservabilityGapTotals(summary);
|
||||
const currentVerificationOutcomeEntries = getBreakdownEntries(
|
||||
summary,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
);
|
||||
const currentRecoveredVerificationEntries =
|
||||
getCurrentRecoveredVerificationEntries(summary);
|
||||
return {
|
||||
suiteCount: normalizeNumber(rawTotals.suiteCount),
|
||||
caseCount: normalizeNumber(rawTotals.caseCount),
|
||||
@@ -274,11 +272,10 @@ function buildNormalizedTotals(summary) {
|
||||
currentObservabilityGapCaseCount: gapTotals.current,
|
||||
degradedObservabilityGapCaseCount: gapTotals.degraded,
|
||||
currentRecoveredVerificationCaseCount:
|
||||
currentVerificationOutcomeEntries.length > 0
|
||||
? getBreakdownCaseCount(
|
||||
summary,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
RECOVERED_VERIFICATION_OUTCOMES,
|
||||
currentRecoveredVerificationEntries.length > 0
|
||||
? currentRecoveredVerificationEntries.reduce(
|
||||
(total, entry) => total + normalizeNumber(entry?.caseCount),
|
||||
0,
|
||||
)
|
||||
: normalizeNumber(rawTotals.currentRecoveredVerificationCaseCount),
|
||||
};
|
||||
@@ -326,6 +323,21 @@ function getBreakdownMap(summary, key) {
|
||||
);
|
||||
}
|
||||
|
||||
function getCurrentRecoveredVerificationEntries(summary) {
|
||||
const explicitEntries = getBreakdownEntries(
|
||||
summary,
|
||||
"currentRecoveredObservabilityVerificationOutcomes",
|
||||
);
|
||||
if (explicitEntries.length > 0) {
|
||||
return explicitEntries;
|
||||
}
|
||||
|
||||
return getBreakdownEntries(
|
||||
summary,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
).filter((entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name));
|
||||
}
|
||||
|
||||
function buildSuiteDeltas(baseline, latest) {
|
||||
const baselineSuites = getSuiteMap(baseline);
|
||||
const latestSuites = getSuiteMap(latest);
|
||||
@@ -428,6 +440,32 @@ function buildFilteredBreakdownDeltas(baseline, latest, key, predicate) {
|
||||
);
|
||||
}
|
||||
|
||||
function buildCurrentRecoveredVerificationDeltas(baseline, latest) {
|
||||
const baselineExplicitEntries = getBreakdownEntries(
|
||||
baseline,
|
||||
"currentRecoveredObservabilityVerificationOutcomes",
|
||||
);
|
||||
const latestExplicitEntries = getBreakdownEntries(
|
||||
latest,
|
||||
"currentRecoveredObservabilityVerificationOutcomes",
|
||||
);
|
||||
|
||||
if (baselineExplicitEntries.length > 0 || latestExplicitEntries.length > 0) {
|
||||
return buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentRecoveredObservabilityVerificationOutcomes",
|
||||
);
|
||||
}
|
||||
|
||||
return buildFilteredBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
|
||||
);
|
||||
}
|
||||
|
||||
function buildStatusSignals(baseline, latest, sampleCount) {
|
||||
const signals = [];
|
||||
|
||||
@@ -507,12 +545,8 @@ function buildStatusSignals(baseline, latest, sampleCount) {
|
||||
);
|
||||
}
|
||||
|
||||
const currentRecoveredVerificationDeltas = buildFilteredBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
|
||||
);
|
||||
const currentRecoveredVerificationDeltas =
|
||||
buildCurrentRecoveredVerificationDeltas(baseline, latest);
|
||||
for (const entry of currentRecoveredVerificationDeltas.filter(
|
||||
(candidate) => candidate.delta.caseCount < 0,
|
||||
)) {
|
||||
@@ -631,12 +665,7 @@ function buildTrendReport(samples, repoRoot) {
|
||||
"observabilityVerificationOutcomes",
|
||||
),
|
||||
currentRecoveredObservabilityVerificationOutcomes:
|
||||
buildFilteredBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"currentObservabilityVerificationOutcomes",
|
||||
(entry) => RECOVERED_VERIFICATION_OUTCOMES.has(entry.name),
|
||||
),
|
||||
buildCurrentRecoveredVerificationDeltas(baseline, latest),
|
||||
currentObservabilityVerificationOutcomes: buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
|
||||
@@ -4,6 +4,22 @@ import {
|
||||
getTextCountStatus,
|
||||
getTextStatus,
|
||||
} from "./legacy-surface-report-summary.mjs";
|
||||
import {
|
||||
buildAdvisoryVerificationRecommendationRationale,
|
||||
buildBlockingVerificationRecommendationRationale,
|
||||
buildObservabilityRecommendationBacklog,
|
||||
buildObservabilityRecommendationRationale,
|
||||
buildVerificationOutcomeSignalMessages,
|
||||
buildRecoveredVerificationRecommendationRationale,
|
||||
buildAdvisoryVerificationFollowUp,
|
||||
buildBlockingVerificationFollowUp,
|
||||
buildRecoveredVerificationFollowUp,
|
||||
buildVerificationOutcomeSummary,
|
||||
deriveVerificationOutcomePresentationFromTrend,
|
||||
formatVerificationOutcomeCompactLabel,
|
||||
formatVerificationOutcomeCompactLabels,
|
||||
getVerificationOutcomeRole,
|
||||
} from "./harness-verification-facts.mjs";
|
||||
|
||||
const PRIORITY_RANK = {
|
||||
P0: 0,
|
||||
@@ -544,277 +560,6 @@ function buildObservabilityFocusEntries(entries, sampleCount) {
|
||||
});
|
||||
}
|
||||
|
||||
const OBSERVABILITY_VERIFICATION_FAILURE_OUTCOMES = new Set([
|
||||
"issues_present",
|
||||
"fallback_used",
|
||||
"failure",
|
||||
"unknown",
|
||||
"failed",
|
||||
]);
|
||||
|
||||
const OBSERVABILITY_VERIFICATION_RECOVERED_OUTCOMES = new Set([
|
||||
"repaired",
|
||||
"success",
|
||||
"passed",
|
||||
"clean",
|
||||
]);
|
||||
|
||||
function isObservabilityVerificationFailureOutcome(outcome) {
|
||||
return OBSERVABILITY_VERIFICATION_FAILURE_OUTCOMES.has(
|
||||
normalizeString(outcome),
|
||||
);
|
||||
}
|
||||
|
||||
function isObservabilityVerificationRecoveredOutcome(outcome) {
|
||||
return OBSERVABILITY_VERIFICATION_RECOVERED_OUTCOMES.has(
|
||||
normalizeString(outcome),
|
||||
);
|
||||
}
|
||||
|
||||
const BLOCKING_VERIFICATION_FAILURES = new Set([
|
||||
"browserVerification:failure",
|
||||
"guiSmoke:failed",
|
||||
]);
|
||||
|
||||
function getObservabilityVerificationOutcomeRole(signal, outcome) {
|
||||
const normalizedSignal = normalizeString(signal);
|
||||
const normalizedOutcome = normalizeString(outcome);
|
||||
const fingerprint = `${normalizedSignal}:${normalizedOutcome}`;
|
||||
|
||||
if (BLOCKING_VERIFICATION_FAILURES.has(fingerprint)) {
|
||||
return "blocking_failure";
|
||||
}
|
||||
|
||||
if (isObservabilityVerificationFailureOutcome(normalizedOutcome)) {
|
||||
return "advisory_failure";
|
||||
}
|
||||
|
||||
if (isObservabilityVerificationRecoveredOutcome(normalizedOutcome)) {
|
||||
return "recovered";
|
||||
}
|
||||
|
||||
return "other";
|
||||
}
|
||||
|
||||
function getObservabilityVerificationOutcomeWeight(outcome) {
|
||||
switch (normalizeString(outcome)) {
|
||||
case "failed":
|
||||
return 140;
|
||||
case "failure":
|
||||
return 130;
|
||||
case "unknown":
|
||||
return 115;
|
||||
case "fallback_used":
|
||||
return 110;
|
||||
case "issues_present":
|
||||
return 100;
|
||||
case "repaired":
|
||||
return 70;
|
||||
default:
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
function buildObservabilityVerificationFocusEntries(entries, sampleCount) {
|
||||
const normalizedEntries = Array.isArray(entries) ? entries : [];
|
||||
|
||||
return normalizedEntries
|
||||
.map((entry) => {
|
||||
const latest = isObject(entry?.latest) ? entry.latest : {};
|
||||
const delta = isObject(entry?.delta) ? entry.delta : {};
|
||||
const baseline = isObject(entry?.baseline) ? entry.baseline : {};
|
||||
const parsed = splitObservabilitySignalName(entry?.name);
|
||||
const positiveDeltaCase = Math.max(0, normalizeNumber(delta.caseCount));
|
||||
const latestCase = normalizeNumber(latest.caseCount);
|
||||
const weight = getObservabilityVerificationOutcomeWeight(parsed.status);
|
||||
const score = positiveDeltaCase * 140 + latestCase * weight;
|
||||
|
||||
let state = "stable";
|
||||
if (sampleCount < 2 && latestCase > 0 && weight > 0) {
|
||||
state = "seed-risk";
|
||||
} else if (positiveDeltaCase > 0 && weight > 0) {
|
||||
state = "regressing";
|
||||
} else if (latestCase > 0 && weight > 0) {
|
||||
state = "present";
|
||||
}
|
||||
|
||||
return {
|
||||
name: normalizeString(entry?.name, "(unknown)"),
|
||||
signal: parsed.signal || "(unknown)",
|
||||
outcome: parsed.status || "unknown",
|
||||
baseline: {
|
||||
caseCount: normalizeNumber(baseline.caseCount),
|
||||
readyCount: normalizeNumber(baseline.readyCount),
|
||||
invalidCount: normalizeNumber(baseline.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(
|
||||
baseline.pendingRequestCaseCount,
|
||||
),
|
||||
needsHumanReviewCount: normalizeNumber(
|
||||
baseline.needsHumanReviewCount,
|
||||
),
|
||||
},
|
||||
latest: {
|
||||
caseCount: latestCase,
|
||||
readyCount: normalizeNumber(latest.readyCount),
|
||||
invalidCount: normalizeNumber(latest.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(
|
||||
latest.pendingRequestCaseCount,
|
||||
),
|
||||
needsHumanReviewCount: normalizeNumber(
|
||||
latest.needsHumanReviewCount,
|
||||
),
|
||||
},
|
||||
delta: {
|
||||
caseCount: normalizeNumber(delta.caseCount),
|
||||
readyCount: normalizeNumber(delta.readyCount),
|
||||
invalidCount: normalizeNumber(delta.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(delta.pendingRequestCaseCount),
|
||||
needsHumanReviewCount: normalizeNumber(delta.needsHumanReviewCount),
|
||||
},
|
||||
state,
|
||||
score,
|
||||
};
|
||||
})
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.score > 0 ||
|
||||
(entry.latest.caseCount > 0 &&
|
||||
isObservabilityVerificationFailureOutcome(entry.outcome)),
|
||||
)
|
||||
.sort((left, right) => {
|
||||
if (right.score !== left.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.name.localeCompare(right.name);
|
||||
});
|
||||
}
|
||||
|
||||
function buildVerificationOutcomeEntriesFromDeltas(entries) {
|
||||
const normalizedEntries = Array.isArray(entries) ? entries : [];
|
||||
|
||||
return normalizedEntries
|
||||
.map((entry) => {
|
||||
const latest = isObject(entry?.latest) ? entry.latest : {};
|
||||
const delta = isObject(entry?.delta) ? entry.delta : {};
|
||||
const baseline = isObject(entry?.baseline) ? entry.baseline : {};
|
||||
const parsed = splitObservabilitySignalName(entry?.name);
|
||||
const latestCase = normalizeNumber(latest.caseCount);
|
||||
const deltaCase = normalizeNumber(delta.caseCount);
|
||||
|
||||
let state = "stable";
|
||||
if (deltaCase > 0) {
|
||||
state = "expanding";
|
||||
} else if (deltaCase < 0) {
|
||||
state = "shrinking";
|
||||
} else if (latestCase > 0) {
|
||||
state = "present";
|
||||
}
|
||||
|
||||
return {
|
||||
name: normalizeString(entry?.name, "(unknown)"),
|
||||
signal: parsed.signal || "(unknown)",
|
||||
outcome: parsed.status || "unknown",
|
||||
baseline: {
|
||||
caseCount: normalizeNumber(baseline.caseCount),
|
||||
readyCount: normalizeNumber(baseline.readyCount),
|
||||
invalidCount: normalizeNumber(baseline.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(
|
||||
baseline.pendingRequestCaseCount,
|
||||
),
|
||||
needsHumanReviewCount: normalizeNumber(
|
||||
baseline.needsHumanReviewCount,
|
||||
),
|
||||
},
|
||||
latest: {
|
||||
caseCount: latestCase,
|
||||
readyCount: normalizeNumber(latest.readyCount),
|
||||
invalidCount: normalizeNumber(latest.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(
|
||||
latest.pendingRequestCaseCount,
|
||||
),
|
||||
needsHumanReviewCount: normalizeNumber(
|
||||
latest.needsHumanReviewCount,
|
||||
),
|
||||
},
|
||||
delta: {
|
||||
caseCount: deltaCase,
|
||||
readyCount: normalizeNumber(delta.readyCount),
|
||||
invalidCount: normalizeNumber(delta.invalidCount),
|
||||
pendingRequestCaseCount: normalizeNumber(delta.pendingRequestCaseCount),
|
||||
needsHumanReviewCount: normalizeNumber(delta.needsHumanReviewCount),
|
||||
},
|
||||
state,
|
||||
score: latestCase * 10 + Math.abs(deltaCase) * 5,
|
||||
};
|
||||
})
|
||||
.filter((entry) => entry.latest.caseCount > 0 || entry.delta.caseCount !== 0)
|
||||
.sort((left, right) => {
|
||||
if (right.latest.caseCount !== left.latest.caseCount) {
|
||||
return right.latest.caseCount - left.latest.caseCount;
|
||||
}
|
||||
if (Math.abs(right.delta.caseCount) !== Math.abs(left.delta.caseCount)) {
|
||||
return Math.abs(right.delta.caseCount) - Math.abs(left.delta.caseCount);
|
||||
}
|
||||
return left.name.localeCompare(right.name);
|
||||
});
|
||||
}
|
||||
|
||||
function buildVerificationOutcomeSummary(focusEntries) {
|
||||
const entries = Array.isArray(focusEntries) ? focusEntries : [];
|
||||
const blockingFailureEntries = entries.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) ===
|
||||
"blocking_failure",
|
||||
);
|
||||
const advisoryFailureEntries = entries.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) ===
|
||||
"advisory_failure",
|
||||
);
|
||||
const failureEntries = [...blockingFailureEntries, ...advisoryFailureEntries];
|
||||
const recoveredEntries = entries.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) ===
|
||||
"recovered",
|
||||
);
|
||||
|
||||
return {
|
||||
focusCount: entries.length,
|
||||
failureFocusCount: failureEntries.length,
|
||||
recoveredFocusCount: recoveredEntries.length,
|
||||
blockingFailureFocusCount: blockingFailureEntries.length,
|
||||
advisoryFailureFocusCount: advisoryFailureEntries.length,
|
||||
failureCaseCount: failureEntries.reduce(
|
||||
(total, entry) => total + normalizeNumber(entry?.latest?.caseCount),
|
||||
0,
|
||||
),
|
||||
blockingFailureCaseCount: blockingFailureEntries.reduce(
|
||||
(total, entry) => total + normalizeNumber(entry?.latest?.caseCount),
|
||||
0,
|
||||
),
|
||||
advisoryFailureCaseCount: advisoryFailureEntries.reduce(
|
||||
(total, entry) => total + normalizeNumber(entry?.latest?.caseCount),
|
||||
0,
|
||||
),
|
||||
recoveredCaseCount: recoveredEntries.reduce(
|
||||
(total, entry) => total + normalizeNumber(entry?.latest?.caseCount),
|
||||
0,
|
||||
),
|
||||
topFailureOutcomes: failureEntries
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal}:${entry.outcome}`),
|
||||
topBlockingFailureOutcomes: blockingFailureEntries
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal}:${entry.outcome}`),
|
||||
topAdvisoryFailureOutcomes: advisoryFailureEntries
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal}:${entry.outcome}`),
|
||||
topRecoveredOutcomes: recoveredEntries
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal}:${entry.outcome}`),
|
||||
};
|
||||
}
|
||||
|
||||
function buildDocFreshnessSummary(docFreshnessReport) {
|
||||
const summary = isObject(docFreshnessReport?.summary)
|
||||
? docFreshnessReport.summary
|
||||
@@ -993,8 +738,8 @@ export function assertGeneratedSlopReportContract(report) {
|
||||
if (
|
||||
verificationFailureFocus.some(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) ===
|
||||
"recovered",
|
||||
getVerificationOutcomeRole(entry?.signal, entry?.outcome) ===
|
||||
"recovered",
|
||||
)
|
||||
) {
|
||||
throw new Error(
|
||||
@@ -1005,183 +750,6 @@ export function assertGeneratedSlopReportContract(report) {
|
||||
return report;
|
||||
}
|
||||
|
||||
function hasObservabilityVerificationOutcome(entries, signal, outcome) {
|
||||
const normalizedEntries = Array.isArray(entries) ? entries : [];
|
||||
const normalizedSignal = normalizeString(signal);
|
||||
const normalizedOutcome = normalizeString(outcome);
|
||||
|
||||
return normalizedEntries.some(
|
||||
(entry) =>
|
||||
normalizeString(entry?.signal) === normalizedSignal &&
|
||||
normalizeString(entry?.outcome) === normalizedOutcome,
|
||||
);
|
||||
}
|
||||
|
||||
function buildCurrentBlockingVerificationFollowUp(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
) {
|
||||
const hasCurrentGuiSmokeFailure = hasObservabilityVerificationOutcome(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
"guiSmoke",
|
||||
"failed",
|
||||
);
|
||||
const hasCurrentBrowserVerificationFailure =
|
||||
hasObservabilityVerificationOutcome(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
"browserVerification",
|
||||
"failure",
|
||||
);
|
||||
const commands = ["npm run harness:eval", "npm run harness:eval:trend"];
|
||||
const backlogTools = [];
|
||||
const rationale = [];
|
||||
|
||||
if (hasCurrentGuiSmokeFailure) {
|
||||
commands.push("npm run verify:gui-smoke");
|
||||
rationale.push(
|
||||
"current 样本已出现 guiSmoke:failed，先恢复 GUI 壳 / DevBridge / Workspace 主路径的最小可启动性。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"优先收敛 GUI 壳 / DevBridge / Workspace 主路径，再复跑 `npm run verify:gui-smoke`。",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasCurrentBrowserVerificationFailure) {
|
||||
rationale.push(
|
||||
"current 样本已出现 browserVerification:failure，应先回看 browser replay / verification 失败样本，把失败断言回挂到受影响主路径。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"回看 browser replay / browser verification 失败样本，并把失败断言回挂到受影响主路径。",
|
||||
);
|
||||
}
|
||||
|
||||
if (backlogTools.length === 0) {
|
||||
backlogTools.push("按受影响主路径追加 `npm run verify:gui-smoke` 或专项 smoke");
|
||||
}
|
||||
|
||||
return {
|
||||
commands: dedupeNonEmptyStrings(commands),
|
||||
backlogTools: dedupeNonEmptyStrings(backlogTools),
|
||||
rationale: dedupeNonEmptyStrings(rationale),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCurrentAdvisoryVerificationFollowUp(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
) {
|
||||
const hasArtifactValidatorIssuesPresent = hasObservabilityVerificationOutcome(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
"artifactValidator",
|
||||
"issues_present",
|
||||
);
|
||||
const hasArtifactValidatorFallbackUsed = hasObservabilityVerificationOutcome(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
"artifactValidator",
|
||||
"fallback_used",
|
||||
);
|
||||
const hasBrowserVerificationUnknown = hasObservabilityVerificationOutcome(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
"browserVerification",
|
||||
"unknown",
|
||||
);
|
||||
const rationale = [];
|
||||
const backlogTools = [];
|
||||
|
||||
if (hasArtifactValidatorIssuesPresent) {
|
||||
rationale.push(
|
||||
"current 样本已出现 artifactValidator:issues_present，应先回看 validator issue 明细，再收敛 artifact 导出字段。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"回看 artifact validator issue 明细，并收敛 evidence pack / artifacts.json / analysis handoff 的 artifact 字段。",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasArtifactValidatorFallbackUsed) {
|
||||
rationale.push(
|
||||
"current 样本已出现 artifactValidator:fallback_used，说明 artifact 主路径仍不稳定，不能继续依赖 fallback 充当事实。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"补齐 artifact 主路径导出与修复链，减少 fallback_used 持续留在 current 样本。",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasBrowserVerificationUnknown) {
|
||||
rationale.push(
|
||||
"current 样本已出现 browserVerification:unknown，需要先把浏览器验证结果收敛成明确 outcome，再继续扩大分析。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"回看 browser verification 导出链，确保 evidence pack / replay / analysis handoff 写出明确 success 或 failure，而不是 unknown。",
|
||||
);
|
||||
}
|
||||
|
||||
if (backlogTools.length === 0) {
|
||||
backlogTools.push(
|
||||
"先对齐 current verification outcome 到 artifact/browser/gui 主路径，再继续补 observability 证据。",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
rationale: dedupeNonEmptyStrings(rationale),
|
||||
backlogTools: dedupeNonEmptyStrings(backlogTools),
|
||||
};
|
||||
}
|
||||
|
||||
function buildCurrentRecoveredVerificationFollowUp(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
) {
|
||||
const hasArtifactValidatorRepaired = hasObservabilityVerificationOutcome(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
"artifactValidator",
|
||||
"repaired",
|
||||
);
|
||||
const hasBrowserVerificationSuccess = hasObservabilityVerificationOutcome(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
"browserVerification",
|
||||
"success",
|
||||
);
|
||||
const hasGuiSmokePassed = hasObservabilityVerificationOutcome(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
"guiSmoke",
|
||||
"passed",
|
||||
);
|
||||
const commands = ["npm run harness:eval", "npm run harness:eval:trend"];
|
||||
const rationale = [];
|
||||
const backlogTools = [];
|
||||
|
||||
if (hasArtifactValidatorRepaired) {
|
||||
rationale.push(
|
||||
"current 样本已出现 artifactValidator:repaired，说明 artifact 修复链已经回到可复用的主路径。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"在 evidence pack / analysis handoff 里同时保留 artifact issue 与 repaired outcome，避免只剩修复结论而丢失修复上下文。",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasBrowserVerificationSuccess) {
|
||||
rationale.push(
|
||||
"current 样本已出现 browserVerification:success，可把浏览器验证成功样本固化成主路径正向基线。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"把 browser verification 成功样本固定进 current replay 基线，后续 failure 或 unknown 直接对比这条正向路径。",
|
||||
);
|
||||
}
|
||||
|
||||
if (hasGuiSmokePassed) {
|
||||
commands.push("npm run verify:gui-smoke");
|
||||
rationale.push(
|
||||
"current 样本已出现 guiSmoke:passed，可继续把 GUI smoke 通过链路当成桌面主路径的正向守卫。",
|
||||
);
|
||||
backlogTools.push(
|
||||
"主路径变更时优先复跑 `npm run verify:gui-smoke`，确认 GUI 壳 / DevBridge / Workspace 不从 passed 回退。",
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
commands: dedupeNonEmptyStrings(commands),
|
||||
rationale: dedupeNonEmptyStrings(rationale),
|
||||
backlogTools: dedupeNonEmptyStrings(backlogTools),
|
||||
};
|
||||
}
|
||||
|
||||
function buildRecommendations({
|
||||
trendSummary,
|
||||
verificationOutcomeSummary,
|
||||
@@ -1213,21 +781,25 @@ function buildRecommendations({
|
||||
const topObservabilitySignals = focusObservabilitySignals
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.status})`);
|
||||
const topVerificationFailureOutcomes = focusVerificationFailureOutcomes
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.outcome})`);
|
||||
const topVerificationFailureOutcomes = formatVerificationOutcomeCompactLabels(
|
||||
focusVerificationFailureOutcomes,
|
||||
3,
|
||||
);
|
||||
const topCurrentVerificationFailureOutcomes =
|
||||
focusCurrentObservabilityVerificationOutcomes
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.outcome})`);
|
||||
formatVerificationOutcomeCompactLabels(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
3,
|
||||
);
|
||||
const topCurrentRecoveredVerificationOutcomes =
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.outcome})`);
|
||||
formatVerificationOutcomeCompactLabels(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
3,
|
||||
);
|
||||
const topDegradedVerificationFailureOutcomes =
|
||||
focusDegradedObservabilityVerificationOutcomes
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.outcome})`);
|
||||
formatVerificationOutcomeCompactLabels(
|
||||
focusDegradedObservabilityVerificationOutcomes,
|
||||
3,
|
||||
);
|
||||
const topRecommendedVerificationFailureOutcomes =
|
||||
topCurrentVerificationFailureOutcomes.length > 0
|
||||
? topCurrentVerificationFailureOutcomes
|
||||
@@ -1241,15 +813,15 @@ function buildRecommendations({
|
||||
const degradedVerificationSummary =
|
||||
verificationOutcomeSummary?.degraded ?? buildVerificationOutcomeSummary([]);
|
||||
const currentBlockingVerificationFollowUp =
|
||||
buildCurrentBlockingVerificationFollowUp(
|
||||
buildBlockingVerificationFollowUp(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
);
|
||||
const currentAdvisoryVerificationFollowUp =
|
||||
buildCurrentAdvisoryVerificationFollowUp(
|
||||
buildAdvisoryVerificationFollowUp(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
);
|
||||
const currentRecoveredVerificationFollowUp =
|
||||
buildCurrentRecoveredVerificationFollowUp(
|
||||
buildRecoveredVerificationFollowUp(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
);
|
||||
|
||||
@@ -1314,16 +886,13 @@ function buildRecommendations({
|
||||
rationale: [
|
||||
`当前 failure mode 焦点：${topFailureModes.join("、") || "暂无"}。`,
|
||||
"先用 replay / eval 固化失败，再按受影响主路径补最小 smoke，而不是直接凭印象清理。",
|
||||
topCurrentVerificationFailureOutcomes.length > 0
|
||||
? `当前 current verification failure outcome 焦点：${topCurrentVerificationFailureOutcomes.join("、")}。`
|
||||
: "当前没有额外的 verification failure outcome 焦点。",
|
||||
...buildBlockingVerificationRecommendationRationale({
|
||||
topCurrentVerificationFailureOutcomes:
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
currentVerificationSummary,
|
||||
degradedVerificationSummary,
|
||||
}),
|
||||
...currentBlockingVerificationFollowUp.rationale,
|
||||
currentVerificationSummary.blockingFailureCaseCount > 0
|
||||
? `其中 current blocking verification failure 共 ${currentVerificationSummary.blockingFailureCaseCount} 个 case：${currentVerificationSummary.topBlockingFailureOutcomes.join("、") || "暂无"}。`
|
||||
: "当前没有额外的 blocking verification failure。",
|
||||
degradedVerificationSummary.blockingFailureCaseCount > 0
|
||||
? `另有 ${degradedVerificationSummary.blockingFailureCaseCount} 个 degraded blocking verification failure 样本作为诊断基线，不直接抬高主线优先级。`
|
||||
: "当前没有额外的 degraded blocking verification baseline。",
|
||||
],
|
||||
commands: currentBlockingVerificationFollowUp.commands,
|
||||
backlogTools: currentBlockingVerificationFollowUp.backlogTools,
|
||||
@@ -1409,38 +978,27 @@ function buildRecommendations({
|
||||
: "P2",
|
||||
title: "先补 observability 证据覆盖，再扩大外部分析与回归",
|
||||
rationale: [
|
||||
trendSummary.latestCurrentObservabilityGapCaseCount > 0
|
||||
? `当前仍有 ${trendSummary.latestCurrentObservabilityGapCaseCount} 个 current case 带着 observability 证据缺口进入 replay/eval。`
|
||||
: "当前 trend 已检测到 observability coverage 漂移，需先修证据而不是空谈根因分析。",
|
||||
trendSummary.latestDegradedObservabilityGapCaseCount > 0
|
||||
? `另有 ${trendSummary.latestDegradedObservabilityGapCaseCount} 个 degraded gap 样本作为诊断基线保留，它们不应直接被当成主线回归。`
|
||||
: "当前没有额外保留的 degraded observability gap 样本。",
|
||||
`当前缺口焦点：${topObservabilitySignals.join("、") || "暂无"}。这些缺口会直接降低 analysis handoff、人工审核和 cleanup report 的判断质量。`,
|
||||
topCurrentVerificationFailureOutcomes.length > 0
|
||||
? `当前 current verification failure outcome 焦点：${topCurrentVerificationFailureOutcomes.join("、")}。可用它们直接定位先补 artifact/browser/gui 哪一层。`
|
||||
: "当前没有额外的 verification failure outcome 焦点。",
|
||||
...buildObservabilityRecommendationRationale({
|
||||
trendSummary,
|
||||
topObservabilitySignals,
|
||||
topCurrentVerificationFailureOutcomes:
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
topDegradedVerificationFailureOutcomes:
|
||||
focusDegradedObservabilityVerificationOutcomes,
|
||||
currentVerificationSummary,
|
||||
}),
|
||||
...currentAdvisoryVerificationFollowUp.rationale,
|
||||
currentVerificationSummary.advisoryFailureCaseCount > 0
|
||||
? `当前 current advisory verification failure 共 ${currentVerificationSummary.advisoryFailureCaseCount} 个 case：${currentVerificationSummary.topAdvisoryFailureOutcomes.join("、") || "暂无"}。`
|
||||
: "当前没有额外的 advisory verification failure。",
|
||||
topDegradedVerificationFailureOutcomes.length > 0
|
||||
? `当前保留的 degraded verification baseline：${topDegradedVerificationFailureOutcomes.join("、")}。`
|
||||
: "当前没有额外的 degraded verification baseline。",
|
||||
],
|
||||
commands: [
|
||||
"npm run harness:eval",
|
||||
"npm run harness:eval:trend",
|
||||
"npm run harness:cleanup-report",
|
||||
],
|
||||
backlogTools: [
|
||||
"优先补 request telemetry 关联键、artifact validator outcome、browser/gui smoke 结果到 evidence pack / analysis handoff / replay。",
|
||||
...(topCurrentVerificationFailureOutcomes.length > 0
|
||||
? [
|
||||
`先对齐 current verification failure outcome：${topCurrentVerificationFailureOutcomes.join("、")}。`,
|
||||
]
|
||||
: []),
|
||||
...currentAdvisoryVerificationFollowUp.backlogTools,
|
||||
],
|
||||
backlogTools: buildObservabilityRecommendationBacklog({
|
||||
topCurrentVerificationFailureOutcomes:
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
advisoryFollowUpBacklogTools: currentAdvisoryVerificationFollowUp.backlogTools,
|
||||
}),
|
||||
focusFailureModes: topFailureModes,
|
||||
focusSuiteTags: topSuiteTags,
|
||||
focusReviewDecisionStatuses: topReviewDecisionStatuses,
|
||||
@@ -1493,9 +1051,11 @@ function buildRecommendations({
|
||||
: "P3",
|
||||
title: "把 recovered verification outcome 固化成 current 正向基线",
|
||||
rationale: [
|
||||
topCurrentRecoveredVerificationOutcomes.length > 0
|
||||
? `当前 current recovered outcome 焦点：${topCurrentRecoveredVerificationOutcomes.join("、")}。`
|
||||
: `当前 current recovered outcome 共 ${currentVerificationSummary.recoveredCaseCount} 个 case。`,
|
||||
...buildRecoveredVerificationRecommendationRationale({
|
||||
topCurrentRecoveredVerificationOutcomes:
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
currentVerificationSummary,
|
||||
}),
|
||||
...currentRecoveredVerificationFollowUp.rationale,
|
||||
"恢复成功的 outcome 不应只停留在统计卡里，还应继续回挂到 replay / smoke / evidence 主链，作为后续回退判断的正向对照。",
|
||||
],
|
||||
@@ -1590,117 +1150,31 @@ export function buildGeneratedSlopReport({
|
||||
trendReport?.classificationDeltas?.observabilitySignals,
|
||||
trendSummary.sampleCount,
|
||||
);
|
||||
const rawObservabilityVerificationOutcomes =
|
||||
buildObservabilityVerificationFocusEntries(
|
||||
trendReport?.classificationDeltas?.observabilityVerificationOutcomes,
|
||||
trendSummary.sampleCount,
|
||||
);
|
||||
const explicitRecoveredObservabilityVerificationOutcomes =
|
||||
buildVerificationOutcomeEntriesFromDeltas(
|
||||
trendReport?.classificationDeltas?.observabilityVerificationOutcomes,
|
||||
).filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) ===
|
||||
"recovered",
|
||||
);
|
||||
const focusVerificationFailureOutcomes =
|
||||
rawObservabilityVerificationOutcomes.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) !==
|
||||
"recovered",
|
||||
);
|
||||
const rawCurrentObservabilityVerificationOutcomes =
|
||||
buildObservabilityVerificationFocusEntries(
|
||||
trendReport?.classificationDeltas?.currentObservabilityVerificationOutcomes,
|
||||
trendSummary.sampleCount,
|
||||
);
|
||||
const explicitCurrentRecoveredObservabilityVerificationOutcomes =
|
||||
buildVerificationOutcomeEntriesFromDeltas(
|
||||
trendReport?.classificationDeltas?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
);
|
||||
const focusCurrentObservabilityVerificationOutcomes =
|
||||
rawCurrentObservabilityVerificationOutcomes.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) !==
|
||||
"recovered",
|
||||
);
|
||||
const focusCurrentRecoveredObservabilityVerificationOutcomes =
|
||||
explicitCurrentRecoveredObservabilityVerificationOutcomes.length > 0
|
||||
? explicitCurrentRecoveredObservabilityVerificationOutcomes
|
||||
: rawCurrentObservabilityVerificationOutcomes.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(
|
||||
entry?.signal,
|
||||
entry?.outcome,
|
||||
) === "recovered",
|
||||
);
|
||||
const rawDegradedObservabilityVerificationOutcomes =
|
||||
buildObservabilityVerificationFocusEntries(
|
||||
trendReport?.classificationDeltas?.degradedObservabilityVerificationOutcomes,
|
||||
trendSummary.sampleCount,
|
||||
);
|
||||
const focusDegradedObservabilityVerificationOutcomes =
|
||||
rawDegradedObservabilityVerificationOutcomes.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(entry?.signal, entry?.outcome) !==
|
||||
"recovered",
|
||||
);
|
||||
const verificationPresentation = deriveVerificationOutcomePresentationFromTrend({
|
||||
trendReport,
|
||||
sampleCount: trendSummary.sampleCount,
|
||||
});
|
||||
const mergedVerificationFailureOutcomes =
|
||||
focusVerificationFailureOutcomes.length > 0
|
||||
? focusVerificationFailureOutcomes
|
||||
: [
|
||||
...focusCurrentObservabilityVerificationOutcomes,
|
||||
...focusDegradedObservabilityVerificationOutcomes,
|
||||
].sort((left, right) => {
|
||||
if (right.score !== left.score) {
|
||||
return right.score - left.score;
|
||||
}
|
||||
return left.name.localeCompare(right.name);
|
||||
});
|
||||
const verificationFailureSummary = buildVerificationOutcomeSummary(
|
||||
mergedVerificationFailureOutcomes,
|
||||
);
|
||||
const recoveredVerificationSummary = buildVerificationOutcomeSummary(
|
||||
explicitRecoveredObservabilityVerificationOutcomes.length > 0
|
||||
? explicitRecoveredObservabilityVerificationOutcomes
|
||||
: mergedVerificationFailureOutcomes.filter(
|
||||
(entry) =>
|
||||
getObservabilityVerificationOutcomeRole(
|
||||
entry?.signal,
|
||||
entry?.outcome,
|
||||
) === "recovered",
|
||||
),
|
||||
);
|
||||
const verificationOutcomeSummary = {
|
||||
...verificationFailureSummary,
|
||||
recoveredFocusCount: recoveredVerificationSummary.recoveredFocusCount,
|
||||
recoveredCaseCount: recoveredVerificationSummary.recoveredCaseCount,
|
||||
topRecoveredOutcomes: recoveredVerificationSummary.topRecoveredOutcomes,
|
||||
};
|
||||
const currentVerificationFailureSummary = buildVerificationOutcomeSummary(
|
||||
focusCurrentObservabilityVerificationOutcomes,
|
||||
);
|
||||
const currentRecoveredVerificationSummary = buildVerificationOutcomeSummary(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
);
|
||||
const currentVerificationOutcomeSummary = {
|
||||
...currentVerificationFailureSummary,
|
||||
recoveredFocusCount: currentRecoveredVerificationSummary.recoveredFocusCount,
|
||||
recoveredCaseCount: currentRecoveredVerificationSummary.recoveredCaseCount,
|
||||
topRecoveredOutcomes: currentRecoveredVerificationSummary.topRecoveredOutcomes,
|
||||
};
|
||||
const degradedVerificationOutcomeSummary = buildVerificationOutcomeSummary(
|
||||
focusDegradedObservabilityVerificationOutcomes,
|
||||
);
|
||||
verificationPresentation.mergedVerificationFailureOutcomes;
|
||||
const focusVerificationFailureOutcomes = mergedVerificationFailureOutcomes;
|
||||
const focusCurrentObservabilityVerificationOutcomes =
|
||||
verificationPresentation.focusCurrentVerificationFailureOutcomes;
|
||||
const focusCurrentRecoveredObservabilityVerificationOutcomes =
|
||||
verificationPresentation.focusCurrentRecoveredVerificationOutcomes;
|
||||
const focusDegradedObservabilityVerificationOutcomes =
|
||||
verificationPresentation.focusDegradedVerificationFailureOutcomes;
|
||||
const combinedVerificationOutcomeSummary =
|
||||
verificationPresentation.verificationOutcomeSummary;
|
||||
const verificationOutcomeSummary = combinedVerificationOutcomeSummary;
|
||||
const currentVerificationOutcomeSummary =
|
||||
combinedVerificationOutcomeSummary.current;
|
||||
const degradedVerificationOutcomeSummary =
|
||||
combinedVerificationOutcomeSummary.degraded;
|
||||
const currentRecoveredVerificationOutcomes =
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.outcome})`);
|
||||
const combinedVerificationOutcomeSummary = {
|
||||
...verificationOutcomeSummary,
|
||||
current: currentVerificationOutcomeSummary,
|
||||
degraded: degradedVerificationOutcomeSummary,
|
||||
};
|
||||
formatVerificationOutcomeCompactLabels(
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
3,
|
||||
);
|
||||
const governanceSurfaces = buildGovernanceSurfaceEntries(governanceReport);
|
||||
const governanceSummary = buildGovernanceSummary(
|
||||
governanceReport,
|
||||
@@ -1771,30 +1245,15 @@ export function buildGeneratedSlopReport({
|
||||
trendSummary.latestDegradedObservabilityGapCaseCount > 0
|
||||
? `当前保留 ${trendSummary.latestDegradedObservabilityGapCaseCount} 个 degraded observability gap 样本作为诊断基线。`
|
||||
: "当前没有额外保留的 degraded observability gap 样本。",
|
||||
focusVerificationFailureOutcomes.length > 0
|
||||
? `当前 verification failure outcome 焦点：${focusVerificationFailureOutcomes
|
||||
.slice(0, 3)
|
||||
.map((entry) => `${entry.signal} (${entry.outcome})`)
|
||||
.join("、")}。`
|
||||
: "当前没有额外的 verification failure outcome 焦点。",
|
||||
verificationOutcomeSummary.failureCaseCount > 0
|
||||
? `当前 verification failure 聚焦 ${verificationOutcomeSummary.failureFocusCount} 类 outcome，共 ${verificationOutcomeSummary.failureCaseCount} 个 case。`
|
||||
: "当前没有额外的 verification failure case。",
|
||||
currentVerificationOutcomeSummary.blockingFailureCaseCount > 0
|
||||
? `当前 current 样本里有 ${currentVerificationOutcomeSummary.blockingFailureCaseCount} 个 blocking verification failure。`
|
||||
: "当前没有额外的 blocking verification failure。",
|
||||
currentVerificationOutcomeSummary.advisoryFailureCaseCount > 0
|
||||
? `当前 current 样本里有 ${currentVerificationOutcomeSummary.advisoryFailureCaseCount} 个 advisory verification failure。`
|
||||
: "当前没有额外的 advisory verification failure。",
|
||||
currentVerificationOutcomeSummary.recoveredCaseCount > 0
|
||||
? `当前 current recovered verification baseline：${currentRecoveredVerificationOutcomes.join("、") || "暂无"}。`
|
||||
: "当前没有额外的 current recovered verification baseline。",
|
||||
degradedVerificationOutcomeSummary.blockingFailureCaseCount > 0
|
||||
? `当前保留 ${degradedVerificationOutcomeSummary.blockingFailureCaseCount} 个 degraded blocking verification failure 样本作为诊断基线。`
|
||||
: "当前没有额外的 degraded blocking verification baseline。",
|
||||
verificationOutcomeSummary.recoveredCaseCount > 0
|
||||
? `当前 verification recovered 聚焦 ${verificationOutcomeSummary.recoveredFocusCount} 类 outcome，共 ${verificationOutcomeSummary.recoveredCaseCount} 个 case。`
|
||||
: "当前没有额外的 verification recovered case。",
|
||||
...buildVerificationOutcomeSignalMessages({
|
||||
focusVerificationFailureOutcomes,
|
||||
verificationOutcomeSummary,
|
||||
currentVerificationOutcomeSummary,
|
||||
degradedVerificationOutcomeSummary,
|
||||
currentRecoveredVerificationOutcomes:
|
||||
focusCurrentRecoveredObservabilityVerificationOutcomes,
|
||||
labelLimit: 3,
|
||||
}),
|
||||
],
|
||||
focus: {
|
||||
failureModes: focusFailureModes.slice(0, 5),
|
||||
@@ -1899,7 +1358,7 @@ export function renderGeneratedSlopText(report) {
|
||||
lines.push("[harness-cleanup] top observability verification outcomes:");
|
||||
for (const entry of report.focus.observabilityVerificationOutcomes) {
|
||||
lines.push(
|
||||
` - ${entry.signal} (${entry.outcome}): state=${entry.state}, latest_case=${entry.latest.caseCount}, delta_case=${entry.delta.caseCount}, score=${entry.score}`,
|
||||
` - ${formatVerificationOutcomeCompactLabel(entry)}: state=${entry.state}, latest_case=${entry.latest.caseCount}, delta_case=${entry.delta.caseCount}, score=${entry.score}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,8 @@
|
||||
import {
|
||||
describeVerificationOutcome,
|
||||
deriveVerificationDashboardPresentation,
|
||||
} from "./harness-verification-facts.mjs";
|
||||
|
||||
function normalizeNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value) ? value : 0;
|
||||
}
|
||||
@@ -141,49 +146,39 @@ function renderRecommendationList(recommendations) {
|
||||
.join("");
|
||||
}
|
||||
|
||||
function describeVerificationOutcome(entry) {
|
||||
const signal = normalizeString(entry?.signal, "unknown");
|
||||
const outcome = normalizeString(entry?.outcome, "unknown");
|
||||
|
||||
if (signal === "artifactValidator" && outcome === "issues_present") {
|
||||
return "当前 evidence 已记录 artifact 校验问题，优先回看 validator issue 明细。";
|
||||
}
|
||||
if (signal === "artifactValidator" && outcome === "fallback_used") {
|
||||
return "当前 artifact 导出仍触发 fallback，说明产物结构或修复链未完全稳定。";
|
||||
}
|
||||
if (signal === "browserVerification" && outcome === "failure") {
|
||||
return "浏览器验证已有明确失败结果，优先回挂到 replay 或 smoke 断言。";
|
||||
}
|
||||
if (signal === "browserVerification" && outcome === "unknown") {
|
||||
return "浏览器验证结果仍不明确，需要先补 outcome 再继续扩分析。";
|
||||
}
|
||||
if (signal === "guiSmoke" && outcome === "failed") {
|
||||
return "GUI smoke 已明确失败，应优先收敛到受影响主路径。";
|
||||
}
|
||||
if (signal === "guiSmoke" && outcome === "passed") {
|
||||
return "GUI smoke 已通过，可继续把注意力放回 gap 与其它失败面。";
|
||||
}
|
||||
if (signal === "artifactValidator" && outcome === "repaired") {
|
||||
return "artifact validator 已执行修复，可结合 issues/fallback 判断是否还需继续治理。";
|
||||
}
|
||||
if (signal === "browserVerification" && outcome === "success") {
|
||||
return "浏览器验证已有成功样本，可作为 current 主线路径的正向基线。";
|
||||
function deriveTrendSummary(trendReport, cleanupReport) {
|
||||
const cleanupTrendSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.trend
|
||||
? cleanupReport.summary.trend
|
||||
: null;
|
||||
if (cleanupTrendSummary) {
|
||||
return cleanupTrendSummary;
|
||||
}
|
||||
|
||||
return "当前 verification outcome 已进入 cleanup 主线，可直接据此定位先修哪层。";
|
||||
}
|
||||
const latestTotals =
|
||||
trendReport && typeof trendReport === "object" && trendReport.latest?.totals
|
||||
? trendReport.latest.totals
|
||||
: {};
|
||||
const delta = trendReport && typeof trendReport === "object" ? trendReport.delta : {};
|
||||
|
||||
const RECOVERED_VERIFICATION_OUTCOMES = new Set([
|
||||
"repaired",
|
||||
"success",
|
||||
"passed",
|
||||
"clean",
|
||||
]);
|
||||
|
||||
function isRecoveredVerificationOutcome(entry) {
|
||||
return RECOVERED_VERIFICATION_OUTCOMES.has(
|
||||
normalizeString(entry?.outcome, "unknown"),
|
||||
);
|
||||
return {
|
||||
sampleCount: normalizeNumber(trendReport?.sampleCount),
|
||||
latestCurrentObservabilityGapCaseCount: normalizeNumber(
|
||||
latestTotals.currentObservabilityGapCaseCount,
|
||||
),
|
||||
latestDegradedObservabilityGapCaseCount: normalizeNumber(
|
||||
latestTotals.degradedObservabilityGapCaseCount,
|
||||
),
|
||||
currentObservabilityGapCaseDelta: normalizeNumber(
|
||||
delta?.currentObservabilityGapCaseCount,
|
||||
),
|
||||
degradedObservabilityGapCaseDelta: normalizeNumber(
|
||||
delta?.degradedObservabilityGapCaseCount,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function renderFocusTable(title, entries, columns) {
|
||||
@@ -240,13 +235,7 @@ export function renderHarnessDashboardHtml({
|
||||
summaryReport && typeof summaryReport === "object" && summaryReport.totals
|
||||
? summaryReport.totals
|
||||
: {};
|
||||
const trendSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.trend
|
||||
? cleanupReport.summary.trend
|
||||
: {};
|
||||
const trendSummary = deriveTrendSummary(trendReport, cleanupReport);
|
||||
const governanceSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
@@ -254,13 +243,12 @@ export function renderHarnessDashboardHtml({
|
||||
cleanupReport.summary.governance
|
||||
? cleanupReport.summary.governance
|
||||
: {};
|
||||
const verificationSummary =
|
||||
cleanupReport &&
|
||||
typeof cleanupReport === "object" &&
|
||||
cleanupReport.summary &&
|
||||
cleanupReport.summary.verificationOutcomes
|
||||
? cleanupReport.summary.verificationOutcomes
|
||||
: {};
|
||||
const verificationPresentation = deriveVerificationDashboardPresentation({
|
||||
summaryReport,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
});
|
||||
const verificationSummary = verificationPresentation.verificationSummary;
|
||||
const currentVerificationSummary =
|
||||
verificationSummary &&
|
||||
typeof verificationSummary.current === "object" &&
|
||||
@@ -281,64 +269,11 @@ export function renderHarnessDashboardHtml({
|
||||
? cleanupReport.recommendations
|
||||
: [];
|
||||
const sampleRows = Array.isArray(trendReport?.samples) ? trendReport.samples : [];
|
||||
const currentVerificationFocusRows = Array.isArray(
|
||||
cleanupReport?.focus?.currentObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentObservabilityVerificationOutcomes.map((entry) => ({
|
||||
...entry,
|
||||
role: "current",
|
||||
}))
|
||||
: [];
|
||||
const degradedVerificationFocusRows = Array.isArray(
|
||||
cleanupReport?.focus?.degradedObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.degradedObservabilityVerificationOutcomes.map(
|
||||
(entry) => ({
|
||||
...entry,
|
||||
role: "degraded",
|
||||
}),
|
||||
)
|
||||
: [];
|
||||
const fallbackVerificationFocusRows = Array.isArray(
|
||||
cleanupReport?.focus?.observabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.observabilityVerificationOutcomes.map((entry) => ({
|
||||
...entry,
|
||||
role: "mixed",
|
||||
}))
|
||||
: [];
|
||||
const explicitCurrentRecoveredVerificationRows = Array.isArray(
|
||||
cleanupReport?.focus?.currentRecoveredObservabilityVerificationOutcomes,
|
||||
)
|
||||
? cleanupReport.focus.currentRecoveredObservabilityVerificationOutcomes.map(
|
||||
(entry) => ({
|
||||
...entry,
|
||||
role: "current",
|
||||
}),
|
||||
)
|
||||
: [];
|
||||
const verificationFocusRows =
|
||||
currentVerificationFocusRows.length > 0 ||
|
||||
degradedVerificationFocusRows.length > 0
|
||||
? [...currentVerificationFocusRows, ...degradedVerificationFocusRows]
|
||||
: fallbackVerificationFocusRows;
|
||||
const verificationFocusRows = verificationPresentation.verificationFocusRows;
|
||||
const currentRecoveredVerificationRows =
|
||||
explicitCurrentRecoveredVerificationRows.length > 0
|
||||
? explicitCurrentRecoveredVerificationRows
|
||||
: currentVerificationFocusRows.length > 0
|
||||
? currentVerificationFocusRows.filter((entry) =>
|
||||
isRecoveredVerificationOutcome(entry),
|
||||
)
|
||||
: fallbackVerificationFocusRows.filter((entry) =>
|
||||
isRecoveredVerificationOutcome(entry),
|
||||
);
|
||||
const currentRecoveredVerificationSummary = currentRecoveredVerificationRows
|
||||
.slice(0, 3)
|
||||
.map(
|
||||
(entry) =>
|
||||
`${normalizeString(entry?.signal, "-")} (${normalizeString(entry?.outcome, "-")})`,
|
||||
)
|
||||
.join("、");
|
||||
verificationPresentation.currentRecoveredRows;
|
||||
const currentRecoveredVerificationSummary =
|
||||
verificationPresentation.currentRecoveredSummaryLabel;
|
||||
|
||||
return `<!doctype html>
|
||||
<html lang="zh-CN">
|
||||
|
||||
@@ -3,6 +3,141 @@ import { describe, expect, it } from "vitest";
|
||||
import { renderHarnessDashboardHtml } from "./harness-dashboard-core.mjs";
|
||||
|
||||
describe("harness-dashboard-core", () => {
|
||||
it("应优先使用 trend 与 summary 的 verification facts，而不是 cleanup 渲染面", () => {
|
||||
const html = renderHarnessDashboardHtml({
|
||||
title: "Harness Engine Dashboard",
|
||||
summaryReport: {
|
||||
generatedAt: "2026-04-12T08:00:00.000Z",
|
||||
totals: {
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
},
|
||||
breakdowns: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{ name: "guiSmoke:failed", caseCount: 1 },
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{ name: "guiSmoke:failed", caseCount: 1 },
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
},
|
||||
trendReport: {
|
||||
generatedAt: "2026-04-12T08:01:00.000Z",
|
||||
sampleCount: 2,
|
||||
delta: {
|
||||
currentObservabilityGapCaseCount: 0,
|
||||
degradedObservabilityGapCaseCount: 0,
|
||||
},
|
||||
latest: {
|
||||
totals: {
|
||||
currentObservabilityGapCaseCount: 0,
|
||||
degradedObservabilityGapCaseCount: 0,
|
||||
currentRecoveredVerificationCaseCount: 1,
|
||||
},
|
||||
},
|
||||
signals: ["current gap 保持为 0。"],
|
||||
samples: [],
|
||||
classificationDeltas: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
},
|
||||
cleanupReport: {
|
||||
generatedAt: "2026-04-12T08:02:00.000Z",
|
||||
signals: [],
|
||||
recommendations: [],
|
||||
focus: {
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
outcome: "fallback_used",
|
||||
state: "regressing",
|
||||
latest: { caseCount: 2 },
|
||||
delta: { caseCount: 2 },
|
||||
},
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
outcome: "repaired",
|
||||
state: "expanding",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
summary: {
|
||||
trend: {
|
||||
sampleCount: 2,
|
||||
latestCurrentObservabilityGapCaseCount: 0,
|
||||
latestDegradedObservabilityGapCaseCount: 0,
|
||||
currentObservabilityGapCaseDelta: 0,
|
||||
degradedObservabilityGapCaseDelta: 0,
|
||||
},
|
||||
verificationOutcomes: {
|
||||
recoveredCaseCount: 0,
|
||||
current: {
|
||||
blockingFailureCaseCount: 0,
|
||||
advisoryFailureCaseCount: 2,
|
||||
recoveredCaseCount: 0,
|
||||
},
|
||||
degraded: {
|
||||
blockingFailureCaseCount: 0,
|
||||
advisoryFailureCaseCount: 0,
|
||||
},
|
||||
},
|
||||
governance: {
|
||||
violationCount: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(html).toMatch(/Current Blocking<\/span>\s*<strong>1<\/strong>/);
|
||||
expect(html).toMatch(/Current Recovered<\/span>\s*<strong>1<\/strong>/);
|
||||
expect(html).toContain("browserVerification (success)");
|
||||
expect(html).toContain("GUI smoke 已明确失败");
|
||||
expect(html).not.toContain("artifactValidator (repaired)、");
|
||||
expect(html).not.toContain("fallback_used");
|
||||
});
|
||||
|
||||
it("应把 summary、trend、cleanup 渲染成单一事实源 dashboard", () => {
|
||||
const html = renderHarnessDashboardHtml({
|
||||
title: "Harness Engine Dashboard",
|
||||
|
||||
@@ -4,6 +4,8 @@ import path from "node:path";
|
||||
import { execFile, execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
import { deriveHistoryRecordVerificationFacts } from "../harness-eval-history-record.mjs";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
@@ -59,6 +61,140 @@ afterEach(() => {
|
||||
});
|
||||
|
||||
describe("harness-eval-history-record", () => {
|
||||
it("应优先使用 summary 与 trend 的 verification facts，而不是 cleanup 反算结果", () => {
|
||||
const summary = {
|
||||
totals: {
|
||||
currentRecoveredVerificationCaseCount: 2,
|
||||
},
|
||||
breakdowns: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{ name: "guiSmoke:failed", caseCount: 2 },
|
||||
{ name: "artifactValidator:issues_present", caseCount: 1 },
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
{ name: "guiSmoke:passed", caseCount: 1 },
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{ name: "guiSmoke:failed", caseCount: 2 },
|
||||
{ name: "artifactValidator:issues_present", caseCount: 1 },
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
{ name: "guiSmoke:passed", caseCount: 1 },
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
{ name: "guiSmoke:passed", caseCount: 1 },
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [
|
||||
{ name: "browserVerification:failure", caseCount: 1 },
|
||||
],
|
||||
},
|
||||
};
|
||||
const trendReport = {
|
||||
latest: {
|
||||
totals: {
|
||||
currentRecoveredVerificationCaseCount: 2,
|
||||
},
|
||||
},
|
||||
classificationDeltas: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 2 },
|
||||
delta: { caseCount: 2 },
|
||||
},
|
||||
{
|
||||
name: "artifactValidator:issues_present",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 2 },
|
||||
delta: { caseCount: 2 },
|
||||
},
|
||||
{
|
||||
name: "artifactValidator:issues_present",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "guiSmoke:passed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "guiSmoke:passed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
},
|
||||
};
|
||||
const cleanupReport = {
|
||||
summary: {
|
||||
verificationOutcomes: {
|
||||
failureCaseCount: 99,
|
||||
recoveredCaseCount: 0,
|
||||
current: {
|
||||
blockingFailureCaseCount: 88,
|
||||
advisoryFailureCaseCount: 77,
|
||||
recoveredCaseCount: 0,
|
||||
},
|
||||
degraded: {
|
||||
blockingFailureCaseCount: 66,
|
||||
},
|
||||
},
|
||||
},
|
||||
focus: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "fallback_used" },
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "fallback_used" },
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "repaired" },
|
||||
],
|
||||
},
|
||||
};
|
||||
|
||||
const result = deriveHistoryRecordVerificationFacts({
|
||||
summary,
|
||||
trendReport,
|
||||
cleanupReport,
|
||||
});
|
||||
|
||||
expect(result.verificationFailureOutcomeFocus).toEqual([
|
||||
"guiSmoke:failed",
|
||||
"artifactValidator:issues_present",
|
||||
]);
|
||||
expect(result.currentRecoveredBaselineFocus).toEqual([
|
||||
"browserVerification:success",
|
||||
"guiSmoke:passed",
|
||||
]);
|
||||
expect(result.verificationOutcomeCounts).toEqual({
|
||||
failureCaseCount: 3,
|
||||
blockingFailureCaseCount: 2,
|
||||
advisoryFailureCaseCount: 1,
|
||||
recoveredCaseCount: 2,
|
||||
currentRecoveredCaseCount: 2,
|
||||
degradedBlockingFailureCaseCount: 1,
|
||||
});
|
||||
});
|
||||
|
||||
it("默认入口应产出完整 harness artifact 套件", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
|
||||
|
||||
@@ -59,6 +59,7 @@ describe("Harness repo fixtures", () => {
|
||||
expect(summary.totals.observabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.currentObservabilityGapCaseCount).toBe(0);
|
||||
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.currentRecoveredVerificationCaseCount).toBe(3);
|
||||
|
||||
const currentCase = repoFixtureSuite.cases.find(
|
||||
(entry: { caseId: string }) =>
|
||||
@@ -148,6 +149,20 @@ describe("Harness repo fixtures", () => {
|
||||
"browserVerification:success",
|
||||
);
|
||||
|
||||
const currentRecoveredVerificationOutcomeBreakdownNames =
|
||||
summary.breakdowns.currentRecoveredObservabilityVerificationOutcomes.map(
|
||||
(entry: { name: string }) => entry.name,
|
||||
);
|
||||
expect(currentRecoveredVerificationOutcomeBreakdownNames).toContain(
|
||||
"artifactValidator:repaired",
|
||||
);
|
||||
expect(currentRecoveredVerificationOutcomeBreakdownNames).toContain(
|
||||
"browserVerification:success",
|
||||
);
|
||||
expect(currentRecoveredVerificationOutcomeBreakdownNames).toContain(
|
||||
"guiSmoke:passed",
|
||||
);
|
||||
|
||||
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
|
||||
[],
|
||||
);
|
||||
|
||||
@@ -350,6 +350,7 @@ describe("Harness review decision / eval integration", () => {
|
||||
expect(summary.totals.observabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.currentObservabilityGapCaseCount).toBe(1);
|
||||
expect(summary.totals.degradedObservabilityGapCaseCount).toBe(0);
|
||||
expect(summary.totals.currentRecoveredVerificationCaseCount).toBe(3);
|
||||
expect(summary.breakdowns.reviewDecisionStatuses).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
@@ -402,6 +403,24 @@ describe("Harness review decision / eval integration", () => {
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
summary.breakdowns.currentRecoveredObservabilityVerificationOutcomes,
|
||||
).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "artifactValidator:repaired",
|
||||
caseCount: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "browserVerification:success",
|
||||
caseCount: 1,
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "guiSmoke:passed",
|
||||
caseCount: 1,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.breakdowns.degradedObservabilityVerificationOutcomes).toEqual(
|
||||
[],
|
||||
);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,524 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildAdvisoryVerificationFollowUp,
|
||||
buildAdvisoryVerificationRecommendationRationale,
|
||||
buildBlockingVerificationFollowUp,
|
||||
buildBlockingVerificationRecommendationRationale,
|
||||
buildObservabilityRecommendationBacklog,
|
||||
buildObservabilityRecommendationRationale,
|
||||
buildVerificationOutcomeSignalMessages,
|
||||
buildRecoveredVerificationRecommendationRationale,
|
||||
deriveVerificationDashboardPresentation,
|
||||
describeVerificationOutcome,
|
||||
formatVerificationOutcomeCompactLabels,
|
||||
buildRecoveredVerificationFollowUp,
|
||||
buildVerificationOutcomeEntriesFromBreakdowns,
|
||||
buildVerificationOutcomeEntriesFromDeltas,
|
||||
buildVerificationFocusEntriesFromDeltas,
|
||||
buildVerificationOutcomeSummary,
|
||||
deriveVerificationOutcomePresentationFromTrend,
|
||||
formatVerificationOutcomeCompactLabel,
|
||||
getVerificationOutcomeRole,
|
||||
getVerificationOutcomeWeight,
|
||||
hasVerificationOutcome,
|
||||
isVerificationFailureOutcome,
|
||||
isVerificationRecoveredOutcome,
|
||||
splitVerificationOutcomeName,
|
||||
} from "./harness-verification-facts.mjs";
|
||||
|
||||
describe("harness-verification-facts", () => {
|
||||
it("应统一解析 outcome 名称、角色与权重", () => {
|
||||
expect(splitVerificationOutcomeName("browserVerification:failure")).toEqual({
|
||||
name: "browserVerification:failure",
|
||||
signal: "browserVerification",
|
||||
outcome: "failure",
|
||||
});
|
||||
expect(splitVerificationOutcomeName("guiSmoke")).toEqual({
|
||||
name: "guiSmoke",
|
||||
signal: "guiSmoke",
|
||||
outcome: "",
|
||||
});
|
||||
|
||||
expect(getVerificationOutcomeRole("browserVerification", "failure")).toBe(
|
||||
"blocking_failure",
|
||||
);
|
||||
expect(getVerificationOutcomeRole("artifactValidator", "issues_present")).toBe(
|
||||
"advisory_failure",
|
||||
);
|
||||
expect(getVerificationOutcomeRole("browserVerification", "success")).toBe(
|
||||
"recovered",
|
||||
);
|
||||
expect(getVerificationOutcomeRole("other", "noop")).toBe("other");
|
||||
|
||||
expect(isVerificationFailureOutcome("fallback_used")).toBe(true);
|
||||
expect(isVerificationRecoveredOutcome("repaired")).toBe(true);
|
||||
expect(getVerificationOutcomeWeight("failed")).toBe(140);
|
||||
expect(getVerificationOutcomeWeight("repaired")).toBe(70);
|
||||
expect(getVerificationOutcomeWeight("noop")).toBe(0);
|
||||
});
|
||||
|
||||
it("应统一从 delta 与 breakdown 派生 verification entries 与 summary", () => {
|
||||
const deltaEntries = buildVerificationOutcomeEntriesFromDeltas([
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 2 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 0 },
|
||||
},
|
||||
]);
|
||||
const breakdownEntries = buildVerificationOutcomeEntriesFromBreakdowns([
|
||||
{
|
||||
name: "artifactValidator:issues_present",
|
||||
caseCount: 1,
|
||||
},
|
||||
]);
|
||||
|
||||
expect(deltaEntries.map((entry) => entry.name)).toEqual([
|
||||
"guiSmoke:failed",
|
||||
"browserVerification:success",
|
||||
]);
|
||||
expect(breakdownEntries[0]).toMatchObject({
|
||||
signal: "artifactValidator",
|
||||
outcome: "issues_present",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 0 },
|
||||
});
|
||||
|
||||
expect(
|
||||
buildVerificationOutcomeSummary([...deltaEntries, ...breakdownEntries]),
|
||||
).toMatchObject({
|
||||
failureCaseCount: 3,
|
||||
blockingFailureCaseCount: 2,
|
||||
advisoryFailureCaseCount: 1,
|
||||
recoveredCaseCount: 1,
|
||||
topBlockingFailureOutcomes: ["guiSmoke:failed"],
|
||||
topRecoveredOutcomes: ["browserVerification:success"],
|
||||
});
|
||||
});
|
||||
|
||||
it("应从 trend classification deltas 派生 cleanup 可复用的 verification presentation", () => {
|
||||
const presentation = deriveVerificationOutcomePresentationFromTrend({
|
||||
sampleCount: 2,
|
||||
trendReport: {
|
||||
classificationDeltas: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "artifactValidator:repaired",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "artifactValidator:repaired",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "artifactValidator:repaired",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:failure",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 0 },
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(
|
||||
buildVerificationFocusEntriesFromDeltas([
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
], 2)[0],
|
||||
).toMatchObject({
|
||||
signal: "guiSmoke",
|
||||
outcome: "failed",
|
||||
state: "regressing",
|
||||
});
|
||||
expect(
|
||||
presentation.focusCurrentVerificationFailureOutcomes.map(
|
||||
(entry) => entry.name,
|
||||
),
|
||||
).toEqual(["guiSmoke:failed"]);
|
||||
expect(
|
||||
presentation.focusCurrentRecoveredVerificationOutcomes.map(
|
||||
(entry) => entry.name,
|
||||
),
|
||||
).toEqual(["artifactValidator:repaired"]);
|
||||
expect(
|
||||
presentation.focusDegradedVerificationFailureOutcomes.map(
|
||||
(entry) => entry.name,
|
||||
),
|
||||
).toEqual(["browserVerification:failure"]);
|
||||
expect(presentation.verificationOutcomeSummary).toMatchObject({
|
||||
failureCaseCount: 1,
|
||||
recoveredCaseCount: 1,
|
||||
current: {
|
||||
blockingFailureCaseCount: 1,
|
||||
recoveredCaseCount: 1,
|
||||
},
|
||||
degraded: {
|
||||
blockingFailureCaseCount: 1,
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("应统一生成 blocking/advisory/recovered follow-up 建议", () => {
|
||||
const blockingEntries = [
|
||||
{
|
||||
signal: "guiSmoke",
|
||||
outcome: "failed",
|
||||
},
|
||||
{
|
||||
signal: "browserVerification",
|
||||
outcome: "failure",
|
||||
},
|
||||
];
|
||||
const advisoryEntries = [
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
outcome: "issues_present",
|
||||
},
|
||||
{
|
||||
signal: "browserVerification",
|
||||
outcome: "unknown",
|
||||
},
|
||||
];
|
||||
const recoveredEntries = [
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
outcome: "repaired",
|
||||
},
|
||||
{
|
||||
signal: "browserVerification",
|
||||
outcome: "success",
|
||||
},
|
||||
{
|
||||
signal: "guiSmoke",
|
||||
outcome: "passed",
|
||||
},
|
||||
];
|
||||
|
||||
expect(
|
||||
hasVerificationOutcome(blockingEntries, "guiSmoke", "failed"),
|
||||
).toBe(true);
|
||||
expect(
|
||||
hasVerificationOutcome(blockingEntries, "guiSmoke", "passed"),
|
||||
).toBe(false);
|
||||
|
||||
expect(buildBlockingVerificationFollowUp(blockingEntries)).toMatchObject({
|
||||
commands: [
|
||||
"npm run harness:eval",
|
||||
"npm run harness:eval:trend",
|
||||
"npm run verify:gui-smoke",
|
||||
],
|
||||
backlogTools: expect.arrayContaining([
|
||||
"优先收敛 GUI 壳 / DevBridge / Workspace 主路径，再复跑 `npm run verify:gui-smoke`。",
|
||||
"回看 browser replay / browser verification 失败样本，并把失败断言回挂到受影响主路径。",
|
||||
]),
|
||||
rationale: expect.arrayContaining([
|
||||
"current 样本已出现 guiSmoke:failed，先恢复 GUI 壳 / DevBridge / Workspace 主路径的最小可启动性。",
|
||||
"current 样本已出现 browserVerification:failure，应先回看 browser replay / verification 失败样本，把失败断言回挂到受影响主路径。",
|
||||
]),
|
||||
});
|
||||
|
||||
expect(buildAdvisoryVerificationFollowUp(advisoryEntries)).toMatchObject({
|
||||
backlogTools: expect.arrayContaining([
|
||||
"回看 artifact validator issue 明细，并收敛 evidence pack / artifacts.json / analysis handoff 的 artifact 字段。",
|
||||
"回看 browser verification 导出链，确保 evidence pack / replay / analysis handoff 写出明确 success 或 failure，而不是 unknown。",
|
||||
]),
|
||||
rationale: expect.arrayContaining([
|
||||
"current 样本已出现 artifactValidator:issues_present，应先回看 validator issue 明细，再收敛 artifact 导出字段。",
|
||||
"current 样本已出现 browserVerification:unknown，需要先把浏览器验证结果收敛成明确 outcome，再继续扩大分析。",
|
||||
]),
|
||||
});
|
||||
|
||||
expect(buildRecoveredVerificationFollowUp(recoveredEntries)).toMatchObject({
|
||||
commands: [
|
||||
"npm run harness:eval",
|
||||
"npm run harness:eval:trend",
|
||||
"npm run verify:gui-smoke",
|
||||
],
|
||||
backlogTools: expect.arrayContaining([
|
||||
"在 evidence pack / analysis handoff 里同时保留 artifact issue 与 repaired outcome，避免只剩修复结论而丢失修复上下文。",
|
||||
"把 browser verification 成功样本固定进 current replay 基线，后续 failure 或 unknown 直接对比这条正向路径。",
|
||||
"主路径变更时优先复跑 `npm run verify:gui-smoke`，确认 GUI 壳 / DevBridge / Workspace 不从 passed 回退。",
|
||||
]),
|
||||
rationale: expect.arrayContaining([
|
||||
"current 样本已出现 artifactValidator:repaired，说明 artifact 修复链已经回到可复用的主路径。",
|
||||
"current 样本已出现 browserVerification:success，可把浏览器验证成功样本固化成主路径正向基线。",
|
||||
"current 样本已出现 guiSmoke:passed，可继续把 GUI smoke 通过链路当成桌面主路径的正向守卫。",
|
||||
]),
|
||||
});
|
||||
});
|
||||
|
||||
it("应统一为 dashboard 派生 verification presentation 与说明文案", () => {
|
||||
const presentation = deriveVerificationDashboardPresentation({
|
||||
summaryReport: {
|
||||
breakdowns: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{ name: "guiSmoke:failed", caseCount: 1 },
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{ name: "guiSmoke:failed", caseCount: 1 },
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{ name: "browserVerification:success", caseCount: 1 },
|
||||
],
|
||||
},
|
||||
},
|
||||
trendReport: {
|
||||
classificationDeltas: {
|
||||
observabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "guiSmoke:failed",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
currentRecoveredObservabilityVerificationOutcomes: [
|
||||
{
|
||||
name: "browserVerification:success",
|
||||
latest: { caseCount: 1 },
|
||||
delta: { caseCount: 1 },
|
||||
},
|
||||
],
|
||||
degradedObservabilityVerificationOutcomes: [],
|
||||
},
|
||||
},
|
||||
cleanupReport: {
|
||||
focus: {
|
||||
currentObservabilityVerificationOutcomes: [
|
||||
{
|
||||
signal: "artifactValidator",
|
||||
outcome: "fallback_used",
|
||||
latest: { caseCount: 2 },
|
||||
delta: { caseCount: 2 },
|
||||
},
|
||||
],
|
||||
},
|
||||
summary: {
|
||||
verificationOutcomes: {
|
||||
current: {
|
||||
advisoryFailureCaseCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
expect(presentation.verificationSummary).toMatchObject({
|
||||
current: {
|
||||
blockingFailureCaseCount: 1,
|
||||
recoveredCaseCount: 1,
|
||||
},
|
||||
});
|
||||
expect(
|
||||
presentation.verificationFocusRows.map((entry) => entry.name),
|
||||
).toEqual(["guiSmoke:failed"]);
|
||||
expect(
|
||||
presentation.currentRecoveredRows.map((entry) => entry.name),
|
||||
).toEqual(["browserVerification:success"]);
|
||||
expect(presentation.currentRecoveredSummaryLabel).toBe(
|
||||
"browserVerification (success)",
|
||||
);
|
||||
expect(
|
||||
formatVerificationOutcomeCompactLabel("artifactValidator:repaired"),
|
||||
).toBe("artifactValidator (repaired)");
|
||||
expect(
|
||||
formatVerificationOutcomeCompactLabels(
|
||||
[
|
||||
{ signal: "artifactValidator", outcome: "repaired" },
|
||||
"browserVerification:success",
|
||||
],
|
||||
2,
|
||||
),
|
||||
).toEqual([
|
||||
"artifactValidator (repaired)",
|
||||
"browserVerification (success)",
|
||||
]);
|
||||
expect(
|
||||
describeVerificationOutcome({
|
||||
signal: "browserVerification",
|
||||
outcome: "success",
|
||||
}),
|
||||
).toContain("浏览器验证已有成功样本");
|
||||
});
|
||||
|
||||
it("应统一生成 verification summary signal 文案", () => {
|
||||
expect(
|
||||
buildVerificationOutcomeSignalMessages({
|
||||
focusVerificationFailureOutcomes: [
|
||||
{ signal: "browserVerification", outcome: "failure" },
|
||||
],
|
||||
verificationOutcomeSummary: {
|
||||
failureFocusCount: 1,
|
||||
failureCaseCount: 1,
|
||||
recoveredFocusCount: 1,
|
||||
recoveredCaseCount: 1,
|
||||
},
|
||||
currentVerificationOutcomeSummary: {
|
||||
blockingFailureCaseCount: 1,
|
||||
advisoryFailureCaseCount: 0,
|
||||
recoveredCaseCount: 1,
|
||||
},
|
||||
degradedVerificationOutcomeSummary: {
|
||||
blockingFailureCaseCount: 0,
|
||||
},
|
||||
currentRecoveredVerificationOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "repaired" },
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
"当前 verification failure outcome 焦点：browserVerification (failure)。",
|
||||
"当前 verification failure 聚焦 1 类 outcome，共 1 个 case。",
|
||||
"当前 current 样本里有 1 个 blocking verification failure。",
|
||||
"当前没有额外的 advisory verification failure。",
|
||||
"当前 current recovered verification baseline：artifactValidator (repaired)。",
|
||||
"当前没有额外的 degraded blocking verification baseline。",
|
||||
"当前 verification recovered 聚焦 1 类 outcome，共 1 个 case。",
|
||||
]);
|
||||
});
|
||||
|
||||
it("应统一生成 recommendation 用的 verification rationale 片段", () => {
|
||||
expect(
|
||||
buildBlockingVerificationRecommendationRationale({
|
||||
topCurrentVerificationFailureOutcomes: [
|
||||
{ signal: "browserVerification", outcome: "failure" },
|
||||
],
|
||||
currentVerificationSummary: {
|
||||
blockingFailureCaseCount: 1,
|
||||
topBlockingFailureOutcomes: ["browserVerification:failure"],
|
||||
},
|
||||
degradedVerificationSummary: {
|
||||
blockingFailureCaseCount: 0,
|
||||
},
|
||||
}),
|
||||
).toEqual([
|
||||
"当前 current verification failure outcome 焦点：browserVerification (failure)。",
|
||||
"其中 current blocking verification failure 共 1 个 case：browserVerification:failure。",
|
||||
"当前没有额外的 degraded blocking verification baseline。",
|
||||
]);
|
||||
|
||||
expect(
|
||||
buildAdvisoryVerificationRecommendationRationale({
|
||||
topCurrentVerificationFailureOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "issues_present" },
|
||||
],
|
||||
topDegradedVerificationFailureOutcomes: [
|
||||
{ signal: "guiSmoke", outcome: "failed" },
|
||||
],
|
||||
currentVerificationSummary: {
|
||||
advisoryFailureCaseCount: 1,
|
||||
topAdvisoryFailureOutcomes: ["artifactValidator:issues_present"],
|
||||
},
|
||||
}),
|
||||
).toEqual([
|
||||
"当前 current verification failure outcome 焦点：artifactValidator (issues_present)。可用它们直接定位先补 artifact/browser/gui 哪一层。",
|
||||
"当前 current advisory verification failure 共 1 个 case：artifactValidator:issues_present。",
|
||||
"当前保留的 degraded verification baseline：guiSmoke (failed)。",
|
||||
]);
|
||||
|
||||
expect(
|
||||
buildRecoveredVerificationRecommendationRationale({
|
||||
topCurrentRecoveredVerificationOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "repaired" },
|
||||
],
|
||||
currentVerificationSummary: {
|
||||
recoveredCaseCount: 1,
|
||||
},
|
||||
}),
|
||||
).toEqual([
|
||||
"当前 current recovered outcome 焦点：artifactValidator (repaired)。",
|
||||
]);
|
||||
});
|
||||
|
||||
it("应统一生成 observability recommendation 的混合文案与待办", () => {
|
||||
expect(
|
||||
buildObservabilityRecommendationRationale({
|
||||
trendSummary: {
|
||||
latestCurrentObservabilityGapCaseCount: 1,
|
||||
latestDegradedObservabilityGapCaseCount: 0,
|
||||
},
|
||||
topObservabilitySignals: ["requestTelemetry (known_gap)"],
|
||||
topCurrentVerificationFailureOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "issues_present" },
|
||||
],
|
||||
topDegradedVerificationFailureOutcomes: [
|
||||
{ signal: "guiSmoke", outcome: "failed" },
|
||||
],
|
||||
currentVerificationSummary: {
|
||||
advisoryFailureCaseCount: 1,
|
||||
topAdvisoryFailureOutcomes: ["artifactValidator:issues_present"],
|
||||
},
|
||||
}),
|
||||
).toEqual([
|
||||
"当前仍有 1 个 current case 带着 observability 证据缺口进入 replay/eval。",
|
||||
"当前没有额外保留的 degraded observability gap 样本。",
|
||||
"当前缺口焦点：requestTelemetry (known_gap)。这些缺口会直接降低 analysis handoff、人工审核和 cleanup report 的判断质量。",
|
||||
"当前 current verification failure outcome 焦点：artifactValidator (issues_present)。可用它们直接定位先补 artifact/browser/gui 哪一层。",
|
||||
"当前 current advisory verification failure 共 1 个 case：artifactValidator:issues_present。",
|
||||
"当前保留的 degraded verification baseline：guiSmoke (failed)。",
|
||||
]);
|
||||
|
||||
expect(
|
||||
buildObservabilityRecommendationBacklog({
|
||||
topCurrentVerificationFailureOutcomes: [
|
||||
{ signal: "artifactValidator", outcome: "issues_present" },
|
||||
],
|
||||
advisoryFollowUpBacklogTools: [
|
||||
"回看 artifact validator issue 明细，并收敛 evidence pack / artifacts.json / analysis handoff 的 artifact 字段。",
|
||||
],
|
||||
}),
|
||||
).toEqual([
|
||||
"优先补 request telemetry 关联键、artifact validator outcome、browser/gui smoke 结果到 evidence pack / analysis handoff / replay。",
|
||||
"先对齐 current verification failure outcome：artifactValidator (issues_present)。",
|
||||
"回看 artifact validator issue 明细，并收敛 evidence pack / artifacts.json / analysis handoff 的 artifact 字段。",
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user