opensource/proxycast
3
0
Fork 0
mirror of https://github.com/aiclientproxy/proxycast.git synced 2026-09-24 23:10:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: finalize latest v0.97.0 release payload

Browse Source
This commit is contained in:
coso
2026-03-27 21:46:17 +08:00
parent dedf46c4be
commit 02ed1cd5b9
42 changed files with 1913 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/harness-analysis-brief.mjs
+51
View File
@@ -374,6 +374,39 @@ function buildAnalysisContext({
title,
workspaceRoot,
}) {
const rawObservabilitySummary =
inputPayload?.observability ?? evidenceJson?.observabilitySummary ?? null;
const observabilitySummary = sanitizeValue(
rawObservabilitySummary ?? {},
workspaceRoot,
options.sanitizedWorkspaceRoot,
);
const observabilityCorrelationKeys = Array.isArray(
observabilitySummary?.correlation?.correlationKeys,
)
? observabilitySummary.correlation.correlationKeys
.map((entry) => (typeof entry === "string" ? entry.trim() : ""))
.filter(Boolean)
: [];
const observabilityGapSignals = Array.isArray(
observabilitySummary?.signalCoverage,
)
? observabilitySummary.signalCoverage
.map((entry) => {
if (!entry || typeof entry !== "object") {
return "";
}
const signal =
typeof entry.signal === "string" ? entry.signal.trim() : "";
const status =
typeof entry.status === "string" ? entry.status.trim() : "";
if (!signal || !status || status === "exported") {
return "";
}
return `${signal} (${status})`;
})
.filter(Boolean)
: [];
const sanitizedInput = sanitizeValue(
{
session: inputPayload?.session ?? {},
@@ -505,6 +538,11 @@ function buildAnalysisContext({
),
),
},
observability: {
summary: observabilitySummary,
correlationKeys: observabilityCorrelationKeys,
gapSignals: observabilityGapSignals,
},
readingOrder: buildReadingOrder(handoffArtifacts, evidenceArtifacts),
externalAnalysisContract: buildExternalAnalysisPromptContract(),
humanReviewChecklist: buildHumanReviewChecklist(inputPayload, expectedPayload),
@@ -555,6 +593,19 @@ function buildAnalysisBrief(context) {
`- pending request：${context.summary.pendingRequestCount}`,
`- queued turn：${context.summary.queuedTurnCount}`,
"",
"## 证据关联与可观测覆盖",
"",
`- 关联键：${
context.observability.correlationKeys.length > 0
? context.observability.correlationKeys.join(", ")
: "无"
}`,
`- 当前缺口：${
context.observability.gapSignals.length > 0
? context.observability.gapSignals.join(", ")
: "无"
}`,
"",
"## 推荐读取顺序",
"",
...context.readingOrder.map((entry, index) => `${index + 1}. ${entry}`),
scripts/harness-eval-history-record.mjs
+32 -9
View File
@@ -16,7 +16,7 @@ function parseArgs(argv) {
cleanupMarkdown: "",
format: "text",
help: false,
historyDir: "./artifacts/history",
historyDir: "./.lime/harness/history",
outputJson: "",
retain: 30,
skipCleanup: false,
@@ -108,10 +108,10 @@ Lime Harness Eval History Record
用法:
node scripts/harness-eval-history-record.mjs
node scripts/harness-eval-history-record.mjs --workspace-root "/path/to/workspace"
node scripts/harness-eval-history-record.mjs --history-dir "./artifacts/history" --output-json "./tmp/harness-history-record.json"
node scripts/harness-eval-history-record.mjs --history-dir "./.lime/harness/history" --output-json "./tmp/harness-history-record.json"
选项:
--history-dir PATH summary 历史目录，默认 ./artifacts/history
--history-dir PATH summary 历史目录，默认 ./.lime/harness/history
--workspace-root PATH 生成当前 summary 时使用的工作区根目录
--retain N 历史窗口保留数量，默认 30
--trend-json PATH trend JSON 输出路径
@@ -173,6 +173,29 @@ function writeJsonFile(filePath, payload) {
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
}
function writeUniqueHistorySummary(historyDir, payload) {
ensureParentDirectory(historyDir);
const json = `${JSON.stringify(payload, null, 2)}\n`;
const baseName = `${timestampForFilename()}-harness-eval-summary`;
for (let index = 0; index < 1000; index += 1) {
const suffix = index === 0 ? "" : `-${index}`;
const filePath = path.join(historyDir, `${baseName}${suffix}.json`);
try {
fs.writeFileSync(filePath, json, { encoding: "utf8", flag: "wx" });
return filePath;
} catch (error) {
if (error?.code === "EEXIST") {
continue;
}
throw error;
}
}
throw new Error(`无法在历史目录中创建唯一 summary 文件: ${historyDir}`);
}
function trimHistoryFiles(historyDir, retain) {
const files = collectHistoryFiles(historyDir).sort((left, right) =>
right.localeCompare(left),
@@ -185,7 +208,11 @@ function trimHistoryFiles(historyDir, retain) {
}
function buildDefaultArtifactPaths(historyDir) {
const artifactsRoot = path.dirname(historyDir);
const historyParent = path.dirname(historyDir);
const artifactsRoot =
path.basename(historyDir) === "history"
? path.join(historyParent, "reports")
: historyParent;
return {
trendJson: path.join(artifactsRoot, "harness-eval-trend.json"),
trendMarkdown: path.join(artifactsRoot, "harness-eval-trend.md"),
@@ -238,11 +265,7 @@ function runHistoryRecordCli() {
fs.mkdirSync(historyDir, { recursive: true });
const summary = buildCurrentSummary(repoRoot, options.workspaceRoot);
const summaryFilePath = path.join(
historyDir,
`${timestampForFilename()}-harness-eval-summary.json`,
);
writeJsonFile(summaryFilePath, summary);
const summaryFilePath = writeUniqueHistorySummary(historyDir, summary);
const trimmedPaths = trimHistoryFiles(historyDir, options.retain);
const historyCount = collectHistoryFiles(historyDir).length;
const defaults = buildDefaultArtifactPaths(historyDir);
scripts/harness-eval-runner.mjs
+140
View File
@@ -388,6 +388,79 @@ function loadReviewDecisionForCase(caseDir, inlineReviewDecision) {
};
}
function normalizeObservabilityCoverage(inputPayload, evidencePayload) {
const inlineSummary =
inputPayload?.observability &&
typeof inputPayload.observability === "object" &&
!Array.isArray(inputPayload.observability)
? inputPayload.observability
: evidencePayload?.observabilitySummary &&
typeof evidencePayload.observabilitySummary === "object" &&
!Array.isArray(evidencePayload.observabilitySummary)
? evidencePayload.observabilitySummary
: null;
if (!inlineSummary) {
return [
{
signal: "observabilitySummary",
status: "missing",
},
];
}
const signalCoverage = Array.isArray(inlineSummary.signalCoverage)
? inlineSummary.signalCoverage
: [];
if (signalCoverage.length === 0) {
return [
{
signal: "observabilitySummary",
status: "missing_signal_coverage",
},
];
}
const seen = new Set();
const coverage = [];
for (const entry of signalCoverage) {
if (!entry || typeof entry !== "object") {
continue;
}
const signal = normalizeOptionalString(entry.signal ?? entry.name);
const status = normalizeOptionalString(entry.status);
if (!signal || !status) {
continue;
}
const fingerprint = `${signal}:${status}`;
if (seen.has(fingerprint)) {
continue;
}
seen.add(fingerprint);
coverage.push({
signal,
status,
source: normalizeOptionalString(entry.source),
detail: normalizeOptionalString(entry.detail),
});
}
if (coverage.length === 0) {
return [
{
signal: "observabilitySummary",
status: "missing_signal_coverage",
},
];
}
return coverage;
}
function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
const requiredArtifacts = normalizeStringList(
caseConfig.requiredArtifacts ?? defaults.requiredArtifacts,
@@ -468,6 +541,16 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
);
reviewDecision = reviewDecisionResult.reviewDecision;
issues.push(...reviewDecisionResult.issues);
const observabilityCoverage = normalizeObservabilityCoverage(
inputPayload,
evidencePayload,
);
const observabilitySignals = observabilityCoverage.map(
(entry) => `${entry.signal}:${entry.status}`,
);
const observabilityGapCount = observabilityCoverage.filter(
(entry) => entry.status !== "exported",
).length;
const pendingRequestCount = Array.isArray(
inputPayload?.runtimeContext?.pendingRequests,
@@ -520,6 +603,9 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
reviewRiskLevel: reviewDecision?.riskLevel ?? "",
reviewHumanReviewer: reviewDecision?.humanReviewer ?? "",
reviewReviewedAt: reviewDecision?.reviewedAt ?? "",
observabilityCoverage,
observabilitySignals,
observabilityGapCount,
preferredMode,
status: issues.length === 0 ? "ready" : "invalid",
issues,
@@ -584,6 +670,11 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
reviewRiskLevel: "",
reviewHumanReviewer: "",
reviewReviewedAt: "",
observabilityCoverage: [
{ signal: "observabilitySummary", status: "missing" },
],
observabilitySignals: ["observabilitySummary:missing"],
observabilityGapCount: 1,
preferredMode: "",
status: "invalid",
issues: [
@@ -633,6 +724,9 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
reviewRiskLevel: "",
reviewHumanReviewer: "",
reviewReviewedAt: "",
observabilityCoverage: [{ signal: "observabilitySummary", status: "missing" }],
observabilitySignals: ["observabilitySummary:missing"],
observabilityGapCount: 1,
preferredMode: "",
status: "invalid",
issues: [`不支持的 case source: ${source || "(empty)"}`],
@@ -676,6 +770,9 @@ function buildSummary(manifest, suites, options) {
const recordedReviewDecisionCases = allCases.filter(
(entry) => entry.reviewDecisionRecorded,
);
const observabilityGapCases = allCases.filter(
(entry) => entry.observabilityGapCount > 0,
);
return {
manifestVersion: String(manifest.manifestVersion ?? "unknown"),
@@ -692,6 +789,7 @@ function buildSummary(manifest, suites, options) {
needsHumanReviewCount: reviewCases.length,
pendingRequestCaseCount: pendingCases.length,
reviewDecisionRecordedCount: recordedReviewDecisionCases.length,
observabilityGapCaseCount: observabilityGapCases.length,
},
breakdowns: {
suiteTags: aggregateCaseBreakdown(allCases, (entry) => entry.tags),
@@ -705,6 +803,10 @@ function buildSummary(manifest, suites, options) {
reviewRiskLevels: aggregateCaseBreakdown(allCases, (entry) =>
entry.reviewRiskLevel ? [entry.reviewRiskLevel] : [],
),
observabilitySignals: aggregateCaseBreakdown(
allCases,
(entry) => entry.observabilitySignals,
),
},
suites,
};
@@ -721,6 +823,7 @@ function renderText(summary) {
`[harness-eval] pending-request cases: ${summary.totals.pendingRequestCaseCount}`,
`[harness-eval] needs-review cases : ${summary.totals.needsHumanReviewCount}`,
`[harness-eval] recorded review decisions: ${summary.totals.reviewDecisionRecordedCount}`,
`[harness-eval] observability-gap cases: ${summary.totals.observabilityGapCaseCount}`,
];
const topFailureModes = summary.breakdowns.failureModes.slice(0, 5);
@@ -753,6 +856,19 @@ function renderText(summary) {
}
}
const topObservabilitySignals = summary.breakdowns.observabilitySignals.slice(
0,
5,
);
if (topObservabilitySignals.length > 0) {
lines.push("[harness-eval] observability signals:");
for (const entry of topObservabilitySignals) {
lines.push(
` - ${entry.name}: case=${entry.caseCount}, ready=${entry.readyCount}, invalid=${entry.invalidCount}`,
);
}
}
for (const suite of summary.suites) {
lines.push(
`[harness-eval] suite ${suite.id}: ready ${suite.stats.readyCount} / ${suite.stats.caseCount}`,
@@ -772,6 +888,11 @@ function renderText(summary) {
` review: ${entry.reviewDecisionStatus} / risk=${entry.reviewRiskLevel || "unknown"}`,
);
}
if (entry.observabilitySignals.length > 0) {
lines.push(
` observability: ${entry.observabilitySignals.join(", ")}`,
);
}
for (const issue of entry.issues) {
lines.push(` * ${issue}`);
}
@@ -795,6 +916,7 @@ function renderMarkdown(summary) {
`- pending request case：${summary.totals.pendingRequestCaseCount}`,
`- needs review case：${summary.totals.needsHumanReviewCount}`,
`- 已记录人工审核：${summary.totals.reviewDecisionRecordedCount}`,
`- observability gap case：${summary.totals.observabilityGapCaseCount}`,
"",
];
@@ -852,6 +974,19 @@ function renderMarkdown(summary) {
lines.push("");
}
if (summary.breakdowns.observabilitySignals.length > 0) {
lines.push("## Observability Coverage 分布");
lines.push("");
lines.push("| Signal | case | ready | invalid |");
lines.push("| --- | --- | --- | --- |");
for (const entry of summary.breakdowns.observabilitySignals) {
lines.push(
`| ${entry.name} | ${entry.caseCount} | ${entry.readyCount} | ${entry.invalidCount} |`,
);
}
lines.push("");
}
for (const suite of summary.suites) {
lines.push(`## ${suite.title}`);
lines.push("");
@@ -885,6 +1020,11 @@ function renderMarkdown(summary) {
if (entry.primaryBlockingKind) {
classificationText.push(`blocking: ${entry.primaryBlockingKind}`);
}
if (entry.observabilitySignals.length > 0) {
classificationText.push(
`observability: ${entry.observabilitySignals.join(", ")}`,
);
}
const reviewText = entry.reviewDecisionStatus
? [
entry.reviewDecisionStatus,
scripts/harness-eval-trend-report.mjs
+43
View File
@@ -324,6 +324,9 @@ function buildStatusSignals(baseline, latest, sampleCount) {
const pendingDelta =
normalizeNumber(latest?.totals?.pendingRequestCaseCount) -
normalizeNumber(baseline?.totals?.pendingRequestCaseCount);
const observabilityGapDelta =
normalizeNumber(latest?.totals?.observabilityGapCaseCount) -
normalizeNumber(baseline?.totals?.observabilityGapCaseCount);
if (invalidDelta > 0) {
signals.push(`invalid case 增加 ${invalidDelta}，存在回归候选。`);
@@ -341,6 +344,12 @@ function buildStatusSignals(baseline, latest, sampleCount) {
);
}
if (observabilityGapDelta > 0) {
signals.push(
`observability gap case 增加 ${observabilityGapDelta}，需先补 evidence / analysis / replay 的证据覆盖。`,
);
}
const failureModeDeltas = buildBreakdownDeltas(
baseline,
latest,
@@ -419,6 +428,9 @@ function buildTrendReport(samples, repoRoot) {
reviewDecisionRecordedCount:
normalizeNumber(latest?.totals?.reviewDecisionRecordedCount) -
normalizeNumber(baseline?.totals?.reviewDecisionRecordedCount),
observabilityGapCaseCount:
normalizeNumber(latest?.totals?.observabilityGapCaseCount) -
normalizeNumber(baseline?.totals?.observabilityGapCaseCount),
readyRate: readyRateDelta,
},
signals: buildStatusSignals(baseline, latest, sortedSamples.length),
@@ -441,6 +453,11 @@ function buildTrendReport(samples, repoRoot) {
latest,
"reviewRiskLevels",
),
observabilitySignals: buildBreakdownDeltas(
baseline,
latest,
"observabilitySignals",
),
},
};
}
@@ -455,6 +472,7 @@ function renderText(report) {
`[harness-eval-trend] delta invalidCount: ${report.delta.invalidCount}`,
`[harness-eval-trend] delta pendingRequestCaseCount: ${report.delta.pendingRequestCaseCount}`,
`[harness-eval-trend] delta reviewDecisionRecordedCount: ${report.delta.reviewDecisionRecordedCount}`,
`[harness-eval-trend] delta observabilityGapCaseCount: ${report.delta.observabilityGapCaseCount}`,
`[harness-eval-trend] delta readyRate: ${(report.delta.readyRate * 100).toFixed(1)}%`,
];
@@ -486,6 +504,17 @@ function renderText(report) {
}
}
const topObservabilityDeltas =
report.classificationDeltas.observabilitySignals.slice(0, 5);
if (topObservabilityDeltas.length > 0) {
lines.push("[harness-eval-trend] observability signal deltas:");
for (const entry of topObservabilityDeltas) {
lines.push(
` - ${entry.name}: delta_case=${entry.delta.caseCount}, latest_case=${entry.latest.caseCount}, latest_invalid=${entry.latest.invalidCount}`,
);
}
}
return `${lines.join("\n")}\n`;
}
@@ -507,6 +536,7 @@ function renderMarkdown(report) {
`- pending request case 变化：${report.delta.pendingRequestCaseCount}`,
`- needs review case 变化：${report.delta.needsHumanReviewCount}`,
`- 已记录人工审核变化：${report.delta.reviewDecisionRecordedCount}`,
`- observability gap case 变化：${report.delta.observabilityGapCaseCount}`,
`- ready rate 变化：${(report.delta.readyRate * 100).toFixed(1)}%`,
"",
"## 信号",
@@ -573,6 +603,19 @@ function renderMarkdown(report) {
}
}
if (report.classificationDeltas.observabilitySignals.length > 0) {
lines.push("");
lines.push("## Observability Coverage 变化");
lines.push("");
lines.push("| Signal | baseline case | latest case | delta case | delta invalid |");
lines.push("| --- | --- | --- | --- | --- |");
for (const entry of report.classificationDeltas.observabilitySignals) {
lines.push(
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} | ${entry.delta.invalidCount} |`,
);
}
}
lines.push("");
lines.push("## 时间线样本");
lines.push("");
scripts/lib/generated-slop-report-core.mjs
+188
View File
@@ -394,11 +394,119 @@ function buildTrendSummary(trendReport) {
0,
latestNeedsReviewCount - latestReviewDecisionRecordedCount,
),
observabilityGapCaseDelta: normalizeNumber(delta.observabilityGapCaseCount),
latestObservabilityGapCaseCount: normalizeNumber(
latestTotals.observabilityGapCaseCount,
),
readyRateDelta: normalizeNumber(delta.readyRate),
signals: Array.isArray(trendReport?.signals) ? trendReport.signals : [],
};
}
function splitObservabilitySignalName(name) {
const normalized = normalizeString(name);
if (!normalized.includes(":")) {
return {
signal: normalized,
status: "",
};
}
const separatorIndex = normalized.indexOf(":");
return {
signal: normalized.slice(0, separatorIndex),
status: normalized.slice(separatorIndex + 1),
};
}
function getObservabilityStatusWeight(status) {
switch (status) {
case "missing":
return 120;
case "missing_signal_coverage":
return 110;
case "unlinked":
return 95;
case "known_gap":
return 85;
case "partial":
return 70;
default:
return 50;
}
}
function buildObservabilityFocusEntries(entries, sampleCount) {
const normalizedEntries = Array.isArray(entries) ? entries : [];
return normalizedEntries
.map((entry) => {
const latest = isObject(entry?.latest) ? entry.latest : {};
const delta = isObject(entry?.delta) ? entry.delta : {};
const baseline = isObject(entry?.baseline) ? entry.baseline : {};
const parsed = splitObservabilitySignalName(entry?.name);
const positiveDeltaCase = Math.max(0, normalizeNumber(delta.caseCount));
const latestCase = normalizeNumber(latest.caseCount);
const score =
positiveDeltaCase * 120 +
latestCase * getObservabilityStatusWeight(parsed.status);
let state = "stable";
if (sampleCount < 2 && latestCase > 0) {
state = "seed-risk";
} else if (positiveDeltaCase > 0) {
state = "regressing";
} else if (latestCase > 0) {
state = "present";
}
return {
name: normalizeString(entry?.name, "(unknown)"),
signal: parsed.signal || "(unknown)",
status: parsed.status || "unknown",
baseline: {
caseCount: normalizeNumber(baseline.caseCount),
readyCount: normalizeNumber(baseline.readyCount),
invalidCount: normalizeNumber(baseline.invalidCount),
pendingRequestCaseCount: normalizeNumber(
baseline.pendingRequestCaseCount,
),
needsHumanReviewCount: normalizeNumber(
baseline.needsHumanReviewCount,
),
},
latest: {
caseCount: latestCase,
readyCount: normalizeNumber(latest.readyCount),
invalidCount: normalizeNumber(latest.invalidCount),
pendingRequestCaseCount: normalizeNumber(
latest.pendingRequestCaseCount,
),
needsHumanReviewCount: normalizeNumber(
latest.needsHumanReviewCount,
),
},
delta: {
caseCount: normalizeNumber(delta.caseCount),
readyCount: normalizeNumber(delta.readyCount),
invalidCount: normalizeNumber(delta.invalidCount),
pendingRequestCaseCount: normalizeNumber(delta.pendingRequestCaseCount),
needsHumanReviewCount: normalizeNumber(delta.needsHumanReviewCount),
},
state,
score,
};
})
.filter((entry) => entry.status && entry.status !== "exported")
.filter((entry) => entry.score > 0 || entry.latest.caseCount > 0)
.sort((left, right) => {
if (right.score !== left.score) {
return right.score - left.score;
}
return left.name.localeCompare(right.name);
});
}
function buildDocFreshnessSummary(docFreshnessReport) {
const summary = isObject(docFreshnessReport?.summary)
? docFreshnessReport.summary
@@ -469,6 +577,7 @@ function buildRecommendations({
focusSuiteTags,
focusReviewDecisionStatuses,
focusReviewRiskLevels,
focusObservabilitySignals,
docFreshnessSummary,
docFreshnessFocus,
governanceSummary,
@@ -485,6 +594,9 @@ function buildRecommendations({
const topReviewRiskLevels = focusReviewRiskLevels
.slice(0, 3)
.map((entry) => entry.name);
const topObservabilitySignals = focusObservabilitySignals
.slice(0, 3)
.map((entry) => `${entry.signal} (${entry.status})`);
const topGovernanceSurfaceIds = governanceSurfaces
.filter((entry) => entry.active && entry.classification !== "current")
.slice(0, 3)
@@ -606,6 +718,47 @@ function buildRecommendations({
});
}
const hasObservabilityPressure =
trendSummary.latestObservabilityGapCaseCount > 0 ||
trendSummary.observabilityGapCaseDelta > 0 ||
focusObservabilitySignals.some(
(entry) => entry.latest.caseCount > 0 || entry.delta.caseCount > 0,
);
if (hasObservabilityPressure) {
maybePushRecommendation(recommendations, {
id: "observability-evidence-follow-up",
priority:
topObservabilitySignals.some(
(entry) =>
entry.includes("observabilitySummary (missing)") ||
entry.includes("requestTelemetry (unlinked)"),
) || trendSummary.observabilityGapCaseDelta > 0
? "P1"
: "P2",
title: "先补 observability 证据覆盖，再扩大外部分析与回归",
rationale: [
trendSummary.latestObservabilityGapCaseCount > 0
? `当前仍有 ${trendSummary.latestObservabilityGapCaseCount} 个 case 带着 observability 证据缺口进入 replay/eval。`
: "当前 trend 已检测到 observability coverage 漂移，需先修证据而不是空谈根因分析。",
`当前缺口焦点：${topObservabilitySignals.join("、") || "暂无"}。这些缺口会直接降低 analysis handoff、人工审核和 cleanup report 的判断质量。`,
],
commands: [
"npm run harness:eval",
"npm run harness:eval:trend",
"npm run harness:cleanup-report",
],
backlogTools: [
"优先补 request telemetry 关联键、artifact validator outcome、browser/gui smoke 结果到 evidence pack / analysis handoff / replay。",
],
focusFailureModes: topFailureModes,
focusSuiteTags: topSuiteTags,
focusReviewDecisionStatuses: topReviewDecisionStatuses,
focusReviewRiskLevels: topReviewRiskLevels,
focusSurfaceIds: [],
});
}
const hasGovernancePressure =
governanceSummary.activeByClassification.compat > 0 ||
governanceSummary.activeByClassification.deprecated > 0 ||
@@ -707,6 +860,10 @@ export function buildGeneratedSlopReport({
trendReport?.classificationDeltas?.reviewRiskLevels,
trendSummary.sampleCount,
);
const focusObservabilitySignals = buildObservabilityFocusEntries(
trendReport?.classificationDeltas?.observabilitySignals,
trendSummary.sampleCount,
);
const governanceSurfaces = buildGovernanceSurfaceEntries(governanceReport);
const governanceSummary = buildGovernanceSummary(
governanceReport,
@@ -718,6 +875,7 @@ export function buildGeneratedSlopReport({
focusSuiteTags,
focusReviewDecisionStatuses,
focusReviewRiskLevels,
focusObservabilitySignals,
docFreshnessSummary,
docFreshnessFocus,
governanceSummary,
@@ -764,12 +922,16 @@ export function buildGeneratedSlopReport({
trendSummary.reviewDecisionBacklogCount > 0
? `仍有 ${trendSummary.reviewDecisionBacklogCount} 个需要人工审核的 case 尚未记录最终 decision。`
: "当前没有待补录的人工审核 backlog。",
trendSummary.latestObservabilityGapCaseCount > 0
? `当前仍有 ${trendSummary.latestObservabilityGapCaseCount} 个 case 缺 observability 证据。`
: "当前没有额外的 observability 证据缺口。",
],
focus: {
failureModes: focusFailureModes.slice(0, 5),
suiteTags: focusSuiteTags.slice(0, 5),
reviewDecisionStatuses: focusReviewDecisionStatuses.slice(0, 5),
reviewRiskLevels: focusReviewRiskLevels.slice(0, 5),
observabilitySignals: focusObservabilitySignals.slice(0, 5),
docFreshness: docFreshnessFocus,
governanceSurfaces: governanceSurfaces
.filter((entry) => entry.active && entry.classification !== "current")
@@ -791,6 +953,7 @@ export function renderGeneratedSlopText(report) {
`[harness-cleanup] delta pending_request: ${report.summary.trend.pendingDelta}`,
`[harness-cleanup] delta review_decision_recorded: ${report.summary.trend.reviewDecisionRecordedDelta}`,
`[harness-cleanup] review backlog: ${report.summary.trend.reviewDecisionBacklogCount}`,
`[harness-cleanup] observability gap cases: ${report.summary.trend.latestObservabilityGapCaseCount}`,
`[harness-cleanup] doc freshness issues: ${report.summary.docFreshness.issueCount}`,
`[harness-cleanup] active compat/deprecated/dead-candidate: ${report.summary.governance.activeByClassification.compat}/${report.summary.governance.activeByClassification.deprecated}/${report.summary.governance.activeByClassification["dead-candidate"]}`,
`[harness-cleanup] governance violations: ${report.summary.governance.violationCount}`,
@@ -836,6 +999,15 @@ export function renderGeneratedSlopText(report) {
}
}
if (report.focus.observabilitySignals.length > 0) {
lines.push("[harness-cleanup] top observability gaps:");
for (const entry of report.focus.observabilitySignals) {
lines.push(
` - ${entry.signal} (${entry.status}): state=${entry.state}, latest_case=${entry.latest.caseCount}, delta_case=${entry.delta.caseCount}, score=${entry.score}`,
);
}
}
if (report.focus.docFreshness.issueKinds.length > 0) {
lines.push("[harness-cleanup] doc freshness issues:");
for (const entry of report.focus.docFreshness.issueKinds) {
@@ -870,6 +1042,7 @@ export function renderGeneratedSlopMarkdown(report) {
`- pending request delta：${report.summary.trend.pendingDelta}`,
`- 已记录人工审核 delta：${report.summary.trend.reviewDecisionRecordedDelta}`,
`- 待补录人工审核 backlog：${report.summary.trend.reviewDecisionBacklogCount}`,
`- observability gap case：${report.summary.trend.latestObservabilityGapCaseCount}`,
`- doc freshness 问题数：${report.summary.docFreshness.issueCount}`,
`- governance 违规数：${report.summary.governance.violationCount}`,
`- compat 活跃 surface：${report.summary.governance.activeByClassification.compat}`,
@@ -955,6 +1128,21 @@ export function renderGeneratedSlopMarkdown(report) {
}
}
if (report.focus.observabilitySignals.length > 0) {
lines.push("");
lines.push("## Observability 证据缺口");
lines.push("");
lines.push(
"| Signal | 状态 | latest case | delta case | score |",
);
lines.push("| --- | --- | --- | --- | --- |");
for (const entry of report.focus.observabilitySignals) {
lines.push(
`| ${entry.signal} | ${entry.status} | ${entry.latest.caseCount} | ${entry.delta.caseCount} | ${entry.score} |`,
);
}
}
if (report.recommendations.length > 0) {
lines.push("");
lines.push("## 推荐动作");
scripts/lib/generated-slop-report-core.test.ts
+68
View File
@@ -13,6 +13,7 @@ describe("generated-slop-report-core", () => {
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
reviewDecisionRecordedCount: 0,
observabilityGapCaseCount: 1,
readyRate: 0,
},
signals: ["样本数不足 2，当前仅形成 trend seed，还不能判断长期退化。"],
@@ -20,12 +21,14 @@ describe("generated-slop-report-core", () => {
totals: {
needsHumanReviewCount: 0,
reviewDecisionRecordedCount: 0,
observabilityGapCaseCount: 0,
},
},
latest: {
totals: {
needsHumanReviewCount: 1,
reviewDecisionRecordedCount: 0,
observabilityGapCaseCount: 1,
},
},
classificationDeltas: {
@@ -133,6 +136,56 @@ describe("generated-slop-report-core", () => {
},
},
],
observabilitySignals: [
{
name: "requestTelemetry:unlinked",
baseline: {
caseCount: 0,
readyCount: 0,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
latest: {
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
delta: {
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
},
{
name: "artifactValidator:known_gap",
baseline: {
caseCount: 0,
readyCount: 0,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
latest: {
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
delta: {
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
},
],
},
},
governanceReport: {
@@ -190,10 +243,15 @@ describe("generated-slop-report-core", () => {
expect(report.summary.trend.isSeed).toBe(true);
expect(report.summary.trend.reviewDecisionBacklogCount).toBe(1);
expect(report.summary.trend.latestObservabilityGapCaseCount).toBe(1);
expect(report.summary.docFreshness.issueCount).toBe(0);
expect(report.focus.failureModes[0].name).toBe("pending_request");
expect(report.focus.reviewDecisionStatuses[0].name).toBe("pending_review");
expect(report.focus.reviewRiskLevels[0].name).toBe("high");
expect(report.focus.observabilitySignals[0]).toMatchObject({
signal: "requestTelemetry",
status: "unlinked",
});
expect(report.focus.governanceSurfaces[0].id).toBe(
"migration-setting-key-leak",
);
@@ -202,6 +260,7 @@ describe("generated-slop-report-core", () => {
"promote-high-value-replay",
"replay-and-smoke-follow-up",
"review-decision-follow-up",
"observability-evidence-follow-up",
"governance-cleanup-priority",
"doc-freshness-review",
]),
@@ -215,6 +274,11 @@ describe("generated-slop-report-core", () => {
report.recommendations.find((entry) => entry.id === "doc-freshness-review")
?.commands,
).toContain("npm run harness:doc-freshness");
expect(
report.recommendations.find(
(entry) => entry.id === "observability-evidence-follow-up",
)?.priority,
).toBe("P1");
});
it("应把 governance 违规提升为 P0 守卫动作", () => {
@@ -227,6 +291,7 @@ describe("generated-slop-report-core", () => {
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
reviewDecisionRecordedCount: 0,
observabilityGapCaseCount: 0,
readyRate: -0.5,
},
signals: ["invalid case 增加 1，存在回归候选。"],
@@ -234,12 +299,14 @@ describe("generated-slop-report-core", () => {
totals: {
needsHumanReviewCount: 0,
reviewDecisionRecordedCount: 0,
observabilityGapCaseCount: 0,
},
},
latest: {
totals: {
needsHumanReviewCount: 0,
reviewDecisionRecordedCount: 0,
observabilityGapCaseCount: 0,
},
},
classificationDeltas: {
@@ -247,6 +314,7 @@ describe("generated-slop-report-core", () => {
suiteTags: [],
reviewDecisionStatuses: [],
reviewRiskLevels: [],
observabilitySignals: [],
},
},
governanceReport: {
scripts/lib/harness-eval-history-record.test.ts
+76 -5
View File
@@ -1,7 +1,7 @@
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { execFileSync } from "node:child_process";
import { execFile, execFileSync } from "node:child_process";
import { afterEach, describe, expect, it } from "vitest";
const repoRoot = process.cwd();
@@ -28,6 +28,27 @@ function runNodeScript(scriptRelativePath: string, args: string[]) {
return JSON.parse(output);
}
function runNodeScriptAsync(scriptRelativePath: string, args: string[]) {
return new Promise<unknown>((resolve, reject) => {
execFile(
process.execPath,
[path.join(repoRoot, scriptRelativePath), ...args],
{
cwd: repoRoot,
encoding: "utf8",
},
(error, stdout) => {
if (error) {
reject(error);
return;
}
resolve(JSON.parse(stdout));
},
);
});
}
afterEach(() => {
while (tempRoots.length > 0) {
const tempRoot = tempRoots.pop();
@@ -40,7 +61,7 @@ afterEach(() => {
describe("harness-eval-history-record", () => {
it("应记录本地 summary 历史，并生成非 seed 的 trend / cleanup", () => {
const tempRoot = createTempRoot();
const historyDir = path.join(tempRoot, "artifacts", "history");
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
const workspaceRoot = path.join(tempRoot, "workspace");
const firstResult = runNodeScript("scripts/harness-eval-history-record.mjs", [
@@ -76,12 +97,62 @@ describe("harness-eval-history-record", () => {
expect(secondResult.cleanup.trendSampleCount).toBe(2);
expect(fs.existsSync(secondResult.recordedSummaryPath)).toBe(true);
expect(
fs.existsSync(path.join(tempRoot, "artifacts", "harness-eval-trend.json")),
fs.existsSync(
path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-eval-trend.json",
),
),
).toBe(true);
expect(
fs.existsSync(
path.join(tempRoot, "artifacts", "harness-cleanup-report.json"),
path.join(
tempRoot,
".lime",
"harness",
"reports",
"harness-cleanup-report.json",
),
),
).toBe(true);
});
}, 30_000);
it("并发记录 history 时不应覆盖已有样本", async () => {
const tempRoot = createTempRoot();
const historyDir = path.join(tempRoot, ".lime", "harness", "history");
const workspaceRoot = path.join(tempRoot, "workspace");
const [firstResult, secondResult] = (await Promise.all([
runNodeScriptAsync("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--history-dir",
historyDir,
"--workspace-root",
workspaceRoot,
]),
runNodeScriptAsync("scripts/harness-eval-history-record.mjs", [
"--format",
"json",
"--history-dir",
historyDir,
"--workspace-root",
workspaceRoot,
]),
])) as Array<{ recordedSummaryPath: string }>;
const historyFiles = fs
.readdirSync(historyDir)
.filter(
(entry) =>
entry.endsWith("-harness-eval-summary.json") ||
/-harness-eval-summary-\d+\.json$/.test(entry),
);
expect(historyFiles).toHaveLength(2);
expect(firstResult.recordedSummaryPath).not.toBe(secondResult.recordedSummaryPath);
}, 30_000);
});
scripts/lib/harness-review-decision-evals.test.ts
+76
View File
@@ -101,6 +101,29 @@ function createWorkspaceSessionArtifacts(tempRoot: string, sessionId: string) {
},
],
},
observability: {
correlation: {
correlationKeys: ["session_id", "thread_id", "turn_id"],
},
signalCoverage: [
{
signal: "correlation",
status: "exported",
},
{
signal: "timeline",
status: "exported",
},
{
signal: "requestTelemetry",
status: "unlinked",
},
{
signal: "artifactValidator",
status: "known_gap",
},
],
},
linkedArtifacts: {
handoffBundle: {
relativeRoot: `.lime/harness/sessions/${sessionId}/handoff`,
@@ -133,6 +156,7 @@ function createWorkspaceSessionArtifacts(tempRoot: string, sessionId: string) {
relativeRoot: `.lime/harness/sessions/${sessionId}/evidence`,
absoluteRoot: evidenceRoot,
},
observabilitySummary: inputPayload.observability,
};
const reviewDecisionPayload = {
schemaVersion: "v1",
@@ -300,6 +324,7 @@ describe("Harness review decision / eval integration", () => {
]);
expect(summary.totals.reviewDecisionRecordedCount).toBe(1);
expect(summary.totals.observabilityGapCaseCount).toBe(1);
expect(summary.breakdowns.reviewDecisionStatuses).toEqual(
expect.arrayContaining([
expect.objectContaining({
@@ -316,10 +341,23 @@ describe("Harness review decision / eval integration", () => {
}),
]),
);
expect(summary.breakdowns.observabilitySignals).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "requestTelemetry:unlinked",
caseCount: 1,
}),
expect.objectContaining({
name: "artifactValidator:known_gap",
caseCount: 1,
}),
]),
);
expect(summary.suites[0].cases[0]).toMatchObject({
reviewDecisionStatus: "accepted",
reviewRiskLevel: "high",
reviewHumanReviewer: "Lime Maintainer",
observabilityGapCount: 2,
});
});
@@ -337,6 +375,7 @@ describe("Harness review decision / eval integration", () => {
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 1,
observabilityGapCaseCount: 1,
},
breakdowns: {
suiteTags: [],
@@ -361,6 +400,16 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 1,
},
],
observabilitySignals: [
{
name: "requestTelemetry:unlinked",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 1,
},
],
},
suites: [],
});
@@ -374,6 +423,7 @@ describe("Harness review decision / eval integration", () => {
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
observabilityGapCaseCount: 1,
},
breakdowns: {
suiteTags: [],
@@ -398,6 +448,16 @@ describe("Harness review decision / eval integration", () => {
needsHumanReviewCount: 0,
},
],
observabilitySignals: [
{
name: "artifactValidator:known_gap",
caseCount: 1,
readyCount: 1,
invalidCount: 0,
pendingRequestCaseCount: 0,
needsHumanReviewCount: 0,
},
],
},
suites: [],
});
@@ -443,5 +503,21 @@ describe("Harness review decision / eval integration", () => {
}),
]),
);
expect(report.classificationDeltas.observabilitySignals).toEqual(
expect.arrayContaining([
expect.objectContaining({
name: "requestTelemetry:unlinked",
delta: expect.objectContaining({
caseCount: -1,
}),
}),
expect.objectContaining({
name: "artifactValidator:known_gap",
delta: expect.objectContaining({
caseCount: 1,
}),
}),
]),
);
});
});
scripts/report-generated-slop.mjs
+18 -8
View File
@@ -144,16 +144,26 @@ function readJsonFile(filePath) {
}
function detectDefaultTrendHistoryDir(repoRoot) {
const candidate = resolvePath(repoRoot, "./artifacts/history");
if (!fs.existsSync(candidate)) {
return "";
const candidates = [
resolvePath(repoRoot, "./.lime/harness/history"),
resolvePath(repoRoot, "./artifacts/history"),
];
for (const candidate of candidates) {
if (!fs.existsSync(candidate)) {
continue;
}
const hasJson = fs
.readdirSync(candidate, { withFileTypes: true })
.some((entry) => entry.isFile() && entry.name.endsWith(".json"));
if (hasJson) {
return candidate;
}
}
const hasJson = fs
.readdirSync(candidate, { withFileTypes: true })
.some((entry) => entry.isFile() && entry.name.endsWith(".json"));
return hasJson ? candidate : "";
return "";
}
function buildTrendReport(repoRoot, options) {