mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-09-24 23:10:56 +08:00
feat: finalize latest v0.97.0 updates
This commit is contained in:
coso
@@ -0,0 +1,146 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import legacySurfaceCatalog from "../src/lib/governance/legacySurfaceCatalog.json" with { type: "json" };
|
||||
import {
|
||||
DOC_FRESHNESS_SPECS,
|
||||
buildDocFreshnessReport,
|
||||
renderDocFreshnessText,
|
||||
} from "./lib/doc-freshness-core.mjs";
|
||||
|
||||
export {
|
||||
DOC_FRESHNESS_SPECS,
|
||||
buildDocFreshnessReport,
|
||||
renderDocFreshnessText,
|
||||
} from "./lib/doc-freshness-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
format: "text",
|
||||
help: false,
|
||||
output: "",
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
options.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--output" && argv[index + 1]) {
|
||||
options.output = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--json") {
|
||||
options.format = "json";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
options.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Doc Freshness Check
|
||||
|
||||
用法:
|
||||
node scripts/check-doc-freshness.mjs
|
||||
node scripts/check-doc-freshness.mjs --json
|
||||
node scripts/check-doc-freshness.mjs --format json --output "./tmp/doc-freshness.json"
|
||||
|
||||
选项:
|
||||
--format FMT 标准输出格式：text | json
|
||||
--json 等价于 --format json
|
||||
--output PATH 将结果写入指定路径
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function buildDeletedSurfaceTargets(repoRoot) {
|
||||
return (legacySurfaceCatalog.imports ?? [])
|
||||
.flatMap((entry) => entry.targets ?? [])
|
||||
.filter((target) => {
|
||||
const absolutePath = path.resolve(repoRoot, target);
|
||||
return !fs.existsSync(absolutePath);
|
||||
});
|
||||
}
|
||||
|
||||
function readMonitoredDocuments(repoRoot) {
|
||||
return DOC_FRESHNESS_SPECS.flatMap((spec) => {
|
||||
const absolutePath = path.resolve(repoRoot, spec.path);
|
||||
if (!fs.existsSync(absolutePath)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return [
|
||||
{
|
||||
path: spec.path,
|
||||
content: fs.readFileSync(absolutePath, "utf8"),
|
||||
},
|
||||
];
|
||||
});
|
||||
}
|
||||
|
||||
export function buildLiveDocFreshnessReport(repoRoot) {
|
||||
return buildDocFreshnessReport({
|
||||
repoRoot,
|
||||
documents: readMonitoredDocuments(repoRoot),
|
||||
deletedSurfaceTargets: buildDeletedSurfaceTargets(repoRoot),
|
||||
pathExists: (absolutePath) => fs.existsSync(absolutePath),
|
||||
});
|
||||
}
|
||||
|
||||
function renderOutput(report, format) {
|
||||
if (format === "json") {
|
||||
return `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
|
||||
return renderDocFreshnessText(report);
|
||||
}
|
||||
|
||||
function runDocFreshnessCli() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const report = buildLiveDocFreshnessReport(repoRoot);
|
||||
|
||||
const output = renderOutput(report, options.format);
|
||||
if (options.output) {
|
||||
const targetPath = path.resolve(repoRoot, options.output);
|
||||
fs.mkdirSync(path.dirname(targetPath), { recursive: true });
|
||||
fs.writeFileSync(targetPath, output, "utf8");
|
||||
console.log(`[lime] doc freshness output: ${targetPath}`);
|
||||
} else {
|
||||
process.stdout.write(output);
|
||||
}
|
||||
|
||||
if (report.summary.issueCount > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
const isMainModule =
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
|
||||
if (isMainModule) {
|
||||
runDocFreshnessCli();
|
||||
}
|
||||
@@ -8,7 +8,7 @@ import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { cruise } from "dependency-cruiser";
|
||||
import extractTSConfig from "dependency-cruiser/config-utl/extract-ts-config";
|
||||
import YAML from "yaml";
|
||||
import { buildLegacySurfaceReport } from "./report-legacy-surfaces.mjs";
|
||||
import { buildLegacySurfaceReport } from "./lib/legacy-surface-report-core.mjs";
|
||||
import {
|
||||
buildRustModuleIndex,
|
||||
buildRustModulePathFromFile,
|
||||
@@ -138,7 +138,10 @@ function parseArgs(argv) {
|
||||
}
|
||||
|
||||
if (arg === "--days" && argv[index + 1]) {
|
||||
result.days = normalizePositiveNumber(argv[index + 1], DEFAULT_SINCE_DAYS);
|
||||
result.days = normalizePositiveNumber(
|
||||
argv[index + 1],
|
||||
DEFAULT_SINCE_DAYS,
|
||||
);
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
@@ -217,8 +220,10 @@ async function buildGovernanceGraphReport({
|
||||
const signalsSummary = summarizeSignals(nodes);
|
||||
const summary = buildSummary(nodes, edges, signalsSummary);
|
||||
const links = {
|
||||
selfHtmlHref: pathToFileURL(path.join(outputDir, "governance-graph.html")).href,
|
||||
selfJsonHref: pathToFileURL(path.join(outputDir, "governance-graph.json")).href,
|
||||
selfHtmlHref: pathToFileURL(path.join(outputDir, "governance-graph.html"))
|
||||
.href,
|
||||
selfJsonHref: pathToFileURL(path.join(outputDir, "governance-graph.json"))
|
||||
.href,
|
||||
heatmapHtmlHref: fs.existsSync(path.join(outputDir, "index.html"))
|
||||
? pathToFileURL(path.join(outputDir, "index.html")).href
|
||||
: "",
|
||||
@@ -280,7 +285,9 @@ function collectGovernanceFiles(repoRoot) {
|
||||
|
||||
const ext = path.extname(relativePath).toLowerCase();
|
||||
const language = ext === ".rs" ? "rs" : ext.replace(/^\./, "");
|
||||
const layer = relativePath.startsWith("src-tauri/src/") ? "rust" : "frontend";
|
||||
const layer = relativePath.startsWith("src-tauri/src/")
|
||||
? "rust"
|
||||
: "frontend";
|
||||
|
||||
records.push({
|
||||
path: relativePath,
|
||||
@@ -399,7 +406,10 @@ function collectGitChurn({ gitCommand, repoRoot, sinceDays, trackedFiles }) {
|
||||
continue;
|
||||
}
|
||||
|
||||
fileChurn.set(normalizedGitPath, (fileChurn.get(normalizedGitPath) || 0) + churn);
|
||||
fileChurn.set(
|
||||
normalizedGitPath,
|
||||
(fileChurn.get(normalizedGitPath) || 0) + churn,
|
||||
);
|
||||
}
|
||||
|
||||
return fileChurn;
|
||||
@@ -469,7 +479,9 @@ async function collectFrontendDependencies(repoRoot, fileIndex) {
|
||||
}
|
||||
|
||||
function collectRustDependencies(repoRoot, fileInventory, fileIndex) {
|
||||
const rustFiles = fileInventory.filter((fileRecord) => fileRecord.layer === "rust");
|
||||
const rustFiles = fileInventory.filter(
|
||||
(fileRecord) => fileRecord.layer === "rust",
|
||||
);
|
||||
const rustFileSet = new Set(rustFiles.map((fileRecord) => fileRecord.path));
|
||||
const moduleIndex = buildRustModuleIndex([...rustFileSet]);
|
||||
const edges = [];
|
||||
@@ -477,7 +489,8 @@ function collectRustDependencies(repoRoot, fileInventory, fileIndex) {
|
||||
for (const fileRecord of rustFiles) {
|
||||
const absolutePath = path.join(repoRoot, fileRecord.path);
|
||||
const sourceCode = fs.readFileSync(absolutePath, "utf8");
|
||||
const currentModulePath = buildRustModulePathFromFile(fileRecord.path) || "";
|
||||
const currentModulePath =
|
||||
buildRustModulePathFromFile(fileRecord.path) || "";
|
||||
|
||||
for (const moduleName of extractRustModDeclarations(sourceCode)) {
|
||||
const targetPath = resolveRustSubmodulePath(
|
||||
@@ -661,7 +674,10 @@ function buildLegacyOverlays(legacyReport) {
|
||||
}
|
||||
}
|
||||
|
||||
for (const result of [...legacyReport.frontendTextResults, ...legacyReport.rustTextResults]) {
|
||||
for (const result of [
|
||||
...legacyReport.frontendTextResults,
|
||||
...legacyReport.rustTextResults,
|
||||
]) {
|
||||
for (const callerPath of result.references) {
|
||||
pushOverlay(callsiteOverlays, callerPath, {
|
||||
source: "legacy-report",
|
||||
@@ -674,7 +690,9 @@ function buildLegacyOverlays(legacyReport) {
|
||||
}
|
||||
|
||||
for (const result of legacyReport.rustTextCountResults) {
|
||||
for (const callerPath of result.runtimeMatches.map((item) => item.relativePath)) {
|
||||
for (const callerPath of result.runtimeMatches.map(
|
||||
(item) => item.relativePath,
|
||||
)) {
|
||||
pushOverlay(callsiteOverlays, callerPath, {
|
||||
source: "legacy-report",
|
||||
overlayType: "callsite",
|
||||
@@ -727,7 +745,8 @@ function buildFileNodes({
|
||||
legacyOverlays.callsiteOverlays.get(fileRecord.path) ?? [];
|
||||
const explicitStatus = matchingRule?.status ?? "unclassified";
|
||||
const overlayStatus = pickOverlayStatus(surfaceOverlays);
|
||||
const status = explicitStatus !== "unclassified" ? explicitStatus : overlayStatus;
|
||||
const status =
|
||||
explicitStatus !== "unclassified" ? explicitStatus : overlayStatus;
|
||||
const ruleSource = matchingRule
|
||||
? {
|
||||
reason: matchingRule.reason || "",
|
||||
@@ -743,12 +762,16 @@ function buildFileNodes({
|
||||
signals.add("unused-file");
|
||||
}
|
||||
|
||||
if ((knipSignals.unusedExportsByFile.get(fileRecord.path) ?? []).length > 0) {
|
||||
if (
|
||||
(knipSignals.unusedExportsByFile.get(fileRecord.path) ?? []).length > 0
|
||||
) {
|
||||
signals.add("unused-export");
|
||||
}
|
||||
|
||||
if (
|
||||
surfaceOverlays.some((overlay) => overlay.classification === "dead-candidate")
|
||||
surfaceOverlays.some(
|
||||
(overlay) => overlay.classification === "dead-candidate",
|
||||
)
|
||||
) {
|
||||
signals.add("dead-candidate");
|
||||
}
|
||||
@@ -801,11 +824,7 @@ function finalizeNodeSignals(nodes, edges, governanceRules) {
|
||||
return nodes.map((node) => {
|
||||
const signals = new Set(node.signals);
|
||||
|
||||
if (
|
||||
!node.reachable &&
|
||||
node.kind === "page" &&
|
||||
node.layer === "frontend"
|
||||
) {
|
||||
if (!node.reachable && node.kind === "page" && node.layer === "frontend") {
|
||||
signals.add("page-unreachable");
|
||||
}
|
||||
|
||||
@@ -861,7 +880,10 @@ function pickOverlayStatus(surfaceOverlays) {
|
||||
}
|
||||
|
||||
function calculateNodeSize(loc) {
|
||||
return Math.max(26, Math.min(86, Math.round(18 + Math.sqrt(Math.max(loc, 1)))));
|
||||
return Math.max(
|
||||
26,
|
||||
Math.min(86, Math.round(18 + Math.sqrt(Math.max(loc, 1)))),
|
||||
);
|
||||
}
|
||||
|
||||
function createParentDirectoryId(relativePath) {
|
||||
@@ -901,7 +923,9 @@ function buildDirectoryNodes(fileNodes) {
|
||||
}
|
||||
}
|
||||
|
||||
return [...dirMap.values()].sort((left, right) => left.path.localeCompare(right.path));
|
||||
return [...dirMap.values()].sort((left, right) =>
|
||||
left.path.localeCompare(right.path),
|
||||
);
|
||||
}
|
||||
|
||||
function collectReachableFiles(entryPaths, edges) {
|
||||
@@ -966,7 +990,10 @@ function isProtectedEntryNode(node, governanceRules) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return resolveMatchingGovernanceRule(node.path, governanceRules)?.status === "current";
|
||||
return (
|
||||
resolveMatchingGovernanceRule(node.path, governanceRules)?.status ===
|
||||
"current"
|
||||
);
|
||||
}
|
||||
|
||||
function calculateCandidateScore(node) {
|
||||
@@ -1003,16 +1030,21 @@ function summarizeSignals(nodes) {
|
||||
|
||||
return [...counts.entries()]
|
||||
.map(([signal, count]) => ({ signal, count }))
|
||||
.sort((left, right) => right.count - left.count || left.signal.localeCompare(right.signal));
|
||||
.sort(
|
||||
(left, right) =>
|
||||
right.count - left.count || left.signal.localeCompare(right.signal),
|
||||
);
|
||||
}
|
||||
|
||||
function buildSummary(nodes, edges, signalsSummary) {
|
||||
const fileNodes = nodes.filter((node) => node.kind !== "dir");
|
||||
const statusCounts = Object.fromEntries(
|
||||
["current", "compat", "deprecated", "dead", "unclassified"].map((status) => [
|
||||
status,
|
||||
fileNodes.filter((node) => node.status === status).length,
|
||||
]),
|
||||
["current", "compat", "deprecated", "dead", "unclassified"].map(
|
||||
(status) => [
|
||||
status,
|
||||
fileNodes.filter((node) => node.status === status).length,
|
||||
],
|
||||
),
|
||||
);
|
||||
const layerCounts = Object.fromEntries(
|
||||
["frontend", "rust"].map((layer) => [
|
||||
|
||||
@@ -0,0 +1,343 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
const RUNNER_PATH = "scripts/harness-eval-runner.mjs";
|
||||
const TREND_PATH = "scripts/harness-eval-trend-report.mjs";
|
||||
const CLEANUP_PATH = "scripts/report-generated-slop.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
cleanupJson: "",
|
||||
cleanupMarkdown: "",
|
||||
format: "text",
|
||||
help: false,
|
||||
historyDir: "./artifacts/history",
|
||||
outputJson: "",
|
||||
retain: 30,
|
||||
skipCleanup: false,
|
||||
skipTrend: false,
|
||||
trendJson: "",
|
||||
trendMarkdown: "",
|
||||
workspaceRoot: process.cwd(),
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
|
||||
if (arg === "--history-dir" && argv[index + 1]) {
|
||||
result.historyDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--workspace-root" && argv[index + 1]) {
|
||||
result.workspaceRoot = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--retain" && argv[index + 1]) {
|
||||
result.retain = Number.parseInt(String(argv[index + 1]).trim(), 10);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--output-json" && argv[index + 1]) {
|
||||
result.outputJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--trend-json" && argv[index + 1]) {
|
||||
result.trendJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--trend-markdown" && argv[index + 1]) {
|
||||
result.trendMarkdown = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--cleanup-json" && argv[index + 1]) {
|
||||
result.cleanupJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--cleanup-markdown" && argv[index + 1]) {
|
||||
result.cleanupMarkdown = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--skip-trend") {
|
||||
result.skipTrend = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--skip-cleanup") {
|
||||
result.skipCleanup = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Harness Eval History Record
|
||||
|
||||
用法:
|
||||
node scripts/harness-eval-history-record.mjs
|
||||
node scripts/harness-eval-history-record.mjs --workspace-root "/path/to/workspace"
|
||||
node scripts/harness-eval-history-record.mjs --history-dir "./artifacts/history" --output-json "./tmp/harness-history-record.json"
|
||||
|
||||
选项:
|
||||
--history-dir PATH summary 历史目录，默认 ./artifacts/history
|
||||
--workspace-root PATH 生成当前 summary 时使用的工作区根目录
|
||||
--retain N 历史窗口保留数量，默认 30
|
||||
--trend-json PATH trend JSON 输出路径
|
||||
--trend-markdown PATH trend Markdown 输出路径
|
||||
--cleanup-json PATH cleanup JSON 输出路径
|
||||
--cleanup-markdown PATH cleanup Markdown 输出路径
|
||||
--skip-trend 只记录 summary，不生成 trend
|
||||
--skip-cleanup 只记录 summary / trend，不生成 cleanup
|
||||
--format FMT 标准输出格式：text | json
|
||||
--output-json PATH 将记录结果写入指定路径
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function resolvePath(baseDir, targetPath) {
|
||||
return path.resolve(baseDir, targetPath);
|
||||
}
|
||||
|
||||
function ensureParentDirectory(filePath) {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function timestampForFilename() {
|
||||
return new Date().toISOString().replace(/[-:]/g, "").replace(/\./g, "");
|
||||
}
|
||||
|
||||
function collectHistoryFiles(historyDir) {
|
||||
if (!fs.existsSync(historyDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
return fs
|
||||
.readdirSync(historyDir)
|
||||
.filter((entry) => entry.endsWith(".json"))
|
||||
.map((entry) => path.join(historyDir, entry))
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function buildCurrentSummary(repoRoot, workspaceRoot) {
|
||||
const runnerPath = resolvePath(repoRoot, RUNNER_PATH);
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[runnerPath, "--format", "json", "--workspace-root", workspaceRoot],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
},
|
||||
);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
function writeJsonFile(filePath, payload) {
|
||||
ensureParentDirectory(filePath);
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function trimHistoryFiles(historyDir, retain) {
|
||||
const files = collectHistoryFiles(historyDir).sort((left, right) =>
|
||||
right.localeCompare(left),
|
||||
);
|
||||
const removable = files.slice(Math.max(retain, 0));
|
||||
for (const filePath of removable) {
|
||||
fs.rmSync(filePath, { force: true });
|
||||
}
|
||||
return removable;
|
||||
}
|
||||
|
||||
function buildDefaultArtifactPaths(historyDir) {
|
||||
const artifactsRoot = path.dirname(historyDir);
|
||||
return {
|
||||
trendJson: path.join(artifactsRoot, "harness-eval-trend.json"),
|
||||
trendMarkdown: path.join(artifactsRoot, "harness-eval-trend.md"),
|
||||
cleanupJson: path.join(artifactsRoot, "harness-cleanup-report.json"),
|
||||
cleanupMarkdown: path.join(artifactsRoot, "harness-cleanup-report.md"),
|
||||
};
|
||||
}
|
||||
|
||||
function renderOutput(result, format) {
|
||||
if (format === "json") {
|
||||
return `${JSON.stringify(result, null, 2)}\n`;
|
||||
}
|
||||
|
||||
const lines = [
|
||||
"[lime] harness eval history record",
|
||||
`[lime] history dir: ${result.historyDir}`,
|
||||
`[lime] recorded summary: ${result.recordedSummaryPath}`,
|
||||
`[lime] history count: ${result.historyCount}`,
|
||||
`[lime] trimmed files: ${result.trimmedPaths.length}`,
|
||||
];
|
||||
|
||||
if (result.trend) {
|
||||
lines.push(`[lime] trend sample count: ${result.trend.sampleCount}`);
|
||||
if (result.trend.outputJsonPath) {
|
||||
lines.push(`[lime] trend json: ${result.trend.outputJsonPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (result.cleanup) {
|
||||
lines.push(
|
||||
`[lime] cleanup trend samples: ${result.cleanup.trendSampleCount}`,
|
||||
);
|
||||
if (result.cleanup.outputJsonPath) {
|
||||
lines.push(`[lime] cleanup json: ${result.cleanup.outputJsonPath}`);
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function runHistoryRecordCli() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const historyDir = resolvePath(repoRoot, options.historyDir);
|
||||
fs.mkdirSync(historyDir, { recursive: true });
|
||||
|
||||
const summary = buildCurrentSummary(repoRoot, options.workspaceRoot);
|
||||
const summaryFilePath = path.join(
|
||||
historyDir,
|
||||
`${timestampForFilename()}-harness-eval-summary.json`,
|
||||
);
|
||||
writeJsonFile(summaryFilePath, summary);
|
||||
const trimmedPaths = trimHistoryFiles(historyDir, options.retain);
|
||||
const historyCount = collectHistoryFiles(historyDir).length;
|
||||
const defaults = buildDefaultArtifactPaths(historyDir);
|
||||
|
||||
const result = {
|
||||
recordedAt: new Date().toISOString(),
|
||||
historyDir,
|
||||
recordedSummaryPath: summaryFilePath,
|
||||
historyCount,
|
||||
trimmedPaths,
|
||||
trend: null,
|
||||
cleanup: null,
|
||||
};
|
||||
|
||||
if (!options.skipTrend) {
|
||||
const trendJsonPath = resolvePath(
|
||||
repoRoot,
|
||||
options.trendJson || defaults.trendJson,
|
||||
);
|
||||
const trendMarkdownPath = resolvePath(
|
||||
repoRoot,
|
||||
options.trendMarkdown || defaults.trendMarkdown,
|
||||
);
|
||||
const trendScriptPath = resolvePath(repoRoot, TREND_PATH);
|
||||
const trendOutput = execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
trendScriptPath,
|
||||
"--format",
|
||||
"json",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--output-json",
|
||||
trendJsonPath,
|
||||
"--output-markdown",
|
||||
trendMarkdownPath,
|
||||
],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
},
|
||||
);
|
||||
const trendReport = JSON.parse(trendOutput);
|
||||
result.trend = {
|
||||
sampleCount: trendReport.sampleCount,
|
||||
outputJsonPath: trendJsonPath,
|
||||
outputMarkdownPath: trendMarkdownPath,
|
||||
};
|
||||
}
|
||||
|
||||
if (!options.skipCleanup) {
|
||||
const cleanupJsonPath = resolvePath(
|
||||
repoRoot,
|
||||
options.cleanupJson || defaults.cleanupJson,
|
||||
);
|
||||
const cleanupMarkdownPath = resolvePath(
|
||||
repoRoot,
|
||||
options.cleanupMarkdown || defaults.cleanupMarkdown,
|
||||
);
|
||||
const cleanupScriptPath = resolvePath(repoRoot, CLEANUP_PATH);
|
||||
const cleanupOutput = execFileSync(
|
||||
process.execPath,
|
||||
[
|
||||
cleanupScriptPath,
|
||||
"--format",
|
||||
"json",
|
||||
"--trend-history-dir",
|
||||
historyDir,
|
||||
"--output-json",
|
||||
cleanupJsonPath,
|
||||
"--output-markdown",
|
||||
cleanupMarkdownPath,
|
||||
],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
},
|
||||
);
|
||||
const cleanupReport = JSON.parse(cleanupOutput);
|
||||
result.cleanup = {
|
||||
trendSampleCount: cleanupReport.summary?.trend?.sampleCount ?? 0,
|
||||
outputJsonPath: cleanupJsonPath,
|
||||
outputMarkdownPath: cleanupMarkdownPath,
|
||||
};
|
||||
}
|
||||
|
||||
const rendered = renderOutput(result, options.format);
|
||||
if (options.outputJson) {
|
||||
const outputPath = resolvePath(repoRoot, options.outputJson);
|
||||
writeJsonFile(outputPath, result);
|
||||
console.log(`[lime] harness eval history record JSON: ${outputPath}`);
|
||||
} else {
|
||||
process.stdout.write(rendered);
|
||||
}
|
||||
}
|
||||
|
||||
runHistoryRecordCli();
|
||||
@@ -5,14 +5,33 @@ import path from "node:path";
|
||||
import process from "node:process";
|
||||
|
||||
const DEFAULT_MANIFEST_PATH = "docs/test/harness-evals.manifest.json";
|
||||
const REVIEW_DECISION_JSON_CANDIDATES = [
|
||||
"review-decision.json",
|
||||
"../review/review-decision.json",
|
||||
];
|
||||
const REVIEW_DECISION_STATUS_SET = new Set([
|
||||
"accepted",
|
||||
"deferred",
|
||||
"rejected",
|
||||
"needs_more_evidence",
|
||||
"pending_review",
|
||||
]);
|
||||
const REVIEW_DECISION_RISK_LEVEL_SET = new Set([
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"unknown",
|
||||
]);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
format: "text",
|
||||
help: false,
|
||||
historyRetain: 30,
|
||||
manifest: DEFAULT_MANIFEST_PATH,
|
||||
outputJson: "",
|
||||
outputMarkdown: "",
|
||||
recordHistoryDir: "",
|
||||
strict: true,
|
||||
workspaceRoot: process.cwd(),
|
||||
};
|
||||
@@ -50,6 +69,18 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--record-history-dir" && argv[index + 1]) {
|
||||
result.recordHistoryDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--history-retain" && argv[index + 1]) {
|
||||
result.historyRetain = Number.parseInt(String(argv[index + 1]), 10);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--no-strict") {
|
||||
result.strict = false;
|
||||
continue;
|
||||
@@ -77,6 +108,7 @@ Lime Harness Eval Runner
|
||||
node scripts/harness-eval-runner.mjs --format json
|
||||
node scripts/harness-eval-runner.mjs --workspace-root "/path/to/workspace"
|
||||
node scripts/harness-eval-runner.mjs --output-json "./tmp/harness-eval-summary.json" --output-markdown "./tmp/harness-eval-summary.md"
|
||||
node scripts/harness-eval-runner.mjs --record-history-dir "./artifacts/history"
|
||||
|
||||
选项:
|
||||
--manifest PATH 指定 manifest，默认 docs/test/harness-evals.manifest.json
|
||||
@@ -84,6 +116,8 @@ Lime Harness Eval Runner
|
||||
--format FMT 控制标准输出格式：text | json | markdown
|
||||
--output-json PATH 将 JSON 摘要写入指定路径
|
||||
--output-markdown PATH 将 Markdown 摘要写入指定路径
|
||||
--record-history-dir PATH 将当前 summary 追加写入历史目录，供 trend/nightly 复用
|
||||
--history-retain N 历史目录最多保留多少条 summary，默认 30
|
||||
--strict 严格模式（默认），发现 invalid case 时返回非 0
|
||||
--no-strict 非严格模式，只输出摘要，不因 invalid case 退出失败
|
||||
-h, --help 显示帮助
|
||||
@@ -102,6 +136,56 @@ function ensureParentDirectory(filePath) {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
}
|
||||
|
||||
function ensureDirectory(dirPath) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function toHistoryTimestamp(generatedAt) {
|
||||
const parsed = new Date(generatedAt);
|
||||
const normalized = Number.isNaN(parsed.getTime())
|
||||
? new Date().toISOString()
|
||||
: parsed.toISOString();
|
||||
return normalized.replace(/[-:]/g, "").replace(/\.(\d{3})Z$/, "$1Z");
|
||||
}
|
||||
|
||||
function trimHistoryDirectory(historyDir, retainCount) {
|
||||
const normalizedRetainCount =
|
||||
Number.isInteger(retainCount) && retainCount > 0 ? retainCount : 30;
|
||||
const historyFiles = fs
|
||||
.readdirSync(historyDir, { withFileTypes: true })
|
||||
.filter(
|
||||
(entry) =>
|
||||
entry.isFile() &&
|
||||
entry.name.endsWith("-harness-eval-summary.json"),
|
||||
)
|
||||
.map((entry) => path.join(historyDir, entry.name))
|
||||
.sort((left, right) => right.localeCompare(left));
|
||||
|
||||
for (const staleFile of historyFiles.slice(normalizedRetainCount)) {
|
||||
fs.rmSync(staleFile, { force: true });
|
||||
}
|
||||
}
|
||||
|
||||
function recordSummaryHistory(repoRoot, summary, options) {
|
||||
if (!options.recordHistoryDir) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const historyDir = resolvePath(repoRoot, options.recordHistoryDir);
|
||||
ensureDirectory(historyDir);
|
||||
const historyFilePath = path.join(
|
||||
historyDir,
|
||||
`${toHistoryTimestamp(summary.generatedAt)}-harness-eval-summary.json`,
|
||||
);
|
||||
fs.writeFileSync(
|
||||
historyFilePath,
|
||||
`${JSON.stringify(summary, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
trimHistoryDirectory(historyDir, options.historyRetain);
|
||||
return historyFilePath;
|
||||
}
|
||||
|
||||
function normalizeStringList(value) {
|
||||
if (!Array.isArray(value)) {
|
||||
return [];
|
||||
@@ -115,6 +199,20 @@ function mergeUniqueStrings(...groups) {
|
||||
return [...new Set(groups.flatMap((group) => normalizeStringList(group)))];
|
||||
}
|
||||
|
||||
function normalizeOptionalString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0
|
||||
? value.trim()
|
||||
: "";
|
||||
}
|
||||
|
||||
function normalizeEnumString(value, allowedValues, fallback = "") {
|
||||
const normalized = normalizeOptionalString(value);
|
||||
if (allowedValues.has(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function createBreakdownEntry(name) {
|
||||
return {
|
||||
name,
|
||||
@@ -208,6 +306,88 @@ function listReplayDirectories(rootPath) {
|
||||
.sort((left, right) => left.localeCompare(right));
|
||||
}
|
||||
|
||||
function resolveExistingReviewDecisionPath(caseDir) {
|
||||
for (const relativePath of REVIEW_DECISION_JSON_CANDIDATES) {
|
||||
const candidate = path.resolve(caseDir, relativePath);
|
||||
if (fs.existsSync(candidate) && fs.statSync(candidate).isFile()) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function normalizeReviewDecisionPayload(payload) {
|
||||
const decision =
|
||||
payload && typeof payload === "object" && payload.decision
|
||||
? payload.decision
|
||||
: {};
|
||||
|
||||
return {
|
||||
decisionStatus: normalizeEnumString(
|
||||
decision.decisionStatus ??
|
||||
decision.decision_status ??
|
||||
payload?.decisionStatus ??
|
||||
payload?.decision_status,
|
||||
REVIEW_DECISION_STATUS_SET,
|
||||
"pending_review",
|
||||
),
|
||||
riskLevel: normalizeEnumString(
|
||||
decision.riskLevel ??
|
||||
decision.risk_level ??
|
||||
payload?.riskLevel ??
|
||||
payload?.risk_level,
|
||||
REVIEW_DECISION_RISK_LEVEL_SET,
|
||||
"unknown",
|
||||
),
|
||||
humanReviewer: normalizeOptionalString(
|
||||
decision.humanReviewer ??
|
||||
decision.human_reviewer ??
|
||||
payload?.humanReviewer ??
|
||||
payload?.human_reviewer,
|
||||
),
|
||||
reviewedAt: normalizeOptionalString(
|
||||
decision.reviewedAt ??
|
||||
decision.reviewed_at ??
|
||||
payload?.reviewedAt ??
|
||||
payload?.reviewed_at,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
function loadReviewDecisionForCase(caseDir, inlineReviewDecision) {
|
||||
const reviewDecisionPath = resolveExistingReviewDecisionPath(caseDir);
|
||||
|
||||
if (reviewDecisionPath) {
|
||||
try {
|
||||
return {
|
||||
reviewDecision: normalizeReviewDecisionPayload(
|
||||
readJsonFile(reviewDecisionPath),
|
||||
),
|
||||
issues: [],
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
reviewDecision: null,
|
||||
issues: [
|
||||
`review-decision.json 解析失败: ${String(error.message ?? error)}`,
|
||||
],
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (inlineReviewDecision && typeof inlineReviewDecision === "object") {
|
||||
return {
|
||||
reviewDecision: normalizeReviewDecisionPayload(inlineReviewDecision),
|
||||
issues: [],
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
reviewDecision: null,
|
||||
issues: [],
|
||||
};
|
||||
}
|
||||
|
||||
function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
const requiredArtifacts = normalizeStringList(
|
||||
caseConfig.requiredArtifacts ?? defaults.requiredArtifacts,
|
||||
@@ -237,6 +417,7 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
let inputPayload = null;
|
||||
let expectedPayload = null;
|
||||
let evidencePayload = null;
|
||||
let reviewDecision = null;
|
||||
|
||||
if (fs.existsSync(files["input.json"] ?? "")) {
|
||||
try {
|
||||
@@ -281,6 +462,13 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
}
|
||||
}
|
||||
|
||||
const reviewDecisionResult = loadReviewDecisionForCase(
|
||||
resolvedCaseDir,
|
||||
caseConfig.reviewDecision,
|
||||
);
|
||||
reviewDecision = reviewDecisionResult.reviewDecision;
|
||||
issues.push(...reviewDecisionResult.issues);
|
||||
|
||||
const pendingRequestCount = Array.isArray(
|
||||
inputPayload?.runtimeContext?.pendingRequests,
|
||||
)
|
||||
@@ -303,6 +491,7 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
typeof expectedPayload?.graderSuggestion?.preferredMode === "string"
|
||||
? expectedPayload.graderSuggestion.preferredMode
|
||||
: "";
|
||||
const reviewDecisionRecorded = reviewDecision != null;
|
||||
|
||||
return {
|
||||
caseId: context.caseId,
|
||||
@@ -326,6 +515,11 @@ function validateCaseDirectory(caseDir, caseConfig, defaults, context) {
|
||||
inputPayload?.task?.goalSummary ?? expectedPayload?.goalSummary ?? "",
|
||||
pendingRequestCount,
|
||||
requiresHumanReview,
|
||||
reviewDecisionRecorded,
|
||||
reviewDecisionStatus: reviewDecision?.decisionStatus ?? "",
|
||||
reviewRiskLevel: reviewDecision?.riskLevel ?? "",
|
||||
reviewHumanReviewer: reviewDecision?.humanReviewer ?? "",
|
||||
reviewReviewedAt: reviewDecision?.reviewedAt ?? "",
|
||||
preferredMode,
|
||||
status: issues.length === 0 ? "ready" : "invalid",
|
||||
issues,
|
||||
@@ -385,6 +579,11 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
|
||||
goalSummary: "",
|
||||
pendingRequestCount: 0,
|
||||
requiresHumanReview: false,
|
||||
reviewDecisionRecorded: false,
|
||||
reviewDecisionStatus: "",
|
||||
reviewRiskLevel: "",
|
||||
reviewHumanReviewer: "",
|
||||
reviewReviewedAt: "",
|
||||
preferredMode: "",
|
||||
status: "invalid",
|
||||
issues: [
|
||||
@@ -429,6 +628,11 @@ function expandSuiteCases(suiteConfig, defaults, repoRoot, workspaceRoot) {
|
||||
goalSummary: "",
|
||||
pendingRequestCount: 0,
|
||||
requiresHumanReview: false,
|
||||
reviewDecisionRecorded: false,
|
||||
reviewDecisionStatus: "",
|
||||
reviewRiskLevel: "",
|
||||
reviewHumanReviewer: "",
|
||||
reviewReviewedAt: "",
|
||||
preferredMode: "",
|
||||
status: "invalid",
|
||||
issues: [`不支持的 case source: ${source || "(empty)"}`],
|
||||
@@ -469,6 +673,9 @@ function buildSummary(manifest, suites, options) {
|
||||
const pendingCases = allCases.filter(
|
||||
(entry) => entry.pendingRequestCount > 0,
|
||||
);
|
||||
const recordedReviewDecisionCases = allCases.filter(
|
||||
(entry) => entry.reviewDecisionRecorded,
|
||||
);
|
||||
|
||||
return {
|
||||
manifestVersion: String(manifest.manifestVersion ?? "unknown"),
|
||||
@@ -484,6 +691,7 @@ function buildSummary(manifest, suites, options) {
|
||||
invalidCount: invalidCases.length,
|
||||
needsHumanReviewCount: reviewCases.length,
|
||||
pendingRequestCaseCount: pendingCases.length,
|
||||
reviewDecisionRecordedCount: recordedReviewDecisionCases.length,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: aggregateCaseBreakdown(allCases, (entry) => entry.tags),
|
||||
@@ -491,6 +699,12 @@ function buildSummary(manifest, suites, options) {
|
||||
allCases,
|
||||
(entry) => entry.failureModes,
|
||||
),
|
||||
reviewDecisionStatuses: aggregateCaseBreakdown(allCases, (entry) =>
|
||||
entry.reviewDecisionStatus ? [entry.reviewDecisionStatus] : [],
|
||||
),
|
||||
reviewRiskLevels: aggregateCaseBreakdown(allCases, (entry) =>
|
||||
entry.reviewRiskLevel ? [entry.reviewRiskLevel] : [],
|
||||
),
|
||||
},
|
||||
suites,
|
||||
};
|
||||
@@ -506,6 +720,7 @@ function renderText(summary) {
|
||||
`[harness-eval] invalid: ${summary.totals.invalidCount}`,
|
||||
`[harness-eval] pending-request cases: ${summary.totals.pendingRequestCaseCount}`,
|
||||
`[harness-eval] needs-review cases : ${summary.totals.needsHumanReviewCount}`,
|
||||
`[harness-eval] recorded review decisions: ${summary.totals.reviewDecisionRecordedCount}`,
|
||||
];
|
||||
|
||||
const topFailureModes = summary.breakdowns.failureModes.slice(0, 5);
|
||||
@@ -528,6 +743,16 @@ function renderText(summary) {
|
||||
}
|
||||
}
|
||||
|
||||
const topReviewStatuses = summary.breakdowns.reviewDecisionStatuses.slice(0, 5);
|
||||
if (topReviewStatuses.length > 0) {
|
||||
lines.push("[harness-eval] review decision statuses:");
|
||||
for (const entry of topReviewStatuses) {
|
||||
lines.push(
|
||||
` - ${entry.name}: case=${entry.caseCount}, ready=${entry.readyCount}, invalid=${entry.invalidCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const suite of summary.suites) {
|
||||
lines.push(
|
||||
`[harness-eval] suite ${suite.id}: ready ${suite.stats.readyCount} / ${suite.stats.caseCount}`,
|
||||
@@ -542,6 +767,11 @@ function renderText(summary) {
|
||||
if (entry.failureModes.length > 0) {
|
||||
lines.push(` failure_modes: ${entry.failureModes.join(", ")}`);
|
||||
}
|
||||
if (entry.reviewDecisionStatus) {
|
||||
lines.push(
|
||||
` review: ${entry.reviewDecisionStatus} / risk=${entry.reviewRiskLevel || "unknown"}`,
|
||||
);
|
||||
}
|
||||
for (const issue of entry.issues) {
|
||||
lines.push(` * ${issue}`);
|
||||
}
|
||||
@@ -564,6 +794,7 @@ function renderMarkdown(summary) {
|
||||
`- invalid：${summary.totals.invalidCount}`,
|
||||
`- pending request case：${summary.totals.pendingRequestCaseCount}`,
|
||||
`- needs review case：${summary.totals.needsHumanReviewCount}`,
|
||||
`- 已记录人工审核：${summary.totals.reviewDecisionRecordedCount}`,
|
||||
"",
|
||||
];
|
||||
|
||||
@@ -595,6 +826,32 @@ function renderMarkdown(summary) {
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (summary.breakdowns.reviewDecisionStatuses.length > 0) {
|
||||
lines.push("## 人工审核状态分布");
|
||||
lines.push("");
|
||||
lines.push("| 审核状态 | case | ready | invalid |");
|
||||
lines.push("| --- | --- | --- | --- |");
|
||||
for (const entry of summary.breakdowns.reviewDecisionStatuses) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.caseCount} | ${entry.readyCount} | ${entry.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
if (summary.breakdowns.reviewRiskLevels.length > 0) {
|
||||
lines.push("## 风险等级分布");
|
||||
lines.push("");
|
||||
lines.push("| 风险等级 | case | ready | invalid |");
|
||||
lines.push("| --- | --- | --- | --- |");
|
||||
for (const entry of summary.breakdowns.reviewRiskLevels) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.caseCount} | ${entry.readyCount} | ${entry.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
}
|
||||
|
||||
for (const suite of summary.suites) {
|
||||
lines.push(`## ${suite.title}`);
|
||||
lines.push("");
|
||||
@@ -613,8 +870,8 @@ function renderMarkdown(summary) {
|
||||
`- ready / total：${suite.stats.readyCount} / ${suite.stats.caseCount}`,
|
||||
);
|
||||
lines.push("");
|
||||
lines.push("| Case | 状态 | 来源 | 分类 | 目录 | 问题 |");
|
||||
lines.push("| --- | --- | --- | --- | --- | --- |");
|
||||
lines.push("| Case | 状态 | 来源 | 分类 | 审核 | 目录 | 问题 |");
|
||||
lines.push("| --- | --- | --- | --- | --- | --- | --- |");
|
||||
for (const entry of suite.cases) {
|
||||
const issueText =
|
||||
entry.issues.length === 0 ? "无" : entry.issues.join("<br>");
|
||||
@@ -628,8 +885,19 @@ function renderMarkdown(summary) {
|
||||
if (entry.primaryBlockingKind) {
|
||||
classificationText.push(`blocking: ${entry.primaryBlockingKind}`);
|
||||
}
|
||||
const reviewText = entry.reviewDecisionStatus
|
||||
? [
|
||||
entry.reviewDecisionStatus,
|
||||
entry.reviewRiskLevel ? `risk: ${entry.reviewRiskLevel}` : "",
|
||||
entry.reviewHumanReviewer
|
||||
? `by: ${entry.reviewHumanReviewer}`
|
||||
: "",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("<br>")
|
||||
: "无";
|
||||
lines.push(
|
||||
`| ${entry.caseId} | ${entry.status} | ${entry.source} | ${classificationText.join("<br>") || "无"} | \`${entry.relativeCaseDir || "."}\` | ${issueText} |`,
|
||||
`| ${entry.caseId} | ${entry.status} | ${entry.source} | ${classificationText.join("<br>") || "无"} | ${reviewText} | \`${entry.relativeCaseDir || "."}\` | ${issueText} |`,
|
||||
);
|
||||
}
|
||||
lines.push("");
|
||||
@@ -670,6 +938,7 @@ function main() {
|
||||
const jsonOutput = `${JSON.stringify(summary, null, 2)}\n`;
|
||||
const markdownOutput = renderMarkdown(summary);
|
||||
const textOutput = renderText(summary);
|
||||
const historyFilePath = recordSummaryHistory(repoRoot, summary, options);
|
||||
|
||||
if (options.outputJson) {
|
||||
const outputPath = resolvePath(repoRoot, options.outputJson);
|
||||
@@ -689,6 +958,11 @@ function main() {
|
||||
process.stdout.write(markdownOutput);
|
||||
} else {
|
||||
process.stdout.write(textOutput);
|
||||
if (historyFilePath) {
|
||||
process.stdout.write(
|
||||
`[harness-eval] history snapshot: ${historyFilePath}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const exitCode = determineExitCode(summary, options);
|
||||
|
||||
@@ -13,6 +13,7 @@ function parseArgs(argv) {
|
||||
help: false,
|
||||
historyDir: "",
|
||||
inputs: [],
|
||||
manifest: "",
|
||||
outputJson: "",
|
||||
outputMarkdown: "",
|
||||
workspaceRoot: process.cwd(),
|
||||
@@ -27,6 +28,12 @@ function parseArgs(argv) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--manifest" && argv[index + 1]) {
|
||||
result.manifest = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--history-dir" && argv[index + 1]) {
|
||||
result.historyDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
@@ -71,11 +78,13 @@ Lime Harness Eval Trend Report
|
||||
|
||||
用法:
|
||||
node scripts/harness-eval-trend-report.mjs
|
||||
node scripts/harness-eval-trend-report.mjs --manifest "./tmp/harness-evals.manifest.json"
|
||||
node scripts/harness-eval-trend-report.mjs --input "./tmp/harness-eval-summary.json"
|
||||
node scripts/harness-eval-trend-report.mjs --history-dir "./artifacts/history"
|
||||
node scripts/harness-eval-trend-report.mjs --output-json "./tmp/harness-eval-trend.json" --output-markdown "./tmp/harness-eval-trend.md"
|
||||
|
||||
选项:
|
||||
--manifest PATH 透传给 harness eval runner，覆盖默认 manifest
|
||||
--input PATH 显式加入一个或多个 harness eval summary JSON
|
||||
--history-dir PATH 扫描目录下的历史 summary JSON
|
||||
--workspace-root PATH 未提供输入时，用该工作区生成当前 summary
|
||||
@@ -138,12 +147,16 @@ function isHarnessEvalSummary(candidate) {
|
||||
);
|
||||
}
|
||||
|
||||
function buildCurrentSummary(repoRoot, workspaceRoot) {
|
||||
function buildCurrentSummary(repoRoot, workspaceRoot, manifestPath) {
|
||||
const nodeCommand = process.execPath;
|
||||
const runnerPath = resolvePath(repoRoot, RUNNER_PATH);
|
||||
const args = [runnerPath, "--format", "json", "--workspace-root", workspaceRoot];
|
||||
if (manifestPath) {
|
||||
args.push("--manifest", manifestPath);
|
||||
}
|
||||
const output = execFileSync(
|
||||
nodeCommand,
|
||||
[runnerPath, "--format", "json", "--workspace-root", workspaceRoot],
|
||||
args,
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
@@ -403,6 +416,9 @@ function buildTrendReport(samples, repoRoot) {
|
||||
needsHumanReviewCount:
|
||||
normalizeNumber(latest?.totals?.needsHumanReviewCount) -
|
||||
normalizeNumber(baseline?.totals?.needsHumanReviewCount),
|
||||
reviewDecisionRecordedCount:
|
||||
normalizeNumber(latest?.totals?.reviewDecisionRecordedCount) -
|
||||
normalizeNumber(baseline?.totals?.reviewDecisionRecordedCount),
|
||||
readyRate: readyRateDelta,
|
||||
},
|
||||
signals: buildStatusSignals(baseline, latest, sortedSamples.length),
|
||||
@@ -415,6 +431,16 @@ function buildTrendReport(samples, repoRoot) {
|
||||
classificationDeltas: {
|
||||
suiteTags: buildBreakdownDeltas(baseline, latest, "suiteTags"),
|
||||
failureModes: buildBreakdownDeltas(baseline, latest, "failureModes"),
|
||||
reviewDecisionStatuses: buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"reviewDecisionStatuses",
|
||||
),
|
||||
reviewRiskLevels: buildBreakdownDeltas(
|
||||
baseline,
|
||||
latest,
|
||||
"reviewRiskLevels",
|
||||
),
|
||||
},
|
||||
};
|
||||
}
|
||||
@@ -428,6 +454,7 @@ function renderText(report) {
|
||||
`[harness-eval-trend] delta readyCount: ${report.delta.readyCount}`,
|
||||
`[harness-eval-trend] delta invalidCount: ${report.delta.invalidCount}`,
|
||||
`[harness-eval-trend] delta pendingRequestCaseCount: ${report.delta.pendingRequestCaseCount}`,
|
||||
`[harness-eval-trend] delta reviewDecisionRecordedCount: ${report.delta.reviewDecisionRecordedCount}`,
|
||||
`[harness-eval-trend] delta readyRate: ${(report.delta.readyRate * 100).toFixed(1)}%`,
|
||||
];
|
||||
|
||||
@@ -448,6 +475,17 @@ function renderText(report) {
|
||||
}
|
||||
}
|
||||
|
||||
const topReviewDecisionDeltas =
|
||||
report.classificationDeltas.reviewDecisionStatuses.slice(0, 5);
|
||||
if (topReviewDecisionDeltas.length > 0) {
|
||||
lines.push("[harness-eval-trend] review decision deltas:");
|
||||
for (const entry of topReviewDecisionDeltas) {
|
||||
lines.push(
|
||||
` - ${entry.name}: delta_case=${entry.delta.caseCount}, delta_invalid=${entry.delta.invalidCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
@@ -468,6 +506,7 @@ function renderMarkdown(report) {
|
||||
`- invalid 数变化：${report.delta.invalidCount}`,
|
||||
`- pending request case 变化：${report.delta.pendingRequestCaseCount}`,
|
||||
`- needs review case 变化：${report.delta.needsHumanReviewCount}`,
|
||||
`- 已记录人工审核变化：${report.delta.reviewDecisionRecordedCount}`,
|
||||
`- ready rate 变化：${(report.delta.readyRate * 100).toFixed(1)}%`,
|
||||
"",
|
||||
"## 信号",
|
||||
@@ -508,6 +547,32 @@ function renderMarkdown(report) {
|
||||
}
|
||||
}
|
||||
|
||||
if (report.classificationDeltas.reviewDecisionStatuses.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## 人工审核状态变化");
|
||||
lines.push("");
|
||||
lines.push("| 审核状态 | baseline case | latest case | delta case | delta invalid |");
|
||||
lines.push("| --- | --- | --- | --- | --- |");
|
||||
for (const entry of report.classificationDeltas.reviewDecisionStatuses) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} | ${entry.delta.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (report.classificationDeltas.reviewRiskLevels.length > 0) {
|
||||
lines.push("");
|
||||
lines.push("## 风险等级变化");
|
||||
lines.push("");
|
||||
lines.push("| 风险等级 | baseline case | latest case | delta case | delta invalid |");
|
||||
lines.push("| --- | --- | --- | --- | --- |");
|
||||
for (const entry of report.classificationDeltas.reviewRiskLevels) {
|
||||
lines.push(
|
||||
`| ${entry.name} | ${entry.baseline.caseCount} | ${entry.latest.caseCount} | ${entry.delta.caseCount} | ${entry.delta.invalidCount} |`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
lines.push("");
|
||||
lines.push("## 时间线样本");
|
||||
lines.push("");
|
||||
@@ -587,6 +652,7 @@ function loadSamples(options, repoRoot) {
|
||||
const currentSummary = buildCurrentSummary(
|
||||
repoRoot,
|
||||
path.resolve(options.workspaceRoot),
|
||||
options.manifest,
|
||||
);
|
||||
sampleEntries.push({
|
||||
sourcePath: "(generated-current-summary)",
|
||||
|
||||
@@ -8,12 +8,27 @@ const DEFAULT_MANIFEST_PATH = "docs/test/harness-evals.manifest.json";
|
||||
const DEFAULT_FIXTURES_ROOT = "docs/test/harness-fixtures/replay";
|
||||
const DEFAULT_SUITE_ID = "repo-promoted-replays";
|
||||
const DEFAULT_SANITIZED_WORKSPACE_ROOT = "/workspace/lime";
|
||||
const REVIEW_DECISION_JSON_FILE_NAME = "review-decision.json";
|
||||
const REVIEW_DECISION_MARKDOWN_FILE_NAME = "review-decision.md";
|
||||
const REQUIRED_ARTIFACTS = [
|
||||
"input.json",
|
||||
"expected.json",
|
||||
"grader.md",
|
||||
"evidence-links.json",
|
||||
];
|
||||
const REVIEW_DECISION_STATUS_SET = new Set([
|
||||
"accepted",
|
||||
"deferred",
|
||||
"rejected",
|
||||
"needs_more_evidence",
|
||||
"pending_review",
|
||||
]);
|
||||
const REVIEW_DECISION_RISK_LEVEL_SET = new Set([
|
||||
"low",
|
||||
"medium",
|
||||
"high",
|
||||
"unknown",
|
||||
]);
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
@@ -150,6 +165,10 @@ function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function readTextFile(filePath) {
|
||||
return fs.readFileSync(filePath, "utf8");
|
||||
}
|
||||
|
||||
function writeJsonFile(filePath, value) {
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
|
||||
}
|
||||
@@ -180,6 +199,20 @@ function mergeUniqueStrings(...groups) {
|
||||
return [...new Set(groups.flatMap((group) => normalizeStringList(group)))];
|
||||
}
|
||||
|
||||
function normalizeEnumString(value, allowedValues, fallback = "") {
|
||||
const normalized = typeof value === "string" ? value.trim() : "";
|
||||
if (allowedValues.has(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function normalizeOptionalString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0
|
||||
? value.trim()
|
||||
: "";
|
||||
}
|
||||
|
||||
function slugify(value) {
|
||||
return String(value)
|
||||
.trim()
|
||||
@@ -322,6 +355,7 @@ function getRelativeIfInside(rootPath, absolutePath) {
|
||||
function buildPromotionMetadata({
|
||||
promotedAt,
|
||||
replayDir,
|
||||
reviewDecision,
|
||||
sessionId,
|
||||
workspaceRoot,
|
||||
sanitizedWorkspaceRoot,
|
||||
@@ -338,6 +372,10 @@ function buildPromotionMetadata({
|
||||
metadata.sourceReplayDir = replayRelativeDir;
|
||||
}
|
||||
|
||||
if (reviewDecision) {
|
||||
metadata.reviewDecision = reviewDecision;
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
@@ -382,9 +420,10 @@ function buildManifestCaseEntry({
|
||||
caseId,
|
||||
caseTitle,
|
||||
inputPayload,
|
||||
reviewDecision,
|
||||
targetCaseDirValue,
|
||||
}) {
|
||||
return {
|
||||
const entry = {
|
||||
id: caseId,
|
||||
title: caseTitle,
|
||||
source: "repo_fixture",
|
||||
@@ -394,6 +433,12 @@ function buildManifestCaseEntry({
|
||||
inputPayload?.classification?.suiteTags,
|
||||
),
|
||||
};
|
||||
|
||||
if (reviewDecision) {
|
||||
entry.reviewDecision = reviewDecision;
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
function updateManifestCase({
|
||||
@@ -448,6 +493,8 @@ function writePromotedArtifacts({
|
||||
expectedPayload,
|
||||
graderMarkdown,
|
||||
inputPayload,
|
||||
reviewDecisionJson,
|
||||
reviewDecisionMarkdown,
|
||||
targetDir,
|
||||
}) {
|
||||
ensureDirectory(targetDir);
|
||||
@@ -455,6 +502,19 @@ function writePromotedArtifacts({
|
||||
writeJsonFile(path.join(targetDir, "expected.json"), expectedPayload);
|
||||
writeJsonFile(path.join(targetDir, "evidence-links.json"), evidencePayload);
|
||||
fs.writeFileSync(path.join(targetDir, "grader.md"), graderMarkdown, "utf8");
|
||||
if (reviewDecisionJson) {
|
||||
writeJsonFile(
|
||||
path.join(targetDir, REVIEW_DECISION_JSON_FILE_NAME),
|
||||
reviewDecisionJson,
|
||||
);
|
||||
}
|
||||
if (reviewDecisionMarkdown) {
|
||||
fs.writeFileSync(
|
||||
path.join(targetDir, REVIEW_DECISION_MARKDOWN_FILE_NAME),
|
||||
reviewDecisionMarkdown,
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function toManifestCaseDirValue(repoRoot, targetDir) {
|
||||
@@ -485,9 +545,100 @@ function renderText(result) {
|
||||
lines.push(`[harness-replay-promote] tags: ${result.tags.join(", ")}`);
|
||||
}
|
||||
|
||||
if (result.reviewDecisionStatus) {
|
||||
lines.push(
|
||||
`[harness-replay-promote] review decision: ${result.reviewDecisionStatus} / risk=${result.reviewRiskLevel || "unknown"}`,
|
||||
);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
function resolveSiblingReviewArtifactPath(replayDir, fileName) {
|
||||
return path.resolve(replayDir, "..", "review", fileName);
|
||||
}
|
||||
|
||||
function normalizeReviewDecisionMetadata(reviewDecisionPayload) {
|
||||
const decision =
|
||||
reviewDecisionPayload &&
|
||||
typeof reviewDecisionPayload === "object" &&
|
||||
reviewDecisionPayload.decision &&
|
||||
typeof reviewDecisionPayload.decision === "object"
|
||||
? reviewDecisionPayload.decision
|
||||
: {};
|
||||
const decisionStatus = normalizeEnumString(
|
||||
decision.decisionStatus ?? decision.decision_status,
|
||||
REVIEW_DECISION_STATUS_SET,
|
||||
"pending_review",
|
||||
);
|
||||
const riskLevel = normalizeEnumString(
|
||||
decision.riskLevel ?? decision.risk_level,
|
||||
REVIEW_DECISION_RISK_LEVEL_SET,
|
||||
"unknown",
|
||||
);
|
||||
const humanReviewer = normalizeOptionalString(
|
||||
decision.humanReviewer ?? decision.human_reviewer,
|
||||
);
|
||||
const reviewedAt = normalizeOptionalString(
|
||||
decision.reviewedAt ?? decision.reviewed_at,
|
||||
);
|
||||
|
||||
return {
|
||||
decisionStatus,
|
||||
riskLevel,
|
||||
humanReviewer,
|
||||
reviewedAt,
|
||||
};
|
||||
}
|
||||
|
||||
function loadOptionalReviewDecisionArtifacts({
|
||||
replayDir,
|
||||
sanitizedWorkspaceRoot,
|
||||
workspaceRoot,
|
||||
}) {
|
||||
const reviewDecisionJsonPath = resolveSiblingReviewArtifactPath(
|
||||
replayDir,
|
||||
REVIEW_DECISION_JSON_FILE_NAME,
|
||||
);
|
||||
const reviewDecisionMarkdownPath = resolveSiblingReviewArtifactPath(
|
||||
replayDir,
|
||||
REVIEW_DECISION_MARKDOWN_FILE_NAME,
|
||||
);
|
||||
const hasJson = fs.existsSync(reviewDecisionJsonPath);
|
||||
const hasMarkdown = fs.existsSync(reviewDecisionMarkdownPath);
|
||||
|
||||
if (!hasJson && !hasMarkdown) {
|
||||
return {
|
||||
metadata: null,
|
||||
reviewDecisionJson: null,
|
||||
reviewDecisionMarkdown: "",
|
||||
};
|
||||
}
|
||||
|
||||
const reviewDecisionJson = hasJson
|
||||
? sanitizePayload(
|
||||
readJsonFile(reviewDecisionJsonPath),
|
||||
workspaceRoot,
|
||||
sanitizedWorkspaceRoot,
|
||||
)
|
||||
: null;
|
||||
const reviewDecisionMarkdown = hasMarkdown
|
||||
? replaceWorkspaceRootInString(
|
||||
readTextFile(reviewDecisionMarkdownPath),
|
||||
workspaceRoot,
|
||||
sanitizedWorkspaceRoot,
|
||||
)
|
||||
: "";
|
||||
|
||||
return {
|
||||
metadata: reviewDecisionJson
|
||||
? normalizeReviewDecisionMetadata(reviewDecisionJson)
|
||||
: null,
|
||||
reviewDecisionJson,
|
||||
reviewDecisionMarkdown,
|
||||
};
|
||||
}
|
||||
|
||||
function main() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
@@ -531,6 +682,7 @@ function main() {
|
||||
const promotionMetadata = buildPromotionMetadata({
|
||||
promotedAt,
|
||||
replayDir,
|
||||
reviewDecision: undefined,
|
||||
sanitizedWorkspaceRoot: options.sanitizedWorkspaceRoot,
|
||||
sessionId,
|
||||
workspaceRoot,
|
||||
@@ -570,6 +722,18 @@ function main() {
|
||||
),
|
||||
promotionMetadata,
|
||||
);
|
||||
const {
|
||||
metadata: reviewDecisionMetadata,
|
||||
reviewDecisionJson,
|
||||
reviewDecisionMarkdown,
|
||||
} = loadOptionalReviewDecisionArtifacts({
|
||||
replayDir,
|
||||
sanitizedWorkspaceRoot: options.sanitizedWorkspaceRoot,
|
||||
workspaceRoot,
|
||||
});
|
||||
if (reviewDecisionMetadata) {
|
||||
promotionMetadata.reviewDecision = reviewDecisionMetadata;
|
||||
}
|
||||
|
||||
const fixturesRoot = resolvePath(repoRoot, options.fixturesRoot);
|
||||
const targetDir = path.join(fixturesRoot, slug);
|
||||
@@ -584,6 +748,7 @@ function main() {
|
||||
caseId,
|
||||
caseTitle: title,
|
||||
inputPayload,
|
||||
reviewDecision: reviewDecisionMetadata,
|
||||
targetCaseDirValue: manifestCaseDir,
|
||||
});
|
||||
|
||||
@@ -597,6 +762,8 @@ function main() {
|
||||
expectedPayload,
|
||||
graderMarkdown,
|
||||
inputPayload,
|
||||
reviewDecisionJson,
|
||||
reviewDecisionMarkdown,
|
||||
targetDir,
|
||||
});
|
||||
const manifestUpdate = updateManifestCase({
|
||||
@@ -623,6 +790,10 @@ function main() {
|
||||
targetCaseDir: toPortablePath(targetDir),
|
||||
title,
|
||||
};
|
||||
if (reviewDecisionMetadata) {
|
||||
result.reviewDecisionStatus = reviewDecisionMetadata.decisionStatus;
|
||||
result.reviewRiskLevel = reviewDecisionMetadata.riskLevel;
|
||||
}
|
||||
|
||||
if (options.format === "json") {
|
||||
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
||||
|
||||
@@ -0,0 +1,380 @@
|
||||
import path from "node:path";
|
||||
|
||||
export const DOC_FRESHNESS_SPECS = [
|
||||
{
|
||||
path: "docs/tech/harness/README.md",
|
||||
requiredMentions: [
|
||||
"iteration-roadmap.md",
|
||||
"review-decision-workflow.md",
|
||||
"tooling-roadmap.md",
|
||||
"entropy-governance-workflow.md",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/tech/harness/iteration-roadmap.md",
|
||||
requiredMentions: [
|
||||
"review-decision-workflow.md",
|
||||
"tooling-roadmap.md",
|
||||
"entropy-governance-workflow.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/check-doc-freshness.mjs",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/tech/harness/tooling-roadmap.md",
|
||||
requiredMentions: [
|
||||
"harness-evals.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/check-doc-freshness.mjs",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/tech/harness/entropy-governance-workflow.md",
|
||||
requiredMentions: [
|
||||
"iteration-roadmap.md",
|
||||
"tooling-roadmap.md",
|
||||
"harness-evals.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/check-doc-freshness.mjs",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/tech/harness/review-decision-workflow.md",
|
||||
requiredMentions: [
|
||||
"external-analysis-handoff.md",
|
||||
"iteration-roadmap.md",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/tech/harness/external-analysis-handoff.md",
|
||||
requiredMentions: [
|
||||
"iteration-roadmap.md",
|
||||
"review-decision-workflow.md",
|
||||
"tooling-roadmap.md",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/tech/harness/implementation-blueprint.md",
|
||||
requiredMentions: [
|
||||
"harness-evals.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/test/harness-evals.md",
|
||||
requiredMentions: [
|
||||
"tooling-roadmap.md",
|
||||
"entropy-governance-workflow.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/check-doc-freshness.mjs",
|
||||
],
|
||||
},
|
||||
{
|
||||
path: "docs/aiprompts/governance.md",
|
||||
requiredMentions: [],
|
||||
},
|
||||
{
|
||||
path: "docs/aiprompts/quality-workflow.md",
|
||||
requiredMentions: [],
|
||||
},
|
||||
];
|
||||
|
||||
const LOCAL_LINK_PATTERN = /\[[^\]]+\]\(([^)]+)\)/g;
|
||||
const REPO_PATH_TOKEN_PATTERN =
|
||||
/(^|[\s("'`])((?:docs|scripts|src|src-tauri|\.github)\/[A-Za-z0-9._/-]+(?:\.[A-Za-z0-9]+)?)/gm;
|
||||
const REPO_ROOT_RELATIVE_PREFIXES = [
|
||||
"docs/",
|
||||
"scripts/",
|
||||
"src/",
|
||||
"src-tauri/",
|
||||
".github/",
|
||||
];
|
||||
|
||||
function normalizePath(filePath) {
|
||||
return filePath.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function stripAnchor(target) {
|
||||
const anchorIndex = target.indexOf("#");
|
||||
return anchorIndex >= 0 ? target.slice(0, anchorIndex) : target;
|
||||
}
|
||||
|
||||
function cleanLinkTarget(target) {
|
||||
const trimmed = String(target ?? "").trim();
|
||||
if (
|
||||
trimmed.length === 0 ||
|
||||
trimmed.startsWith("http://") ||
|
||||
trimmed.startsWith("https://") ||
|
||||
trimmed.startsWith("mailto:") ||
|
||||
trimmed.startsWith("#") ||
|
||||
trimmed.startsWith("vscode:") ||
|
||||
trimmed.startsWith("file://")
|
||||
) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const withoutAngleBrackets =
|
||||
trimmed.startsWith("<") && trimmed.endsWith(">")
|
||||
? trimmed.slice(1, -1).trim()
|
||||
: trimmed;
|
||||
const withoutTitle = withoutAngleBrackets.split(/\s+"/, 1)[0];
|
||||
const withoutAnchor = stripAnchor(withoutTitle);
|
||||
|
||||
return withoutAnchor.includes("<") ? "" : withoutAnchor;
|
||||
}
|
||||
|
||||
function resolveRepoPath({ repoRoot, documentPath, rawPath }) {
|
||||
const cleanedPath = cleanLinkTarget(rawPath);
|
||||
if (!cleanedPath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const normalizedRepoRoot = path.resolve(repoRoot);
|
||||
let absolutePath = "";
|
||||
|
||||
if (path.isAbsolute(cleanedPath)) {
|
||||
const normalizedAbsolute = path.resolve(cleanedPath);
|
||||
if (!normalizedAbsolute.startsWith(normalizedRepoRoot)) {
|
||||
return null;
|
||||
}
|
||||
absolutePath = normalizedAbsolute;
|
||||
} else if (
|
||||
REPO_ROOT_RELATIVE_PREFIXES.some((prefix) => cleanedPath.startsWith(prefix))
|
||||
) {
|
||||
absolutePath = path.resolve(normalizedRepoRoot, cleanedPath);
|
||||
} else {
|
||||
absolutePath = path.resolve(
|
||||
normalizedRepoRoot,
|
||||
path.dirname(documentPath),
|
||||
cleanedPath,
|
||||
);
|
||||
}
|
||||
|
||||
const repoRelativePath = normalizePath(
|
||||
path.relative(normalizedRepoRoot, absolutePath),
|
||||
);
|
||||
if (repoRelativePath.startsWith("..")) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
absolutePath,
|
||||
repoRelativePath,
|
||||
};
|
||||
}
|
||||
|
||||
function collectLocalLinks(content) {
|
||||
const results = [];
|
||||
for (const match of content.matchAll(LOCAL_LINK_PATTERN)) {
|
||||
const rawTarget = match[1];
|
||||
if (!rawTarget) {
|
||||
continue;
|
||||
}
|
||||
results.push(rawTarget);
|
||||
}
|
||||
return [...new Set(results)];
|
||||
}
|
||||
|
||||
function collectRepoPathTokens(content) {
|
||||
const results = [];
|
||||
for (const match of content.matchAll(REPO_PATH_TOKEN_PATTERN)) {
|
||||
const rawTarget = match[2];
|
||||
if (!rawTarget) {
|
||||
continue;
|
||||
}
|
||||
results.push(rawTarget.trim());
|
||||
}
|
||||
return [...new Set(results)];
|
||||
}
|
||||
|
||||
function createIssue(kind, documentPath, detail) {
|
||||
return {
|
||||
kind,
|
||||
documentPath,
|
||||
detail,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDocFreshnessReport({
|
||||
repoRoot,
|
||||
documents,
|
||||
deletedSurfaceTargets = [],
|
||||
pathExists,
|
||||
specs = DOC_FRESHNESS_SPECS,
|
||||
}) {
|
||||
const documentsByPath = new Map(
|
||||
(Array.isArray(documents) ? documents : []).map((entry) => [
|
||||
normalizePath(entry.path),
|
||||
String(entry.content ?? ""),
|
||||
]),
|
||||
);
|
||||
const issues = [];
|
||||
const documentReports = [];
|
||||
|
||||
for (const spec of specs) {
|
||||
const documentPath = normalizePath(spec.path);
|
||||
const content = documentsByPath.get(documentPath);
|
||||
|
||||
if (content == null) {
|
||||
issues.push(createIssue("missing-document", documentPath, documentPath));
|
||||
documentReports.push({
|
||||
path: documentPath,
|
||||
exists: false,
|
||||
requiredMentions: [],
|
||||
localLinks: [],
|
||||
codePathMentions: [],
|
||||
deletedSurfaceReferences: [],
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
const requiredMentions = (spec.requiredMentions ?? []).map((needle) => ({
|
||||
needle,
|
||||
found: content.includes(needle),
|
||||
}));
|
||||
for (const entry of requiredMentions) {
|
||||
if (!entry.found) {
|
||||
issues.push(
|
||||
createIssue("missing-required-reference", documentPath, entry.needle),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const localLinks = collectLocalLinks(content).map((rawTarget) => {
|
||||
const resolved = resolveRepoPath({
|
||||
repoRoot,
|
||||
documentPath,
|
||||
rawPath: rawTarget,
|
||||
});
|
||||
if (!resolved) {
|
||||
return {
|
||||
rawTarget,
|
||||
repoRelativePath: "",
|
||||
exists: true,
|
||||
};
|
||||
}
|
||||
|
||||
const exists = pathExists(resolved.absolutePath, resolved.repoRelativePath);
|
||||
if (!exists) {
|
||||
issues.push(
|
||||
createIssue(
|
||||
"broken-markdown-link",
|
||||
documentPath,
|
||||
`${rawTarget} -> ${resolved.repoRelativePath}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
rawTarget,
|
||||
repoRelativePath: resolved.repoRelativePath,
|
||||
exists,
|
||||
};
|
||||
});
|
||||
|
||||
const codePathMentions = collectRepoPathTokens(content).map((rawTarget) => {
|
||||
const resolved = resolveRepoPath({
|
||||
repoRoot,
|
||||
documentPath,
|
||||
rawPath: rawTarget,
|
||||
});
|
||||
if (!resolved) {
|
||||
return {
|
||||
rawTarget,
|
||||
repoRelativePath: "",
|
||||
exists: true,
|
||||
};
|
||||
}
|
||||
|
||||
const exists = pathExists(resolved.absolutePath, resolved.repoRelativePath);
|
||||
if (!exists) {
|
||||
issues.push(
|
||||
createIssue(
|
||||
"broken-code-path-reference",
|
||||
documentPath,
|
||||
`${rawTarget} -> ${resolved.repoRelativePath}`,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
rawTarget,
|
||||
repoRelativePath: resolved.repoRelativePath,
|
||||
exists,
|
||||
};
|
||||
});
|
||||
|
||||
const deletedSurfaceReferences = [...new Set(deletedSurfaceTargets)]
|
||||
.filter(Boolean)
|
||||
.filter((target) => content.includes(target));
|
||||
for (const target of deletedSurfaceReferences) {
|
||||
issues.push(
|
||||
createIssue("deleted-surface-reference", documentPath, target),
|
||||
);
|
||||
}
|
||||
|
||||
documentReports.push({
|
||||
path: documentPath,
|
||||
exists: true,
|
||||
requiredMentions,
|
||||
localLinks,
|
||||
codePathMentions,
|
||||
deletedSurfaceReferences,
|
||||
});
|
||||
}
|
||||
|
||||
const summary = {
|
||||
monitoredDocumentCount: specs.length,
|
||||
existingDocumentCount: documentReports.filter((entry) => entry.exists).length,
|
||||
issueCount: issues.length,
|
||||
missingDocumentCount: issues.filter((entry) => entry.kind === "missing-document")
|
||||
.length,
|
||||
missingRequiredReferenceCount: issues.filter(
|
||||
(entry) => entry.kind === "missing-required-reference",
|
||||
).length,
|
||||
brokenMarkdownLinkCount: issues.filter(
|
||||
(entry) => entry.kind === "broken-markdown-link",
|
||||
).length,
|
||||
brokenCodePathReferenceCount: issues.filter(
|
||||
(entry) => entry.kind === "broken-code-path-reference",
|
||||
).length,
|
||||
deletedSurfaceReferenceCount: issues.filter(
|
||||
(entry) => entry.kind === "deleted-surface-reference",
|
||||
).length,
|
||||
};
|
||||
|
||||
return {
|
||||
reportVersion: "v1",
|
||||
generatedAt: new Date().toISOString(),
|
||||
repoRoot: path.resolve(repoRoot),
|
||||
summary,
|
||||
documents: documentReports,
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
export function renderDocFreshnessText(report) {
|
||||
const lines = [
|
||||
"[lime] doc freshness report",
|
||||
`[lime] monitored docs: ${report.summary.monitoredDocumentCount}`,
|
||||
`[lime] existing docs: ${report.summary.existingDocumentCount}`,
|
||||
`[lime] issues: ${report.summary.issueCount}`,
|
||||
`[lime] missing docs: ${report.summary.missingDocumentCount}`,
|
||||
`[lime] missing required refs: ${report.summary.missingRequiredReferenceCount}`,
|
||||
`[lime] broken markdown links: ${report.summary.brokenMarkdownLinkCount}`,
|
||||
`[lime] broken code path refs: ${report.summary.brokenCodePathReferenceCount}`,
|
||||
`[lime] deleted surface refs: ${report.summary.deletedSurfaceReferenceCount}`,
|
||||
];
|
||||
|
||||
if (report.issues.length === 0) {
|
||||
lines.push("[lime] doc freshness: clean");
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
|
||||
lines.push("[lime] issues:");
|
||||
for (const issue of report.issues) {
|
||||
lines.push(` - [${issue.kind}] ${issue.documentPath} -> ${issue.detail}`);
|
||||
}
|
||||
|
||||
return `${lines.join("\n")}\n`;
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { buildDocFreshnessReport } from "./doc-freshness-core.mjs";
|
||||
|
||||
describe("doc-freshness-core", () => {
|
||||
it("应在回链与路径都正常时返回 clean 报告", () => {
|
||||
const report = buildDocFreshnessReport({
|
||||
repoRoot: "/tmp/lime",
|
||||
specs: [
|
||||
{
|
||||
path: "docs/tech/harness/entropy-governance-workflow.md",
|
||||
requiredMentions: [
|
||||
"iteration-roadmap.md",
|
||||
"tooling-roadmap.md",
|
||||
"harness-evals.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/check-doc-freshness.mjs",
|
||||
],
|
||||
},
|
||||
],
|
||||
documents: [
|
||||
{
|
||||
path: "docs/tech/harness/entropy-governance-workflow.md",
|
||||
content: `
|
||||
[Roadmap](iteration-roadmap.md)
|
||||
[Tooling](tooling-roadmap.md)
|
||||
[Evals](../../test/harness-evals.md)
|
||||
\`scripts/report-generated-slop.mjs\`
|
||||
\`scripts/check-doc-freshness.mjs\`
|
||||
`,
|
||||
},
|
||||
],
|
||||
deletedSurfaceTargets: ["src/lib/api/agentCompat.ts"],
|
||||
pathExists: (_absolutePath, repoRelativePath) =>
|
||||
[
|
||||
"docs/tech/harness/iteration-roadmap.md",
|
||||
"docs/tech/harness/tooling-roadmap.md",
|
||||
"docs/test/harness-evals.md",
|
||||
"scripts/report-generated-slop.mjs",
|
||||
"scripts/check-doc-freshness.mjs",
|
||||
].includes(repoRelativePath),
|
||||
});
|
||||
|
||||
expect(report.summary.issueCount).toBe(0);
|
||||
expect(report.documents[0].requiredMentions.every((entry) => entry.found)).toBe(
|
||||
true,
|
||||
);
|
||||
});
|
||||
|
||||
it("应识别缺失回链、坏链接、坏路径与已删除表面引用", () => {
|
||||
const report = buildDocFreshnessReport({
|
||||
repoRoot: "/tmp/lime",
|
||||
specs: [
|
||||
{
|
||||
path: "docs/tech/harness/review-decision-workflow.md",
|
||||
requiredMentions: [
|
||||
"external-analysis-handoff.md",
|
||||
"iteration-roadmap.md",
|
||||
],
|
||||
},
|
||||
],
|
||||
documents: [
|
||||
{
|
||||
path: "docs/tech/harness/review-decision-workflow.md",
|
||||
content: `
|
||||
[Bad Link](missing-doc.md)
|
||||
\`scripts/missing-tool.mjs\`
|
||||
旧入口：src/lib/api/agentCompat.ts
|
||||
`,
|
||||
},
|
||||
],
|
||||
deletedSurfaceTargets: ["src/lib/api/agentCompat.ts"],
|
||||
pathExists: () => false,
|
||||
});
|
||||
|
||||
expect(report.summary.issueCount).toBe(5);
|
||||
expect(report.issues.map((entry) => entry.kind)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"missing-required-reference",
|
||||
"broken-markdown-link",
|
||||
"broken-code-path-reference",
|
||||
"deleted-surface-reference",
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,324 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { buildGeneratedSlopReport } from "./generated-slop-report-core.mjs";
|
||||
|
||||
describe("generated-slop-report-core", () => {
|
||||
it("应把 trend seed 与活跃 legacy surface 转成治理建议", () => {
|
||||
const report = buildGeneratedSlopReport({
|
||||
repoRoot: "/tmp/lime",
|
||||
trendReport: {
|
||||
sampleCount: 1,
|
||||
delta: {
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
reviewDecisionRecordedCount: 0,
|
||||
readyRate: 0,
|
||||
},
|
||||
signals: ["样本数不足 2，当前仅形成 trend seed，还不能判断长期退化。"],
|
||||
baseline: {
|
||||
totals: {
|
||||
needsHumanReviewCount: 0,
|
||||
reviewDecisionRecordedCount: 0,
|
||||
},
|
||||
},
|
||||
latest: {
|
||||
totals: {
|
||||
needsHumanReviewCount: 1,
|
||||
reviewDecisionRecordedCount: 0,
|
||||
},
|
||||
},
|
||||
classificationDeltas: {
|
||||
failureModes: [
|
||||
{
|
||||
name: "pending_request",
|
||||
baseline: {
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 1,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 1,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 0,
|
||||
readyCount: 0,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
suiteTags: [
|
||||
{
|
||||
name: "conversation-runtime",
|
||||
baseline: {
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 1,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 1,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 0,
|
||||
readyCount: 0,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
},
|
||||
],
|
||||
reviewDecisionStatuses: [
|
||||
{
|
||||
name: "pending_review",
|
||||
baseline: {
|
||||
caseCount: 0,
|
||||
readyCount: 0,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
readyCount: 0,
|
||||
invalidCount: 1,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 1,
|
||||
readyCount: 0,
|
||||
invalidCount: 1,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
reviewRiskLevels: [
|
||||
{
|
||||
name: "high",
|
||||
baseline: {
|
||||
caseCount: 0,
|
||||
readyCount: 0,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
latest: {
|
||||
caseCount: 1,
|
||||
readyCount: 0,
|
||||
invalidCount: 1,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
delta: {
|
||||
caseCount: 1,
|
||||
readyCount: 0,
|
||||
invalidCount: 1,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
governanceReport: {
|
||||
summary: {
|
||||
zeroReferenceCandidates: [],
|
||||
classificationDriftCandidates: [],
|
||||
violations: [],
|
||||
},
|
||||
importResults: [
|
||||
{
|
||||
id: "team-subagent-scheduler-hook",
|
||||
classification: "compat",
|
||||
description: "compat hook",
|
||||
existingTargets: ["src/hooks/useSubAgentScheduler.ts"],
|
||||
references: [
|
||||
"src/components/agent/chat/hooks/useCompatSubagentRuntime.ts",
|
||||
],
|
||||
testReferences: ["src/hooks/useSubAgentScheduler.test.tsx"],
|
||||
violations: [],
|
||||
},
|
||||
],
|
||||
commandResults: [],
|
||||
frontendTextResults: [
|
||||
{
|
||||
id: "migration-setting-key-leak",
|
||||
classification: "deprecated",
|
||||
description: "migration keys",
|
||||
references: ["src-tauri/crates/core/src/database/migration/foo.rs"],
|
||||
testReferences: [],
|
||||
violations: [],
|
||||
},
|
||||
],
|
||||
rustTextResults: [],
|
||||
rustTextCountResults: [],
|
||||
},
|
||||
docFreshnessReport: {
|
||||
summary: {
|
||||
monitoredDocumentCount: 10,
|
||||
existingDocumentCount: 10,
|
||||
issueCount: 0,
|
||||
missingDocumentCount: 0,
|
||||
missingRequiredReferenceCount: 0,
|
||||
brokenMarkdownLinkCount: 0,
|
||||
brokenCodePathReferenceCount: 0,
|
||||
deletedSurfaceReferenceCount: 0,
|
||||
},
|
||||
issues: [],
|
||||
},
|
||||
sources: {
|
||||
trend: { kind: "generated-current" },
|
||||
docFreshness: { kind: "live-scan" },
|
||||
governance: { kind: "live-scan" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.summary.trend.isSeed).toBe(true);
|
||||
expect(report.summary.trend.reviewDecisionBacklogCount).toBe(1);
|
||||
expect(report.summary.docFreshness.issueCount).toBe(0);
|
||||
expect(report.focus.failureModes[0].name).toBe("pending_request");
|
||||
expect(report.focus.reviewDecisionStatuses[0].name).toBe("pending_review");
|
||||
expect(report.focus.reviewRiskLevels[0].name).toBe("high");
|
||||
expect(report.focus.governanceSurfaces[0].id).toBe(
|
||||
"migration-setting-key-leak",
|
||||
);
|
||||
expect(report.recommendations.map((entry) => entry.id)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"promote-high-value-replay",
|
||||
"replay-and-smoke-follow-up",
|
||||
"review-decision-follow-up",
|
||||
"governance-cleanup-priority",
|
||||
"doc-freshness-review",
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
report.recommendations.find(
|
||||
(entry) => entry.id === "review-decision-follow-up",
|
||||
)?.focusReviewRiskLevels,
|
||||
).toContain("high");
|
||||
expect(
|
||||
report.recommendations.find((entry) => entry.id === "doc-freshness-review")
|
||||
?.commands,
|
||||
).toContain("npm run harness:doc-freshness");
|
||||
});
|
||||
|
||||
it("应把 governance 违规提升为 P0 守卫动作", () => {
|
||||
const report = buildGeneratedSlopReport({
|
||||
repoRoot: "/tmp/lime",
|
||||
trendReport: {
|
||||
sampleCount: 2,
|
||||
delta: {
|
||||
invalidCount: 1,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
reviewDecisionRecordedCount: 0,
|
||||
readyRate: -0.5,
|
||||
},
|
||||
signals: ["invalid case 增加 1，存在回归候选。"],
|
||||
baseline: {
|
||||
totals: {
|
||||
needsHumanReviewCount: 0,
|
||||
reviewDecisionRecordedCount: 0,
|
||||
},
|
||||
},
|
||||
latest: {
|
||||
totals: {
|
||||
needsHumanReviewCount: 0,
|
||||
reviewDecisionRecordedCount: 0,
|
||||
},
|
||||
},
|
||||
classificationDeltas: {
|
||||
failureModes: [],
|
||||
suiteTags: [],
|
||||
reviewDecisionStatuses: [],
|
||||
reviewRiskLevels: [],
|
||||
},
|
||||
},
|
||||
governanceReport: {
|
||||
summary: {
|
||||
zeroReferenceCandidates: [],
|
||||
classificationDriftCandidates: [],
|
||||
violations: ["chat-compat -> src/foo.ts"],
|
||||
},
|
||||
importResults: [],
|
||||
commandResults: [
|
||||
{
|
||||
id: "chat-compat-command",
|
||||
classification: "deprecated",
|
||||
description: "legacy command",
|
||||
referencesByCommand: {
|
||||
chat_create_session: ["src/lib/api/legacy.ts"],
|
||||
},
|
||||
testReferencesByCommand: {},
|
||||
violations: ["chat_create_session -> src/lib/api/legacy.ts"],
|
||||
},
|
||||
],
|
||||
frontendTextResults: [],
|
||||
rustTextResults: [],
|
||||
rustTextCountResults: [],
|
||||
},
|
||||
docFreshnessReport: {
|
||||
summary: {
|
||||
monitoredDocumentCount: 10,
|
||||
existingDocumentCount: 9,
|
||||
issueCount: 2,
|
||||
missingDocumentCount: 0,
|
||||
missingRequiredReferenceCount: 1,
|
||||
brokenMarkdownLinkCount: 0,
|
||||
brokenCodePathReferenceCount: 1,
|
||||
deletedSurfaceReferenceCount: 0,
|
||||
},
|
||||
issues: [
|
||||
{
|
||||
kind: "missing-required-reference",
|
||||
documentPath: "docs/tech/harness/entropy-governance-workflow.md",
|
||||
detail: "harness-evals.md",
|
||||
},
|
||||
{
|
||||
kind: "broken-code-path-reference",
|
||||
documentPath: "docs/tech/harness/tooling-roadmap.md",
|
||||
detail: "scripts/missing-tool.mjs",
|
||||
},
|
||||
],
|
||||
},
|
||||
sources: {
|
||||
trend: { kind: "input-json" },
|
||||
docFreshness: { kind: "input-json" },
|
||||
governance: { kind: "input-json" },
|
||||
},
|
||||
});
|
||||
|
||||
expect(report.recommendations[0].id).toBe("contracts-and-boundary-guards");
|
||||
expect(report.recommendations[0].priority).toBe("P0");
|
||||
expect(report.signals).toContain("doc freshness 发现 2 个问题。");
|
||||
expect(report.focus.docFreshness.issueKinds[0]).toEqual({
|
||||
kind: "broken-code-path-reference",
|
||||
count: 1,
|
||||
});
|
||||
expect(report.recommendations[0].commands).toEqual(
|
||||
expect.arrayContaining([
|
||||
"npm run governance:legacy-report",
|
||||
"npm run test:contracts",
|
||||
]),
|
||||
);
|
||||
expect(
|
||||
report.recommendations.find((entry) => entry.id === "doc-freshness-review")
|
||||
?.priority,
|
||||
).toBe("P1");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,87 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
function createTempRoot() {
|
||||
const tempRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "lime-harness-history-record-"),
|
||||
);
|
||||
tempRoots.push(tempRoot);
|
||||
return tempRoot;
|
||||
}
|
||||
|
||||
function runNodeScript(scriptRelativePath: string, args: string[]) {
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[path.join(repoRoot, scriptRelativePath), ...args],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (tempRoots.length > 0) {
|
||||
const tempRoot = tempRoots.pop();
|
||||
if (tempRoot) {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("harness-eval-history-record", () => {
|
||||
it("应记录本地 summary 历史，并生成非 seed 的 trend / cleanup", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const historyDir = path.join(tempRoot, "artifacts", "history");
|
||||
const workspaceRoot = path.join(tempRoot, "workspace");
|
||||
|
||||
const firstResult = runNodeScript("scripts/harness-eval-history-record.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--output-json",
|
||||
path.join(tempRoot, "first.json"),
|
||||
]);
|
||||
|
||||
const secondResult = runNodeScript(
|
||||
"scripts/harness-eval-history-record.mjs",
|
||||
[
|
||||
"--format",
|
||||
"json",
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--output-json",
|
||||
path.join(tempRoot, "second.json"),
|
||||
],
|
||||
);
|
||||
|
||||
expect(firstResult.historyCount).toBe(1);
|
||||
expect(firstResult.trend.sampleCount).toBe(1);
|
||||
|
||||
expect(secondResult.historyCount).toBe(2);
|
||||
expect(secondResult.trend.sampleCount).toBe(2);
|
||||
expect(secondResult.cleanup.trendSampleCount).toBe(2);
|
||||
expect(fs.existsSync(secondResult.recordedSummaryPath)).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(path.join(tempRoot, "artifacts", "harness-eval-trend.json")),
|
||||
).toBe(true);
|
||||
expect(
|
||||
fs.existsSync(
|
||||
path.join(tempRoot, "artifacts", "harness-cleanup-report.json"),
|
||||
),
|
||||
).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
function createTempRoot() {
|
||||
const tempRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "lime-harness-history-window-"),
|
||||
);
|
||||
tempRoots.push(tempRoot);
|
||||
return tempRoot;
|
||||
}
|
||||
|
||||
function ensureDir(dirPath: string) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function writeJson(filePath: string, payload: unknown) {
|
||||
ensureDir(path.dirname(filePath));
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function writeText(filePath: string, value: string) {
|
||||
ensureDir(path.dirname(filePath));
|
||||
fs.writeFileSync(filePath, value, "utf8");
|
||||
}
|
||||
|
||||
function sleepMs(durationMs: number) {
|
||||
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, durationMs);
|
||||
}
|
||||
|
||||
function createHarnessManifest(caseDir: string) {
|
||||
return {
|
||||
manifestVersion: "v1",
|
||||
title: "Harness History Window Test Manifest",
|
||||
defaults: {
|
||||
requiredArtifacts: [
|
||||
"input.json",
|
||||
"expected.json",
|
||||
"grader.md",
|
||||
"evidence-links.json",
|
||||
],
|
||||
requiredInputFields: [
|
||||
"session.sessionId",
|
||||
"session.threadId",
|
||||
"task.goalSummary",
|
||||
"classification.suiteTags",
|
||||
"classification.failureModes",
|
||||
"linkedArtifacts.handoffBundle.relativeRoot",
|
||||
"linkedArtifacts.evidencePack.relativeRoot",
|
||||
],
|
||||
requiredExpectedFields: [
|
||||
"successCriteria",
|
||||
"blockingChecks",
|
||||
"artifactChecks",
|
||||
"graderSuggestion.preferredMode",
|
||||
],
|
||||
requiredEvidenceFields: [
|
||||
"handoffBundle.relativeRoot",
|
||||
"evidencePack.relativeRoot",
|
||||
],
|
||||
},
|
||||
suites: [
|
||||
{
|
||||
id: "repo-fixtures",
|
||||
title: "仓库固定 Replay 样本",
|
||||
cases: [
|
||||
{
|
||||
id: "fixture-history-window",
|
||||
title: "Harness 历史窗口样本",
|
||||
source: "repo_fixture",
|
||||
caseDir,
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function createReplayFixture(tempRoot: string) {
|
||||
const caseDir = path.join(tempRoot, "fixture-case");
|
||||
const handoffRoot = path.join(tempRoot, "handoff");
|
||||
const evidenceRoot = path.join(tempRoot, "evidence");
|
||||
|
||||
writeJson(path.join(caseDir, "input.json"), {
|
||||
session: {
|
||||
sessionId: "fixture-history-session",
|
||||
threadId: "fixture-history-thread",
|
||||
},
|
||||
task: {
|
||||
goalSummary: "验证 harness eval history window",
|
||||
},
|
||||
classification: {
|
||||
suiteTags: ["conversation-runtime", "replay"],
|
||||
failureModes: ["pending_request"],
|
||||
},
|
||||
linkedArtifacts: {
|
||||
handoffBundle: {
|
||||
relativeRoot: ".lime/handoff",
|
||||
absoluteRoot: handoffRoot,
|
||||
},
|
||||
evidencePack: {
|
||||
relativeRoot: ".lime/evidence",
|
||||
absoluteRoot: evidenceRoot,
|
||||
},
|
||||
},
|
||||
});
|
||||
writeJson(path.join(caseDir, "expected.json"), {
|
||||
sessionId: "fixture-history-session",
|
||||
threadId: "fixture-history-thread",
|
||||
goalSummary: "验证 harness eval history window",
|
||||
successCriteria: ["history snapshot 可以被 trend 使用"],
|
||||
blockingChecks: ["summary history 目录会保留最近窗口"],
|
||||
artifactChecks: ["history dir 中存在 harness-eval-summary.json"],
|
||||
graderSuggestion: {
|
||||
preferredMode: "summary_only",
|
||||
requiresHumanReview: false,
|
||||
},
|
||||
});
|
||||
writeJson(path.join(caseDir, "evidence-links.json"), {
|
||||
handoffBundle: {
|
||||
relativeRoot: ".lime/handoff",
|
||||
absoluteRoot: handoffRoot,
|
||||
},
|
||||
evidencePack: {
|
||||
relativeRoot: ".lime/evidence",
|
||||
absoluteRoot: evidenceRoot,
|
||||
},
|
||||
});
|
||||
writeText(path.join(caseDir, "grader.md"), "# Grader\n");
|
||||
|
||||
return caseDir;
|
||||
}
|
||||
|
||||
function runNodeScript(scriptRelativePath: string, args: string[]) {
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[path.join(repoRoot, scriptRelativePath), ...args],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (tempRoots.length > 0) {
|
||||
const tempRoot = tempRoots.pop();
|
||||
if (tempRoot) {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("Harness eval history window", () => {
|
||||
it("runner 应记录并裁剪历史窗口，trend 应复用该目录", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const caseDir = createReplayFixture(tempRoot);
|
||||
const manifestPath = path.join(tempRoot, "manifest.json");
|
||||
const historyDir = path.join(tempRoot, "history");
|
||||
|
||||
writeJson(manifestPath, createHarnessManifest(caseDir));
|
||||
|
||||
runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--record-history-dir",
|
||||
historyDir,
|
||||
"--history-retain",
|
||||
"2",
|
||||
"--no-strict",
|
||||
]);
|
||||
sleepMs(10);
|
||||
runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--record-history-dir",
|
||||
historyDir,
|
||||
"--history-retain",
|
||||
"2",
|
||||
"--no-strict",
|
||||
]);
|
||||
sleepMs(10);
|
||||
runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--record-history-dir",
|
||||
historyDir,
|
||||
"--history-retain",
|
||||
"2",
|
||||
"--no-strict",
|
||||
]);
|
||||
|
||||
const historyFiles = fs
|
||||
.readdirSync(historyDir)
|
||||
.filter((entry) => entry.endsWith("-harness-eval-summary.json"))
|
||||
.sort();
|
||||
expect(historyFiles).toHaveLength(2);
|
||||
|
||||
const report = runNodeScript("scripts/harness-eval-trend-report.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--history-dir",
|
||||
historyDir,
|
||||
]);
|
||||
|
||||
expect(report.sampleCount).toBe(2);
|
||||
expect(report.signals).not.toContain(
|
||||
"样本数不足 2，当前仅形成 trend seed，还不能判断长期退化。",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,447 @@
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const tempRoots: string[] = [];
|
||||
|
||||
function createTempRoot() {
|
||||
const tempRoot = fs.mkdtempSync(
|
||||
path.join(os.tmpdir(), "lime-harness-review-evals-"),
|
||||
);
|
||||
tempRoots.push(tempRoot);
|
||||
return tempRoot;
|
||||
}
|
||||
|
||||
function ensureDir(dirPath: string) {
|
||||
fs.mkdirSync(dirPath, { recursive: true });
|
||||
}
|
||||
|
||||
function writeJson(filePath: string, payload: unknown) {
|
||||
ensureDir(path.dirname(filePath));
|
||||
fs.writeFileSync(filePath, `${JSON.stringify(payload, null, 2)}\n`, "utf8");
|
||||
}
|
||||
|
||||
function writeText(filePath: string, value: string) {
|
||||
ensureDir(path.dirname(filePath));
|
||||
fs.writeFileSync(filePath, value, "utf8");
|
||||
}
|
||||
|
||||
function createHarnessManifest(overrides?: Record<string, unknown>) {
|
||||
return {
|
||||
manifestVersion: "v1",
|
||||
title: "Test Harness Eval Manifest",
|
||||
defaults: {
|
||||
requiredArtifacts: [
|
||||
"input.json",
|
||||
"expected.json",
|
||||
"grader.md",
|
||||
"evidence-links.json",
|
||||
],
|
||||
requiredInputFields: [
|
||||
"session.sessionId",
|
||||
"session.threadId",
|
||||
"task.goalSummary",
|
||||
"classification.suiteTags",
|
||||
"classification.failureModes",
|
||||
"linkedArtifacts.handoffBundle.relativeRoot",
|
||||
"linkedArtifacts.evidencePack.relativeRoot",
|
||||
],
|
||||
requiredExpectedFields: [
|
||||
"successCriteria",
|
||||
"blockingChecks",
|
||||
"artifactChecks",
|
||||
"graderSuggestion.preferredMode",
|
||||
],
|
||||
requiredEvidenceFields: [
|
||||
"handoffBundle.relativeRoot",
|
||||
"evidencePack.relativeRoot",
|
||||
],
|
||||
},
|
||||
suites: [],
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function createWorkspaceSessionArtifacts(tempRoot: string, sessionId: string) {
|
||||
const workspaceRoot = path.join(tempRoot, "workspace");
|
||||
const sessionRoot = path.join(
|
||||
workspaceRoot,
|
||||
".lime",
|
||||
"harness",
|
||||
"sessions",
|
||||
sessionId,
|
||||
);
|
||||
const replayDir = path.join(sessionRoot, "replay");
|
||||
const reviewDir = path.join(sessionRoot, "review");
|
||||
ensureDir(replayDir);
|
||||
ensureDir(reviewDir);
|
||||
|
||||
const handoffRoot = path.join(sessionRoot, "handoff");
|
||||
const evidenceRoot = path.join(sessionRoot, "evidence");
|
||||
const inputPayload = {
|
||||
session: {
|
||||
sessionId: sessionId,
|
||||
threadId: `${sessionId}-thread`,
|
||||
},
|
||||
task: {
|
||||
goalSummary: "验证 review decision 会进入 replay/eval 资产",
|
||||
},
|
||||
classification: {
|
||||
suiteTags: ["conversation-runtime", "replay"],
|
||||
failureModes: ["pending_request"],
|
||||
primaryBlockingKind: "pending_request",
|
||||
},
|
||||
runtimeContext: {
|
||||
pendingRequests: [
|
||||
{
|
||||
id: "req-1",
|
||||
},
|
||||
],
|
||||
},
|
||||
linkedArtifacts: {
|
||||
handoffBundle: {
|
||||
relativeRoot: `.lime/harness/sessions/${sessionId}/handoff`,
|
||||
absoluteRoot: handoffRoot,
|
||||
},
|
||||
evidencePack: {
|
||||
relativeRoot: `.lime/harness/sessions/${sessionId}/evidence`,
|
||||
absoluteRoot: evidenceRoot,
|
||||
},
|
||||
},
|
||||
};
|
||||
const expectedPayload = {
|
||||
sessionId: sessionId,
|
||||
threadId: `${sessionId}-thread`,
|
||||
goalSummary: "验证 review decision 会进入 replay/eval 资产",
|
||||
successCriteria: ["人工审核状态进入 eval 摘要"],
|
||||
blockingChecks: ["review decision 不再停留在工作区孤岛"],
|
||||
artifactChecks: ["review-decision.json 被识别"],
|
||||
graderSuggestion: {
|
||||
preferredMode: "manual_review",
|
||||
requiresHumanReview: true,
|
||||
},
|
||||
};
|
||||
const evidencePayload = {
|
||||
handoffBundle: {
|
||||
relativeRoot: `.lime/harness/sessions/${sessionId}/handoff`,
|
||||
absoluteRoot: handoffRoot,
|
||||
},
|
||||
evidencePack: {
|
||||
relativeRoot: `.lime/harness/sessions/${sessionId}/evidence`,
|
||||
absoluteRoot: evidenceRoot,
|
||||
},
|
||||
};
|
||||
const reviewDecisionPayload = {
|
||||
schemaVersion: "v1",
|
||||
contractShape: "lime_review_decision_template",
|
||||
decision: {
|
||||
decisionStatus: "accepted",
|
||||
decisionSummary: "确认该失败应沉淀为长期回归样本。",
|
||||
chosenFixStrategy: "继续把人工审核结果挂到 replay/eval 主链。",
|
||||
riskLevel: "high",
|
||||
riskTags: ["runtime", "eval"],
|
||||
humanReviewer: "Lime Maintainer",
|
||||
reviewedAt: "2026-03-27T12:00:00Z",
|
||||
followupActions: ["补 trend 汇总"],
|
||||
regressionRequirements: ["npm run harness:eval"],
|
||||
notes: `工作区目录 ${workspaceRoot} 已参与人工审核。`,
|
||||
},
|
||||
};
|
||||
|
||||
writeJson(path.join(replayDir, "input.json"), inputPayload);
|
||||
writeJson(path.join(replayDir, "expected.json"), expectedPayload);
|
||||
writeJson(path.join(replayDir, "evidence-links.json"), evidencePayload);
|
||||
writeText(
|
||||
path.join(replayDir, "grader.md"),
|
||||
`# Grader\n\n- 工作区：\`${workspaceRoot}\`\n`,
|
||||
);
|
||||
writeJson(
|
||||
path.join(reviewDir, "review-decision.json"),
|
||||
reviewDecisionPayload,
|
||||
);
|
||||
writeText(
|
||||
path.join(reviewDir, "review-decision.md"),
|
||||
`# Review\n\n- 工作区：\`${workspaceRoot}\`\n`,
|
||||
);
|
||||
|
||||
return {
|
||||
workspaceRoot,
|
||||
replayDir,
|
||||
reviewDir,
|
||||
};
|
||||
}
|
||||
|
||||
function runNodeScript(scriptRelativePath: string, args: string[]) {
|
||||
const output = execFileSync(
|
||||
process.execPath,
|
||||
[path.join(repoRoot, scriptRelativePath), ...args],
|
||||
{
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "pipe"],
|
||||
},
|
||||
);
|
||||
return JSON.parse(output);
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (tempRoots.length > 0) {
|
||||
const tempRoot = tempRoots.pop();
|
||||
if (tempRoot) {
|
||||
fs.rmSync(tempRoot, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
describe("Harness review decision / eval integration", () => {
|
||||
it("harness-replay-promote 应复制并脱敏 review decision 制品", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const { workspaceRoot } = createWorkspaceSessionArtifacts(
|
||||
tempRoot,
|
||||
"session-promote-1",
|
||||
);
|
||||
const manifestPath = path.join(tempRoot, "manifest.json");
|
||||
const fixturesRoot = path.join(tempRoot, "fixtures");
|
||||
|
||||
writeJson(
|
||||
manifestPath,
|
||||
createHarnessManifest({
|
||||
suites: [
|
||||
{
|
||||
id: "repo-promoted-replays",
|
||||
title: "仓库沉淀 Replay 样本",
|
||||
cases: [],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const result = runNodeScript("scripts/harness-replay-promote.mjs", [
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--session-id",
|
||||
"session-promote-1",
|
||||
"--slug",
|
||||
"review-promoted-case",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--fixtures-root",
|
||||
fixturesRoot,
|
||||
"--format",
|
||||
"json",
|
||||
]);
|
||||
|
||||
const promotedCaseDir = path.join(fixturesRoot, "review-promoted-case");
|
||||
const promotedReviewJson = JSON.parse(
|
||||
fs.readFileSync(
|
||||
path.join(promotedCaseDir, "review-decision.json"),
|
||||
"utf8",
|
||||
),
|
||||
);
|
||||
const promotedReviewMarkdown = fs.readFileSync(
|
||||
path.join(promotedCaseDir, "review-decision.md"),
|
||||
"utf8",
|
||||
);
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
|
||||
const caseEntry = manifest.suites[0].cases[0];
|
||||
|
||||
expect(result.reviewDecisionStatus).toBe("accepted");
|
||||
expect(promotedReviewJson.decision.decisionStatus).toBe("accepted");
|
||||
expect(promotedReviewJson.decision.notes).toContain("/workspace/lime");
|
||||
expect(promotedReviewMarkdown).toContain("/workspace/lime");
|
||||
expect(caseEntry.reviewDecision).toMatchObject({
|
||||
decisionStatus: "accepted",
|
||||
riskLevel: "high",
|
||||
humanReviewer: "Lime Maintainer",
|
||||
});
|
||||
});
|
||||
|
||||
it("harness-eval-runner 应在 workspace replay discovery 中读取 sibling review decision", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const { workspaceRoot } = createWorkspaceSessionArtifacts(
|
||||
tempRoot,
|
||||
"session-runner-1",
|
||||
);
|
||||
const manifestPath = path.join(tempRoot, "manifest.json");
|
||||
|
||||
writeJson(
|
||||
manifestPath,
|
||||
createHarnessManifest({
|
||||
suites: [
|
||||
{
|
||||
id: "workspace-replay-discovery",
|
||||
title: "工作区 Replay 自动发现",
|
||||
cases: [
|
||||
{
|
||||
id: "workspace-session-replays",
|
||||
title: "工作区会话 Replay 样本",
|
||||
source: "workspace_replay_discovery",
|
||||
root: ".lime/harness/sessions",
|
||||
allowZeroMatches: false,
|
||||
tags: ["workspace", "replay"],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
const summary = runNodeScript("scripts/harness-eval-runner.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--manifest",
|
||||
manifestPath,
|
||||
"--workspace-root",
|
||||
workspaceRoot,
|
||||
"--no-strict",
|
||||
]);
|
||||
|
||||
expect(summary.totals.reviewDecisionRecordedCount).toBe(1);
|
||||
expect(summary.breakdowns.reviewDecisionStatuses).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "accepted",
|
||||
caseCount: 1,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.breakdowns.reviewRiskLevels).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "high",
|
||||
caseCount: 1,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(summary.suites[0].cases[0]).toMatchObject({
|
||||
reviewDecisionStatus: "accepted",
|
||||
reviewRiskLevel: "high",
|
||||
reviewHumanReviewer: "Lime Maintainer",
|
||||
});
|
||||
});
|
||||
|
||||
it("harness-eval-trend-report 应聚合人工审核状态与风险等级变化", () => {
|
||||
const tempRoot = createTempRoot();
|
||||
const baselinePath = path.join(tempRoot, "baseline.json");
|
||||
const latestPath = path.join(tempRoot, "latest.json");
|
||||
|
||||
writeJson(baselinePath, {
|
||||
generatedAt: "2026-03-27T10:00:00Z",
|
||||
totals: {
|
||||
suiteCount: 1,
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: [],
|
||||
failureModes: [],
|
||||
reviewDecisionStatuses: [
|
||||
{
|
||||
name: "needs_more_evidence",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
],
|
||||
reviewRiskLevels: [
|
||||
{
|
||||
name: "high",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 1,
|
||||
},
|
||||
],
|
||||
},
|
||||
suites: [],
|
||||
});
|
||||
|
||||
writeJson(latestPath, {
|
||||
generatedAt: "2026-03-27T12:00:00Z",
|
||||
totals: {
|
||||
suiteCount: 1,
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
breakdowns: {
|
||||
suiteTags: [],
|
||||
failureModes: [],
|
||||
reviewDecisionStatuses: [
|
||||
{
|
||||
name: "accepted",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
reviewRiskLevels: [
|
||||
{
|
||||
name: "medium",
|
||||
caseCount: 1,
|
||||
readyCount: 1,
|
||||
invalidCount: 0,
|
||||
pendingRequestCaseCount: 0,
|
||||
needsHumanReviewCount: 0,
|
||||
},
|
||||
],
|
||||
},
|
||||
suites: [],
|
||||
});
|
||||
|
||||
const report = runNodeScript("scripts/harness-eval-trend-report.mjs", [
|
||||
"--format",
|
||||
"json",
|
||||
"--input",
|
||||
baselinePath,
|
||||
"--input",
|
||||
latestPath,
|
||||
]);
|
||||
|
||||
expect(report.classificationDeltas.reviewDecisionStatuses).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "accepted",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "needs_more_evidence",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: -1,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(report.classificationDeltas.reviewRiskLevels).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
name: "high",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: -1,
|
||||
}),
|
||||
}),
|
||||
expect.objectContaining({
|
||||
name: "medium",
|
||||
delta: expect.objectContaining({
|
||||
caseCount: 1,
|
||||
}),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,688 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import legacySurfaceCatalog from "../../src/lib/governance/legacySurfaceCatalog.json" with { type: "json" };
|
||||
import {
|
||||
buildLegacySurfaceSummary,
|
||||
getCommandStatus,
|
||||
getImportStatus,
|
||||
getTextCountStatus,
|
||||
getTextStatus,
|
||||
serializeMapEntries,
|
||||
} from "./legacy-surface-report-summary.mjs";
|
||||
|
||||
const repoRoot = path.resolve(process.cwd());
|
||||
const sourceRoots = ["src"];
|
||||
const sourceExtensions = new Set([
|
||||
".ts",
|
||||
".tsx",
|
||||
".js",
|
||||
".jsx",
|
||||
".mjs",
|
||||
".cjs",
|
||||
]);
|
||||
const rustSourceRoots = ["src-tauri/src", "src-tauri/crates"];
|
||||
const rustSourceExtensions = new Set([".rs"]);
|
||||
const ignoredDirs = new Set([
|
||||
"node_modules",
|
||||
"dist",
|
||||
"build",
|
||||
"coverage",
|
||||
"target",
|
||||
".git",
|
||||
".turbo",
|
||||
".next",
|
||||
]);
|
||||
|
||||
const {
|
||||
imports: importSurfaceMonitors,
|
||||
commands: commandSurfaceMonitors,
|
||||
frontendText: frontendTextSurfaceMonitors,
|
||||
rustText: rustTextSurfaceMonitors,
|
||||
rustTextCounts: rustTextCountMonitors,
|
||||
} = legacySurfaceCatalog;
|
||||
|
||||
function normalizePath(filePath) {
|
||||
return filePath.split(path.sep).join("/");
|
||||
}
|
||||
|
||||
function resolveExistingSourcePath(absolutePath) {
|
||||
if (fs.existsSync(absolutePath)) {
|
||||
const stats = fs.statSync(absolutePath);
|
||||
if (stats.isFile()) {
|
||||
return absolutePath;
|
||||
}
|
||||
}
|
||||
|
||||
if (!path.extname(absolutePath)) {
|
||||
for (const extension of sourceExtensions) {
|
||||
const fileCandidate = `${absolutePath}${extension}`;
|
||||
if (fs.existsSync(fileCandidate) && fs.statSync(fileCandidate).isFile()) {
|
||||
return fileCandidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const extension of sourceExtensions) {
|
||||
const indexCandidate = path.join(absolutePath, `index${extension}`);
|
||||
if (fs.existsSync(indexCandidate) && fs.statSync(indexCandidate).isFile()) {
|
||||
return indexCandidate;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function resolveImportPath(importerRelativePath, specifier) {
|
||||
let absoluteCandidate = null;
|
||||
|
||||
if (specifier.startsWith("@/")) {
|
||||
absoluteCandidate = path.join(repoRoot, "src", specifier.slice(2));
|
||||
} else if (specifier.startsWith(".")) {
|
||||
absoluteCandidate = path.resolve(
|
||||
path.dirname(path.join(repoRoot, importerRelativePath)),
|
||||
specifier,
|
||||
);
|
||||
}
|
||||
|
||||
if (!absoluteCandidate) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const resolvedPath = resolveExistingSourcePath(absoluteCandidate);
|
||||
if (!resolvedPath) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return normalizePath(path.relative(repoRoot, resolvedPath));
|
||||
}
|
||||
|
||||
function isTestFile(relativePath) {
|
||||
return (
|
||||
/(^|\/)tests(\/|$)/.test(relativePath) ||
|
||||
/(^|\/)(__tests__|__mocks__)(\/|$)/.test(relativePath) ||
|
||||
/\.(test|spec)\.[^/.]+$/.test(relativePath)
|
||||
);
|
||||
}
|
||||
|
||||
function walkDirectory(directoryPath, extensions) {
|
||||
const files = [];
|
||||
|
||||
for (const entry of fs.readdirSync(directoryPath, { withFileTypes: true })) {
|
||||
if (ignoredDirs.has(entry.name)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fullPath = path.join(directoryPath, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...walkDirectory(fullPath, extensions));
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!extensions.has(path.extname(entry.name))) {
|
||||
continue;
|
||||
}
|
||||
|
||||
files.push(fullPath);
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
function extractImportSpecifiers(sourceCode) {
|
||||
const specifiers = new Set();
|
||||
const patterns = [
|
||||
/\bimport\s+(?:type\s+)?(?:[\s\S]*?\s+from\s+)?["'`]([^"'`]+)["'`]/g,
|
||||
/\bexport\s+(?:type\s+)?[\s\S]*?\s+from\s+["'`]([^"'`]+)["'`]/g,
|
||||
/\bimport\s*\(\s*["'`]([^"'`]+)["'`]\s*\)/g,
|
||||
/\brequire\s*\(\s*["'`]([^"'`]+)["'`]\s*\)/g,
|
||||
/\b(?:vi|jest)\.mock\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
for (const match of sourceCode.matchAll(pattern)) {
|
||||
specifiers.add(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return specifiers;
|
||||
}
|
||||
|
||||
function extractInvokeCommands(sourceCode) {
|
||||
const commands = new Set();
|
||||
const patterns = [
|
||||
/\bsafeInvoke(?:<[^>]+>)?\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
||||
/\binvoke(?:<[^>]+>)?\s*\(\s*["'`]([^"'`]+)["'`]/g,
|
||||
];
|
||||
|
||||
for (const pattern of patterns) {
|
||||
for (const match of sourceCode.matchAll(pattern)) {
|
||||
commands.add(match[1]);
|
||||
}
|
||||
}
|
||||
|
||||
return commands;
|
||||
}
|
||||
|
||||
function stripRustTestModules(sourceCode) {
|
||||
return sourceCode.replace(
|
||||
/(?:^|\n)\s*#\s*\[\s*cfg\s*\(\s*test\s*\)\s*\]\s*(?:pub\s+)?mod\s+\w+\s*(?:\{[\s\S]*$|;)/m,
|
||||
"\n",
|
||||
);
|
||||
}
|
||||
|
||||
function collectSources() {
|
||||
const runtimeSources = [];
|
||||
const testSources = [];
|
||||
|
||||
for (const root of sourceRoots) {
|
||||
const absoluteRoot = path.join(repoRoot, root);
|
||||
if (!fs.existsSync(absoluteRoot)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const filePath of walkDirectory(absoluteRoot, sourceExtensions)) {
|
||||
const relativePath = normalizePath(path.relative(repoRoot, filePath));
|
||||
const sourceCode = fs.readFileSync(filePath, "utf8");
|
||||
const imports = extractImportSpecifiers(sourceCode);
|
||||
const collectedSource = {
|
||||
relativePath,
|
||||
imports,
|
||||
resolvedImports: new Set(
|
||||
[...imports]
|
||||
.map((specifier) => resolveImportPath(relativePath, specifier))
|
||||
.filter(Boolean),
|
||||
),
|
||||
commands: extractInvokeCommands(sourceCode),
|
||||
};
|
||||
|
||||
if (isTestFile(relativePath)) {
|
||||
testSources.push(collectedSource);
|
||||
continue;
|
||||
}
|
||||
|
||||
runtimeSources.push(collectedSource);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
runtimeSources,
|
||||
testSources,
|
||||
};
|
||||
}
|
||||
|
||||
function collectTextSources(roots, extensions) {
|
||||
const runtimeSources = [];
|
||||
const testSources = [];
|
||||
|
||||
for (const root of roots) {
|
||||
const absoluteRoot = path.join(repoRoot, root);
|
||||
if (!fs.existsSync(absoluteRoot)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (const filePath of walkDirectory(absoluteRoot, extensions)) {
|
||||
const relativePath = normalizePath(path.relative(repoRoot, filePath));
|
||||
const sourceCode = fs.readFileSync(filePath, "utf8");
|
||||
const collectedSource = {
|
||||
relativePath,
|
||||
sourceCode:
|
||||
path.extname(relativePath) === ".rs"
|
||||
? stripRustTestModules(sourceCode)
|
||||
: sourceCode,
|
||||
rawSourceCode: sourceCode,
|
||||
};
|
||||
|
||||
if (isTestFile(relativePath)) {
|
||||
testSources.push(collectedSource);
|
||||
continue;
|
||||
}
|
||||
|
||||
runtimeSources.push(collectedSource);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
runtimeSources,
|
||||
testSources,
|
||||
};
|
||||
}
|
||||
|
||||
function formatPaths(paths) {
|
||||
if (paths.length === 0) {
|
||||
return "无";
|
||||
}
|
||||
|
||||
return paths.map((item) => ` - ${item}`).join("\n");
|
||||
}
|
||||
|
||||
function evaluateImportMonitor(monitor, runtimeSources, testSources) {
|
||||
const existingTargets = monitor.targets.filter((target) =>
|
||||
fs.existsSync(path.join(repoRoot, target)),
|
||||
);
|
||||
const missingTargets = monitor.targets.filter(
|
||||
(target) => !fs.existsSync(path.join(repoRoot, target)),
|
||||
);
|
||||
const references = runtimeSources
|
||||
.filter((file) =>
|
||||
[...file.resolvedImports].some((resolvedPath) =>
|
||||
monitor.targets.includes(resolvedPath),
|
||||
),
|
||||
)
|
||||
.map((file) => file.relativePath)
|
||||
.sort();
|
||||
const testReferences = testSources
|
||||
.filter((file) =>
|
||||
[...file.resolvedImports].some((resolvedPath) =>
|
||||
monitor.targets.includes(resolvedPath),
|
||||
),
|
||||
)
|
||||
.map((file) => file.relativePath)
|
||||
.sort();
|
||||
|
||||
const violations = references.filter(
|
||||
(relativePath) => !monitor.allowedPaths.includes(relativePath),
|
||||
);
|
||||
|
||||
return {
|
||||
...monitor,
|
||||
existingTargets,
|
||||
missingTargets,
|
||||
references,
|
||||
testReferences,
|
||||
violations,
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateCommandMonitor(monitor, runtimeSources, testSources) {
|
||||
const referencesByCommand = new Map();
|
||||
const testReferencesByCommand = new Map();
|
||||
|
||||
for (const command of monitor.commands) {
|
||||
referencesByCommand.set(
|
||||
command,
|
||||
runtimeSources
|
||||
.filter((file) => file.commands.has(command))
|
||||
.map((file) => file.relativePath)
|
||||
.sort(),
|
||||
);
|
||||
testReferencesByCommand.set(
|
||||
command,
|
||||
testSources
|
||||
.filter((file) => file.commands.has(command))
|
||||
.map((file) => file.relativePath)
|
||||
.sort(),
|
||||
);
|
||||
}
|
||||
|
||||
const violations = [];
|
||||
for (const [command, references] of referencesByCommand.entries()) {
|
||||
for (const relativePath of references) {
|
||||
if (!monitor.allowedPaths.includes(relativePath)) {
|
||||
violations.push(`${command} -> ${relativePath}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
...monitor,
|
||||
referencesByCommand,
|
||||
testReferencesByCommand,
|
||||
violations,
|
||||
};
|
||||
}
|
||||
|
||||
function evaluateTextMonitor(monitor, runtimeSources, testSources) {
|
||||
const filteredRuntimeSources = monitor.includePathPrefixes
|
||||
? runtimeSources.filter((file) =>
|
||||
monitor.includePathPrefixes.some((prefix) =>
|
||||
file.relativePath.startsWith(prefix),
|
||||
),
|
||||
)
|
||||
: runtimeSources;
|
||||
const filteredTestSources = monitor.includePathPrefixes
|
||||
? testSources.filter((file) =>
|
||||
monitor.includePathPrefixes.some((prefix) =>
|
||||
file.relativePath.startsWith(prefix),
|
||||
),
|
||||
)
|
||||
: testSources;
|
||||
const matchesPattern = (sourceCode) =>
|
||||
monitor.patterns.some((pattern) => sourceCode.includes(pattern)) ||
|
||||
(monitor.regexPatterns ?? []).some((pattern) =>
|
||||
new RegExp(pattern, "m").test(sourceCode),
|
||||
);
|
||||
|
||||
const references = filteredRuntimeSources
|
||||
.filter((file) => matchesPattern(file.sourceCode))
|
||||
.map((file) => file.relativePath)
|
||||
.sort();
|
||||
const testReferences = filteredTestSources
|
||||
.filter((file) => matchesPattern(file.rawSourceCode ?? file.sourceCode))
|
||||
.map((file) => file.relativePath)
|
||||
.sort();
|
||||
const violations = references.filter(
|
||||
(relativePath) => !monitor.allowedPaths.includes(relativePath),
|
||||
);
|
||||
|
||||
return {
|
||||
...monitor,
|
||||
references,
|
||||
testReferences,
|
||||
violations,
|
||||
};
|
||||
}
|
||||
|
||||
function countOccurrences(sourceCode, pattern) {
|
||||
if (!pattern) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
let count = 0;
|
||||
let startIndex = 0;
|
||||
|
||||
while (true) {
|
||||
const matchIndex = sourceCode.indexOf(pattern, startIndex);
|
||||
if (matchIndex === -1) {
|
||||
return count;
|
||||
}
|
||||
count += 1;
|
||||
startIndex = matchIndex + pattern.length;
|
||||
}
|
||||
}
|
||||
|
||||
function evaluateTextCountMonitor(monitor, runtimeSources, testSources) {
|
||||
const filteredRuntimeSources = monitor.includePathPrefixes
|
||||
? runtimeSources.filter((file) =>
|
||||
monitor.includePathPrefixes.some((prefix) =>
|
||||
file.relativePath.startsWith(prefix),
|
||||
),
|
||||
)
|
||||
: runtimeSources;
|
||||
const filteredTestSources = monitor.includePathPrefixes
|
||||
? testSources.filter((file) =>
|
||||
monitor.includePathPrefixes.some((prefix) =>
|
||||
file.relativePath.startsWith(prefix),
|
||||
),
|
||||
)
|
||||
: testSources;
|
||||
const runtimeMatches = [];
|
||||
const testMatches = [];
|
||||
const violations = [];
|
||||
|
||||
for (const file of filteredRuntimeSources) {
|
||||
const counts = monitor.occurrences
|
||||
.map((rule) => ({
|
||||
...rule,
|
||||
count: countOccurrences(file.sourceCode, rule.pattern),
|
||||
}))
|
||||
.filter((rule) => rule.count > 0);
|
||||
|
||||
if (counts.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
runtimeMatches.push({
|
||||
relativePath: file.relativePath,
|
||||
counts,
|
||||
});
|
||||
|
||||
for (const rule of counts) {
|
||||
if (rule.count > rule.maxCount) {
|
||||
violations.push(
|
||||
`${file.relativePath} -> ${rule.pattern} (${rule.count} > ${rule.maxCount})`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const file of filteredTestSources) {
|
||||
const counts = monitor.occurrences
|
||||
.map((rule) => ({
|
||||
...rule,
|
||||
count: countOccurrences(
|
||||
file.rawSourceCode ?? file.sourceCode,
|
||||
rule.pattern,
|
||||
),
|
||||
}))
|
||||
.filter((rule) => rule.count > 0);
|
||||
|
||||
if (counts.length === 0) {
|
||||
continue;
|
||||
}
|
||||
|
||||
testMatches.push({
|
||||
relativePath: file.relativePath,
|
||||
counts,
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
...monitor,
|
||||
runtimeMatches,
|
||||
testMatches,
|
||||
violations,
|
||||
};
|
||||
}
|
||||
|
||||
function printImportReport(result) {
|
||||
const status = getImportStatus(result);
|
||||
|
||||
console.log(
|
||||
`- [${status}] ${result.id} (${result.classification})：${result.description}`,
|
||||
);
|
||||
console.log(` 目标文件：${result.targets.join(", ")}`);
|
||||
console.log(` 允许引用：${result.allowedPaths.join(", ") || "无"}`);
|
||||
if (result.missingTargets.length > 0) {
|
||||
console.log(` 已删除目标：\n${formatPaths(result.missingTargets)}`);
|
||||
}
|
||||
console.log(` 实际引用：\n${formatPaths(result.references)}`);
|
||||
console.log(` 测试引用：\n${formatPaths(result.testReferences)}`);
|
||||
|
||||
if (result.violations.length > 0) {
|
||||
console.log(` 违规引用：\n${formatPaths(result.violations)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function printCommandReport(result) {
|
||||
const status = getCommandStatus(result);
|
||||
|
||||
console.log(
|
||||
`- [${status}] ${result.id} (${result.classification})：${result.description}`,
|
||||
);
|
||||
console.log(` 命令：${result.commands.join(", ")}`);
|
||||
console.log(` 允许引用：${result.allowedPaths.join(", ") || "无"}`);
|
||||
|
||||
for (const command of result.commands) {
|
||||
const references = result.referencesByCommand.get(command) ?? [];
|
||||
const testReferences = result.testReferencesByCommand.get(command) ?? [];
|
||||
console.log(` ${command}：\n${formatPaths(references)}`);
|
||||
console.log(` ${command}（测试）：\n${formatPaths(testReferences)}`);
|
||||
}
|
||||
|
||||
if (result.violations.length > 0) {
|
||||
console.log(` 违规引用：\n${formatPaths(result.violations)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function printTextReport(result) {
|
||||
const status = getTextStatus(result);
|
||||
|
||||
console.log(
|
||||
`- [${status}] ${result.id} (${result.classification})：${result.description}`,
|
||||
);
|
||||
const keywords = [
|
||||
...result.patterns,
|
||||
...(result.regexPatterns ?? []).map((pattern) => `regex:${pattern}`),
|
||||
];
|
||||
console.log(` 关键字：${keywords.join(", ")}`);
|
||||
console.log(` 允许引用：${result.allowedPaths.join(", ") || "无"}`);
|
||||
console.log(` 实际引用：\n${formatPaths(result.references)}`);
|
||||
console.log(` 测试引用：\n${formatPaths(result.testReferences)}`);
|
||||
|
||||
if (result.violations.length > 0) {
|
||||
console.log(` 违规引用：\n${formatPaths(result.violations)}`);
|
||||
}
|
||||
}
|
||||
|
||||
function printTextCountReport(result) {
|
||||
const status = getTextCountStatus(result);
|
||||
|
||||
console.log(
|
||||
`- [${status}] ${result.id} (${result.classification})：${result.description}`,
|
||||
);
|
||||
console.log(
|
||||
` 次数规则：${result.occurrences
|
||||
.map((rule) => `${rule.pattern} <= ${rule.maxCount}`)
|
||||
.join("；")}`,
|
||||
);
|
||||
console.log(
|
||||
` 实际命中：\n${formatPaths(
|
||||
result.runtimeMatches.map(
|
||||
(item) =>
|
||||
`${item.relativePath} -> ${item.counts
|
||||
.map((rule) => `${rule.pattern} (${rule.count})`)
|
||||
.join("；")}`,
|
||||
),
|
||||
)}`,
|
||||
);
|
||||
console.log(
|
||||
` 测试命中：\n${formatPaths(
|
||||
result.testMatches.map(
|
||||
(item) =>
|
||||
`${item.relativePath} -> ${item.counts
|
||||
.map((rule) => `${rule.pattern} (${rule.count})`)
|
||||
.join("；")}`,
|
||||
),
|
||||
)}`,
|
||||
);
|
||||
|
||||
if (result.violations.length > 0) {
|
||||
console.log(` 违规引用：\n${formatPaths(result.violations)}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function buildLegacySurfaceReport() {
|
||||
const { runtimeSources, testSources } = collectSources();
|
||||
const {
|
||||
runtimeSources: frontendRuntimeTextSources,
|
||||
testSources: frontendTestTextSources,
|
||||
} = collectTextSources(sourceRoots, sourceExtensions);
|
||||
const { runtimeSources: rustRuntimeSources, testSources: rustTestSources } =
|
||||
collectTextSources(rustSourceRoots, rustSourceExtensions);
|
||||
const importResults = importSurfaceMonitors.map((monitor) =>
|
||||
evaluateImportMonitor(monitor, runtimeSources, testSources),
|
||||
);
|
||||
const commandResults = commandSurfaceMonitors.map((monitor) =>
|
||||
evaluateCommandMonitor(monitor, runtimeSources, testSources),
|
||||
);
|
||||
const frontendTextResults = frontendTextSurfaceMonitors.map((monitor) =>
|
||||
evaluateTextMonitor(
|
||||
monitor,
|
||||
frontendRuntimeTextSources,
|
||||
frontendTestTextSources,
|
||||
),
|
||||
);
|
||||
const rustTextResults = rustTextSurfaceMonitors.map((monitor) =>
|
||||
evaluateTextMonitor(monitor, rustRuntimeSources, rustTestSources),
|
||||
);
|
||||
const rustTextCountResults = rustTextCountMonitors.map((monitor) =>
|
||||
evaluateTextCountMonitor(monitor, rustRuntimeSources, rustTestSources),
|
||||
);
|
||||
const summary = buildLegacySurfaceSummary({
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
});
|
||||
|
||||
return {
|
||||
repoRoot,
|
||||
runtimeSources,
|
||||
testSources,
|
||||
rustRuntimeSources,
|
||||
rustTestSources,
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
...summary,
|
||||
};
|
||||
}
|
||||
|
||||
export function toSerializableLegacySurfaceReport(report) {
|
||||
return {
|
||||
repoRoot: report.repoRoot,
|
||||
summary: {
|
||||
runtimeSourceCount: report.runtimeSources.length,
|
||||
testSourceCount: report.testSources.length,
|
||||
rustRuntimeSourceCount: report.rustRuntimeSources.length,
|
||||
rustTestSourceCount: report.rustTestSources.length,
|
||||
zeroReferenceCandidates: report.zeroReferenceCandidates,
|
||||
classificationDriftCandidates: report.classificationDriftCandidates,
|
||||
violations: report.violations,
|
||||
},
|
||||
importResults: report.importResults,
|
||||
commandResults: report.commandResults.map((result) => ({
|
||||
...result,
|
||||
referencesByCommand: serializeMapEntries(result.referencesByCommand),
|
||||
testReferencesByCommand: serializeMapEntries(
|
||||
result.testReferencesByCommand,
|
||||
),
|
||||
})),
|
||||
frontendTextResults: report.frontendTextResults,
|
||||
rustTextResults: report.rustTextResults,
|
||||
rustTextCountResults: report.rustTextCountResults,
|
||||
};
|
||||
}
|
||||
|
||||
export function printLegacySurfaceReport(report) {
|
||||
console.log("[lime] legacy surface report");
|
||||
console.log("");
|
||||
console.log("## 入口引用");
|
||||
for (const result of report.importResults) {
|
||||
printImportReport(result);
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("## 命令边界");
|
||||
for (const result of report.commandResults) {
|
||||
printCommandReport(result);
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("## 前端护栏");
|
||||
for (const result of report.frontendTextResults) {
|
||||
printTextReport(result);
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("## Rust 护栏");
|
||||
for (const result of report.rustTextResults) {
|
||||
printTextReport(result);
|
||||
}
|
||||
for (const result of report.rustTextCountResults) {
|
||||
printTextCountReport(result);
|
||||
}
|
||||
|
||||
console.log("");
|
||||
console.log("## 摘要");
|
||||
console.log(`- 扫描文件数：${report.runtimeSources.length}`);
|
||||
console.log(`- 测试文件数：${report.testSources.length}`);
|
||||
console.log(`- Rust 扫描文件数：${report.rustRuntimeSources.length}`);
|
||||
console.log(`- Rust 测试文件数：${report.rustTestSources.length}`);
|
||||
console.log(`- 零引用候选：${report.zeroReferenceCandidates.length}`);
|
||||
for (const candidate of report.zeroReferenceCandidates) {
|
||||
console.log(` - ${candidate}`);
|
||||
}
|
||||
console.log(`- 分类漂移候选：${report.classificationDriftCandidates.length}`);
|
||||
for (const candidate of report.classificationDriftCandidates) {
|
||||
console.log(` - ${candidate}`);
|
||||
}
|
||||
console.log(`- 边界违规：${report.violations.length}`);
|
||||
for (const violation of report.violations) {
|
||||
console.log(` - ${violation}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { toSerializableLegacySurfaceReport } from "./legacy-surface-report-core.mjs";
|
||||
|
||||
describe("legacy-surface-report-core", () => {
|
||||
it("应把命令引用 Map 序列化成普通对象", () => {
|
||||
const serialized = toSerializableLegacySurfaceReport({
|
||||
repoRoot: "/tmp/lime",
|
||||
runtimeSources: [{ relativePath: "src/foo.ts" }],
|
||||
testSources: [{ relativePath: "src/foo.test.ts" }],
|
||||
rustRuntimeSources: [{ relativePath: "src-tauri/src/lib.rs" }],
|
||||
rustTestSources: [],
|
||||
importResults: [],
|
||||
commandResults: [
|
||||
{
|
||||
id: "agent-runtime",
|
||||
classification: "current",
|
||||
description: "runtime command",
|
||||
commands: ["agent_runtime_submit_turn"],
|
||||
allowedPaths: ["src/lib/api/agentRuntime.ts"],
|
||||
referencesByCommand: new Map([
|
||||
["agent_runtime_submit_turn", ["src/lib/api/agentRuntime.ts"]],
|
||||
]),
|
||||
testReferencesByCommand: new Map([
|
||||
["agent_runtime_submit_turn", ["src/lib/api/agentRuntime.test.ts"]],
|
||||
]),
|
||||
violations: [],
|
||||
},
|
||||
],
|
||||
frontendTextResults: [],
|
||||
rustTextResults: [],
|
||||
rustTextCountResults: [],
|
||||
zeroReferenceCandidates: [],
|
||||
classificationDriftCandidates: [],
|
||||
violations: [],
|
||||
});
|
||||
|
||||
expect(serialized.summary.runtimeSourceCount).toBe(1);
|
||||
expect(serialized.commandResults[0].referencesByCommand).toEqual({
|
||||
agent_runtime_submit_turn: ["src/lib/api/agentRuntime.ts"],
|
||||
});
|
||||
expect(serialized.commandResults[0].testReferencesByCommand).toEqual({
|
||||
agent_runtime_submit_turn: ["src/lib/api/agentRuntime.test.ts"],
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,176 @@
|
||||
function flattenCommandReferences(result) {
|
||||
return [...result.referencesByCommand.values()].flat();
|
||||
}
|
||||
|
||||
export function getImportStatus(result) {
|
||||
return result.violations.length > 0
|
||||
? "违规"
|
||||
: result.references.length === 0 && result.existingTargets.length === 0
|
||||
? "已删除"
|
||||
: result.references.length === 0
|
||||
? "零引用"
|
||||
: "受控";
|
||||
}
|
||||
|
||||
export function getCommandStatus(result) {
|
||||
const uniqueReferences = [
|
||||
...new Set(flattenCommandReferences(result)),
|
||||
].sort();
|
||||
return result.violations.length > 0
|
||||
? "违规"
|
||||
: uniqueReferences.length === 0
|
||||
? "零引用"
|
||||
: "受控";
|
||||
}
|
||||
|
||||
export function getTextStatus(result) {
|
||||
return result.violations.length > 0
|
||||
? "违规"
|
||||
: result.references.length === 0
|
||||
? "零引用"
|
||||
: "受控";
|
||||
}
|
||||
|
||||
export function getTextCountStatus(result) {
|
||||
return result.violations.length > 0
|
||||
? "违规"
|
||||
: result.runtimeMatches.length === 0
|
||||
? "零引用"
|
||||
: "受控";
|
||||
}
|
||||
|
||||
export function isStatusClassificationDrift(status, classification) {
|
||||
return (
|
||||
(status === "已删除" || status === "零引用") &&
|
||||
classification !== "dead-candidate"
|
||||
);
|
||||
}
|
||||
|
||||
function collectClassificationDriftCandidates({
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
}) {
|
||||
return [
|
||||
...importResults
|
||||
.filter((result) =>
|
||||
isStatusClassificationDrift(
|
||||
getImportStatus(result),
|
||||
result.classification,
|
||||
),
|
||||
)
|
||||
.map(
|
||||
(result) =>
|
||||
`${result.id} -> ${result.classification} / ${getImportStatus(result)}`,
|
||||
),
|
||||
...commandResults
|
||||
.filter((result) =>
|
||||
isStatusClassificationDrift(
|
||||
getCommandStatus(result),
|
||||
result.classification,
|
||||
),
|
||||
)
|
||||
.map(
|
||||
(result) =>
|
||||
`${result.id} -> ${result.classification} / ${getCommandStatus(result)}`,
|
||||
),
|
||||
...frontendTextResults
|
||||
.filter((result) =>
|
||||
isStatusClassificationDrift(
|
||||
getTextStatus(result),
|
||||
result.classification,
|
||||
),
|
||||
)
|
||||
.map(
|
||||
(result) =>
|
||||
`${result.id} -> ${result.classification} / ${getTextStatus(result)}`,
|
||||
),
|
||||
...rustTextResults
|
||||
.filter((result) =>
|
||||
isStatusClassificationDrift(
|
||||
getTextStatus(result),
|
||||
result.classification,
|
||||
),
|
||||
)
|
||||
.map(
|
||||
(result) =>
|
||||
`${result.id} -> ${result.classification} / ${getTextStatus(result)}`,
|
||||
),
|
||||
...rustTextCountResults
|
||||
.filter((result) =>
|
||||
isStatusClassificationDrift(
|
||||
getTextCountStatus(result),
|
||||
result.classification,
|
||||
),
|
||||
)
|
||||
.map(
|
||||
(result) =>
|
||||
`${result.id} -> ${result.classification} / ${getTextCountStatus(result)}`,
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function collectViolations({
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
}) {
|
||||
return [
|
||||
...importResults.flatMap((result) =>
|
||||
result.violations.map((item) => `${result.id} -> ${item}`),
|
||||
),
|
||||
...commandResults.flatMap((result) =>
|
||||
result.violations.map((item) => `${result.id} -> ${item}`),
|
||||
),
|
||||
...frontendTextResults.flatMap((result) =>
|
||||
result.violations.map((item) => `${result.id} -> ${item}`),
|
||||
),
|
||||
...rustTextResults.flatMap((result) =>
|
||||
result.violations.map((item) => `${result.id} -> ${item}`),
|
||||
),
|
||||
...rustTextCountResults.flatMap((result) =>
|
||||
result.violations.map((item) => `${result.id} -> ${item}`),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
export function buildLegacySurfaceSummary({
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
}) {
|
||||
const zeroReferenceCandidates = importResults
|
||||
.filter(
|
||||
(result) =>
|
||||
result.references.length === 0 && result.existingTargets.length > 0,
|
||||
)
|
||||
.map((result) => `${result.id} (${result.description})`);
|
||||
|
||||
return {
|
||||
zeroReferenceCandidates,
|
||||
classificationDriftCandidates: collectClassificationDriftCandidates({
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
}),
|
||||
violations: collectViolations({
|
||||
importResults,
|
||||
commandResults,
|
||||
frontendTextResults,
|
||||
rustTextResults,
|
||||
rustTextCountResults,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export function serializeMapEntries(map) {
|
||||
return Object.fromEntries(map.entries());
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
buildLegacySurfaceSummary,
|
||||
getCommandStatus,
|
||||
getImportStatus,
|
||||
getTextCountStatus,
|
||||
getTextStatus,
|
||||
} from "./legacy-surface-report-summary.mjs";
|
||||
|
||||
describe("legacy-surface-report-summary", () => {
|
||||
it("应正确判定各类 surface 状态", () => {
|
||||
expect(
|
||||
getImportStatus({
|
||||
violations: [],
|
||||
references: [],
|
||||
existingTargets: [],
|
||||
}),
|
||||
).toBe("已删除");
|
||||
|
||||
expect(
|
||||
getCommandStatus({
|
||||
violations: [],
|
||||
referencesByCommand: new Map([["cmd", []]]),
|
||||
}),
|
||||
).toBe("零引用");
|
||||
|
||||
expect(
|
||||
getTextStatus({
|
||||
violations: [],
|
||||
references: ["src/foo.ts"],
|
||||
}),
|
||||
).toBe("受控");
|
||||
|
||||
expect(
|
||||
getTextCountStatus({
|
||||
violations: [],
|
||||
runtimeMatches: [],
|
||||
}),
|
||||
).toBe("零引用");
|
||||
});
|
||||
|
||||
it("应汇总零引用候选、分类漂移与违规", () => {
|
||||
const summary = buildLegacySurfaceSummary({
|
||||
importResults: [
|
||||
{
|
||||
id: "import-monitor",
|
||||
classification: "compat",
|
||||
description: "import drift",
|
||||
references: [],
|
||||
existingTargets: ["src/legacy.ts"],
|
||||
violations: [],
|
||||
},
|
||||
],
|
||||
commandResults: [
|
||||
{
|
||||
id: "command-monitor",
|
||||
classification: "current",
|
||||
referencesByCommand: new Map([["legacy_cmd", ["src/foo.ts"]]]),
|
||||
violations: ["legacy_cmd -> src/foo.ts"],
|
||||
},
|
||||
],
|
||||
frontendTextResults: [
|
||||
{
|
||||
id: "frontend-monitor",
|
||||
classification: "deprecated",
|
||||
references: [],
|
||||
violations: [],
|
||||
},
|
||||
],
|
||||
rustTextResults: [],
|
||||
rustTextCountResults: [],
|
||||
});
|
||||
|
||||
expect(summary.zeroReferenceCandidates).toEqual([
|
||||
"import-monitor (import drift)",
|
||||
]);
|
||||
expect(summary.classificationDriftCandidates).toEqual([
|
||||
"import-monitor -> compat / 零引用",
|
||||
"frontend-monitor -> deprecated / 零引用",
|
||||
]);
|
||||
expect(summary.violations).toEqual([
|
||||
"command-monitor -> legacy_cmd -> src/foo.ts",
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,293 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { execFileSync } from "node:child_process";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import process from "node:process";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
buildGeneratedSlopReport,
|
||||
renderGeneratedSlopMarkdown,
|
||||
renderGeneratedSlopText,
|
||||
} from "./lib/generated-slop-report-core.mjs";
|
||||
import { buildLiveDocFreshnessReport } from "./check-doc-freshness.mjs";
|
||||
import {
|
||||
buildLegacySurfaceReport,
|
||||
toSerializableLegacySurfaceReport,
|
||||
} from "./report-legacy-surfaces.mjs";
|
||||
|
||||
const TREND_REPORT_PATH = "scripts/harness-eval-trend-report.mjs";
|
||||
|
||||
export {
|
||||
buildGeneratedSlopReport,
|
||||
renderGeneratedSlopMarkdown,
|
||||
renderGeneratedSlopText,
|
||||
} from "./lib/generated-slop-report-core.mjs";
|
||||
|
||||
function parseArgs(argv) {
|
||||
const result = {
|
||||
format: "text",
|
||||
help: false,
|
||||
docFreshnessInput: "",
|
||||
legacyInput: "",
|
||||
outputJson: "",
|
||||
outputMarkdown: "",
|
||||
trendHistoryDir: "",
|
||||
trendInput: "",
|
||||
workspaceRoot: process.cwd(),
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
|
||||
if (arg === "--trend-input" && argv[index + 1]) {
|
||||
result.trendInput = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--doc-freshness-input" && argv[index + 1]) {
|
||||
result.docFreshnessInput = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--legacy-input" && argv[index + 1]) {
|
||||
result.legacyInput = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--trend-history-dir" && argv[index + 1]) {
|
||||
result.trendHistoryDir = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--workspace-root" && argv[index + 1]) {
|
||||
result.workspaceRoot = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--format" && argv[index + 1]) {
|
||||
result.format = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--output-json" && argv[index + 1]) {
|
||||
result.outputJson = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--output-markdown" && argv[index + 1]) {
|
||||
result.outputMarkdown = String(argv[index + 1]).trim();
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg === "--help" || arg === "-h") {
|
||||
result.help = true;
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`
|
||||
Lime Harness Cleanup / Slop Report
|
||||
|
||||
用法:
|
||||
node scripts/report-generated-slop.mjs
|
||||
node scripts/report-generated-slop.mjs --trend-input "./tmp/harness-eval-trend.json"
|
||||
node scripts/report-generated-slop.mjs --doc-freshness-input "./tmp/doc-freshness.json"
|
||||
node scripts/report-generated-slop.mjs --legacy-input "./tmp/legacy-surface-report.json"
|
||||
node scripts/report-generated-slop.mjs --trend-history-dir "./artifacts/history"
|
||||
node scripts/report-generated-slop.mjs --output-json "./tmp/harness-cleanup-report.json" --output-markdown "./tmp/harness-cleanup-report.md"
|
||||
|
||||
选项:
|
||||
--trend-input PATH 显式指定 harness eval trend JSON
|
||||
--doc-freshness-input PATH 显式指定 doc freshness JSON
|
||||
--legacy-input PATH 显式指定 governance / legacy surface report JSON
|
||||
--trend-history-dir PATH 透传给 trend report，用历史 summary 生成趋势
|
||||
--workspace-root PATH 未提供 trend-input 时，用该工作区生成当前 trend 报告
|
||||
--format FMT 标准输出格式：text | json | markdown
|
||||
--output-json PATH 将 cleanup/slop 报告 JSON 写入指定路径
|
||||
--output-markdown PATH 将 cleanup/slop 报告 Markdown 写入指定路径
|
||||
-h, --help 显示帮助
|
||||
`);
|
||||
}
|
||||
|
||||
function resolvePath(baseDir, targetPath) {
|
||||
return path.resolve(baseDir, targetPath);
|
||||
}
|
||||
|
||||
function ensureParentDirectory(filePath) {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
}
|
||||
|
||||
function readJsonFile(filePath) {
|
||||
return JSON.parse(fs.readFileSync(filePath, "utf8"));
|
||||
}
|
||||
|
||||
function detectDefaultTrendHistoryDir(repoRoot) {
|
||||
const candidate = resolvePath(repoRoot, "./artifacts/history");
|
||||
if (!fs.existsSync(candidate)) {
|
||||
return "";
|
||||
}
|
||||
|
||||
const hasJson = fs
|
||||
.readdirSync(candidate, { withFileTypes: true })
|
||||
.some((entry) => entry.isFile() && entry.name.endsWith(".json"));
|
||||
|
||||
return hasJson ? candidate : "";
|
||||
}
|
||||
|
||||
function buildTrendReport(repoRoot, options) {
|
||||
if (options.trendInput) {
|
||||
const resolvedPath = resolvePath(repoRoot, options.trendInput);
|
||||
return {
|
||||
report: readJsonFile(resolvedPath),
|
||||
source: {
|
||||
kind: "input-json",
|
||||
path: resolvedPath,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const nodeCommand = process.execPath;
|
||||
const trendScriptPath = resolvePath(repoRoot, TREND_REPORT_PATH);
|
||||
const args = [trendScriptPath, "--format", "json"];
|
||||
const effectiveHistoryDir =
|
||||
options.trendHistoryDir || detectDefaultTrendHistoryDir(repoRoot);
|
||||
|
||||
if (effectiveHistoryDir) {
|
||||
args.push("--history-dir", effectiveHistoryDir);
|
||||
}
|
||||
|
||||
if (options.workspaceRoot) {
|
||||
args.push("--workspace-root", options.workspaceRoot);
|
||||
}
|
||||
|
||||
const output = execFileSync(nodeCommand, args, {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
|
||||
return {
|
||||
report: JSON.parse(output),
|
||||
source: {
|
||||
kind: effectiveHistoryDir ? "generated-history" : "generated-current",
|
||||
path: effectiveHistoryDir
|
||||
? resolvePath(repoRoot, effectiveHistoryDir)
|
||||
: "",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildGovernanceReport(repoRoot, options) {
|
||||
if (options.legacyInput) {
|
||||
const resolvedPath = resolvePath(repoRoot, options.legacyInput);
|
||||
return {
|
||||
report: readJsonFile(resolvedPath),
|
||||
source: {
|
||||
kind: "input-json",
|
||||
path: resolvedPath,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
report: toSerializableLegacySurfaceReport(buildLegacySurfaceReport()),
|
||||
source: {
|
||||
kind: "live-scan",
|
||||
path: "",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function buildDocFreshnessReport(repoRoot, options) {
|
||||
if (options.docFreshnessInput) {
|
||||
const resolvedPath = resolvePath(repoRoot, options.docFreshnessInput);
|
||||
return {
|
||||
report: readJsonFile(resolvedPath),
|
||||
source: {
|
||||
kind: "input-json",
|
||||
path: resolvedPath,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
report: buildLiveDocFreshnessReport(repoRoot),
|
||||
source: {
|
||||
kind: "live-scan",
|
||||
path: "",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function renderOutput(report, format) {
|
||||
if (format === "json") {
|
||||
return `${JSON.stringify(report, null, 2)}\n`;
|
||||
}
|
||||
|
||||
if (format === "markdown") {
|
||||
return renderGeneratedSlopMarkdown(report);
|
||||
}
|
||||
|
||||
return renderGeneratedSlopText(report);
|
||||
}
|
||||
|
||||
function runGeneratedSlopReportCli() {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
printHelp();
|
||||
return;
|
||||
}
|
||||
|
||||
const repoRoot = process.cwd();
|
||||
const trendResult = buildTrendReport(repoRoot, options);
|
||||
const docFreshnessResult = buildDocFreshnessReport(repoRoot, options);
|
||||
const governanceResult = buildGovernanceReport(repoRoot, options);
|
||||
const report = buildGeneratedSlopReport({
|
||||
repoRoot,
|
||||
trendReport: trendResult.report,
|
||||
docFreshnessReport: docFreshnessResult.report,
|
||||
governanceReport: governanceResult.report,
|
||||
sources: {
|
||||
trend: trendResult.source,
|
||||
docFreshness: docFreshnessResult.source,
|
||||
governance: governanceResult.source,
|
||||
},
|
||||
});
|
||||
|
||||
if (options.outputJson) {
|
||||
const targetPath = resolvePath(repoRoot, options.outputJson);
|
||||
ensureParentDirectory(targetPath);
|
||||
fs.writeFileSync(targetPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
||||
console.log(`[lime] cleanup/slop report JSON: ${targetPath}`);
|
||||
}
|
||||
|
||||
if (options.outputMarkdown) {
|
||||
const targetPath = resolvePath(repoRoot, options.outputMarkdown);
|
||||
ensureParentDirectory(targetPath);
|
||||
fs.writeFileSync(targetPath, renderGeneratedSlopMarkdown(report), "utf8");
|
||||
console.log(`[lime] cleanup/slop report Markdown: ${targetPath}`);
|
||||
}
|
||||
|
||||
process.stdout.write(renderOutput(report, options.format));
|
||||
}
|
||||
|
||||
const isMainModule =
|
||||
process.argv[1] &&
|
||||
path.resolve(process.argv[1]) === fileURLToPath(import.meta.url);
|
||||
|
||||
if (isMainModule) {
|
||||
runGeneratedSlopReportCli();
|
||||
}
|
||||
+11
-2108
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user