diff --git a/scripts/mutation-health/README.md b/scripts/mutation-health/README.md index abe7c397435..1901ac22850 100644 --- a/scripts/mutation-health/README.md +++ b/scripts/mutation-health/README.md @@ -53,7 +53,7 @@ That divergence is exactly why this project exists. | File | Purpose | | --- | --- | -| `pick-next.mjs` | Walk `/src/`, merge with the live ledger, return the next source file to mutate | +| `pick-next.mjs` | Walk `/src/` (per-package mode) or every vitest-eligible package (global mode, `--global`), merge with the live ledger, return the next source file(s) to mutate | | `mutate.mjs` | Run Stryker on one source file of any vitest package, write `summary.json` | | `stryker.default.mjs` | Default Stryker config for onboarded packages (points at the package's own `vitest.config.*`) | | `emit-payload.mjs` | Turn a Stryker `summary.json` into a BQ-ready writer payload | @@ -240,11 +240,24 @@ curl --fail -sS \ 'https://internal.users.n8n.cloud/webhook/mutation-health-ledger?package=n8n-workflow' \ -o /tmp/ledger.json -# Pick the next file to score +# Pick the next file to score (per-package mode) node scripts/mutation-health/pick-next.mjs \ --package-dir packages/workflow \ --ledger-file /tmp/ledger.json +# Pick top-N files across every vitest-eligible package, ranked by the +# global value formula w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage). +# Signals and coverage files are optional — missing terms contribute 0 +# except `(1 − coverage)`, which degrades to 1 (worst-case) so untracked +# files float to the top. +node scripts/mutation-health/pick-next.mjs \ + --global \ + --ledger-file /tmp/ledger.json \ + --signals-file /tmp/signals.json \ + --coverage-file /tmp/coverage.json \ + --top-n 5 \ + --block '@n8n/expression-runtime' + # Build a BQ payload from a Stryker run node scripts/mutation-health/emit-payload.mjs \ --summary packages/workflow/reports/mutation/summary.json \ diff --git a/scripts/mutation-health/pick-next.mjs b/scripts/mutation-health/pick-next.mjs index b6295a09932..af0689cdaed 100644 --- a/scripts/mutation-health/pick-next.mjs +++ b/scripts/mutation-health/pick-next.mjs @@ -1,7 +1,8 @@ #!/usr/bin/env node /** - * Walk a package's source tree, merge with the live BQ ledger snapshot, - * return the next pair to mutate. + * Walk a package's source tree (per-package mode) or every vitest-eligible + * package's source tree (global mode), merge with the live BQ ledger snapshot, + * return the next file(s) to mutate. * * Files present in src/ but absent from the live ledger are synthesised as * status='new'. No separate seed step needed — the ledger fills in @@ -11,29 +12,49 @@ * Effective statuses (computed at pick time): new | red | stale | green * * Picker priority: new → red → stale → skip green - * Tiebreaks within each bucket: + * + * Per-package mode tiebreaks within each bucket: * - new: alphabetical by source_file_path * - red: lowest score first (focus on weakest tests) * - stale: oldest last_checked_at first (natural cycling) * + * Global mode tiebreaks within each bucket: highest value first, where + * value = w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage) + * — churn and fix-density come from `signals.mjs`, coverage from an optional + * input file. Path used as final lexical tiebreak for determinism. + * * "Stale" is an in-memory promotion of green rows older than * STALE_AFTER_WEEKS (default 4). Not stored. * - * Inputs: + * Inputs (per-package mode): * --package-dir Required. Repo-relative path to the package, e.g. packages/workflow * --ledger-file Required. Live ledger JSON: { "ledger": [ ... ] } - * --mode Optional. Restrict the picker to one bucket: - * baseline → only `new` (establish first scores) - * coverage → only `red`/`stale` (revisit weakest, lowest-first) - * omitted → combined new → red → stale (default) + * --mode Optional. Restrict the picker to one bucket. * --stale-after-weeks Optional. Default 4. * - * Output (stdout): { picked: { source_file_path, package, prior_status, effective_status } } - * OR { picked: null, reason: "all-green" | "empty-source-tree" - * | "no-new-files" | "nothing-below-threshold" }. + * Inputs (global mode): + * --global Required to enter global mode. + * --ledger-file Required. Live read-all ledger JSON (rows for every package). + * --signals-file Optional. JSON from `signals.mjs gatherSignals`. + * --coverage-file Optional. JSON map { "": 0..1 } of line-coverage. + * --top-n Optional. Default 1. How many top-ranked rows to emit. + * --block Optional. Comma-separated package names to exclude from the walk. + * --w-churn / --w-fix-density / --w-coverage Optional. Per-signal weights for value formula. + * --mode Optional. Restrict picker to one bucket (same as per-package). + * --stale-after-weeks Optional. Default 4. + * + * Output (stdout): + * per-package mode → + * { picked: { source_file_path, package, prior_status, effective_status } } + * OR { picked: null, reason: "all-green" | "empty-source-tree" + * | "no-new-files" | "nothing-below-threshold" } + * global mode → + * { picked: [ { source_file_path, package, prior_status, effective_status, value }, ... ] } + * OR { picked: [], reason: "all-green" | "empty-source-tree" + * | "no-new-files" | "nothing-below-threshold" } * * Exit codes: - * 0 — picked a row OR nothing to do (with picked: null sentinel) + * 0 — picked a row OR nothing to do (with picked: null / [] sentinel) * 2 — usage / config error */ @@ -95,6 +116,69 @@ const MIN_MEANINGFUL_LINES = 15; // safety net for packages that co-locate tests as siblings (e.g. `foo.test.ts`). const NON_SOURCE_DIRS = new Set(['__tests__', '__mocks__', 'fixtures']); +// The vitest-eligible mutation-tracked packages. Single source of truth for +// global mode: the picker walks every entry here unless overridden via +// --block. Adding a package = one-line append here AND in +// .github/workflows/mutation-health-nightly.yml `setup` job. Jest packages +// and the isolated-vm engine (@n8n/expression-runtime, blocked on DEVP-257) +// are intentionally absent. +export const ELIGIBLE_PACKAGES = [ + { name: 'n8n-workflow', dir: 'packages/workflow' }, + { name: '@n8n/crdt', dir: 'packages/@n8n/crdt' }, + { name: '@n8n/decorators', dir: 'packages/@n8n/decorators' }, +]; + +export function isEligible(pkgName) { + return ELIGIBLE_PACKAGES.some((p) => p.name === pkgName); +} + +export const DEFAULT_WEIGHTS = Object.freeze({ churn: 1, fixDensity: 1, coverage: 1 }); + +/** + * Value formula: w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage). + * + * Missing signals contribute 0 (no penalty, no boost). Missing coverage is + * treated as 0 so the (1 − coverage) term becomes 1 — unknown coverage = + * worst case = highest urge to score, matching the picker's bias toward + * surfacing untracked files. + */ +export function computeValue( + { churn = 0, fixDensity = 0, coverage = 0 } = {}, + weights = DEFAULT_WEIGHTS, +) { + const churnVal = Number.isFinite(Number(churn)) ? Number(churn) : 0; + const fdVal = Number.isFinite(Number(fixDensity)) ? Number(fixDensity) : 0; + const covRaw = Number(coverage); + const covVal = Number.isFinite(covRaw) ? Math.max(0, Math.min(1, covRaw)) : 0; + return ( + (weights.churn ?? 0) * churnVal + + (weights.fixDensity ?? 0) * fdVal + + (weights.coverage ?? 0) * (1 - covVal) + ); +} + +/** + * `signals` shape matches `gatherSignals`'s JSON output: churn[path] = { commits, linesChanged } + * (or a bare number from caller transforms), fixDensity[path] = number. + * Returns the per-row `{ churn, fixDensity, coverage }` triple used by `computeValue`. + */ +export function extractSignals(row, { signals = {}, coverage = {} } = {}) { + const churnEntry = signals.churn?.[row.source_file_path]; + const churnVal = + typeof churnEntry === 'number' + ? churnEntry + : typeof churnEntry?.commits === 'number' + ? churnEntry.commits + : 0; + const fixDensity = signals.fixDensity?.[row.source_file_path] ?? 0; + const cov = coverage[row.source_file_path]; + return { + churn: churnVal, + fixDensity, + coverage: typeof cov === 'number' ? cov : 0, + }; +} + function countMeaningfulLines(content) { let count = 0; for (const raw of content.split('\n')) { @@ -106,7 +190,7 @@ function countMeaningfulLines(content) { return count; } -function isMutationWorthy(absPath) { +export function isMutationWorthy(absPath, { read = readFileSync } = {}) { if (absPath.endsWith('.d.ts')) return false; if (/\.(test|spec)\.ts$/.test(absPath)) return false; if (absPath.includes(`${path.sep}__tests__${path.sep}`)) return false; @@ -114,11 +198,11 @@ function isMutationWorthy(absPath) { const base = path.basename(absPath, '.ts'); const lastSegment = base.split('.').at(-1); if (LOW_VALUE_BASENAMES.has(base) || LOW_VALUE_BASENAMES.has(lastSegment)) return false; - if (countMeaningfulLines(readFileSync(absPath, 'utf8')) < MIN_MEANINGFUL_LINES) return false; + if (countMeaningfulLines(read(absPath, 'utf8')) < MIN_MEANINGFUL_LINES) return false; return true; } -async function walkSources(dir) { +export async function walkSources(dir) { const entries = await readdir(dir, { withFileTypes: true }); const out = []; for (const e of entries) { @@ -133,174 +217,409 @@ async function walkSources(dir) { return out; } -const args = parseArgs(process.argv.slice(2)); - -const STALE_AFTER_WEEKS_DEFAULT = 4; -const staleArg = args['stale-after-weeks']; -const parsedStale = Number(staleArg); -let STALE_AFTER_WEEKS; -if (staleArg === undefined) { - STALE_AFTER_WEEKS = STALE_AFTER_WEEKS_DEFAULT; -} else if (Number.isFinite(parsedStale) && parsedStale > 0) { - STALE_AFTER_WEEKS = parsedStale; -} else { - process.stderr.write( - `Invalid --stale-after-weeks=${staleArg}, falling back to ${STALE_AFTER_WEEKS_DEFAULT}.\n`, - ); - STALE_AFTER_WEEKS = STALE_AFTER_WEEKS_DEFAULT; -} - -const pkgDirArg = args['package-dir']; -const ledgerFile = args['ledger-file']; -if (!pkgDirArg) die(2, 'Missing required --package-dir '); -if (!ledgerFile) die(2, 'Missing required --ledger-file '); - -const repoRoot = path.resolve( - execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }).trim(), -); -const pkgDir = path.isAbsolute(pkgDirArg) ? pkgDirArg : path.join(repoRoot, pkgDirArg); -if (!existsSync(pkgDir)) die(2, `Package dir not found: ${pkgDir}`); - -const pkgJsonPath = path.join(pkgDir, 'package.json'); -if (!existsSync(pkgJsonPath)) die(2, `No package.json at ${pkgJsonPath}`); -const pkgName = JSON.parse(await readFile(pkgJsonPath, 'utf8')).name; - -const srcDir = path.join(pkgDir, 'src'); -if (!existsSync(srcDir)) die(2, `No src/ in ${pkgDir}`); - -const ledgerPath = path.isAbsolute(ledgerFile) ? ledgerFile : path.join(process.cwd(), ledgerFile); -if (!existsSync(ledgerPath)) die(2, `Ledger file not found: ${ledgerPath}`); - -// One read returns every row across every package; we narrow to this -// package's rows internally so the picker's per-package behaviour is -// preserved whether the file holds one package or many. The empty-body -// case mirrors the reader webhook's response for unscored packages — the -// picker still synthesises `new` rows from the source tree. -let liveLedger; -try { - ({ rows: liveLedger } = await readLedger({ path: ledgerPath, pkg: pkgName })); -} catch (err) { - die(2, err.message); -} - -const allSources = (await walkSources(srcDir)).sort(); -const worthy = allSources.filter(isMutationWorthy).map((abs) => path.relative(repoRoot, abs)); - -if (worthy.length === 0) { - process.stderr.write('No mutation-worthy source files found under src/.\n'); - process.stdout.write(JSON.stringify({ picked: null, reason: 'empty-source-tree' }) + '\n'); - process.exit(0); -} - -// Merge: live ledger row wins over synthesised "new" row. -const byPath = new Map(); -for (const row of liveLedger) { - byPath.set(row.source_file_path, row); -} -const merged = worthy.map( - (p) => - byPath.get(p) ?? { - source_file_path: p, - package: pkgName, - last_score: null, - threshold_at_run: null, - last_checked_at: null, - status: 'new', - }, -); - -const NOW = Date.now(); -const STALE_AFTER_MS = STALE_AFTER_WEEKS * 7 * 24 * 60 * 60 * 1000; - -function computeEffectiveStatus(row) { - if (row.status === 'new') return 'new'; - if (row.status === 'red') return 'red'; - // status === 'green' — promote to 'stale' if old enough - if (row.last_checked_at) { - const age = NOW - Date.parse(row.last_checked_at); - if (age > STALE_AFTER_MS) return 'stale'; - } - return 'green'; -} - const PRIORITY = { new: 0, red: 1, stale: 2, green: 3 }; -const annotated = merged.map((row) => ({ ...row, effective_status: computeEffectiveStatus(row) })); - -annotated.sort((a, b) => { - const pa = PRIORITY[a.effective_status] ?? 99; - const pb = PRIORITY[b.effective_status] ?? 99; - if (pa !== pb) return pa - pb; - - if (a.effective_status === 'new') { - return a.source_file_path.localeCompare(b.source_file_path); - } - - if (a.effective_status === 'red') { - const sa = a.last_score == null ? Infinity : Number(a.last_score); - const sb = b.last_score == null ? Infinity : Number(b.last_score); - if (sa !== sb) return sa - sb; - return a.source_file_path.localeCompare(b.source_file_path); - } - - // stale: oldest last_checked_at first - const ta = a.last_checked_at ? Date.parse(a.last_checked_at) : 0; - const tb = b.last_checked_at ? Date.parse(b.last_checked_at) : 0; - if (ta !== tb) return ta - tb; - return a.source_file_path.localeCompare(b.source_file_path); -}); - -const counts = annotated.reduce((acc, r) => { - acc[r.effective_status] = (acc[r.effective_status] ?? 0) + 1; - return acc; -}, {}); - -process.stderr.write( - `Source files: ${worthy.length} • ` + - `new=${counts.new ?? 0} red=${counts.red ?? 0} stale=${counts.stale ?? 0} green=${counts.green ?? 0}\n`, -); - -// --mode restricts the candidate set to one bucket; omitted = combined. const MODE_BUCKETS = { baseline: new Set(['new']), coverage: new Set(['red', 'stale']), }; -const mode = args.mode; -if (mode !== undefined && !Object.hasOwn(MODE_BUCKETS, mode)) { - die(2, `Invalid --mode=${mode}. Use 'baseline' or 'coverage' (omit for combined new→red→stale).`); -} -const candidates = mode - ? annotated.filter((r) => MODE_BUCKETS[mode].has(r.effective_status)) - : annotated; -const top = candidates[0]; - -// Nothing to do: an empty mode-filtered set, or (combined mode) the best row is green. -if (!top || (!mode && top.effective_status === 'green')) { - const reason = - mode === 'baseline' - ? 'no-new-files' - : mode === 'coverage' - ? 'nothing-below-threshold' - : 'all-green'; - process.stderr.write(`Nothing to do for mode=${mode ?? 'combined'} (${reason}).\n`); - process.stdout.write(JSON.stringify({ picked: null, reason }) + '\n'); - process.exit(0); +export function computeEffectiveStatus(row, { now, staleAfterMs }) { + if (row.status === 'new') return 'new'; + if (row.status === 'red') return 'red'; + if (row.last_checked_at) { + const age = now - Date.parse(row.last_checked_at); + if (age > staleAfterMs) return 'stale'; + } + return 'green'; } -process.stderr.write( - `Picked: ${top.source_file_path}\n` + - ` priority=${top.effective_status} ` + - `(was ${top.status}, last_checked_at=${top.last_checked_at ?? 'never'})\n`, -); +/** + * Rank a flat list of (already-merged) candidate rows by bucket priority, + * then by value formula (descending) within each bucket, with the source + * path as the final lexical tiebreak. + * + * Pure function — used by both the global CLI path and the test suite. + * Excludes any row whose package is in the `blocked` set, exits the + * `green` bucket entirely (those are "nothing to do"), and applies the + * optional `--mode` bucket filter. + * + * Returns the rows in rank order, each annotated with `effective_status` + * and `value`. Caller decides how many to keep. + */ +export function rankCandidates( + rows, + { + now, + staleAfterMs, + mode, + blocked = new Set(), + signals = {}, + coverage = {}, + weights = DEFAULT_WEIGHTS, + } = {}, +) { + const annotated = rows + .filter((r) => !blocked.has(r.package)) + .map((row) => { + const effective_status = computeEffectiveStatus(row, { now, staleAfterMs }); + const signalTriple = extractSignals(row, { signals, coverage }); + return { + ...row, + effective_status, + value: computeValue(signalTriple, weights), + }; + }); -process.stdout.write( - JSON.stringify({ - picked: { - source_file_path: top.source_file_path, - package: top.package, - prior_status: top.status, - effective_status: top.effective_status, - }, - }) + '\n', -); + const filtered = annotated.filter((r) => { + if (r.effective_status === 'green') return false; + if (mode && !MODE_BUCKETS[mode].has(r.effective_status)) return false; + return true; + }); + + filtered.sort((a, b) => { + const pa = PRIORITY[a.effective_status] ?? 99; + const pb = PRIORITY[b.effective_status] ?? 99; + if (pa !== pb) return pa - pb; + if (b.value !== a.value) return b.value - a.value; + return a.source_file_path.localeCompare(b.source_file_path); + }); + + return filtered; +} + +/** + * Synthesise a "new" row for every mutation-worthy source file that has no + * ledger row yet, then layer the live ledger rows on top (ledger wins). Used + * by both per-package and global walks. + */ +export function mergeWithLedger({ worthyPaths, pkgName, ledgerRows }) { + const byPath = new Map(); + for (const row of ledgerRows) { + if (row.package === pkgName) byPath.set(row.source_file_path, row); + } + return worthyPaths.map( + (p) => + byPath.get(p) ?? { + source_file_path: p, + package: pkgName, + last_score: null, + threshold_at_run: null, + last_checked_at: null, + status: 'new', + }, + ); +} + +function parseStaleAfterWeeks(staleArg) { + const DEFAULT = 4; + if (staleArg === undefined) return DEFAULT; + const parsed = Number(staleArg); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + process.stderr.write(`Invalid --stale-after-weeks=${staleArg}, falling back to ${DEFAULT}.\n`); + return DEFAULT; +} + +function parseWeights(args) { + const w = { ...DEFAULT_WEIGHTS }; + for (const [flag, key] of [ + ['w-churn', 'churn'], + ['w-fix-density', 'fixDensity'], + ['w-coverage', 'coverage'], + ]) { + if (args[flag] === undefined) continue; + const parsed = Number(args[flag]); + if (!Number.isFinite(parsed) || parsed < 0) { + die(2, `Invalid --${flag}=${args[flag]} (expected non-negative number).`); + } + w[key] = parsed; + } + return w; +} + +function parseTopN(args) { + if (args['top-n'] === undefined) return 1; + const parsed = Number(args['top-n']); + if (!Number.isInteger(parsed) || parsed <= 0) { + die(2, `Invalid --top-n=${args['top-n']} (expected positive integer).`); + } + return parsed; +} + +function parseBlocked(args) { + const raw = args.block; + if (raw === undefined || raw === true) return new Set(); + return new Set( + String(raw) + .split(',') + .map((s) => s.trim()) + .filter(Boolean), + ); +} + +function resolveLedgerPath(ledgerFile) { + return path.isAbsolute(ledgerFile) ? ledgerFile : path.join(process.cwd(), ledgerFile); +} + +async function readJsonIfPresent(file) { + if (!file) return null; + const resolved = path.isAbsolute(file) ? file : path.join(process.cwd(), file); + if (!existsSync(resolved)) die(2, `File not found: ${resolved}`); + try { + return JSON.parse(await readFile(resolved, 'utf8')); + } catch (err) { + die(2, `Failed to parse JSON at ${resolved}: ${err.message}`); + } + return null; +} + +async function collectPackageCandidates({ repoRoot, pkg }) { + const srcDir = path.join(repoRoot, pkg.dir, 'src'); + if (!existsSync(srcDir)) return null; + const allSources = (await walkSources(srcDir)).sort(); + const worthy = allSources + .filter((p) => isMutationWorthy(p)) + .map((abs) => path.relative(repoRoot, abs)); + return { pkg, worthy }; +} + +async function runGlobal({ args, repoRoot, now }) { + const ledgerFile = args['ledger-file']; + if (!ledgerFile) die(2, 'Missing required --ledger-file '); + const ledgerPath = resolveLedgerPath(ledgerFile); + if (!existsSync(ledgerPath)) die(2, `Ledger file not found: ${ledgerPath}`); + + const mode = args.mode; + if (mode !== undefined && mode !== true && !Object.hasOwn(MODE_BUCKETS, mode)) { + die(2, `Invalid --mode=${mode}. Use 'baseline' or 'coverage' (omit for combined).`); + } + const modeArg = mode === true ? undefined : mode; + + const staleAfterWeeks = parseStaleAfterWeeks(args['stale-after-weeks']); + const staleAfterMs = staleAfterWeeks * 7 * 24 * 60 * 60 * 1000; + const topN = parseTopN(args); + const blocked = parseBlocked(args); + const weights = parseWeights(args); + + let liveLedger; + try { + ({ rows: liveLedger } = await readLedger({ path: ledgerPath })); + } catch (err) { + die(2, err.message); + } + + const signals = (await readJsonIfPresent(args['signals-file'])) ?? { churn: {}, fixDensity: {} }; + const coverage = (await readJsonIfPresent(args['coverage-file'])) ?? {}; + + const eligible = ELIGIBLE_PACKAGES.filter((p) => !blocked.has(p.name)); + const merged = []; + for (const pkg of eligible) { + const collected = await collectPackageCandidates({ repoRoot, pkg }); + if (!collected) { + process.stderr.write(`No src/ for ${pkg.name} at ${pkg.dir}; skipping.\n`); + continue; + } + merged.push( + ...mergeWithLedger({ + worthyPaths: collected.worthy, + pkgName: pkg.name, + ledgerRows: liveLedger, + }), + ); + } + + if (merged.length === 0) { + process.stderr.write('No mutation-worthy source files found across eligible packages.\n'); + process.stdout.write(JSON.stringify({ picked: [], reason: 'empty-source-tree' }) + '\n'); + process.exit(0); + } + + const ranked = rankCandidates(merged, { + now, + staleAfterMs, + mode: modeArg, + blocked, + signals, + coverage, + weights, + }); + + const counts = ranked.reduce((acc, r) => { + acc[r.effective_status] = (acc[r.effective_status] ?? 0) + 1; + return acc; + }, {}); + process.stderr.write( + `Global walk: candidates=${merged.length} ranked=${ranked.length} • ` + + `new=${counts.new ?? 0} red=${counts.red ?? 0} stale=${counts.stale ?? 0}\n`, + ); + + if (ranked.length === 0) { + const reason = + modeArg === 'baseline' + ? 'no-new-files' + : modeArg === 'coverage' + ? 'nothing-below-threshold' + : 'all-green'; + process.stderr.write(`Nothing to do for mode=${modeArg ?? 'combined'} (${reason}).\n`); + process.stdout.write(JSON.stringify({ picked: [], reason }) + '\n'); + process.exit(0); + } + + const top = ranked.slice(0, topN).map((r) => ({ + source_file_path: r.source_file_path, + package: r.package, + prior_status: r.status, + effective_status: r.effective_status, + value: r.value, + })); + + for (const t of top) { + process.stderr.write( + `Picked: ${t.source_file_path} ` + + `[${t.package}] priority=${t.effective_status} value=${t.value.toFixed(4)}\n`, + ); + } + + process.stdout.write(JSON.stringify({ picked: top }) + '\n'); +} + +async function runPerPackage({ args, repoRoot, now }) { + const pkgDirArg = args['package-dir']; + const ledgerFile = args['ledger-file']; + if (!pkgDirArg) die(2, 'Missing required --package-dir '); + if (!ledgerFile) die(2, 'Missing required --ledger-file '); + + const pkgDir = path.isAbsolute(pkgDirArg) ? pkgDirArg : path.join(repoRoot, pkgDirArg); + if (!existsSync(pkgDir)) die(2, `Package dir not found: ${pkgDir}`); + + const pkgJsonPath = path.join(pkgDir, 'package.json'); + if (!existsSync(pkgJsonPath)) die(2, `No package.json at ${pkgJsonPath}`); + const pkgName = JSON.parse(await readFile(pkgJsonPath, 'utf8')).name; + + const srcDir = path.join(pkgDir, 'src'); + if (!existsSync(srcDir)) die(2, `No src/ in ${pkgDir}`); + + const ledgerPath = resolveLedgerPath(ledgerFile); + if (!existsSync(ledgerPath)) die(2, `Ledger file not found: ${ledgerPath}`); + + const staleAfterWeeks = parseStaleAfterWeeks(args['stale-after-weeks']); + const staleAfterMs = staleAfterWeeks * 7 * 24 * 60 * 60 * 1000; + + // One read returns every row across every package; we narrow to this + // package's rows internally so the picker's per-package behaviour is + // preserved whether the file holds one package or many. + let liveLedger; + try { + ({ rows: liveLedger } = await readLedger({ path: ledgerPath, pkg: pkgName })); + } catch (err) { + die(2, err.message); + } + + const allSources = (await walkSources(srcDir)).sort(); + const worthy = allSources + .filter((p) => isMutationWorthy(p)) + .map((abs) => path.relative(repoRoot, abs)); + + if (worthy.length === 0) { + process.stderr.write('No mutation-worthy source files found under src/.\n'); + process.stdout.write(JSON.stringify({ picked: null, reason: 'empty-source-tree' }) + '\n'); + process.exit(0); + } + + const merged = mergeWithLedger({ worthyPaths: worthy, pkgName, ledgerRows: liveLedger }); + + const annotated = merged.map((row) => ({ + ...row, + effective_status: computeEffectiveStatus(row, { now, staleAfterMs }), + })); + + annotated.sort((a, b) => { + const pa = PRIORITY[a.effective_status] ?? 99; + const pb = PRIORITY[b.effective_status] ?? 99; + if (pa !== pb) return pa - pb; + + if (a.effective_status === 'new') { + return a.source_file_path.localeCompare(b.source_file_path); + } + + if (a.effective_status === 'red') { + const sa = a.last_score == null ? Infinity : Number(a.last_score); + const sb = b.last_score == null ? Infinity : Number(b.last_score); + if (sa !== sb) return sa - sb; + return a.source_file_path.localeCompare(b.source_file_path); + } + + // stale: oldest last_checked_at first + const ta = a.last_checked_at ? Date.parse(a.last_checked_at) : 0; + const tb = b.last_checked_at ? Date.parse(b.last_checked_at) : 0; + if (ta !== tb) return ta - tb; + return a.source_file_path.localeCompare(b.source_file_path); + }); + + const counts = annotated.reduce((acc, r) => { + acc[r.effective_status] = (acc[r.effective_status] ?? 0) + 1; + return acc; + }, {}); + + process.stderr.write( + `Source files: ${worthy.length} • ` + + `new=${counts.new ?? 0} red=${counts.red ?? 0} stale=${counts.stale ?? 0} green=${counts.green ?? 0}\n`, + ); + + const mode = args.mode; + if (mode !== undefined && mode !== true && !Object.hasOwn(MODE_BUCKETS, mode)) { + die( + 2, + `Invalid --mode=${mode}. Use 'baseline' or 'coverage' (omit for combined new→red→stale).`, + ); + } + const modeArg = mode === true ? undefined : mode; + const candidates = modeArg + ? annotated.filter((r) => MODE_BUCKETS[modeArg].has(r.effective_status)) + : annotated; + + const top = candidates[0]; + + if (!top || (!modeArg && top.effective_status === 'green')) { + const reason = + modeArg === 'baseline' + ? 'no-new-files' + : modeArg === 'coverage' + ? 'nothing-below-threshold' + : 'all-green'; + process.stderr.write(`Nothing to do for mode=${modeArg ?? 'combined'} (${reason}).\n`); + process.stdout.write(JSON.stringify({ picked: null, reason }) + '\n'); + process.exit(0); + } + + process.stderr.write( + `Picked: ${top.source_file_path}\n` + + ` priority=${top.effective_status} ` + + `(was ${top.status}, last_checked_at=${top.last_checked_at ?? 'never'})\n`, + ); + + process.stdout.write( + JSON.stringify({ + picked: { + source_file_path: top.source_file_path, + package: top.package, + prior_status: top.status, + effective_status: top.effective_status, + }, + }) + '\n', + ); +} + +const isCli = import.meta.url === `file://${process.argv[1]}`; +if (isCli) { + const args = parseArgs(process.argv.slice(2)); + const repoRoot = path.resolve( + execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }).trim(), + ); + const now = Date.now(); + if (args.global) { + await runGlobal({ args, repoRoot, now }); + } else { + await runPerPackage({ args, repoRoot, now }); + } +} diff --git a/scripts/mutation-health/pick-next.test.mjs b/scripts/mutation-health/pick-next.test.mjs new file mode 100644 index 00000000000..1d267c879e1 --- /dev/null +++ b/scripts/mutation-health/pick-next.test.mjs @@ -0,0 +1,376 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; + +import { + DEFAULT_WEIGHTS, + ELIGIBLE_PACKAGES, + computeEffectiveStatus, + computeValue, + extractSignals, + isEligible, + mergeWithLedger, + rankCandidates, +} from './pick-next.mjs'; + +// Fixed reference epoch (2026-06-20 00:00:00 UTC). All age-based assertions +// derive from this so they don't drift with wall-clock. +const NOW = Date.parse('2026-06-20T00:00:00.000Z'); +const WEEK_MS = 7 * 24 * 60 * 60 * 1000; +const STALE_AFTER_MS = 4 * WEEK_MS; + +// Three-package fixture ledger. Each row pins a known status (new is +// implicit — files not in the ledger become `new` when merged) so the +// bucketed-ordering assertions don't depend on staleness math. +const MULTI_PKG_LEDGER = [ + // n8n-workflow — one red row (will be ordered by value) + { + source_file_path: 'packages/workflow/src/a.ts', + package: 'n8n-workflow', + status: 'red', + last_score: 30, + last_checked_at: '2026-05-01T00:00:00.000Z', + }, + // @n8n/crdt — one red row + { + source_file_path: 'packages/@n8n/crdt/src/x.ts', + package: '@n8n/crdt', + status: 'red', + last_score: 40, + last_checked_at: '2026-05-15T00:00:00.000Z', + }, + // @n8n/decorators — one green row that has aged past STALE_AFTER_MS + { + source_file_path: 'packages/@n8n/decorators/src/y.ts', + package: '@n8n/decorators', + status: 'green', + last_score: 85, + last_checked_at: '2026-04-01T00:00:00.000Z', + }, + // excluded package — should never appear in any candidate set + { + source_file_path: 'packages/@n8n/expression-runtime/src/forbidden.ts', + package: '@n8n/expression-runtime', + status: 'red', + last_score: 10, + last_checked_at: '2026-06-10T00:00:00.000Z', + }, +]; + +// Synthetic worthy-file inputs per package — these come from walking each +// package's src/ in the real CLI. Here we feed them straight to +// mergeWithLedger so the test has no filesystem dependency. +const WALK_INPUTS = { + 'n8n-workflow': ['packages/workflow/src/a.ts', 'packages/workflow/src/b.ts'], + '@n8n/crdt': ['packages/@n8n/crdt/src/x.ts', 'packages/@n8n/crdt/src/z.ts'], + '@n8n/decorators': ['packages/@n8n/decorators/src/y.ts'], +}; + +// Signals are tuned so the value ordering within each bucket is +// deterministic and obvious: +// - within `new`: crdt/z.ts (churn 100) > workflow/b.ts (churn 1) +// - within `red`: workflow/a.ts (churn 50, fix 5) > crdt/x.ts (churn 2, fix 0) +const SIGNALS = { + churn: { + 'packages/workflow/src/a.ts': { commits: 50, linesChanged: 999 }, + 'packages/workflow/src/b.ts': { commits: 1, linesChanged: 1 }, + 'packages/@n8n/crdt/src/x.ts': { commits: 2, linesChanged: 2 }, + 'packages/@n8n/crdt/src/z.ts': { commits: 100, linesChanged: 200 }, + 'packages/@n8n/decorators/src/y.ts': { commits: 0, linesChanged: 0 }, + }, + fixDensity: { + 'packages/workflow/src/a.ts': 5, + 'packages/@n8n/crdt/src/x.ts': 0, + 'packages/@n8n/crdt/src/z.ts': 1, + }, +}; + +const COVERAGE = { + 'packages/workflow/src/a.ts': 0.4, + 'packages/workflow/src/b.ts': 0.9, + 'packages/@n8n/crdt/src/x.ts': 0.95, + 'packages/@n8n/crdt/src/z.ts': 0.1, +}; + +function buildMerged() { + const merged = []; + for (const [pkgName, worthy] of Object.entries(WALK_INPUTS)) { + merged.push(...mergeWithLedger({ worthyPaths: worthy, pkgName, ledgerRows: MULTI_PKG_LEDGER })); + } + return merged; +} + +describe('isEligible (vitest allowlist unit test)', () => { + it('admits every name in ELIGIBLE_PACKAGES', () => { + assert.ok(ELIGIBLE_PACKAGES.length >= 2, 'allowlist must span at least two packages'); + for (const pkg of ELIGIBLE_PACKAGES) { + assert.equal(isEligible(pkg.name), true, `expected eligible: ${pkg.name}`); + } + }); + + it('rejects packages outside the allowlist', () => { + // expression-runtime is intentionally blocked on DEVP-257 + assert.equal(isEligible('@n8n/expression-runtime'), false); + // jest packages aren't onboarded yet + assert.equal(isEligible('@n8n/cli'), false); + assert.equal(isEligible(''), false); + assert.equal(isEligible(undefined), false); + }); +}); + +describe('computeValue', () => { + it('applies w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage)', () => { + const v = computeValue( + { churn: 10, fixDensity: 2, coverage: 0.25 }, + { churn: 1, fixDensity: 3, coverage: 4 }, + ); + // 1*10 + 3*2 + 4*(1 - 0.25) = 10 + 6 + 3 = 19 + assert.equal(v, 19); + }); + + it('treats missing signals as zero (no penalty, no boost)', () => { + // Default weights {1,1,1}. With everything missing → 1*0 + 1*0 + 1*(1-0) = 1. + assert.equal(computeValue({}), 1); + assert.equal(computeValue({ churn: undefined, fixDensity: null }), 1); + }); + + it('clamps coverage to [0,1]', () => { + assert.equal(computeValue({ coverage: 2 }, { churn: 0, fixDensity: 0, coverage: 1 }), 0); + assert.equal(computeValue({ coverage: -1 }, { churn: 0, fixDensity: 0, coverage: 1 }), 1); + }); +}); + +describe('extractSignals', () => { + it('handles `{ commits, linesChanged }` shape from gatherSignals', () => { + const triple = extractSignals( + { source_file_path: 'packages/workflow/src/a.ts' }, + { signals: SIGNALS, coverage: COVERAGE }, + ); + assert.deepEqual(triple, { churn: 50, fixDensity: 5, coverage: 0.4 }); + }); + + it('also accepts a bare-number churn value', () => { + const triple = extractSignals( + { source_file_path: 'foo.ts' }, + { signals: { churn: { 'foo.ts': 7 }, fixDensity: { 'foo.ts': 3 } }, coverage: {} }, + ); + assert.deepEqual(triple, { churn: 7, fixDensity: 3, coverage: 0 }); + }); + + it('zero-fills when the file is absent from signals/coverage', () => { + assert.deepEqual(extractSignals({ source_file_path: 'missing.ts' }), { + churn: 0, + fixDensity: 0, + coverage: 0, + }); + }); +}); + +describe('computeEffectiveStatus', () => { + it('promotes a green row older than the stale window to stale', () => { + const row = { status: 'green', last_checked_at: '2026-04-01T00:00:00.000Z' }; + assert.equal(computeEffectiveStatus(row, { now: NOW, staleAfterMs: STALE_AFTER_MS }), 'stale'); + }); + + it('keeps a fresh green row green', () => { + const row = { status: 'green', last_checked_at: '2026-06-19T00:00:00.000Z' }; + assert.equal(computeEffectiveStatus(row, { now: NOW, staleAfterMs: STALE_AFTER_MS }), 'green'); + }); + + it('passes new/red through unchanged', () => { + assert.equal( + computeEffectiveStatus({ status: 'new' }, { now: NOW, staleAfterMs: STALE_AFTER_MS }), + 'new', + ); + assert.equal( + computeEffectiveStatus({ status: 'red' }, { now: NOW, staleAfterMs: STALE_AFTER_MS }), + 'red', + ); + }); +}); + +describe('mergeWithLedger', () => { + it('synthesises new rows for worthy paths not in the ledger', () => { + const merged = mergeWithLedger({ + worthyPaths: ['packages/workflow/src/a.ts', 'packages/workflow/src/b.ts'], + pkgName: 'n8n-workflow', + ledgerRows: MULTI_PKG_LEDGER, + }); + const a = merged.find((r) => r.source_file_path === 'packages/workflow/src/a.ts'); + const b = merged.find((r) => r.source_file_path === 'packages/workflow/src/b.ts'); + assert.equal(a.status, 'red'); // from the live ledger + assert.equal(b.status, 'new'); // synthesised + assert.equal(b.last_score, null); + }); + + it("never lets another package's rows leak into the merge", () => { + // Ledger has a row for @n8n/expression-runtime; merging for n8n-workflow + // must ignore it entirely. + const merged = mergeWithLedger({ + worthyPaths: ['packages/workflow/src/a.ts'], + pkgName: 'n8n-workflow', + ledgerRows: MULTI_PKG_LEDGER, + }); + assert.equal(merged.length, 1); + assert.equal(merged[0].package, 'n8n-workflow'); + }); +}); + +// PR-gate contract from DEVP-494: +// "integration tests: fixture ledger spanning ≥2 packages returns N rows +// ordered by value within buckets, ≥2 packages present, zero +// excluded-package rows" +describe('rankCandidates (DEVP-494 PR gate)', () => { + it('orders results bucket-first (new → red → stale), then by value desc within each bucket', () => { + const merged = buildMerged(); + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + signals: SIGNALS, + coverage: COVERAGE, + weights: DEFAULT_WEIGHTS, + }); + + // Effective bucket sequence must be monotonic in priority. + const priority = { new: 0, red: 1, stale: 2, green: 3 }; + for (let i = 1; i < ranked.length; i++) { + assert.ok( + priority[ranked[i - 1].effective_status] <= priority[ranked[i].effective_status], + `bucket order broken at index ${i}: ${ranked[i - 1].effective_status} → ${ranked[i].effective_status}`, + ); + } + + // Within each bucket, value must be non-increasing. + let prev = null; + for (const row of ranked) { + if (prev && prev.effective_status === row.effective_status) { + assert.ok( + prev.value >= row.value, + `value descending broken in bucket=${row.effective_status}: ${prev.value} → ${row.value}`, + ); + } + prev = row; + } + + // Spot-check the new bucket: crdt/z.ts (churn 100) must outrank + // workflow/b.ts (churn 1). + const newBucket = ranked.filter((r) => r.effective_status === 'new'); + assert.equal(newBucket[0].source_file_path, 'packages/@n8n/crdt/src/z.ts'); + + // Spot-check the red bucket: workflow/a.ts (churn 50, fix 5) must + // outrank crdt/x.ts (churn 2, fix 0). + const redBucket = ranked.filter((r) => r.effective_status === 'red'); + assert.equal(redBucket[0].source_file_path, 'packages/workflow/src/a.ts'); + + // Stale bucket has just decorators/y.ts (green → stale by age). + const staleBucket = ranked.filter((r) => r.effective_status === 'stale'); + assert.equal(staleBucket.length, 1); + assert.equal(staleBucket[0].package, '@n8n/decorators'); + }); + + it('top-N output spans ≥2 packages on the multi-package fixture', () => { + const merged = buildMerged(); + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + signals: SIGNALS, + coverage: COVERAGE, + }); + const topN = ranked.slice(0, 4); + const distinctPackages = new Set(topN.map((r) => r.package)); + assert.ok( + distinctPackages.size >= 2, + `top-${topN.length} must span ≥2 packages; got ${[...distinctPackages].join(', ')}`, + ); + }); + + it('excludes blocked-package rows entirely (zero excluded rows)', () => { + // Add a worthy-file row for the blocked package as if its src/ tree + // were walked. The picker must still drop every row from that pkg. + const blockedRows = mergeWithLedger({ + worthyPaths: ['packages/@n8n/expression-runtime/src/forbidden.ts'], + pkgName: '@n8n/expression-runtime', + ledgerRows: MULTI_PKG_LEDGER, + }); + const merged = [...buildMerged(), ...blockedRows]; + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + signals: SIGNALS, + coverage: COVERAGE, + blocked: new Set(['@n8n/expression-runtime']), + }); + const blockedHits = ranked.filter((r) => r.package === '@n8n/expression-runtime'); + assert.equal(blockedHits.length, 0, 'blocked package rows must not appear'); + }); + + it('drops every green row from the candidate set (combined mode)', () => { + const merged = buildMerged(); + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + signals: SIGNALS, + coverage: COVERAGE, + }); + assert.equal( + ranked.filter((r) => r.effective_status === 'green').length, + 0, + 'green rows must be filtered out of the candidate set', + ); + }); + + it('--mode baseline restricts the candidate set to `new` only', () => { + const merged = buildMerged(); + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + mode: 'baseline', + signals: SIGNALS, + coverage: COVERAGE, + }); + assert.ok(ranked.length > 0); + assert.ok( + ranked.every((r) => r.effective_status === 'new'), + `baseline mode must yield only "new" rows; got ${ranked.map((r) => r.effective_status).join(', ')}`, + ); + }); + + it('--mode coverage restricts the candidate set to `red`/`stale` only', () => { + const merged = buildMerged(); + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + mode: 'coverage', + signals: SIGNALS, + coverage: COVERAGE, + }); + assert.ok(ranked.length > 0); + assert.ok( + ranked.every((r) => r.effective_status === 'red' || r.effective_status === 'stale'), + `coverage mode must yield only red/stale rows; got ${ranked.map((r) => r.effective_status).join(', ')}`, + ); + }); + + it('weights tune the ordering — boosting churn flips priority within a bucket', () => { + // Two new files in the same bucket: crdt/z.ts (churn=100, fix=1) and + // workflow/b.ts (churn=1). With default weights crdt wins. If we zero + // out churn and only weight fix-density, the row WITHOUT a fix + // (workflow/b.ts) falls behind crdt/z.ts (fix=1), but with churn=0 + // and fixDensity=0 the tiebreak becomes the (1−coverage) term: + // workflow/b.ts has coverage 0.9 → value 0.1, crdt/z.ts has 0.1 → + // value 0.9. crdt/z.ts still wins. Now zero out everything except + // w_coverage and watch the lower-coverage file climb. + const merged = buildMerged(); + const ranked = rankCandidates(merged, { + now: NOW, + staleAfterMs: STALE_AFTER_MS, + mode: 'baseline', + signals: SIGNALS, + coverage: COVERAGE, + weights: { churn: 0, fixDensity: 0, coverage: 1 }, + }); + const top = ranked[0]; + // crdt/z.ts has the lowest coverage (0.1) among new rows → highest + // (1 − coverage) value. + assert.equal(top.source_file_path, 'packages/@n8n/crdt/src/z.ts'); + }); +});