mirror of
https://github.com/n8n-io/n8n.git
synced 2026-08-28 09:12:12 +08:00
test: Pin the diff parsing and source filter in mutate.mjs (#36746)
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com> Co-authored-by: n8n-cat-bot[bot] <n8n-cat-bot[bot]@users.noreply.github.com>
This commit is contained in:
Declan Carroll
@@ -1,218 +0,0 @@
|
||||
name: 'Mutation Health (nightly)'
|
||||
run-name: "${{ github.event_name == 'workflow_dispatch' && format('Mutation Health (mode={0}, top_n={1}, source_file={2})', inputs.mode, inputs.top_n, inputs.source_file) || '' }}"
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 03:30 UTC daily — outside CI rush, before EU morning. Runs both passes.
|
||||
- cron: '30 3 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
mode:
|
||||
description: 'Which pass to run'
|
||||
type: choice
|
||||
options:
|
||||
- both
|
||||
- baseline # score files with no result yet (the `new` bucket)
|
||||
- coverage # revisit the weakest scored files (`red`/`stale`, lowest first)
|
||||
default: both
|
||||
top_n:
|
||||
description: 'How many top-ranked rows the global picker schedules for this run (--top-n).'
|
||||
type: string
|
||||
default: '6'
|
||||
source_file:
|
||||
description: 'Optional: re-score this exact repo-relative file, skipping the picker (e.g. packages/workflow/src/common/get-node-by-name.ts). Used by the lane:mutation-increase close handler to refresh the ledger.'
|
||||
type: string
|
||||
default: ''
|
||||
bootstrap_packages:
|
||||
description: |
|
||||
Comma-separated ELIGIBLE_PACKAGES.name list to skip in the divergence guard for this run.
|
||||
Use after onboarding a new entry to ELIGIBLE_PACKAGES (the guard otherwise refuses to schedule for a package with zero prior-status rows). Use '*' to acknowledge a genuine cold-start (empty ledger). Leave empty otherwise — empty ledger then fails loud on the scheduled path.
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
env:
|
||||
READER_URL: https://internal.users.n8n.cloud/webhook/mutation-health-ledger
|
||||
|
||||
jobs:
|
||||
# Build the matrix once. The package list is the picker's ELIGIBLE_PACKAGES
|
||||
# export (single source of truth — DEVP-497); the workflow no longer
|
||||
# redeclares it. fetch-depth: 0 here so signals.mjs has full git history
|
||||
# for churn / fix-density when ranking globally. The picker's --global mode
|
||||
# walks every eligible package, ranks rows by value, and returns the top-N
|
||||
# picks across all packages; the matrix maps one job per pick.
|
||||
setup:
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
matrix: ${{ steps.build.outputs.matrix }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
# Full history feeds signals.mjs (churn + fix-density). A shallow
|
||||
# clone yields zero signals and falls back to a coverage-only
|
||||
# ranking, which is fine but loses the value formula's leverage.
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0
|
||||
with:
|
||||
node-version: 24.18.1
|
||||
|
||||
- name: Fetch read-all live ledger from BigQuery
|
||||
# Read-all (no ?package= param): one curl returns every row across
|
||||
# every eligible package. Per-package narrowing happens inside the
|
||||
# picker.
|
||||
run: |
|
||||
mkdir -p .mutation-health
|
||||
curl --fail -sS "$READER_URL" -o .mutation-health/live-ledger.json
|
||||
|
||||
- name: Gather git-derived signals (churn + fix-density)
|
||||
# Drives the value formula inside the global picker. Skipped when
|
||||
# SOURCE_FILE is set — that path bypasses the picker.
|
||||
if: ${{ github.event.inputs.source_file == '' }}
|
||||
run: |
|
||||
node scripts/mutation-health/signals.mjs \
|
||||
--since 180.days.ago \
|
||||
> .mutation-health/signals.json
|
||||
|
||||
- name: Build matrix
|
||||
id: build
|
||||
env:
|
||||
REQUESTED_MODE: ${{ github.event.inputs.mode || 'both' }}
|
||||
# TOP_N intentionally has no `|| '6'` fallback here — the default lives
|
||||
# in build-matrix.mjs so the script stays self-contained for local
|
||||
# invocation. Scheduled runs (no inputs) hit the script default.
|
||||
TOP_N: ${{ github.event.inputs.top_n }}
|
||||
SOURCE_FILE: ${{ github.event.inputs.source_file || '' }}
|
||||
BOOTSTRAP_PACKAGES: ${{ github.event.inputs.bootstrap_packages || '' }}
|
||||
LEDGER_FILE: .mutation-health/live-ledger.json
|
||||
SIGNALS_FILE: .mutation-health/signals.json
|
||||
run: node scripts/mutation-health/build-matrix.mjs >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Upload picker artefacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: mutation-health-picker-${{ github.run_id }}
|
||||
path: |
|
||||
.mutation-health/live-ledger.json
|
||||
.mutation-health/signals.json
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
|
||||
mutate:
|
||||
needs: setup
|
||||
# Gate on setup success so a die-loud failure in build-matrix.mjs (e.g.
|
||||
# the divergence guard) surfaces cleanly via setup, instead of via
|
||||
# `fromJSON('')` here.
|
||||
if: ${{ needs.setup.result == 'success' && needs.setup.outputs.matrix && fromJSON(needs.setup.outputs.matrix).include[0] != null }}
|
||||
name: ${{ matrix.mode }} · ${{ matrix.name }} · ${{ matrix.source_file }}
|
||||
runs-on: blacksmith-4vcpu-ubuntu-2204
|
||||
timeout-minutes: 60
|
||||
strategy:
|
||||
fail-fast: false # one leg's failure must not skip the others
|
||||
matrix: ${{ fromJSON(needs.setup.outputs.matrix) }}
|
||||
# Concurrency-group key is per-row (mode + package slug + file slug) so
|
||||
# multiple top-N picks within the same (mode, package) bucket each get
|
||||
# their own group. With `cancel-in-progress: false` Actions otherwise
|
||||
# keeps at most one running + one pending job per group and cancels any
|
||||
# older pending — which would silently drop middle picks every nightly
|
||||
# and leave their ledger rows stuck on `new`. The original double-write
|
||||
# protection (scheduled vs manual on the same row) is unchanged because
|
||||
# the same `file_slug` resolves to the same group key.
|
||||
concurrency:
|
||||
group: mutation-health-${{ matrix.mode }}-${{ matrix.slug }}-${{ matrix.file_slug }}
|
||||
cancel-in-progress: false
|
||||
env:
|
||||
PACKAGE_NAME: ${{ matrix.name }}
|
||||
PKG_DIR: ${{ matrix.dir }}
|
||||
REPORTS_DIR: ${{ matrix.dir }}/reports/mutation
|
||||
SOURCE_FILE: ${{ matrix.source_file || '' }}
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
# Shallow clone is fine — the file to mutate has already been picked
|
||||
# in the setup job, and Stryker only needs the working tree.
|
||||
|
||||
- name: Setup Environment
|
||||
uses: ./.github/actions/setup-nodejs
|
||||
|
||||
- name: Download picker artefacts (signals.json)
|
||||
# The setup job computed churn/fix-density over full history and uploaded
|
||||
# signals.json. This shallow checkout can't recompute them from git, so
|
||||
# emit-payload reads them from here instead — keeping the ledger's churn
|
||||
# and fix_density columns populated and consistent with what the picker
|
||||
# ranked on. Best-effort: the on-demand source_file path skips the gather
|
||||
# step, so signals.json may be absent; emit-payload falls back gracefully.
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
|
||||
continue-on-error: true
|
||||
with:
|
||||
name: mutation-health-picker-${{ github.run_id }}
|
||||
path: .mutation-health
|
||||
|
||||
- name: Mutate the picked source file
|
||||
# Exit code semantics from mutate.mjs:
|
||||
# 0 — gate passed: score >= threshold AND no unjustified survivors (green)
|
||||
# 1 — gate failed: score < threshold OR >=1 Survived/NoCoverage mutant (red)
|
||||
# — emit step still runs, this is normal
|
||||
# 2 — usage error
|
||||
# 3 — Stryker hard failure (no summary.json) — must fail the job
|
||||
# We capture rc explicitly so we can distinguish "gate failed" from
|
||||
# "Stryker crashed." continue-on-error: true would collapse them.
|
||||
run: |
|
||||
mkdir -p "$REPORTS_DIR"
|
||||
src_rel="${SOURCE_FILE#"$PKG_DIR"/}"
|
||||
set +e
|
||||
node scripts/mutation-health/mutate.mjs "$src_rel" --package-dir "$PKG_DIR"
|
||||
rc=$?
|
||||
set -e
|
||||
if [ "$rc" -gt 1 ]; then
|
||||
echo "::error::Stryker hard-failed with exit code $rc."
|
||||
exit "$rc"
|
||||
fi
|
||||
echo "Mutate exit $rc ($([ "$rc" = "0" ] && echo green || echo red))."
|
||||
|
||||
- name: Emit BQ payload
|
||||
# Pass --signals only when the setup job produced it (skipped on the
|
||||
# source_file re-score path). Without it emit-payload falls back to git,
|
||||
# which yields null churn/fix_density on this shallow clone.
|
||||
run: |
|
||||
signals_arg=()
|
||||
if [ -f .mutation-health/signals.json ]; then
|
||||
signals_arg=(--signals .mutation-health/signals.json)
|
||||
else
|
||||
echo "::notice::signals.json not present — churn/fix_density will fall back to git (null on shallow clone)."
|
||||
fi
|
||||
node scripts/mutation-health/emit-payload.mjs \
|
||||
--summary "$REPORTS_DIR/summary.json" \
|
||||
--package "$PACKAGE_NAME" \
|
||||
"${signals_arg[@]}"
|
||||
|
||||
- name: POST result payload
|
||||
env:
|
||||
MUTATION_HEALTH_WEBHOOK: ${{ secrets.MUTATION_HEALTH_WEBHOOK }}
|
||||
run: |
|
||||
if [ -z "$MUTATION_HEALTH_WEBHOOK" ]; then
|
||||
echo "::notice::MUTATION_HEALTH_WEBHOOK not set — dry-run, POST skipped (payload uploaded as artefact)."
|
||||
exit 0
|
||||
fi
|
||||
curl --fail -sS -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data @"$REPORTS_DIR/bq-payload.json" \
|
||||
"$MUTATION_HEALTH_WEBHOOK"
|
||||
|
||||
- name: Upload artefacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: mutation-health-${{ matrix.mode }}-${{ matrix.slug }}-${{ matrix.file_slug }}-${{ github.run_id }}
|
||||
path: |
|
||||
${{ env.REPORTS_DIR }}/raw.json
|
||||
${{ env.REPORTS_DIR }}/raw.html
|
||||
${{ env.REPORTS_DIR }}/summary.json
|
||||
${{ env.REPORTS_DIR }}/bq-payload.json
|
||||
retention-days: 14
|
||||
if-no-files-found: warn
|
||||
@@ -58,6 +58,7 @@
|
||||
"setup-backend-module": "node scripts/ensure-zx.mjs && zx scripts/backend-module/setup.mjs",
|
||||
"start": "node scripts/os-normalize.mjs --dir packages/cli/bin n8n",
|
||||
"mutate": "node scripts/mutation-health/mutate.mjs",
|
||||
"mutate:diff": "node scripts/mutation-health/mutate.mjs --diff",
|
||||
"test": "turbo run test",
|
||||
"test:ci": "turbo run test --continue --concurrency=1",
|
||||
"test:ci:frontend": "turbo run test --continue --filter='./packages/frontend/**' --filter='./packages/modules/**'",
|
||||
@@ -84,6 +85,8 @@
|
||||
"@dotenvx/dotenvx": "^1.40.0",
|
||||
"@n8n/eslint-config": "workspace:*",
|
||||
"@n8n/module-cli": "workspace:*",
|
||||
"@stryker-mutator/core": "catalog:",
|
||||
"@stryker-mutator/vitest-runner": "catalog:",
|
||||
"@types/node": "*",
|
||||
"@types/supertest": "^6.0.3",
|
||||
"babel-plugin-transform-import-meta": "^2.3.2",
|
||||
|
||||
Generated
+491
-293
File diff suppressed because it is too large
Load Diff
+3
-3
@@ -123,8 +123,8 @@ catalog:
|
||||
'@pinecone-database/pinecone': ^5.0.2
|
||||
'@qdrant/js-client-rest': ^1.16.2
|
||||
'@rudderstack/rudder-sdk-node': 3.0.5
|
||||
'@stryker-mutator/core': 9.6.1
|
||||
'@stryker-mutator/vitest-runner': 9.6.1
|
||||
'@stryker-mutator/core': 10.0.0
|
||||
'@stryker-mutator/vitest-runner': 10.0.0
|
||||
'@supabase/supabase-js': 2.112.3
|
||||
'@testcontainers/k3s': ^11.13.0
|
||||
'@testcontainers/kafka': ^11.13.0
|
||||
@@ -452,7 +452,7 @@ overrides:
|
||||
undici@6: catalog:undici-v6
|
||||
undici@7: catalog:undici-v7
|
||||
node-gyp>undici: catalog:undici-v7
|
||||
'@babel/traverse': ^7.23.2
|
||||
'@babel/traverse@<7.23.2': ^7.23.2
|
||||
'@vitest/browser@<4.1.10': 4.1.10
|
||||
immutable: 5.1.8
|
||||
nanoid@<3.3.18: 'catalog:'
|
||||
|
||||
@@ -1,92 +0,0 @@
|
||||
# Demo handover: stacked PR on #30956
|
||||
|
||||
Use this prompt to drive the mutant-fix loop end-to-end and open a stacked PR that demonstrates the trial.
|
||||
|
||||
---
|
||||
|
||||
## Prompt
|
||||
|
||||
> I want to demo the mutation-health mutant-fix loop from PR #30956. Drive the whole flow from a fresh branch and open a stacked PR.
|
||||
>
|
||||
> **Base branch**: `devp-stryker-mvp-spike` (the PR's branch — not master yet).
|
||||
>
|
||||
> **Steps**:
|
||||
>
|
||||
> 1. `git fetch origin && git checkout devp-stryker-mvp-spike && git pull && git checkout -b demo/strengthen-<file-basename>`
|
||||
>
|
||||
> 2. Query the live ledger to find the lowest-score red file:
|
||||
> ```bash
|
||||
> curl -sS 'https://internal.users.n8n.cloud/webhook/mutation-health-ledger?package=n8n-workflow' \
|
||||
> | jq '.ledger | map(select(.status == "red")) | sort_by(.last_score | tonumber) | .[0]'
|
||||
> ```
|
||||
> Use whatever it returns. As of 2026-05-22, that's `src/workflow-checksum.ts` at 38.64% — but check live state first.
|
||||
>
|
||||
> 3. Run the local mutation-testing skill on that file:
|
||||
> `/mutant-score packages/workflow/src/<picked-file>`
|
||||
>
|
||||
> Confirm the output JSON shows the score and a list of survivors with mutator + location + covering tests.
|
||||
>
|
||||
> 4. Run the strengthen skill:
|
||||
> `/mutant-fix`
|
||||
>
|
||||
> It'll triage survivors (HIGH/MODERATE/LOW), edit the covering test file with targeted assertions, and re-run `mutant-score` to verify the score climbed. Max 5 survivors per pass.
|
||||
>
|
||||
> 5. Review the diff yourself: `git diff packages/workflow/test/`
|
||||
>
|
||||
> Sanity-check each new assertion. Reject anything that's mocking-the-mock, asserting trivia, or pinning behaviour the source doesn't actually have. The skill is supposed to refuse to fabricate but humans verify.
|
||||
>
|
||||
> 6. If you want to push further, re-invoke `/mutant-fix` for the next 5 survivors. Or move on.
|
||||
>
|
||||
> 7. Final verification:
|
||||
> `/mutant-score packages/workflow/src/<picked-file>`
|
||||
>
|
||||
> Capture the before/after score for the PR body.
|
||||
>
|
||||
> 8. Push and open a stacked PR **against `devp-stryker-mvp-spike`** (not master):
|
||||
> ```bash
|
||||
> git push -u origin demo/strengthen-<file-basename>
|
||||
> gh pr create --draft --base devp-stryker-mvp-spike \
|
||||
> --title "test(core): strengthen <file-basename> assertions (demo) (no-changelog)" \
|
||||
> --body "<see template below>"
|
||||
> ```
|
||||
>
|
||||
> **PR body template:**
|
||||
>
|
||||
> ```markdown
|
||||
> ## Summary
|
||||
>
|
||||
> Demo PR for #30956. Drives the `mutant-fix` loop against `packages/workflow/src/<file>` to show the trial loop end-to-end.
|
||||
>
|
||||
> **Before**: <X>% mutation score, <N> survivors
|
||||
> **After**: <Y>% mutation score, <M> survivors
|
||||
>
|
||||
> Survivors addressed (with rationale):
|
||||
> 1. <mutator at location> — added <which assertion> to <which existing test>
|
||||
> 2. ...
|
||||
>
|
||||
> ## Test plan
|
||||
> - [ ] `pnpm mutate packages/workflow/src/<file>` reproduces the post-score locally
|
||||
> - [ ] `pnpm --filter=n8n-workflow test test/<file>.test.ts` passes
|
||||
> - [ ] Each new assertion has a clear "this would have caught X bug" justification
|
||||
> ```
|
||||
>
|
||||
> **Goals of the demo:**
|
||||
>
|
||||
> - Reviewer sees a real diff with surgical assertion edits, not big-bang test rewrites
|
||||
> - The before/after numbers are reproducible (`pnpm mutate` gives the same answer to anyone)
|
||||
> - The skill refused to fabricate or split low-leverage survivors out — what landed is what mattered
|
||||
>
|
||||
> **What NOT to do:**
|
||||
>
|
||||
> - Don't rewrite whole test files. The skill should only add/extend covering tests.
|
||||
> - Don't bypass the verify step. Every change must be backed by a re-run that shows the score moved.
|
||||
> - Don't auto-merge. This is a draft demo; the reviewer takes it forward.
|
||||
|
||||
---
|
||||
|
||||
## Why this is a good first PR for a reviewer
|
||||
|
||||
- Small (a handful of assertion lines)
|
||||
- Numerically verifiable (run `pnpm mutate` yourself, see the same number)
|
||||
- Demonstrates the full loop without committing to the AI auto-PR pipeline yet
|
||||
- Stacked on `devp-stryker-mvp-spike` so the loop's machinery is already on the branch
|
||||
@@ -1,6 +1,6 @@
|
||||
# `scripts/mutation-health/`
|
||||
|
||||
Phase 1 substrate for the Mutation Health Observability initiative.
|
||||
Patch-scoped mutation testing for n8n: prove the tests covering your change actually assert its behaviour.
|
||||
|
||||
## What is mutation testing?
|
||||
|
||||
@@ -49,201 +49,83 @@ That divergence is exactly why this project exists.
|
||||
|
||||
---
|
||||
|
||||
|
||||
## What's in this directory
|
||||
|
||||
| File | Purpose |
|
||||
| File | Role |
|
||||
| --- | --- |
|
||||
| `pick-next.mjs` | Walk `<pkg>/src/` (per-package mode) or every vitest-eligible package (global mode, `--global`), merge with the live ledger, return the next source file(s) to mutate |
|
||||
| `mutate.mjs` | Run Stryker on one source file of any vitest package, write `summary.json` |
|
||||
| `stryker.default.mjs` | Default Stryker config for onboarded packages (points at the package's own `vitest.config.*`) |
|
||||
| `emit-payload.mjs` | Turn a Stryker `summary.json` into a BQ-ready writer payload |
|
||||
| `ledger.mjs` | Read-all ledger access: `readLedger({ path, pkg? })` returns every row across every package in one pass, optionally narrowed to one package without re-reading |
|
||||
| `signals.mjs` | Per-file git-derived risk signals (churn + fix-density) used by the global picker's value formula |
|
||||
| `build-matrix.mjs` | Convert the global picker's top-N output (or an on-demand `SOURCE_FILE`) into the GHA `mutate` matrix; runs the die-loud `ELIGIBLE_PACKAGES` ↔ ledger guard |
|
||||
| `mutate.mjs` | The whole engine. Runs Stryker over a package and emits an actionable summary. Exposed as `pnpm mutate`. |
|
||||
| `mutate.test.mjs` | Unit tests for its pure helpers (`node --test 'scripts/mutation-health/*.test.mjs'`). |
|
||||
| `stryker.default.mjs` | Shared Stryker config for any vitest package. A package that needs special handling ships its own `stryker.config.mjs`, which `mutate.mjs` prefers. |
|
||||
|
||||
`mutate.mjs` is package-agnostic — run `pnpm mutate <repo-relative-file>` from the repo root and the package is inferred from the path (or pass `--package-dir <pkg>` for a package-relative target, as the nightly does). It uses the package's own `stryker.config.mjs` if one exists (e.g. `packages/workflow` carves out the isolated-vm engine), otherwise `stryker.default.mjs`.
|
||||
Outputs land in `<package>/reports/mutation/` (gitignored):
|
||||
|
||||
The reader and writer webhooks are plain HTTP — the GHA hits them with `curl`. There is no fetch/post wrapper script; if you want to call them locally, see [Local usage](#local-usage).
|
||||
- `raw.json` — the full Stryker Mutation Testing Elements report (600 KB+; don't read it directly).
|
||||
- `summary.json` — the compact actionable summary: every survivor's location, mutator, replacement, and covering tests. **This is the file to read.**
|
||||
|
||||
The BQ table schema lives with the writer workflow (in n8n's internal Quality project), not in this repo — the writer owns the MERGE statement and is the single source of truth.
|
||||
## Usage
|
||||
|
||||
## End-to-end pipeline
|
||||
The primary mode is `--diff`: mutate only the lines this branch changed.
|
||||
|
||||
```
|
||||
[GHA nightly cron, .github/workflows/mutation-health-nightly.yml]
|
||||
│
|
||||
├─► setup job (fetch-depth: 0, one process per night)
|
||||
│ │
|
||||
│ ├─► curl GET reader webhook → live-ledger.json (read-all BQ state)
|
||||
│ │ │
|
||||
│ │ └─► [n8n: QA Mutation Health Reader] ──► SELECT from BQ ledger
|
||||
│ │
|
||||
│ ├─► signals.mjs → signals.json (churn + fix-density)
|
||||
│ │
|
||||
│ ├─► build-matrix.mjs → matrix={ include: [top-N picks] }
|
||||
│ │ │
|
||||
│ │ ├─► die-loud guard: empty ledger throws (strict mode); each
|
||||
│ │ │ ELIGIBLE_PACKAGES.name must have ≥1 non-`new` row.
|
||||
│ │ │ Escape hatch: `bootstrap_packages` workflow_dispatch input.
|
||||
│ │ │
|
||||
│ │ └─► pick-next.mjs --global --top-n N
|
||||
│ │ walks every eligible package's src/, merges with ledger,
|
||||
│ │ ranks: w_churn·churn + w_fix·fixDensity + w_cov·(1−cov)
|
||||
│ │ priority: new → red → stale → skip green
|
||||
│ │
|
||||
│ └─► outputs.matrix = { include: [{name, dir, slug, mode, source_file, file_slug}, ...] }
|
||||
│
|
||||
└─► mutate job (one per matrix include)
|
||||
│
|
||||
├─► mutate.mjs --package-dir <pkg> <source_file> → summary.json
|
||||
│
|
||||
├─► emit-payload.mjs → bq-payload.json
|
||||
│
|
||||
└─► curl POST writer webhook → INSERTs event + MERGEs ledger row
|
||||
↓
|
||||
[n8n writer workflow: QA: Mutation Health Writer]
|
||||
↓
|
||||
┌───────────────────────────────────┐
|
||||
│ qa_mutation_health_ledger (MERGE) │
|
||||
│ qa_performance_metrics (INSERT) │
|
||||
└───────────────────────────────────┘
|
||||
```bash
|
||||
# Everything you changed vs origin/master — committed and uncommitted —
|
||||
# batched into one Stryker run per package.
|
||||
pnpm mutate --diff
|
||||
pnpm mutate --diff --base upstream/master
|
||||
|
||||
# One file, whole.
|
||||
pnpm mutate packages/@n8n/crdt/src/utils.ts
|
||||
|
||||
# One file, only lines 40-75.
|
||||
pnpm mutate packages/@n8n/crdt/src/utils.ts:40-75
|
||||
|
||||
# Package-relative target.
|
||||
pnpm mutate src/cron.ts --package-dir packages/workflow
|
||||
```
|
||||
|
||||
The writer workflow lives in n8n's internal Quality project. It's created and maintained outside this repo. This README documents the contract it implements.
|
||||
Exit codes: `0` gate passed · `1` gate failed (summary.json still written — this is the
|
||||
iterate signal) · `2` usage error · `3` Stryker could not run. A toolchain failure is
|
||||
**never** `1`, so a broken checkout can't be mistaken for a score of zero.
|
||||
|
||||
## Passes, packages & onboarding
|
||||
### Why `--diff` is fast
|
||||
|
||||
The nightly runs a **dynamic matrix of top-N picks** (built once in the `setup` job of `mutation-health-nightly.yml`). The setup job fetches the read-all live ledger, gathers git-derived signals, and calls `pick-next.mjs --global --top-n N` once across every eligible package. Each picked row becomes one `mutate` job; jobs run independently, the ledger is keyed by package + file, so they don't collide. Two passes, selectable via the `mode` dispatch input (`both` on schedule):
|
||||
Two things do the work:
|
||||
|
||||
- **baseline** — files with no result yet (the `new` bucket). Builds out coverage. Maps from `effective_status: new` on a picked row.
|
||||
- **coverage** — revisits the weakest scored files (`red`/`stale`, lowest first). Strengthens existing tests. Maps from `effective_status: red | stale` on a picked row.
|
||||
1. **Patch scoping.** Stryker's mutation-range syntax (`file.ts:13-16`) means only the mutants
|
||||
inside your changed lines are generated. You're scored on the lines you touched, not on
|
||||
inherited debt.
|
||||
2. **One dry run per package.** Targets are comma-joined into a single `--mutate` argument.
|
||||
Repeated `--mutate` flags silently *overwrite* each other in Stryker's CLI, so comma-joining
|
||||
is the only way to batch — and it means a package pays for its dry run once, not once per file.
|
||||
|
||||
To onboard a **vitest** package: add one `{ name, dir }` entry to `ELIGIBLE_PACKAGES` in `pick-next.mjs`. The nightly setup job derives its matrix from that single source of truth, so the picker and the workflow can't drift. No per-package config needed — `stryker.default.mjs` auto-resolves the package's own `vitest.config.*` (verified on plain and DI-decorator packages). Add a local `stryker.config.mjs` only if the package needs special handling.
|
||||
On top of that, Stryker's vitest runner only loads the tests *related* to the mutated files, so
|
||||
cost tracks the related suite rather than package size. Measured end-to-end, whole-file:
|
||||
`@n8n/decorators` 1s · `@n8n/scheduler` 3s · `packages/workflow` 13s · `nodes-base` 26s ·
|
||||
`packages/cli` 88s. Line-scoping cuts these further.
|
||||
|
||||
Then, **once**, trigger the nightly with `workflow_dispatch` and set `bootstrap_packages` to the new package name (e.g. `@n8n/decorators`). The divergence guard otherwise refuses to schedule for a package that has zero non-`new` ledger rows — that's how it catches an `ELIGIBLE_PACKAGES.name` ↔ `ledger.package` rename mismatch, but it can't tell that from a legitimate new onboarding. The one-off `bootstrap_packages` value skips the guard for that package on that run only; subsequent scheduled nightlies revert to strict mode once the new package's rows have been populated.
|
||||
### In-place mutation
|
||||
|
||||
For a **genuine cold-start** (first run after the ledger is provisioned, the read-all returns `[]`), trigger with `bootstrap_packages: '*'` instead — it acknowledges the empty ledger and skips the per-package divergence check for every entry.
|
||||
Runs use Stryker's `--inPlace`. Its default sandbox copy breaks on any package whose vitest
|
||||
config resolves a workspace dependency through a path alias — the alias doesn't survive the
|
||||
copy, and `packages/cli` dies on `ERR_LOAD_URL … .stryker-tmp/@n8n/backend-test-utils`.
|
||||
|
||||
Not yet covered: `@n8n/expression-runtime` (it _is_ the isolated-vm engine; blocked on the patch in DEVP-257).
|
||||
Stryker restores your files on a clean exit and on `Ctrl-C`. Because a hard crash would not,
|
||||
and because in `--diff` mode those files hold *uncommitted work* (so `git checkout --` is not a
|
||||
safe undo), `mutate.mjs` snapshots the exact bytes of every target before the run and writes
|
||||
them back if they differ afterwards.
|
||||
|
||||
## State transitions
|
||||
## Which packages can be scored
|
||||
|
||||
| Trigger | Stored `status` |
|
||||
| --- | --- |
|
||||
| Source file in `src/` but no row yet | synthesised as `new` at pick time; not stored |
|
||||
| Last run passed the gate (score ≥ `threshold_at_run` AND no unjustified survivors) | `green` |
|
||||
| Last run failed the gate (score < `threshold_at_run` OR ≥1 `Survived`/`NoCoverage` mutant) | `red` |
|
||||
Any package whose `test` script runs **vitest** — which, since the Jest migration, is nearly all
|
||||
of them, including `nodes-base`, `nodes-langchain`, `cli` and `db`. `--diff` derives eligibility
|
||||
per file and prints a one-line reason for anything it skips; there is no curated list to
|
||||
maintain.
|
||||
|
||||
Stored statuses are just two: `red` and `green`. `new` is computed in-memory by the picker for any file in the source tree that has no ledger row yet — the row is only persisted after that file's first scored run. The picker also computes a transient `stale` state — any `green` row whose `last_checked_at` is older than 4 weeks is treated as `stale` for that pick. No `last_checked_sha` is needed; no git history is consulted.
|
||||
Not scored:
|
||||
|
||||
Picker priority: `new` → `red` → `stale` → skip fresh `green`.
|
||||
|
||||
- Within `new`: alphabetical (rows exit the bucket as they're scored)
|
||||
- Within `red`: lowest score first (weakest tests revisited first)
|
||||
- Within `stale`: oldest `last_checked_at` first (natural cycling of long-stable files)
|
||||
|
||||
If every row is green and fresh, the picker exits 0 with `{"picked": null, "reason": "all-green"}` — a healthy "nothing to do" state, not a failure.
|
||||
|
||||
## Webhook contracts
|
||||
|
||||
Two n8n workflows back the pipeline. Both live in the internal Quality project (`L8csxtEbFpFOWlf8`) and are created/maintained outside this repo. Both run unauthenticated (URL-as-secret pattern, matching existing `qa_*` workers).
|
||||
|
||||
| Endpoint | Method | Workflow | Purpose |
|
||||
| --- | --- | --- | --- |
|
||||
| `https://internal.users.n8n.cloud/webhook/mutation-health-writer` | POST | `QA: Mutation Health Writer` (`iYEBmBat8OscRTVq`) | INSERT events + MERGE ledger |
|
||||
| `https://internal.users.n8n.cloud/webhook/mutation-health-ledger?package=<name>` | GET | `QA: Mutation Health Reader` (`ZmRsNUwvgfCSq0JI`) | Read current ledger state |
|
||||
|
||||
### Writer webhook
|
||||
|
||||
`POST https://internal.users.n8n.cloud/webhook/mutation-health-writer` with `Content-Type: application/json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"ledger": [
|
||||
{
|
||||
"source_file_path": "packages/workflow/src/cron.ts",
|
||||
"package": "n8n-workflow",
|
||||
"last_score": 95.12,
|
||||
"coverage": 0.93,
|
||||
"churn": 7,
|
||||
"fix_density": 2.41,
|
||||
"threshold_at_run": 80,
|
||||
"last_checked_at": "2026-05-22T10:03:55.660Z",
|
||||
"status": "green",
|
||||
"mutants_killed": 39,
|
||||
"mutants_survived": 2,
|
||||
"mutants_no_coverage": 0,
|
||||
"mutants_timeout": 0
|
||||
}
|
||||
],
|
||||
"events": [
|
||||
{
|
||||
"benchmark_name": "mutation_health",
|
||||
"value": 95.12,
|
||||
"timestamp": "2026-05-22T10:03:55.660Z",
|
||||
"dimensions": {
|
||||
"package": "n8n-workflow",
|
||||
"source_file": "packages/workflow/src/cron.ts",
|
||||
"sha": "095239e175",
|
||||
"status_after": "green",
|
||||
"threshold": 80,
|
||||
"coverage": 0.93,
|
||||
"churn": 7,
|
||||
"fix_density": 2.41,
|
||||
"mutants_killed": 39,
|
||||
"mutants_survived": 2,
|
||||
"mutants_no_coverage": 0,
|
||||
"mutants_timeout": 0
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
Either array may be empty (manual smoke tests sometimes send only `events`).
|
||||
|
||||
Each ledger row also carries `coverage` — the scored file's line-coverage proxy in `[0,1]`, the share of mutants a test actually exercised — written back by `mutate.mjs` after each run. The global picker reads it next cycle as the `(1 − coverage)` term of its value formula, so a file no test touches (`coverage` 0) gets the strongest urge to be scored.
|
||||
|
||||
Rows also carry `churn` — the count of commits touching the source file within a recent window (default `90 days`, override with `--churn-window`), derived from git in `emit-payload.mjs`. It feeds the picker's `churn` term so hot files outrank cold ones. On a shallow clone history is truncated and the count would undercount, so `churn` is emitted as `null` rather than a misleading low number.
|
||||
|
||||
The third value-formula term, `fix_density`, is emitted the same way — the file's time-decayed, delta-weighted fix-density (the same git-derived signal `signals.mjs` computes: lines changed by `fix:`-shaped commits, decayed by a half-life on commit age). `emit-payload.mjs` reads one per-package `git log --numstat` pass (bounded by `--fix-density-window`, default `1 year`; half-life via `--fix-density-half-life`, default `90` days) and scores each file. A file with no fix commits scores `0` (known: no fixes); a shallow clone or git failure emits `null` (unknown), the same guard as `churn`. A richer variant joining against the bug taxonomy is a possible future enhancement on the writer side, but is not required to feed the picker.
|
||||
|
||||
The writer:
|
||||
|
||||
1. For each `events[]` row → `INSERT` into `qa_performance_metrics`.
|
||||
2. For each `ledger[]` row → `MERGE` into `qa_mutation_health_ledger` on `source_file_path`. Status is always `red` or `green` — the picker synthesises `new` in-memory and never posts it.
|
||||
|
||||
The webhook URL is delivered to GHA via the `MUTATION_HEALTH_WEBHOOK` repo secret. The secret URL itself is the only auth (matches existing `qa_*` writer pattern); rotate the secret if leaked.
|
||||
|
||||
### Reader webhook
|
||||
|
||||
`GET https://internal.users.n8n.cloud/webhook/mutation-health-ledger?package=<pkg>`:
|
||||
|
||||
```json
|
||||
{
|
||||
"ledger": [
|
||||
{
|
||||
"source_file_path": "packages/workflow/src/cron.ts",
|
||||
"package": "n8n-workflow",
|
||||
"last_score": 95.12,
|
||||
"coverage": 0.93,
|
||||
"churn": 7,
|
||||
"fix_density": 2.41,
|
||||
"threshold_at_run": 80,
|
||||
"last_checked_at": "2026-05-22T10:03:55.660Z",
|
||||
"status": "green",
|
||||
"mutants_killed": 39,
|
||||
"mutants_survived": 2,
|
||||
"mutants_no_coverage": 0,
|
||||
"mutants_timeout": 0
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
The `package` query param is validated server-side against the same pnpm-workspace allowlist regex used elsewhere in the pipeline; invalid input returns 500. No SQL is constructed or accepted on the client side — the SELECT is hardcoded in the workflow.
|
||||
|
||||
Unauthenticated — the URL is not a secret. The data isn't sensitive (file paths + integer scores), but treat the URL as low-trust: anyone with it can read all current ledger state for the queried package.
|
||||
- `@n8n/expression-runtime` — Stryker's dry run SIGABRTs on the isolated-vm engine ([DEVP-257](https://linear.app/n8n/issue/DEVP-257)).
|
||||
- `.vue` single-file components — every SFC package crashed Stryker's mutate step in the 2026-06 sweep, and the component layer is low-value to mutate.
|
||||
- Tests, declarations, stories, configs, migrations and build output.
|
||||
|
||||
## Gate semantics
|
||||
|
||||
@@ -256,49 +138,12 @@ Stryker excludes `Ignored` mutants from both numerator and denominator of the sc
|
||||
|
||||
`summary.json` surfaces every `Ignored` mutant alongside its disable-comment reason so reviewers can spot-check the justifications — those become the high-signal review artifact rather than N padding tests.
|
||||
|
||||
A **partial** run never passes: if Stryker exits non-zero but left a salvageable `raw.json`,
|
||||
the summary is kept (survivors found so far are still useful) and flagged `partial`, because
|
||||
mutants it never got to could be survivors.
|
||||
|
||||
### Threshold (provisional)
|
||||
|
||||
Runs use `STRYKER_THRESHOLD=80` as a placeholder. The threshold moves to evidence-based after ~4 weeks of accumulated data. Until then, treat `red`/`green` verdicts as preliminary.
|
||||
|
||||
## Local usage
|
||||
|
||||
```bash
|
||||
# Run Stryker on one file (the inner loop — also invokable via /mutant-score skill).
|
||||
# Package is inferred from the repo-relative path; works for any vitest package.
|
||||
pnpm mutate packages/workflow/src/cron.ts
|
||||
pnpm mutate packages/@n8n/crdt/src/utils.ts
|
||||
|
||||
# Pull current ledger from BQ
|
||||
curl --fail -sS \
|
||||
'https://internal.users.n8n.cloud/webhook/mutation-health-ledger?package=n8n-workflow' \
|
||||
-o /tmp/ledger.json
|
||||
|
||||
# Pick the next file to score (per-package mode)
|
||||
node scripts/mutation-health/pick-next.mjs \
|
||||
--package-dir packages/workflow \
|
||||
--ledger-file /tmp/ledger.json
|
||||
|
||||
# Pick top-N files across every vitest-eligible package, ranked by the
|
||||
# global value formula w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage).
|
||||
# Signals and coverage files are optional — missing terms contribute 0
|
||||
# except `(1 − coverage)`, which degrades to 1 (worst-case) so untracked
|
||||
# files float to the top.
|
||||
node scripts/mutation-health/pick-next.mjs \
|
||||
--global \
|
||||
--ledger-file /tmp/ledger.json \
|
||||
--signals-file /tmp/signals.json \
|
||||
--coverage-file /tmp/coverage.json \
|
||||
--top-n 5 \
|
||||
--block '@n8n/expression-runtime'
|
||||
|
||||
# Build a BQ payload from a Stryker run
|
||||
node scripts/mutation-health/emit-payload.mjs \
|
||||
--summary packages/workflow/reports/mutation/summary.json \
|
||||
--package n8n-workflow
|
||||
|
||||
# POST the result (requires MUTATION_HEALTH_WEBHOOK to be set)
|
||||
curl --fail -sS -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
--data @packages/workflow/reports/mutation/bq-payload.json \
|
||||
"$MUTATION_HEALTH_WEBHOOK"
|
||||
```
|
||||
Runs use `STRYKER_THRESHOLD=80` as a placeholder. Scoped to a patch the number is coarse — a
|
||||
handful of mutants makes for a jumpy percentage — so the load-bearing half of the gate is
|
||||
"no unjustified survivors", not the score.
|
||||
|
||||
@@ -1,314 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build the nightly mutation-health (package × pass) matrix from one global
|
||||
* picker invocation, instead of redeclaring the package list in the workflow.
|
||||
*
|
||||
* ELIGIBLE_PACKAGES is sourced from `pick-next.mjs` (the picker's single
|
||||
* source of truth — DEVP-497), so the nightly setup job never drifts from
|
||||
* what the picker walks.
|
||||
*
|
||||
* Inputs (env):
|
||||
* LEDGER_FILE Required (unless SOURCE_FILE is set). Read-all live ledger JSON.
|
||||
* REQUESTED_MODE Optional. 'both' | 'baseline' | 'coverage'. Default 'both'.
|
||||
* TOP_N Optional. Positive integer. Default 6.
|
||||
* SOURCE_FILE Optional. Skip picker, emit a single matrix entry for a
|
||||
* repo-relative file (used by the on-demand re-score path).
|
||||
* SIGNALS_FILE Optional. Passed to picker as --signals-file.
|
||||
* COVERAGE_FILE Optional. Explicit coverage map passed to picker as
|
||||
* --coverage-file. When unset, a sparse map is derived
|
||||
* from the read-all ledger rows (those carrying a
|
||||
* `coverage` value) and written to a temp file.
|
||||
* BOOTSTRAP_PACKAGES Optional. Comma-separated ELIGIBLE_PACKAGES.name list to
|
||||
* exempt from the divergence guard (use after onboarding
|
||||
* a new package — its ledger rows don't exist yet). Use
|
||||
* '*' to acknowledge a genuine cold-start (also allows an
|
||||
* empty ledger). Empty (default) = strict mode: empty
|
||||
* ledger AND any per-package divergence both throw.
|
||||
*
|
||||
* Output (stdout): exactly one line, `matrix=<json>`, suitable for appending
|
||||
* to $GITHUB_OUTPUT. `<json>` matches GHA matrix shape: { include: [...] }.
|
||||
* The picker's global mode returns an array; this script branches on that
|
||||
* shape so a future per-package fallback in the picker stays compatible.
|
||||
*
|
||||
* Each matrix row exposes:
|
||||
* name workspace package name (e.g. n8n-workflow)
|
||||
* dir repo-relative package dir (e.g. packages/workflow)
|
||||
* slug short package slug (e.g. workflow)
|
||||
* mode baseline | coverage | file (preserves the original
|
||||
* concurrency-group key shape)
|
||||
* source_file repo-relative .ts to mutate
|
||||
* file_slug artefact-safe slug derived from `source_file` — keeps the
|
||||
* upload-artifact name unique when multiple top-N picks share
|
||||
* the same (mode, slug) pair
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 — wrote a matrix line (include[] may be empty when picker has no work)
|
||||
* 2 — usage / config error, ledger divergence, or picker spawn failure
|
||||
*/
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
import { existsSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import os from 'node:os';
|
||||
import path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
import { parseLedgerBody } from './ledger.mjs';
|
||||
import { ELIGIBLE_PACKAGES } from './pick-next.mjs';
|
||||
|
||||
function die(code, msg) {
|
||||
process.stderr.write(`${msg}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
export function slugForPackage(pkg) {
|
||||
return pkg.dir.split('/').pop();
|
||||
}
|
||||
|
||||
export function fileSlug(repoRelPath) {
|
||||
return repoRelPath
|
||||
.replace(/[^a-zA-Z0-9]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
}
|
||||
|
||||
export function modeForEffectiveStatus(effectiveStatus) {
|
||||
if (effectiveStatus === 'new') return 'baseline';
|
||||
if (effectiveStatus === 'red' || effectiveStatus === 'stale') return 'coverage';
|
||||
throw new Error(`Cannot map effective_status="${effectiveStatus}" to a matrix mode`);
|
||||
}
|
||||
|
||||
export function findPackageForSourceFile(sourceFile, eligible = ELIGIBLE_PACKAGES) {
|
||||
return eligible.find((p) => sourceFile.startsWith(`${p.dir}/`));
|
||||
}
|
||||
|
||||
/**
|
||||
* Die-loud guard against `ELIGIBLE_PACKAGES.name` ↔ ledger `.package`
|
||||
* divergence and against silently re-baselining the whole tree on a
|
||||
* transient/degraded read.
|
||||
*
|
||||
* Strict mode (`allowEmptyLedger: false`, empty `skipPackages`):
|
||||
* - Empty ledger throws — a `200` from the reader with `[]` (BQ hiccup,
|
||||
* degraded fetch) would otherwise look identical to "every file is new"
|
||||
* and re-baseline the whole eligible tree in one night.
|
||||
* - For each eligible package, fewer than one non-`new` row throws — the
|
||||
* stored ledger only ever holds `red`/`green` rows (`new` is synthesised
|
||||
* by the picker at pick time and never written), so "zero prior-status
|
||||
* rows for an eligible package in a non-empty ledger" is the signature
|
||||
* of an ELIGIBLE_PACKAGES.name ↔ ledger.package mismatch.
|
||||
*
|
||||
* Escape hatch (operator-driven via the workflow_dispatch
|
||||
* `bootstrap_packages` input):
|
||||
* - `allowEmptyLedger: true` is the genuine cold-start acknowledgement.
|
||||
* - Packages in `skipPackages` are exempt from the divergence check —
|
||||
* used after onboarding a new entry to ELIGIBLE_PACKAGES (its rows
|
||||
* don't exist yet) so the first nightly can populate them. Subsequent
|
||||
* scheduled nightlies revert to strict mode.
|
||||
*/
|
||||
export function assertNoLedgerDivergence(
|
||||
ledgerRows,
|
||||
eligible = ELIGIBLE_PACKAGES,
|
||||
{ allowEmptyLedger = false, skipPackages = [] } = {},
|
||||
) {
|
||||
if (!Array.isArray(ledgerRows) || ledgerRows.length === 0) {
|
||||
if (allowEmptyLedger) return;
|
||||
throw new Error(
|
||||
'Ledger divergence: read-all ledger is empty. Either the reader webhook ' +
|
||||
'returned [] for an unexpected reason (BQ hiccup / degraded fetch) and the ' +
|
||||
'whole eligible tree would be silently re-baselined, or this is a genuine ' +
|
||||
"cold start. Re-run with workflow_dispatch input `bootstrap_packages: '*'` " +
|
||||
'to acknowledge the cold start; otherwise investigate the reader webhook.',
|
||||
);
|
||||
}
|
||||
const skip = new Set(skipPackages);
|
||||
for (const pkg of eligible) {
|
||||
if (skip.has(pkg.name)) continue;
|
||||
const priorHits = ledgerRows.filter(
|
||||
(r) => r && r.package === pkg.name && r.status && r.status !== 'new',
|
||||
);
|
||||
if (priorHits.length === 0) {
|
||||
throw new Error(
|
||||
`Ledger divergence: read-all ledger has ${ledgerRows.length} row(s) but zero ` +
|
||||
`prior-status (non-"new") rows for eligible package "${pkg.name}". ` +
|
||||
'If this is a newly-onboarded package, re-run with workflow_dispatch input ' +
|
||||
`\`bootstrap_packages: "${pkg.name}"\` to skip the guard once. Otherwise, ` +
|
||||
'suspected ELIGIBLE_PACKAGES.name ↔ ledger.package mismatch — refusing to ' +
|
||||
'silently re-baseline the package. Verify the package name and ledger contents.',
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function buildMatrixRowForFile(sourceFile, eligible = ELIGIBLE_PACKAGES) {
|
||||
const pkg = findPackageForSourceFile(sourceFile, eligible);
|
||||
if (!pkg) throw new Error(`No mutation-tracked package owns ${sourceFile}`);
|
||||
return {
|
||||
name: pkg.name,
|
||||
dir: pkg.dir,
|
||||
slug: slugForPackage(pkg),
|
||||
mode: 'file',
|
||||
source_file: sourceFile,
|
||||
file_slug: fileSlug(sourceFile),
|
||||
};
|
||||
}
|
||||
|
||||
export function buildMatrixFromPicked(pickedRows, eligible = ELIGIBLE_PACKAGES) {
|
||||
if (!Array.isArray(pickedRows)) {
|
||||
throw new Error('Picker output is not an array — expected global-mode shape `{ picked: [...] }`');
|
||||
}
|
||||
return {
|
||||
include: pickedRows.map((row) => {
|
||||
const pkg = eligible.find((p) => p.name === row.package);
|
||||
if (!pkg) {
|
||||
throw new Error(
|
||||
`Picker returned package "${row.package}" which is not in ELIGIBLE_PACKAGES — refusing to schedule a mutate job for an unknown package`,
|
||||
);
|
||||
}
|
||||
return {
|
||||
name: pkg.name,
|
||||
dir: pkg.dir,
|
||||
slug: slugForPackage(pkg),
|
||||
mode: modeForEffectiveStatus(row.effective_status),
|
||||
source_file: row.source_file_path,
|
||||
file_slug: fileSlug(row.source_file_path),
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Derive the picker's coverage map `{ <repo-rel source_file_path>: 0..1 }` from
|
||||
* the read-all ledger rows. Only rows that carry a finite numeric `coverage`
|
||||
* contribute — rows the writer left without a coverage value (never re-scored
|
||||
* since the column landed) are omitted on purpose.
|
||||
*
|
||||
* A sparse map is correct and safe: `pick-next.mjs` treats an absent path as
|
||||
* coverage 0, which makes the `(1 − coverage)` value term 1 (worst case =
|
||||
* highest urge). The map fills in over nightly cycles as files are re-scored.
|
||||
*/
|
||||
export function buildCoverageMap(ledgerRows) {
|
||||
const map = {};
|
||||
if (!Array.isArray(ledgerRows)) return map;
|
||||
for (const row of ledgerRows) {
|
||||
if (!row || typeof row.source_file_path !== 'string') continue;
|
||||
if (typeof row.coverage === 'number' && Number.isFinite(row.coverage)) {
|
||||
map[row.source_file_path] = row.coverage;
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function writeCoverageMapToTemp(coverageMap) {
|
||||
const dir = mkdtempSync(path.join(os.tmpdir(), 'mutation-coverage-'));
|
||||
const file = path.join(dir, 'coverage.json');
|
||||
writeFileSync(file, JSON.stringify(coverageMap));
|
||||
return file;
|
||||
}
|
||||
|
||||
function readLedgerRows(ledgerFile) {
|
||||
if (!ledgerFile) die(2, 'Missing required LEDGER_FILE env var');
|
||||
if (!existsSync(ledgerFile)) die(2, `Ledger file not found: ${ledgerFile}`);
|
||||
try {
|
||||
return parseLedgerBody(readFileSync(ledgerFile, 'utf8')).rows;
|
||||
} catch (err) {
|
||||
die(2, `Failed to read ledger at ${ledgerFile}: ${err.message}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function runPicker({ ledgerFile, requestedMode, topN, signalsFile, coverageFile }) {
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const pickerPath = path.join(here, 'pick-next.mjs');
|
||||
const args = [pickerPath, '--global', '--top-n', String(topN), '--ledger-file', ledgerFile];
|
||||
if (requestedMode === 'baseline' || requestedMode === 'coverage') {
|
||||
args.push('--mode', requestedMode);
|
||||
}
|
||||
if (signalsFile) args.push('--signals-file', signalsFile);
|
||||
if (coverageFile) args.push('--coverage-file', coverageFile);
|
||||
|
||||
const res = spawnSync(process.execPath, args, { encoding: 'utf8' });
|
||||
if (res.error) die(2, `Failed to spawn picker: ${res.error.message}`);
|
||||
if (res.stderr) process.stderr.write(res.stderr);
|
||||
if (res.status !== 0) die(res.status ?? 2, `Picker exited with status ${res.status}`);
|
||||
try {
|
||||
return JSON.parse(res.stdout);
|
||||
} catch (err) {
|
||||
die(2, `Failed to parse picker output as JSON: ${err.message}\nstdout: ${res.stdout}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const isCli = import.meta.url === `file://${process.argv[1]}`;
|
||||
if (isCli) {
|
||||
const sourceFile = (process.env.SOURCE_FILE ?? '').trim();
|
||||
const requestedMode = (process.env.REQUESTED_MODE ?? 'both').trim() || 'both';
|
||||
const topNRaw = (process.env.TOP_N ?? '6').trim() || '6';
|
||||
const ledgerFile = (process.env.LEDGER_FILE ?? '').trim();
|
||||
const signalsFile = (process.env.SIGNALS_FILE ?? '').trim();
|
||||
const coverageFile = (process.env.COVERAGE_FILE ?? '').trim();
|
||||
|
||||
if (sourceFile) {
|
||||
try {
|
||||
const row = buildMatrixRowForFile(sourceFile);
|
||||
process.stdout.write(`matrix=${JSON.stringify({ include: [row] })}\n`);
|
||||
process.exit(0);
|
||||
} catch (err) {
|
||||
die(2, err.message);
|
||||
}
|
||||
}
|
||||
|
||||
if (!['both', 'baseline', 'coverage'].includes(requestedMode)) {
|
||||
die(2, `Invalid REQUESTED_MODE="${requestedMode}". Use 'both', 'baseline', or 'coverage'.`);
|
||||
}
|
||||
|
||||
const topN = Number(topNRaw);
|
||||
if (!Number.isInteger(topN) || topN <= 0) {
|
||||
die(2, `Invalid TOP_N="${topNRaw}" (expected positive integer).`);
|
||||
}
|
||||
|
||||
const bootstrapRaw = (process.env.BOOTSTRAP_PACKAGES ?? '').trim();
|
||||
const bootstrapWildcard = bootstrapRaw === '*';
|
||||
const skipPackages = bootstrapWildcard
|
||||
? ELIGIBLE_PACKAGES.map((p) => p.name)
|
||||
: bootstrapRaw
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
const allowEmptyLedger = bootstrapWildcard;
|
||||
|
||||
const ledgerRows = readLedgerRows(ledgerFile);
|
||||
try {
|
||||
assertNoLedgerDivergence(ledgerRows, ELIGIBLE_PACKAGES, {
|
||||
allowEmptyLedger,
|
||||
skipPackages,
|
||||
});
|
||||
} catch (err) {
|
||||
die(2, err.message);
|
||||
}
|
||||
|
||||
// Feed ledger coverage into the global picker so the `(1 − coverage)` value
|
||||
// term goes live. An explicit COVERAGE_FILE env wins (operator override);
|
||||
// otherwise derive a sparse map from the read-all ledger rows we already
|
||||
// hold and hand it to the picker via a temp file.
|
||||
let effectiveCoverageFile = coverageFile;
|
||||
if (!effectiveCoverageFile) {
|
||||
const coverageMap = buildCoverageMap(ledgerRows);
|
||||
if (Object.keys(coverageMap).length > 0) {
|
||||
effectiveCoverageFile = writeCoverageMapToTemp(coverageMap);
|
||||
}
|
||||
}
|
||||
|
||||
const pickerOutput = runPicker({
|
||||
ledgerFile,
|
||||
requestedMode,
|
||||
topN,
|
||||
signalsFile,
|
||||
coverageFile: effectiveCoverageFile,
|
||||
});
|
||||
|
||||
const pickedRows = Array.isArray(pickerOutput?.picked) ? pickerOutput.picked : [];
|
||||
try {
|
||||
const matrix = buildMatrixFromPicked(pickedRows);
|
||||
process.stdout.write(`matrix=${JSON.stringify(matrix)}\n`);
|
||||
} catch (err) {
|
||||
die(2, err.message);
|
||||
}
|
||||
}
|
||||
@@ -1,343 +0,0 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
assertNoLedgerDivergence,
|
||||
buildCoverageMap,
|
||||
buildMatrixFromPicked,
|
||||
buildMatrixRowForFile,
|
||||
fileSlug,
|
||||
findPackageForSourceFile,
|
||||
modeForEffectiveStatus,
|
||||
slugForPackage,
|
||||
} from './build-matrix.mjs';
|
||||
import { ELIGIBLE_PACKAGES } from './pick-next.mjs';
|
||||
|
||||
const ELIGIBLE = ELIGIBLE_PACKAGES;
|
||||
|
||||
describe('slugForPackage', () => {
|
||||
it('returns the trailing path segment of `dir`', () => {
|
||||
assert.equal(slugForPackage({ name: 'n8n-workflow', dir: 'packages/workflow' }), 'workflow');
|
||||
assert.equal(slugForPackage({ name: '@n8n/crdt', dir: 'packages/@n8n/crdt' }), 'crdt');
|
||||
assert.equal(
|
||||
slugForPackage({ name: '@n8n/decorators', dir: 'packages/@n8n/decorators' }),
|
||||
'decorators',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('fileSlug', () => {
|
||||
it('sanitises a repo-relative path into an artefact-safe slug', () => {
|
||||
assert.equal(
|
||||
fileSlug('packages/workflow/src/cron.ts'),
|
||||
'packages-workflow-src-cron-ts',
|
||||
);
|
||||
assert.equal(
|
||||
fileSlug('packages/@n8n/crdt/src/utils.ts'),
|
||||
'packages-n8n-crdt-src-utils-ts',
|
||||
);
|
||||
});
|
||||
|
||||
it('collapses runs of separators and trims edges', () => {
|
||||
assert.equal(fileSlug('a//b///c.ts'), 'a-b-c-ts');
|
||||
assert.equal(fileSlug('.hidden/file.ts'), 'hidden-file-ts');
|
||||
});
|
||||
});
|
||||
|
||||
describe('modeForEffectiveStatus', () => {
|
||||
it('maps new → baseline and red/stale → coverage', () => {
|
||||
assert.equal(modeForEffectiveStatus('new'), 'baseline');
|
||||
assert.equal(modeForEffectiveStatus('red'), 'coverage');
|
||||
assert.equal(modeForEffectiveStatus('stale'), 'coverage');
|
||||
});
|
||||
|
||||
it('throws on green or unknown — green never enters the candidate set', () => {
|
||||
assert.throws(() => modeForEffectiveStatus('green'), /Cannot map/);
|
||||
assert.throws(() => modeForEffectiveStatus(undefined), /Cannot map/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('findPackageForSourceFile', () => {
|
||||
it('matches by exact dir prefix with a trailing separator', () => {
|
||||
const pkg = findPackageForSourceFile('packages/workflow/src/cron.ts', ELIGIBLE);
|
||||
assert.ok(pkg);
|
||||
assert.equal(pkg.name, 'n8n-workflow');
|
||||
});
|
||||
|
||||
it('returns undefined for unowned paths', () => {
|
||||
assert.equal(findPackageForSourceFile('README.md', ELIGIBLE), undefined);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMatrixRowForFile (on-demand re-score path)', () => {
|
||||
it('emits a single matrix row with mode=file for an owned file', () => {
|
||||
const row = buildMatrixRowForFile('packages/@n8n/crdt/src/utils.ts', ELIGIBLE);
|
||||
assert.deepEqual(row, {
|
||||
name: '@n8n/crdt',
|
||||
dir: 'packages/@n8n/crdt',
|
||||
slug: 'crdt',
|
||||
mode: 'file',
|
||||
source_file: 'packages/@n8n/crdt/src/utils.ts',
|
||||
file_slug: 'packages-n8n-crdt-src-utils-ts',
|
||||
});
|
||||
});
|
||||
|
||||
it('throws for files outside ELIGIBLE_PACKAGES', () => {
|
||||
assert.throws(
|
||||
() => buildMatrixRowForFile('packages/nodes-base/utils/binary.ts', ELIGIBLE),
|
||||
/No mutation-tracked package owns/,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('assertNoLedgerDivergence (die-loud guard)', () => {
|
||||
it('throws loudly on empty ledger in strict mode (transient/degraded read masquerades as cold-start)', () => {
|
||||
// Default mode: an empty ledger could be a genuine cold-start OR a BQ
|
||||
// hiccup that re-baselines the whole eligible tree silently. We can't
|
||||
// tell which, so we fail loud and require an explicit operator gesture.
|
||||
assert.throws(
|
||||
() => assertNoLedgerDivergence([], ELIGIBLE),
|
||||
/read-all ledger is empty/,
|
||||
);
|
||||
});
|
||||
|
||||
it('allows empty ledger when allowEmptyLedger is set (genuine cold-start acknowledgement)', () => {
|
||||
// bootstrap_packages: '*' on workflow_dispatch sets allowEmptyLedger:true.
|
||||
assert.doesNotThrow(() =>
|
||||
assertNoLedgerDivergence([], ELIGIBLE, { allowEmptyLedger: true }),
|
||||
);
|
||||
});
|
||||
|
||||
it('passes when every eligible package has at least one non-`new` row', () => {
|
||||
const rows = ELIGIBLE.map((pkg, i) => ({
|
||||
source_file_path: `${pkg.dir}/src/seed-${i}.ts`,
|
||||
package: pkg.name,
|
||||
status: 'green',
|
||||
}));
|
||||
assert.doesNotThrow(() => assertNoLedgerDivergence(rows, ELIGIBLE));
|
||||
});
|
||||
|
||||
it('fails loudly when an eligible package has zero non-`new` rows in a non-empty ledger', () => {
|
||||
// First package has a real row; second has zero. Mimics ELIGIBLE_PACKAGES
|
||||
// vs ledger.package divergence (e.g. a rename like n8n-workflow vs workflow).
|
||||
const rows = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/cron.ts',
|
||||
package: 'n8n-workflow',
|
||||
status: 'green',
|
||||
},
|
||||
];
|
||||
assert.throws(
|
||||
() => assertNoLedgerDivergence(rows, ELIGIBLE),
|
||||
/zero prior-status \(non-"new"\) rows for eligible package/,
|
||||
);
|
||||
});
|
||||
|
||||
it('error message hints at the bootstrap_packages escape hatch and names the missing package', () => {
|
||||
// The error must be actionable: an operator needs to know which input
|
||||
// to set and which package to put in it.
|
||||
const fakeEligible = [
|
||||
{ name: 'n8n-workflow', dir: 'packages/workflow' },
|
||||
{ name: '@n8n/newly-onboarded', dir: 'packages/@n8n/newly-onboarded' },
|
||||
];
|
||||
const rows = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/cron.ts',
|
||||
package: 'n8n-workflow',
|
||||
status: 'green',
|
||||
},
|
||||
];
|
||||
assert.throws(() => assertNoLedgerDivergence(rows, fakeEligible), (err) => {
|
||||
return (
|
||||
/bootstrap_packages/.test(err.message) &&
|
||||
/@n8n\/newly-onboarded/.test(err.message)
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
it('skips packages listed in skipPackages — newly-onboarded entry boots without a red nightly', () => {
|
||||
// Operator adds `@n8n/newly-onboarded` to ELIGIBLE_PACKAGES (one-line
|
||||
// onboarding per the README), and runs the next nightly with
|
||||
// `bootstrap_packages: "@n8n/newly-onboarded"`. The new package has zero
|
||||
// ledger rows; the guard must skip it for that one run so the nightly
|
||||
// can populate its rows.
|
||||
const fakeEligible = [
|
||||
{ name: 'n8n-workflow', dir: 'packages/workflow' },
|
||||
{ name: '@n8n/newly-onboarded', dir: 'packages/@n8n/newly-onboarded' },
|
||||
];
|
||||
const rows = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/cron.ts',
|
||||
package: 'n8n-workflow',
|
||||
status: 'green',
|
||||
},
|
||||
];
|
||||
assert.doesNotThrow(() =>
|
||||
assertNoLedgerDivergence(rows, fakeEligible, {
|
||||
skipPackages: ['@n8n/newly-onboarded'],
|
||||
}),
|
||||
);
|
||||
// Defence in depth: the other eligible packages are still guarded.
|
||||
const onlyNewRows = [
|
||||
{
|
||||
source_file_path: 'packages/@n8n/newly-onboarded/src/foo.ts',
|
||||
package: '@n8n/newly-onboarded',
|
||||
status: 'green',
|
||||
},
|
||||
];
|
||||
assert.throws(
|
||||
() =>
|
||||
assertNoLedgerDivergence(onlyNewRows, fakeEligible, {
|
||||
skipPackages: ['@n8n/newly-onboarded'],
|
||||
}),
|
||||
/zero prior-status .* "n8n-workflow"/,
|
||||
);
|
||||
});
|
||||
|
||||
it('ignores `new` rows when counting prior-status hits', () => {
|
||||
// Even with rows present, if they're all `new` it counts as zero prior-status.
|
||||
const rows = ELIGIBLE.map((pkg, i) => ({
|
||||
source_file_path: `${pkg.dir}/src/seed-${i}.ts`,
|
||||
package: pkg.name,
|
||||
status: 'new',
|
||||
}));
|
||||
assert.throws(() => assertNoLedgerDivergence(rows, ELIGIBLE), /zero prior-status/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildCoverageMap (ledger coverage → picker --coverage-file map)', () => {
|
||||
it('produces a { path: 0..1 } map from rows that carry a numeric coverage', () => {
|
||||
const rows = [
|
||||
{ source_file_path: 'packages/workflow/src/a.ts', coverage: 0.4 },
|
||||
{ source_file_path: 'packages/@n8n/crdt/src/z.ts', coverage: 0.1 },
|
||||
];
|
||||
assert.deepEqual(buildCoverageMap(rows), {
|
||||
'packages/workflow/src/a.ts': 0.4,
|
||||
'packages/@n8n/crdt/src/z.ts': 0.1,
|
||||
});
|
||||
});
|
||||
|
||||
it('omits rows without a usable coverage value (sparse map = correct)', () => {
|
||||
// Unscored files have no coverage; the picker treats an absent path as
|
||||
// 0 (worst case). So omitting them is safe and expected.
|
||||
const rows = [
|
||||
{ source_file_path: 'packages/workflow/src/a.ts', coverage: 0.4 },
|
||||
{ source_file_path: 'packages/workflow/src/b.ts', coverage: null },
|
||||
{ source_file_path: 'packages/workflow/src/c.ts' },
|
||||
{ source_file_path: 'packages/workflow/src/d.ts', coverage: 'nope' },
|
||||
{ source_file_path: 'packages/workflow/src/e.ts', coverage: Number.NaN },
|
||||
];
|
||||
assert.deepEqual(buildCoverageMap(rows), {
|
||||
'packages/workflow/src/a.ts': 0.4,
|
||||
});
|
||||
});
|
||||
|
||||
it('keeps boundary coverage values 0 and 1', () => {
|
||||
const rows = [
|
||||
{ source_file_path: 'a.ts', coverage: 0 },
|
||||
{ source_file_path: 'b.ts', coverage: 1 },
|
||||
];
|
||||
assert.deepEqual(buildCoverageMap(rows), { 'a.ts': 0, 'b.ts': 1 });
|
||||
});
|
||||
|
||||
it('skips malformed rows and tolerates a non-array input', () => {
|
||||
const rows = [null, {}, { coverage: 0.5 }, { source_file_path: 'ok.ts', coverage: 0.7 }];
|
||||
assert.deepEqual(buildCoverageMap(rows), { 'ok.ts': 0.7 });
|
||||
assert.deepEqual(buildCoverageMap(undefined), {});
|
||||
assert.deepEqual(buildCoverageMap(null), {});
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildMatrixFromPicked (global picker → matrix shape)', () => {
|
||||
it('consumes the global-mode array shape and emits one row per pick', () => {
|
||||
const picked = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/cron.ts',
|
||||
package: 'n8n-workflow',
|
||||
prior_status: 'new',
|
||||
effective_status: 'new',
|
||||
value: 3.5,
|
||||
},
|
||||
{
|
||||
source_file_path: 'packages/@n8n/crdt/src/utils.ts',
|
||||
package: '@n8n/crdt',
|
||||
prior_status: 'red',
|
||||
effective_status: 'red',
|
||||
value: 2.1,
|
||||
},
|
||||
];
|
||||
const matrix = buildMatrixFromPicked(picked, ELIGIBLE);
|
||||
assert.equal(matrix.include.length, 2);
|
||||
assert.deepEqual(matrix.include[0], {
|
||||
name: 'n8n-workflow',
|
||||
dir: 'packages/workflow',
|
||||
slug: 'workflow',
|
||||
mode: 'baseline',
|
||||
source_file: 'packages/workflow/src/cron.ts',
|
||||
file_slug: 'packages-workflow-src-cron-ts',
|
||||
});
|
||||
assert.equal(matrix.include[1].mode, 'coverage');
|
||||
assert.equal(matrix.include[1].slug, 'crdt');
|
||||
});
|
||||
|
||||
it('returns an empty include[] when the picker has no work (picked: [])', () => {
|
||||
assert.deepEqual(buildMatrixFromPicked([], ELIGIBLE), { include: [] });
|
||||
});
|
||||
|
||||
it('throws when the picker output is not an array (per-package shape would leak through)', () => {
|
||||
assert.throws(
|
||||
() => buildMatrixFromPicked({ source_file_path: 'x.ts' }, ELIGIBLE),
|
||||
/not an array/,
|
||||
);
|
||||
assert.throws(() => buildMatrixFromPicked(null, ELIGIBLE), /not an array/);
|
||||
});
|
||||
|
||||
it('throws when a picked row carries `green` effective_status (green should never enter the candidate set)', () => {
|
||||
// Defence in depth: the picker's contract is that `green` rows are
|
||||
// skipped before ranking. If a future bug leaks a green row through,
|
||||
// the matrix builder dies via modeForEffectiveStatus rather than
|
||||
// scheduling a mutate job for a file that doesn't need re-scoring.
|
||||
const picked = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/cron.ts',
|
||||
package: 'n8n-workflow',
|
||||
prior_status: 'green',
|
||||
effective_status: 'green',
|
||||
value: 0,
|
||||
},
|
||||
];
|
||||
assert.throws(() => buildMatrixFromPicked(picked, ELIGIBLE), /Cannot map effective_status/);
|
||||
});
|
||||
|
||||
it('throws when a picked row carries an unknown effective_status', () => {
|
||||
const picked = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/cron.ts',
|
||||
package: 'n8n-workflow',
|
||||
prior_status: 'red',
|
||||
effective_status: 'totally-bogus',
|
||||
value: 1,
|
||||
},
|
||||
];
|
||||
assert.throws(() => buildMatrixFromPicked(picked, ELIGIBLE), /Cannot map effective_status/);
|
||||
});
|
||||
|
||||
it('refuses to schedule a job for a package outside ELIGIBLE_PACKAGES', () => {
|
||||
// Defence in depth: if the picker ever leaked a row for a non-eligible
|
||||
// package (e.g. via a future bug or a stale ledger row), the matrix
|
||||
// builder dies rather than spawning a mutate job we can't service.
|
||||
const picked = [
|
||||
{
|
||||
source_file_path: 'packages/nodes-base/utils/binary.ts',
|
||||
package: 'n8n-nodes-base',
|
||||
prior_status: 'new',
|
||||
effective_status: 'new',
|
||||
value: 1,
|
||||
},
|
||||
];
|
||||
assert.throws(
|
||||
() => buildMatrixFromPicked(picked, ELIGIBLE),
|
||||
/not in ELIGIBLE_PACKAGES/,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,372 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Emit a BQ payload from a mutate.mjs summary.json.
|
||||
*
|
||||
* Input: packages/<pkg>/reports/mutation/summary.json (from `pnpm mutate <file>`)
|
||||
* Output: JSON document with two keys, `ledger` (rows for qa_mutation_health_ledger)
|
||||
* and `events` (rows for qa_performance_metrics, benchmark_name="mutation_health").
|
||||
*
|
||||
* Each ledger row carries all three of the global picker's value-formula terms:
|
||||
*
|
||||
* coverage — the scored file's coverage fraction (clamped to [0,1])
|
||||
* written back by mutate.mjs, used as the `(1 − coverage)`
|
||||
* term (DEVP-496).
|
||||
* churn — commit count touching the source file within a recent
|
||||
* window, derived here from git, used as the `churn` term
|
||||
* (DEVP-546).
|
||||
* fix_density — the file's time-decayed, delta-weighted fix-density (the
|
||||
* same git-derived signal `signals.mjs` computes), used as the
|
||||
* `fix_density` term (DEVP-546).
|
||||
*
|
||||
* (A richer fix-density variant joined against the bug taxonomy is a possible
|
||||
* future enhancement on the writer side, but the git-derived signal here is
|
||||
* what feeds the picker's value formula.)
|
||||
*
|
||||
* The output is what the n8n writer workflow consumes via webhook. This script
|
||||
* intentionally does NOT call BigQuery directly — the writer workflow owns
|
||||
* BQ credentials and the MERGE statement for the ledger upsert.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/mutation-health/emit-payload.mjs \
|
||||
* --summary packages/workflow/reports/mutation/summary.json \
|
||||
* --package n8n-workflow \
|
||||
* [--signals <path>] # signals.json from `signals.mjs gatherSignals`
|
||||
* # (the setup job's full-history signals). When
|
||||
* # present, churn/fix_density are read from it per
|
||||
* # file instead of computed from git — the only
|
||||
* # path that works under the mutate job's shallow
|
||||
* # clone, and keeps stored values consistent with
|
||||
* # what the picker ranked on.
|
||||
* [--churn-window "90 days"] # git approxidate for the churn count (git fallback only)
|
||||
* [--fix-density-window "1 year"] # git approxidate bounding the fix-density log read (git fallback only)
|
||||
* [--fix-density-half-life 90] # half-life in days for the fix-density decay (git fallback only)
|
||||
* [--out <path>] # default: <pkg>/reports/mutation/bq-payload.json
|
||||
*/
|
||||
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import path from 'node:path';
|
||||
|
||||
import {
|
||||
GIT_LOG_FORMAT,
|
||||
parseGitLog,
|
||||
computeFixDensity,
|
||||
DEFAULT_HALF_LIFE_DAYS,
|
||||
} from './signals.mjs';
|
||||
|
||||
function die(code, msg) {
|
||||
process.stderr.write(`${msg}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (!a.startsWith('--')) continue;
|
||||
const key = a.slice(2);
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith('--')) {
|
||||
out[key] = true;
|
||||
} else {
|
||||
out[key] = next;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Coverage fraction for a summary file row, clamped to [0,1]. Prefers the
|
||||
* `coverage` mutate.mjs wrote back; falls back to deriving it from the mutant
|
||||
* counts for summaries produced before the writeback existed. Returns null only
|
||||
* when neither source is usable, so the ledger column degrades gracefully.
|
||||
*/
|
||||
export function coverageForLedger(f) {
|
||||
if (typeof f.coverage === 'number' && Number.isFinite(f.coverage)) {
|
||||
return +Math.min(1, Math.max(0, f.coverage)).toFixed(4);
|
||||
}
|
||||
const c = f.counts;
|
||||
if (!c) return null;
|
||||
const covered = (c.killed ?? 0) + (c.survived ?? 0) + (c.timeout ?? 0) + (c.runtimeError ?? 0);
|
||||
const total = covered + (c.noCoverage ?? 0);
|
||||
if (total === 0) return 0;
|
||||
return +Math.min(1, Math.max(0, covered / total)).toFixed(4);
|
||||
}
|
||||
|
||||
/** Default git approxidate window for the per-file churn count. */
|
||||
export const DEFAULT_CHURN_WINDOW = '90 days';
|
||||
|
||||
/**
|
||||
* Default git approxidate window bounding the fix-density log read. A 90-day
|
||||
* half-life means commits older than ~1 year contribute negligibly, so this
|
||||
* bounds the per-package log read without affecting the signal in practice.
|
||||
*/
|
||||
export const DEFAULT_FIX_DENSITY_WINDOW = '1 year';
|
||||
|
||||
// 256MB: a per-package `git log --numstat` over year-scale history can blow the
|
||||
// default 1MB stdout cap; `rev-list --count` output is tiny so this is harmless
|
||||
// for churn, letting both factories share one runner.
|
||||
const GIT_MAX_BUFFER = 256 * 1024 * 1024;
|
||||
|
||||
function defaultRunGit(args, cwd) {
|
||||
return execFileSync('git', args, { cwd, encoding: 'utf8', maxBuffer: GIT_MAX_BUFFER });
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `churnFor(sourceRel)` that returns the number of commits touching a
|
||||
* file within `since` (a git approxidate, e.g. "90 days") via
|
||||
* `git rev-list --count`.
|
||||
*
|
||||
* Shallow clones are detected up-front: their history is truncated, so the
|
||||
* count would be misleadingly low — we return null for every file rather than
|
||||
* emit a wrong signal. A git failure on an individual file (e.g. a path that
|
||||
* never existed) also degrades to null, leaving the ledger column empty rather
|
||||
* than guessing.
|
||||
*
|
||||
* `runGit` is injectable so the shallow-guard and count parsing are unit
|
||||
* testable without a real repo; in the pipeline it shells out to git.
|
||||
*/
|
||||
export function makeChurnFor({
|
||||
cwd = process.cwd(),
|
||||
since = DEFAULT_CHURN_WINDOW,
|
||||
runGit = defaultRunGit,
|
||||
} = {}) {
|
||||
let shallow = false;
|
||||
try {
|
||||
shallow = runGit(['rev-parse', '--is-shallow-repository'], cwd).trim() === 'true';
|
||||
} catch {
|
||||
// Can't ask (e.g. not a git checkout) — treat every file as unknown.
|
||||
shallow = true;
|
||||
}
|
||||
return (sourceRel) => {
|
||||
if (shallow) return null;
|
||||
try {
|
||||
const out = runGit(['rev-list', '--count', `--since=${since}`, 'HEAD', '--', sourceRel], cwd);
|
||||
const n = Number(out.trim());
|
||||
return Number.isFinite(n) ? n : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a `fixDensityFor(sourceRel)` returning a file's time-decayed,
|
||||
* delta-weighted fix-density — the same git-derived signal `signals.mjs`
|
||||
* computes — scoped to a single package's history.
|
||||
*
|
||||
* One `git log --numstat` pass (scoped to `pathspec`, bounded by `since`) is
|
||||
* parsed with `signals.mjs`'s pure helpers, so churn and fix-density share the
|
||||
* same fix-detection + decay logic rather than reimplementing it here. Files
|
||||
* with no fix commits score 0 (known: no fixes), distinct from the `null` a
|
||||
* shallow clone or git failure emits (unknown) — mirroring `makeChurnFor`'s
|
||||
* shallow guard so a truncated history never emits a misleadingly low signal.
|
||||
*
|
||||
* `now`/`halfLifeDays` are injectable for deterministic tests; `runGit` is
|
||||
* injectable so the parse + guard are unit testable without a real repo.
|
||||
*/
|
||||
export function makeFixDensityFor({
|
||||
cwd = process.cwd(),
|
||||
since = DEFAULT_FIX_DENSITY_WINDOW,
|
||||
halfLifeDays = DEFAULT_HALF_LIFE_DAYS,
|
||||
now = Math.floor(Date.now() / 1000),
|
||||
pathspec,
|
||||
runGit = defaultRunGit,
|
||||
} = {}) {
|
||||
let density = null;
|
||||
try {
|
||||
if (runGit(['rev-parse', '--is-shallow-repository'], cwd).trim() !== 'true') {
|
||||
const args = ['log', '--no-merges', `--pretty=format:${GIT_LOG_FORMAT}`, '--numstat'];
|
||||
if (since) args.push(`--since=${since}`);
|
||||
if (pathspec) args.push('--', pathspec);
|
||||
density = computeFixDensity(parseGitLog(runGit(args, cwd)), { halfLifeDays, now });
|
||||
}
|
||||
} catch {
|
||||
// Shallow clone, not a git checkout, or a git/parse failure — every file
|
||||
// unknown, so the column degrades to null rather than guessing.
|
||||
density = null;
|
||||
}
|
||||
return (sourceRel) => {
|
||||
if (density === null) return null;
|
||||
return +(density.get(sourceRel) ?? 0).toFixed(4);
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build `{ churnFor, fixDensityFor }` that read from the setup job's
|
||||
* `signals.json` (the full-history signals `signals.mjs gatherSignals` wrote and
|
||||
* the picker ranked on) instead of from git.
|
||||
*
|
||||
* This is the path the nightly `mutate` job uses: its checkout is shallow, so
|
||||
* the git-derived factories above would emit `null` for every file. Reusing the
|
||||
* already-computed signals keeps the stored ledger columns consistent with the
|
||||
* exact churn/fix-density the global picker scored — and matches how the picker
|
||||
* reads them (`extractSignals` in `pick-next.mjs`): churn is the per-file commit
|
||||
* count, fix-density the decayed score.
|
||||
*
|
||||
* A file absent from the signals map had no commits (churn) / no fix commits
|
||||
* (fix-density) in the gather window, so both lookups return 0 — known-zero, not
|
||||
* the `null` (unknown) the shallow-clone git path emits. The whole map is keyed
|
||||
* by repo-relative path, the same key `buildPayload` joins on.
|
||||
*/
|
||||
export function makeSignalLookups(signals) {
|
||||
const churn = signals?.churn ?? {};
|
||||
const fixDensity = signals?.fixDensity ?? {};
|
||||
return {
|
||||
churnFor: (sourceRel) => {
|
||||
const entry = churn[sourceRel];
|
||||
if (typeof entry === 'number') return Number.isFinite(entry) ? entry : 0;
|
||||
if (typeof entry?.commits === 'number' && Number.isFinite(entry.commits)) {
|
||||
return entry.commits;
|
||||
}
|
||||
return 0;
|
||||
},
|
||||
fixDensityFor: (sourceRel) => {
|
||||
const v = fixDensity[sourceRel];
|
||||
return typeof v === 'number' && Number.isFinite(v) ? +v.toFixed(4) : 0;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the `{ ledger, events }` payload from a parsed summary. Pure given its
|
||||
* signal dependencies: takes the summary plus run metadata, returns the rows
|
||||
* the writer webhook consumes. `churnFor(sourceRel)` / `fixDensityFor(sourceRel)`
|
||||
* yield the per-file churn count and fix-density (or null); both default to a
|
||||
* no-op so callers without git stay pure.
|
||||
*/
|
||||
export function buildPayload(
|
||||
summary,
|
||||
{ pkg, sha, pkgRelToRepo, churnFor = () => null, fixDensityFor = () => null },
|
||||
) {
|
||||
const threshold = Number(summary.threshold);
|
||||
const timestamp = summary.generatedAt;
|
||||
|
||||
const ledger = [];
|
||||
const events = [];
|
||||
|
||||
for (const f of summary.files) {
|
||||
const sourceRel = path.posix.join(pkgRelToRepo, f.file);
|
||||
const status = f.thresholdMet ? 'green' : 'red';
|
||||
const coverage = coverageForLedger(f);
|
||||
const churn = churnFor(sourceRel);
|
||||
const fixDensity = fixDensityFor(sourceRel);
|
||||
|
||||
ledger.push({
|
||||
source_file_path: sourceRel,
|
||||
package: pkg,
|
||||
last_score: f.score,
|
||||
coverage,
|
||||
churn,
|
||||
fix_density: fixDensity,
|
||||
threshold_at_run: threshold,
|
||||
last_checked_at: timestamp,
|
||||
status,
|
||||
mutants_killed: f.counts.killed,
|
||||
mutants_survived: f.counts.survived,
|
||||
mutants_no_coverage: f.counts.noCoverage,
|
||||
mutants_timeout: f.counts.timeout,
|
||||
});
|
||||
|
||||
events.push({
|
||||
benchmark_name: 'mutation_health',
|
||||
value: f.score,
|
||||
timestamp,
|
||||
dimensions: {
|
||||
package: pkg,
|
||||
source_file: sourceRel,
|
||||
sha,
|
||||
status_after: status,
|
||||
threshold,
|
||||
coverage,
|
||||
churn,
|
||||
fix_density: fixDensity,
|
||||
mutants_killed: f.counts.killed,
|
||||
mutants_survived: f.counts.survived,
|
||||
mutants_no_coverage: f.counts.noCoverage,
|
||||
mutants_timeout: f.counts.timeout,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return { ledger, events };
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
|
||||
const summaryPath = args.summary;
|
||||
const pkg = args.package;
|
||||
|
||||
if (!summaryPath) die(2, 'Missing required --summary <path>');
|
||||
if (!pkg) die(2, 'Missing required --package <name>');
|
||||
if (!existsSync(summaryPath)) die(2, `Summary not found: ${summaryPath}`);
|
||||
|
||||
const sha = execFileSync('git', ['rev-parse', 'HEAD'], { encoding: 'utf8' }).trim();
|
||||
|
||||
const summary = JSON.parse(await readFile(summaryPath, 'utf8'));
|
||||
|
||||
if (!Array.isArray(summary.files)) {
|
||||
die(2, 'Summary missing `files` array.');
|
||||
}
|
||||
|
||||
// pkg-root = two dirs up from the summary (reports/mutation/summary.json)
|
||||
const pkgRoot = path.resolve(path.dirname(summaryPath), '../..');
|
||||
const repoRoot = path.resolve(
|
||||
execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }).trim(),
|
||||
);
|
||||
const pkgRelToRepo = path.relative(repoRoot, pkgRoot);
|
||||
|
||||
// Prefer the setup job's full-history signals.json when provided: the nightly
|
||||
// mutate job's checkout is shallow, so the git-derived factories below would
|
||||
// emit null for every file. Falls back to git for local/standalone runs that
|
||||
// have full history and no signals file.
|
||||
let churnFor;
|
||||
let fixDensityFor;
|
||||
const signalsPath = typeof args.signals === 'string' ? args.signals : undefined;
|
||||
if (signalsPath) {
|
||||
if (!existsSync(signalsPath)) die(2, `Signals file not found: ${signalsPath}`);
|
||||
const signals = JSON.parse(await readFile(signalsPath, 'utf8'));
|
||||
({ churnFor, fixDensityFor } = makeSignalLookups(signals));
|
||||
} else {
|
||||
churnFor = makeChurnFor({
|
||||
cwd: repoRoot,
|
||||
since: typeof args['churn-window'] === 'string' ? args['churn-window'] : DEFAULT_CHURN_WINDOW,
|
||||
});
|
||||
|
||||
const halfLifeArg = Number(args['fix-density-half-life']);
|
||||
fixDensityFor = makeFixDensityFor({
|
||||
cwd: repoRoot,
|
||||
since:
|
||||
typeof args['fix-density-window'] === 'string'
|
||||
? args['fix-density-window']
|
||||
: DEFAULT_FIX_DENSITY_WINDOW,
|
||||
halfLifeDays:
|
||||
Number.isFinite(halfLifeArg) && halfLifeArg > 0 ? halfLifeArg : DEFAULT_HALF_LIFE_DAYS,
|
||||
pathspec: pkgRelToRepo,
|
||||
});
|
||||
}
|
||||
|
||||
const { ledger, events } = buildPayload(summary, {
|
||||
pkg,
|
||||
sha,
|
||||
pkgRelToRepo,
|
||||
churnFor,
|
||||
fixDensityFor,
|
||||
});
|
||||
|
||||
const outPath = args.out ?? path.join(pkgRoot, 'reports/mutation/bq-payload.json');
|
||||
await mkdir(path.dirname(outPath), { recursive: true });
|
||||
await writeFile(outPath, JSON.stringify({ ledger, events }, null, 2));
|
||||
|
||||
process.stderr.write(
|
||||
`Emitted ${ledger.length} ledger row(s) + ${events.length} event row(s) → ${outPath}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
const isCli = import.meta.url === `file://${process.argv[1]}`;
|
||||
if (isCli) {
|
||||
await main();
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
buildPayload,
|
||||
coverageForLedger,
|
||||
makeChurnFor,
|
||||
makeFixDensityFor,
|
||||
makeSignalLookups,
|
||||
} from './emit-payload.mjs';
|
||||
|
||||
const summaryFixture = () => ({
|
||||
threshold: 60,
|
||||
generatedAt: '2026-06-24T03:30:00.000Z',
|
||||
files: [
|
||||
{
|
||||
file: 'src/hot.ts',
|
||||
score: 42,
|
||||
thresholdMet: false,
|
||||
coverage: 0.5,
|
||||
counts: { killed: 3, survived: 2, noCoverage: 1, timeout: 0, runtimeError: 0 },
|
||||
},
|
||||
{
|
||||
file: 'src/cold.ts',
|
||||
score: 90,
|
||||
thresholdMet: true,
|
||||
coverage: 0.95,
|
||||
counts: { killed: 9, survived: 0, noCoverage: 0, timeout: 1, runtimeError: 0 },
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
describe('coverageForLedger', () => {
|
||||
it('prefers the written-back coverage, clamped to [0,1]', () => {
|
||||
assert.equal(coverageForLedger({ coverage: 0.5 }), 0.5);
|
||||
assert.equal(coverageForLedger({ coverage: 1.7 }), 1);
|
||||
assert.equal(coverageForLedger({ coverage: -0.3 }), 0);
|
||||
});
|
||||
|
||||
it('derives from mutant counts when coverage is absent', () => {
|
||||
assert.equal(
|
||||
coverageForLedger({ counts: { killed: 3, survived: 1, timeout: 0, noCoverage: 0 } }),
|
||||
1,
|
||||
);
|
||||
assert.equal(
|
||||
coverageForLedger({ counts: { killed: 1, survived: 0, timeout: 0, noCoverage: 1 } }),
|
||||
0.5,
|
||||
);
|
||||
});
|
||||
|
||||
it('returns null when neither source is usable', () => {
|
||||
assert.equal(coverageForLedger({}), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('makeSignalLookups', () => {
|
||||
const signals = {
|
||||
churn: {
|
||||
'packages/workflow/src/hot.ts': { commits: 7, linesChanged: 120 },
|
||||
'packages/workflow/src/bare.ts': 4,
|
||||
},
|
||||
fixDensity: {
|
||||
'packages/workflow/src/hot.ts': 3.14159,
|
||||
},
|
||||
};
|
||||
|
||||
it('reads the per-file commit count for churn', () => {
|
||||
const { churnFor } = makeSignalLookups(signals);
|
||||
assert.equal(churnFor('packages/workflow/src/hot.ts'), 7);
|
||||
});
|
||||
|
||||
it('accepts a bare numeric churn entry', () => {
|
||||
const { churnFor } = makeSignalLookups(signals);
|
||||
assert.equal(churnFor('packages/workflow/src/bare.ts'), 4);
|
||||
});
|
||||
|
||||
it('reads and rounds fix-density', () => {
|
||||
const { fixDensityFor } = makeSignalLookups(signals);
|
||||
assert.equal(fixDensityFor('packages/workflow/src/hot.ts'), 3.1416);
|
||||
});
|
||||
|
||||
it('returns known-zero (not null) for a file absent from the signals map', () => {
|
||||
const { churnFor, fixDensityFor } = makeSignalLookups(signals);
|
||||
assert.equal(churnFor('packages/workflow/src/cold.ts'), 0);
|
||||
assert.equal(fixDensityFor('packages/workflow/src/cold.ts'), 0);
|
||||
});
|
||||
|
||||
it('tolerates an empty or partial signals object', () => {
|
||||
const { churnFor, fixDensityFor } = makeSignalLookups({});
|
||||
assert.equal(churnFor('any.ts'), 0);
|
||||
assert.equal(fixDensityFor('any.ts'), 0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('buildPayload with signal lookups', () => {
|
||||
it('lands non-null churn/fix_density when signals carry the file (DEVP-552)', () => {
|
||||
const signals = {
|
||||
churn: { 'packages/workflow/src/hot.ts': { commits: 7, linesChanged: 120 } },
|
||||
fixDensity: { 'packages/workflow/src/hot.ts': 2.5 },
|
||||
};
|
||||
const { churnFor, fixDensityFor } = makeSignalLookups(signals);
|
||||
|
||||
const { ledger, events } = buildPayload(summaryFixture(), {
|
||||
pkg: 'n8n-workflow',
|
||||
sha: 'deadbeef',
|
||||
pkgRelToRepo: 'packages/workflow',
|
||||
churnFor,
|
||||
fixDensityFor,
|
||||
});
|
||||
|
||||
const hot = ledger.find((r) => r.source_file_path === 'packages/workflow/src/hot.ts');
|
||||
assert.equal(hot.churn, 7);
|
||||
assert.equal(hot.fix_density, 2.5);
|
||||
assert.notEqual(hot.churn, null);
|
||||
assert.notEqual(hot.fix_density, null);
|
||||
|
||||
// A scored file with no signal entry is known-zero, never null.
|
||||
const cold = ledger.find((r) => r.source_file_path === 'packages/workflow/src/cold.ts');
|
||||
assert.equal(cold.churn, 0);
|
||||
assert.equal(cold.fix_density, 0);
|
||||
|
||||
// Same values flow into the perf-metric event dimensions.
|
||||
const hotEvent = events.find(
|
||||
(e) => e.dimensions.source_file === 'packages/workflow/src/hot.ts',
|
||||
);
|
||||
assert.equal(hotEvent.dimensions.churn, 7);
|
||||
assert.equal(hotEvent.dimensions.fix_density, 2.5);
|
||||
});
|
||||
|
||||
it('defaults to null churn/fix_density without signal lookups', () => {
|
||||
const { ledger } = buildPayload(summaryFixture(), {
|
||||
pkg: 'n8n-workflow',
|
||||
sha: 'deadbeef',
|
||||
pkgRelToRepo: 'packages/workflow',
|
||||
});
|
||||
assert.equal(ledger[0].churn, null);
|
||||
assert.equal(ledger[0].fix_density, null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('git fallback factories degrade to null on a shallow clone', () => {
|
||||
// The shallow-clone signature emit-payload originally hit in the mutate job:
|
||||
// both factories must emit null rather than a misleadingly low signal.
|
||||
const shallowRunGit = (args) => {
|
||||
if (args[0] === 'rev-parse' && args.includes('--is-shallow-repository')) return 'true\n';
|
||||
return '';
|
||||
};
|
||||
|
||||
it('makeChurnFor returns null for every file under a shallow clone', () => {
|
||||
const churnFor = makeChurnFor({ runGit: shallowRunGit });
|
||||
assert.equal(churnFor('packages/workflow/src/hot.ts'), null);
|
||||
});
|
||||
|
||||
it('makeFixDensityFor returns null for every file under a shallow clone', () => {
|
||||
const fixDensityFor = makeFixDensityFor({ runGit: shallowRunGit });
|
||||
assert.equal(fixDensityFor('packages/workflow/src/hot.ts'), null);
|
||||
});
|
||||
});
|
||||
@@ -1,50 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Read-all ledger access.
|
||||
*
|
||||
* One call returns every row in the live BigQuery ledger across every
|
||||
* package — call sites no longer need to invoke per package. Per-package
|
||||
* behaviour is preserved by passing `pkg`, which narrows the same
|
||||
* single-pass read to one package without re-fetching.
|
||||
*
|
||||
* Accepted body shapes (matching the reader-webhook contract):
|
||||
*
|
||||
* '' → { rows: [] }
|
||||
* '{"ledger":[]}' → { rows: [] }
|
||||
* '{"ledger":[ {row}, ... ]}' → { rows: [ {row}, ... ] }
|
||||
*
|
||||
* The empty-body case mirrors the reader webhook's response for packages
|
||||
* it has never scored — the picker must still synthesise `new` rows from
|
||||
* the source tree in that situation.
|
||||
*
|
||||
* Malformed JSON or a missing `ledger` array throw, leaving the caller
|
||||
* to decide whether to die or fall back.
|
||||
*/
|
||||
|
||||
import { readFile } from 'node:fs/promises';
|
||||
|
||||
/**
|
||||
* Pure parser. Used directly by tests and via the `readLedger` wrapper.
|
||||
*/
|
||||
export function parseLedgerBody(raw, { pkg } = {}) {
|
||||
const trimmed = typeof raw === 'string' ? raw.trim() : '';
|
||||
const payload = trimmed === '' ? { ledger: [] } : JSON.parse(trimmed);
|
||||
if (!payload || !Array.isArray(payload.ledger)) {
|
||||
throw new Error('Ledger payload missing `ledger` array.');
|
||||
}
|
||||
const all = payload.ledger;
|
||||
if (pkg === undefined) return { rows: all };
|
||||
return { rows: all.filter((r) => r && r.package === pkg) };
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the ledger file and return all rows in one pass. With `pkg`, the
|
||||
* same single read is narrowed to one package — existing per-package call
|
||||
* sites stay structurally identical, the pipeline can later switch to a
|
||||
* single fetch shared across packages without touching the call site.
|
||||
*/
|
||||
export async function readLedger({ path, pkg } = {}) {
|
||||
if (!path) throw new TypeError('readLedger requires a `path`');
|
||||
const raw = await readFile(path, 'utf8');
|
||||
return parseLedgerBody(raw, { pkg });
|
||||
}
|
||||
@@ -1,140 +0,0 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs';
|
||||
import { tmpdir } from 'node:os';
|
||||
import path from 'node:path';
|
||||
|
||||
import { parseLedgerBody, readLedger } from './ledger.mjs';
|
||||
|
||||
const body = (rows) => JSON.stringify({ ledger: rows });
|
||||
|
||||
// A multi-package fixture store: rows for three onboarded vitest packages
|
||||
// sit side-by-side. Shape mirrors what a read-all reader webhook would
|
||||
// return when no `?package=` filter is applied.
|
||||
const MULTI_PKG_FIXTURE = [
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/a.ts',
|
||||
package: 'n8n-workflow',
|
||||
status: 'red',
|
||||
last_score: 30,
|
||||
last_checked_at: '2026-05-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/b.ts',
|
||||
package: 'n8n-workflow',
|
||||
status: 'green',
|
||||
last_score: 95,
|
||||
last_checked_at: '2026-06-01T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
source_file_path: 'packages/@n8n/crdt/src/x.ts',
|
||||
package: '@n8n/crdt',
|
||||
status: 'red',
|
||||
last_score: 40,
|
||||
last_checked_at: '2026-05-15T00:00:00.000Z',
|
||||
},
|
||||
{
|
||||
source_file_path: 'packages/@n8n/decorators/src/y.ts',
|
||||
package: '@n8n/decorators',
|
||||
status: 'green',
|
||||
last_score: 85,
|
||||
last_checked_at: '2026-06-10T00:00:00.000Z',
|
||||
},
|
||||
];
|
||||
|
||||
describe('parseLedgerBody', () => {
|
||||
it('treats an empty body as a zero-row ledger', () => {
|
||||
assert.deepEqual(parseLedgerBody(''), { rows: [] });
|
||||
assert.deepEqual(parseLedgerBody(' '), { rows: [] });
|
||||
});
|
||||
|
||||
it('treats `{"ledger":[]}` as a zero-row ledger', () => {
|
||||
assert.deepEqual(parseLedgerBody('{"ledger":[]}'), { rows: [] });
|
||||
});
|
||||
|
||||
it('throws on a payload missing the `ledger` array', () => {
|
||||
assert.throws(() => parseLedgerBody('{}'), /missing `ledger` array/);
|
||||
assert.throws(() => parseLedgerBody('{"ledger":"oops"}'), /missing `ledger` array/);
|
||||
assert.throws(() => parseLedgerBody('null'), /missing `ledger` array/);
|
||||
});
|
||||
|
||||
it('lets malformed JSON throw via JSON.parse', () => {
|
||||
assert.throws(() => parseLedgerBody('not json'), SyntaxError);
|
||||
});
|
||||
});
|
||||
|
||||
// PR-gate contract from DEVP-495:
|
||||
// "multi-package fixture store returns all rows in one call"
|
||||
//
|
||||
// The fixture holds rows for three packages. A single call must return
|
||||
// every row, with no per-package invocation at the call site.
|
||||
describe('read-all contract (DEVP-495 PR gate)', () => {
|
||||
it('returns every row across every package in one call (no filter)', () => {
|
||||
const { rows } = parseLedgerBody(body(MULTI_PKG_FIXTURE));
|
||||
assert.equal(rows.length, MULTI_PKG_FIXTURE.length);
|
||||
assert.deepEqual([...new Set(rows.map((r) => r.package))].sort(), [
|
||||
'@n8n/crdt',
|
||||
'@n8n/decorators',
|
||||
'n8n-workflow',
|
||||
]);
|
||||
});
|
||||
|
||||
it('preserves row order from the underlying store', () => {
|
||||
const { rows } = parseLedgerBody(body(MULTI_PKG_FIXTURE));
|
||||
assert.deepEqual(
|
||||
rows.map((r) => r.source_file_path),
|
||||
MULTI_PKG_FIXTURE.map((r) => r.source_file_path),
|
||||
);
|
||||
});
|
||||
|
||||
it('narrows to one package without re-reading when `pkg` is supplied', () => {
|
||||
const { rows } = parseLedgerBody(body(MULTI_PKG_FIXTURE), { pkg: 'n8n-workflow' });
|
||||
assert.equal(rows.length, 2);
|
||||
assert.ok(rows.every((r) => r.package === 'n8n-workflow'));
|
||||
});
|
||||
|
||||
it('returns an empty set when narrowing to a package with no rows', () => {
|
||||
const { rows } = parseLedgerBody(body(MULTI_PKG_FIXTURE), { pkg: 'n8n-nonesuch' });
|
||||
assert.deepEqual(rows, []);
|
||||
});
|
||||
|
||||
it('drops rows missing a `package` field when narrowing', () => {
|
||||
const { rows } = parseLedgerBody(
|
||||
body([...MULTI_PKG_FIXTURE, { source_file_path: 'stray.ts' }]),
|
||||
{ pkg: 'n8n-workflow' },
|
||||
);
|
||||
assert.equal(rows.length, 2);
|
||||
assert.ok(rows.every((r) => r.package === 'n8n-workflow'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('readLedger (file I/O)', () => {
|
||||
const tmp = mkdtempSync(path.join(tmpdir(), 'mh-ledger-'));
|
||||
|
||||
it('reads a multi-package fixture file and returns every row in one pass', async () => {
|
||||
const file = path.join(tmp, 'multi.json');
|
||||
writeFileSync(file, body(MULTI_PKG_FIXTURE));
|
||||
const { rows } = await readLedger({ path: file });
|
||||
assert.equal(rows.length, MULTI_PKG_FIXTURE.length);
|
||||
});
|
||||
|
||||
it('reads an empty-body file as a zero-row ledger (reader-webhook contract for unscored packages)', async () => {
|
||||
const file = path.join(tmp, 'empty.json');
|
||||
writeFileSync(file, '');
|
||||
const { rows } = await readLedger({ path: file });
|
||||
assert.deepEqual(rows, []);
|
||||
});
|
||||
|
||||
it('filters to a single package on top of the single-pass read', async () => {
|
||||
const file = path.join(tmp, 'filtered.json');
|
||||
writeFileSync(file, body(MULTI_PKG_FIXTURE));
|
||||
const { rows } = await readLedger({ path: file, pkg: '@n8n/crdt' });
|
||||
assert.equal(rows.length, 1);
|
||||
assert.equal(rows[0].package, '@n8n/crdt');
|
||||
});
|
||||
|
||||
it('requires a `path`', async () => {
|
||||
await assert.rejects(readLedger(), /requires a `path`/);
|
||||
await assert.rejects(readLedger({}), /requires a `path`/);
|
||||
});
|
||||
});
|
||||
+453
-160
@@ -1,16 +1,16 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Run Stryker on a single source file of a workspace package and emit an
|
||||
* actionable summary. Package-agnostic: the nightly matrix and the per-package
|
||||
* `mutate` npm scripts both call this one script.
|
||||
* Run Stryker over a workspace package and write an actionable summary. This
|
||||
* script works for any package. Run it as `pnpm mutate` from the repo root.
|
||||
*
|
||||
* Usage (also exposed as `pnpm mutate <file>` from the repo root):
|
||||
* node scripts/mutation-health/mutate.mjs <file> [--package-dir <repo-rel-path>] [--config <path>]
|
||||
* pnpm mutate <file>[:<start>-<end>] [--package-dir <dir>] [--config <path>]
|
||||
* pnpm mutate --diff [--base <ref>] [--config <path>]
|
||||
*
|
||||
* The package is inferred from a repo-relative file path; pass --package-dir when
|
||||
* the target is package-relative (the nightly does this).
|
||||
* node scripts/mutation-health/mutate.mjs packages/@n8n/crdt/src/utils.ts # inferred
|
||||
* node scripts/mutation-health/mutate.mjs src/cron.ts --package-dir packages/workflow
|
||||
* Use `--diff` before you merge. It reads the changed line ranges from
|
||||
* `git diff -U0 $(git merge-base <base> HEAD)`, which covers committed branch
|
||||
* work and uncommitted edits. It mutates only those lines, in one Stryker run
|
||||
* per package. This makes the gate apply to the patch: it scores the lines you
|
||||
* changed, not the debt you inherited.
|
||||
*
|
||||
* Stryker config resolution (first match wins):
|
||||
* 1. --config <path> explicit override
|
||||
@@ -27,40 +27,31 @@
|
||||
* the survivors it found rather than dying with nothing.
|
||||
*
|
||||
* Each summary file row also carries a `coverage` fraction in [0,1] — the share
|
||||
* of mutants a test actually exercised — so the global picker can read
|
||||
* `(1 − coverage)` from the ledger next cycle (DEVP-496). `emit-payload.mjs`
|
||||
* forwards it onto the ledger row.
|
||||
* of mutants a test actually exercised.
|
||||
*
|
||||
* Gate semantics:
|
||||
* A run passes only when the mutation score meets `STRYKER_THRESHOLD` AND
|
||||
* every remaining mutant is either killed or explicitly justified via a
|
||||
* `// Stryker disable …` comment (status `Ignored`). Any `Survived` or
|
||||
* `NoCoverage` mutant is an unjustified survivor and fails the gate even
|
||||
* above the threshold — raw score alone counts low-value and equivalent
|
||||
* mutants in the denominator and lets agents pad the suite to 80%. See
|
||||
* DEVP-442.
|
||||
* A run passes only when the score meets `STRYKER_THRESHOLD` and no mutant
|
||||
* survives without a reason. To give a reason, add a `// Stryker disable …`
|
||||
* comment. Stryker then reports the mutant as `Ignored` and keeps it out of
|
||||
* the score. Each `Survived` or `NoCoverage` mutant fails the gate, also
|
||||
* above the threshold. The score alone lets an author add weak tests to reach
|
||||
* 80% and leave real gaps. See DEVP-442.
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 — gate passed (score ≥ threshold AND no unjustified survivors)
|
||||
* 1 — gate failed: score < threshold OR at least one Survived/NoCoverage
|
||||
* mutant remains (AI loop should iterate), OR the run was partial (a
|
||||
* non-zero Stryker exit with a salvaged raw.json — untested mutants
|
||||
* could be survivors, so a partial run never passes). Also used when
|
||||
* Stryker reports "No tests were executed" — the file has no covering
|
||||
* tests, so we synthesise a score-0 red summary rather than hard-failing
|
||||
* the job. The ledger then records the gap and the picker advances.
|
||||
* 2 — usage / config error
|
||||
* 3 — Stryker run failed for any other reason (instrumentation crash etc.)
|
||||
* 0 — the gate passed.
|
||||
* 1 — the gate failed. Iterate: read summary.json and strengthen the tests.
|
||||
* 2 — usage or config error.
|
||||
* 3 — Stryker did not resolve or did not run. Never 1: a broken toolchain
|
||||
* must stay distinct from a score of zero.
|
||||
*/
|
||||
|
||||
import { spawn } from 'node:child_process';
|
||||
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
||||
import { existsSync } from 'node:fs';
|
||||
import { spawn, spawnSync } from 'node:child_process';
|
||||
import { readFile, writeFile, mkdir, rm } from 'node:fs/promises';
|
||||
import { existsSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { createRequire } from 'node:module';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import path from 'node:path';
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const repoRoot = path.resolve(__dirname, '../..');
|
||||
|
||||
@@ -110,9 +101,7 @@ export function scoreFromCounts(c) {
|
||||
* could be covered (ran + no-coverage). Ignored and compile-error mutants
|
||||
* never ran for reasons unrelated to coverage, so they sit outside the ratio.
|
||||
*
|
||||
* Returns a fraction in [0,1] — clamped defensively. The global picker reads it
|
||||
* back from the ledger as the `(1 − coverage)` term of its value formula, so a
|
||||
* file no test touches (coverage 0) gets the strongest urge to be scored.
|
||||
* Returns a fraction in [0,1]. The result is clamped.
|
||||
*/
|
||||
export function coverageFromCounts(c) {
|
||||
const covered = (c.killed ?? 0) + (c.survived ?? 0) + (c.timeout ?? 0) + (c.runtimeError ?? 0);
|
||||
@@ -121,6 +110,25 @@ export function coverageFromCounts(c) {
|
||||
return +Math.min(1, Math.max(0, covered / total)).toFixed(4);
|
||||
}
|
||||
|
||||
/**
|
||||
* What a finished Stryker run produced. `hasReport` must describe THIS run:
|
||||
* the caller deletes the previous reports first, because a file left by an
|
||||
* earlier run makes a crashed run look complete and report the earlier target.
|
||||
*
|
||||
* complete — the run finished and wrote a report.
|
||||
* partial — the run wrote a report, then exited non-zero. Untested mutants
|
||||
* can still be survivors, so this never passes the gate.
|
||||
* no-tests — no test covers the target. A result, not an error: the score is
|
||||
* zero and every mutant has no coverage. See DEVP-414.
|
||||
* failed — no report. The caller reports a toolchain failure.
|
||||
*/
|
||||
export function classifyRun({ exitCode, output, hasReport }) {
|
||||
if (!hasReport) {
|
||||
return /no tests were executed/i.test(output) ? 'no-tests' : 'failed';
|
||||
}
|
||||
return exitCode === 0 ? 'complete' : 'partial';
|
||||
}
|
||||
|
||||
// A run is only "passing" when the score meets the floor AND every unkilled
|
||||
// mutant has been explicitly justified (Ignored via a Stryker disable
|
||||
// comment). Any Survived/NoCoverage mutant is unjustified by definition.
|
||||
@@ -131,7 +139,7 @@ export function gatePassed(score, counts, threshold) {
|
||||
/**
|
||||
* Build the compact summary from a raw Stryker Mutation Testing Elements
|
||||
* report. Pure: takes the parsed report plus run metadata, returns the summary
|
||||
* object written to summary.json (and consumed by emit-payload.mjs).
|
||||
* object written to summary.json.
|
||||
*/
|
||||
export function buildSummary(raw, { threshold, target, generatedAt }) {
|
||||
// test-id → test-name lookup so survivors can name the tests that covered
|
||||
@@ -266,142 +274,308 @@ function findPackageRoot(fromAbs) {
|
||||
return null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
// --- args: one positional target + --package-dir (required) + --config (optional)
|
||||
const argv = process.argv.slice(2);
|
||||
let packageDirArg;
|
||||
let configArg;
|
||||
let targetArg;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--package-dir') packageDirArg = argv[++i];
|
||||
else if (a === '--config') configArg = argv[++i];
|
||||
else if (!a.startsWith('--') && targetArg === undefined) targetArg = a;
|
||||
// --- diff-mode planning (pure helpers exported for the unit tests) ---
|
||||
|
||||
// Stryker's dry run stops with SIGABRT on the isolated-vm engine. See DEVP-257.
|
||||
const BLOCKED_PACKAGES = new Set(['@n8n/expression-runtime']);
|
||||
|
||||
const NON_SOURCE = [
|
||||
/\.d\.ts$/,
|
||||
/\.(test|spec)\.[cm]?tsx?$/,
|
||||
/(^|\/)__(tests|mocks)__\//,
|
||||
/\.stories\.[cm]?tsx?$/,
|
||||
/\.config\.[cm]?[jt]s$/,
|
||||
/(^|\/)(dist|node_modules|coverage)\//,
|
||||
/(^|\/)tests?\//,
|
||||
/(^|\/)migrations\//,
|
||||
];
|
||||
|
||||
// This is an exclusion list, not a `src/`-only allowlist. nodes-base and
|
||||
// nodes-langchain keep their code in `nodes/` and `credentials/`. An allowlist
|
||||
// drops the largest mutable surface in the repo.
|
||||
export function isMutableSource(repoRelPath) {
|
||||
if (!/\.[cm]?tsx?$/.test(repoRelPath)) return false;
|
||||
return !NON_SOURCE.some((re) => re.test(repoRelPath));
|
||||
}
|
||||
|
||||
// Merge overlapping and adjacent ranges. Stryker then gets one span per region.
|
||||
export function mergeRanges(ranges) {
|
||||
const out = [];
|
||||
for (const r of [...ranges].sort((a, b) => a.start - b.start)) {
|
||||
const last = out.at(-1);
|
||||
if (last && r.start <= last.end + 1) last.end = Math.max(last.end, r.end);
|
||||
else out.push({ ...r });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Read the new-side line ranges from `git diff -U0` hunk headers.
|
||||
// `@@ -12,0 +13,4 @@` gives `{ start: 13, end: 16 }`. A new-side count of zero
|
||||
// is a deletion. No code stays there to mutate, so this drops it.
|
||||
export function parseHunkRanges(diffText) {
|
||||
const ranges = [];
|
||||
for (const line of diffText.split('\n')) {
|
||||
const m = /^@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/.exec(line);
|
||||
if (!m) continue;
|
||||
const start = Number(m[1]);
|
||||
const count = m[2] === undefined ? 1 : Number(m[2]);
|
||||
if (count === 0) continue;
|
||||
ranges.push({ start, end: start + count - 1 });
|
||||
}
|
||||
return mergeRanges(ranges);
|
||||
}
|
||||
|
||||
// Join the targets with commas. Stryker keeps only the last `--mutate` flag,
|
||||
// thus repeated flags mutate one target and the package pays for a dry run
|
||||
// again for each file.
|
||||
export function formatMutateArg(targets) {
|
||||
return targets.join(',');
|
||||
}
|
||||
|
||||
export function splitRange(target) {
|
||||
const m = /^(.*):(\d+)-(\d+)$/.exec(target);
|
||||
return m ? { file: m[1], range: `${m[2]}-${m[3]}` } : { file: target, range: null };
|
||||
}
|
||||
|
||||
function git(args) {
|
||||
const res = spawnSync('git', args, {
|
||||
cwd: repoRoot,
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 64 * 1024 * 1024,
|
||||
});
|
||||
if (res.error) die(2, `git ${args[0]} failed to start: ${res.error.message}`);
|
||||
return res;
|
||||
}
|
||||
|
||||
function packageNameOf(pkgRoot) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(path.join(pkgRoot, 'package.json'), 'utf8')).name ?? '';
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
// Stryker runs the package's own vitest. If the `test` script runs something
|
||||
// else, the package is skipped, not failed.
|
||||
function packageUsesVitest(pkgRoot) {
|
||||
try {
|
||||
const pkg = JSON.parse(readFileSync(path.join(pkgRoot, 'package.json'), 'utf8'));
|
||||
return /\bvitest\b/.test(pkg.scripts?.test ?? '');
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Why a package cannot be scored, or null when it can. Both planners call this,
|
||||
// so a named target and a --diff target always get the same answer.
|
||||
function ineligibleReason(pkgRoot) {
|
||||
const pkgName = packageNameOf(pkgRoot) || path.relative(repoRoot, pkgRoot);
|
||||
if (BLOCKED_PACKAGES.has(packageNameOf(pkgRoot))) {
|
||||
return `${pkgName} is blocked: the isolated-vm engine crashes Stryker's dry run (DEVP-257)`;
|
||||
}
|
||||
if (!packageUsesVitest(pkgRoot)) return `${pkgName} is not a vitest package`;
|
||||
return null;
|
||||
}
|
||||
|
||||
function planFromDiff(base) {
|
||||
// Diff the merge base against the working tree, not against HEAD. This also
|
||||
// scores uncommitted edits. On a PR checkout the tree is clean, thus the
|
||||
// result is the same as the branch diff.
|
||||
const mergeBase = git(['merge-base', base, 'HEAD']);
|
||||
if (mergeBase.status !== 0) {
|
||||
die(2, `No merge base with '${base}' — is the ref fetched?\n${mergeBase.stderr.trim()}`);
|
||||
}
|
||||
const from = mergeBase.stdout.trim();
|
||||
|
||||
const names = git(['diff', '--name-only', from]);
|
||||
if (names.status !== 0) {
|
||||
die(2, `git diff against '${base}' failed.\n${names.stderr.trim()}`);
|
||||
}
|
||||
|
||||
const usage =
|
||||
'Usage: node scripts/mutation-health/mutate.mjs <file> [--package-dir <repo-rel-path>] [--config <path>]\n' +
|
||||
' - repo-relative file → package is inferred: node scripts/mutation-health/mutate.mjs packages/@n8n/crdt/src/utils.ts\n' +
|
||||
' - package-relative file → pass --package-dir: node scripts/mutation-health/mutate.mjs src/cron.ts --package-dir packages/workflow';
|
||||
const byPackage = new Map();
|
||||
const skipped = [];
|
||||
for (const file of names.stdout
|
||||
.split('\n')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean)) {
|
||||
if (!isMutableSource(file)) continue;
|
||||
const abs = path.resolve(repoRoot, file);
|
||||
if (!existsSync(abs)) continue; // the branch deleted the file
|
||||
|
||||
if (!targetArg) die(2, `Missing mutate target.\n${usage}`);
|
||||
const pkgRoot = findPackageRoot(abs);
|
||||
if (!pkgRoot) {
|
||||
skipped.push([file, 'no enclosing package']);
|
||||
continue;
|
||||
}
|
||||
const reason = ineligibleReason(pkgRoot);
|
||||
if (reason) {
|
||||
skipped.push([file, reason]);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Resolve pkgRoot + the src-relative target, supporting two call styles:
|
||||
// 1. --package-dir given → target is package-relative (or absolute). (the nightly's style)
|
||||
// 2. no --package-dir → target is a repo-relative file; infer the package from it.
|
||||
let pkgRoot;
|
||||
let target;
|
||||
if (packageDirArg) {
|
||||
pkgRoot = path.resolve(repoRoot, packageDirArg);
|
||||
if (!existsSync(pkgRoot)) die(2, `Package dir not found: ${pkgRoot}`);
|
||||
target = path.isAbsolute(targetArg) ? path.relative(pkgRoot, targetArg) : targetArg;
|
||||
} else {
|
||||
const abs = path.resolve(repoRoot, targetArg);
|
||||
if (!existsSync(abs)) die(2, `Target not found: ${abs}\n${usage}`);
|
||||
const found = findPackageRoot(abs);
|
||||
if (!found)
|
||||
die(2, `Could not infer the package for ${targetArg} — pass --package-dir.\n${usage}`);
|
||||
pkgRoot = found;
|
||||
target = path.relative(pkgRoot, abs);
|
||||
const ranges = parseHunkRanges(git(['diff', '-U0', from, '--', file]).stdout);
|
||||
if (ranges.length === 0) continue;
|
||||
|
||||
const rel = path.relative(pkgRoot, abs);
|
||||
const packageDir = path.relative(repoRoot, pkgRoot);
|
||||
const job = byPackage.get(pkgRoot) ?? { pkgRoot, packageDir, targets: [] };
|
||||
for (const r of ranges) job.targets.push(`${rel}:${r.start}-${r.end}`);
|
||||
byPackage.set(pkgRoot, job);
|
||||
}
|
||||
return { jobs: [...byPackage.values()], skipped };
|
||||
}
|
||||
|
||||
if (!target.startsWith('src/') || target.includes('..')) {
|
||||
die(2, `Target must be under the package's src/. Got: ${target}`);
|
||||
// --- running ---
|
||||
|
||||
function resolveConfig(pkgRoot, configArg) {
|
||||
if (configArg) return path.resolve(repoRoot, configArg);
|
||||
const local = path.join(pkgRoot, 'stryker.config.mjs');
|
||||
return existsSync(local) ? local : path.join(__dirname, 'stryker.default.mjs');
|
||||
}
|
||||
|
||||
// Try the package's own copy first, then the root devDep. A package that pins
|
||||
// Stryker thus gets the version it pinned. A miss is a broken checkout, not a
|
||||
// red gate: exit 3 keeps it distinct from a score of zero.
|
||||
function resolveStrykerBin(pkgRoot, packageDir) {
|
||||
for (const from of [path.join(pkgRoot, 'package.json'), import.meta.url]) {
|
||||
try {
|
||||
const resolved = createRequire(from).resolve('@stryker-mutator/core/package.json');
|
||||
return path.join(path.dirname(resolved), 'bin/stryker.js');
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (!existsSync(path.join(pkgRoot, target))) {
|
||||
die(2, `Target not found: ${path.join(pkgRoot, target)}`);
|
||||
}
|
||||
const packageDir = path.relative(repoRoot, pkgRoot);
|
||||
|
||||
// --- resolve the Stryker config: override → package-local → shared default
|
||||
const localConfig = path.join(pkgRoot, 'stryker.config.mjs');
|
||||
const defaultConfig = path.join(__dirname, 'stryker.default.mjs');
|
||||
const configPath = configArg
|
||||
? path.resolve(repoRoot, configArg)
|
||||
: existsSync(localConfig)
|
||||
? localConfig
|
||||
: defaultConfig;
|
||||
|
||||
// --- resolve the Stryker binary from the hoisted store (works for any package)
|
||||
const strykerBin = path.join(
|
||||
path.dirname(require.resolve('@stryker-mutator/core/package.json')),
|
||||
'bin/stryker.js',
|
||||
return die(
|
||||
3,
|
||||
`Could not resolve @stryker-mutator/core from ${packageDir} or the repo root. ` +
|
||||
'Run `pnpm install` — it is a root devDep.',
|
||||
);
|
||||
}
|
||||
|
||||
// Runs use `--inPlace`, because the sandbox copy breaks each package whose
|
||||
// vitest config finds a workspace dependency through a path alias. The alias
|
||||
// does not stay correct in the copy. See the README for the failure.
|
||||
//
|
||||
// Stryker restores the files after a usual exit and after SIGINT, but not after
|
||||
// a crash. In diff mode the files hold uncommitted work, thus `git checkout --`
|
||||
// is not a safe undo. Keep a copy of the bytes and write them back instead.
|
||||
function snapshotFiles(absPaths) {
|
||||
const snap = new Map();
|
||||
for (const p of absPaths) {
|
||||
try {
|
||||
snap.set(p, readFileSync(p));
|
||||
} catch {
|
||||
// The file is unreadable. There is nothing to restore.
|
||||
}
|
||||
}
|
||||
return snap;
|
||||
}
|
||||
|
||||
function restoreFiles(snap) {
|
||||
const restored = [];
|
||||
for (const [p, original] of snap) {
|
||||
try {
|
||||
if (!readFileSync(p).equals(original)) {
|
||||
writeFileSync(p, original);
|
||||
restored.push(path.relative(repoRoot, p));
|
||||
}
|
||||
} catch {
|
||||
// The file is gone. There is nothing to restore.
|
||||
}
|
||||
}
|
||||
return restored;
|
||||
}
|
||||
|
||||
async function runJob({ pkgRoot, packageDir, targets }, { configArg }) {
|
||||
const configPath = resolveConfig(pkgRoot, configArg);
|
||||
const strykerBin = resolveStrykerBin(pkgRoot, packageDir);
|
||||
const mutateArg = formatMutateArg(targets);
|
||||
|
||||
const reportDir = path.join(pkgRoot, 'reports/mutation');
|
||||
const rawJsonPath = path.join(reportDir, 'raw.json');
|
||||
const summaryJsonPath = path.join(reportDir, 'summary.json');
|
||||
await mkdir(reportDir, { recursive: true });
|
||||
|
||||
// Delete the previous reports first. The code below reads "raw.json exists"
|
||||
// as "this run wrote a report". A file from an earlier run makes a crashed
|
||||
// run report the earlier target and its score, and it also hides the
|
||||
// no-covering-tests result. After this, a report on disk is always this run's.
|
||||
await Promise.all([rm(rawJsonPath, { force: true }), rm(summaryJsonPath, { force: true })]);
|
||||
|
||||
process.stderr.write(
|
||||
`Running Stryker on ${packageDir}/${target} (config: ${path.relative(repoRoot, configPath)}, threshold: ${THRESHOLD}%)\n`,
|
||||
`\nRunning Stryker on ${packageDir} — ${targets.length} target(s) ` +
|
||||
`(config: ${path.relative(repoRoot, configPath)}, threshold: ${THRESHOLD}%)\n`,
|
||||
);
|
||||
|
||||
// Capture Stryker's combined output while still forwarding it to the parent
|
||||
// stdio (so CI logs look unchanged). We need the buffer to detect the
|
||||
// "No tests were executed" dry-run case below.
|
||||
const snap = snapshotFiles([
|
||||
...new Set(targets.map((t) => path.join(pkgRoot, splitRange(t).file))),
|
||||
]);
|
||||
|
||||
const strykerOutputChunks = [];
|
||||
const strykerExitCode = await new Promise((resolve) => {
|
||||
const child = spawn(process.execPath, [strykerBin, 'run', configPath, '--mutate', target], {
|
||||
cwd: pkgRoot,
|
||||
stdio: ['inherit', 'pipe', 'pipe'],
|
||||
let child;
|
||||
const onSignal = () => {
|
||||
child?.kill('SIGINT');
|
||||
};
|
||||
process.on('SIGINT', onSignal);
|
||||
process.on('SIGTERM', onSignal);
|
||||
let strykerExitCode;
|
||||
try {
|
||||
strykerExitCode = await new Promise((resolve) => {
|
||||
child = spawn(
|
||||
process.execPath,
|
||||
[strykerBin, 'run', configPath, '--inPlace', '--mutate', mutateArg],
|
||||
{ cwd: pkgRoot, stdio: ['inherit', 'pipe', 'pipe'] },
|
||||
);
|
||||
child.stdout.on('data', (chunk) => {
|
||||
strykerOutputChunks.push(chunk);
|
||||
process.stdout.write(chunk);
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
strykerOutputChunks.push(chunk);
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
child.on('exit', (code) => resolve(code ?? 1));
|
||||
child.on('error', (err) => die(3, `Stryker failed to start: ${err.message}`));
|
||||
});
|
||||
child.stdout.on('data', (chunk) => {
|
||||
strykerOutputChunks.push(chunk);
|
||||
process.stdout.write(chunk);
|
||||
});
|
||||
child.stderr.on('data', (chunk) => {
|
||||
strykerOutputChunks.push(chunk);
|
||||
process.stderr.write(chunk);
|
||||
});
|
||||
child.on('exit', (code) => resolve(code ?? 1));
|
||||
child.on('error', (err) => die(3, `Stryker failed to start: ${err.message}`));
|
||||
});
|
||||
} finally {
|
||||
process.off('SIGINT', onSignal);
|
||||
process.off('SIGTERM', onSignal);
|
||||
const restored = restoreFiles(snap);
|
||||
if (restored.length > 0) {
|
||||
process.stderr.write(
|
||||
`⚠ Stryker left mutants behind; restored ${restored.length} file(s): ${restored.join(', ')}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const strykerOutput = Buffer.concat(strykerOutputChunks).toString('utf8');
|
||||
const target = mutateArg;
|
||||
|
||||
// "No tests were executed" is Stryker's dry-run verdict when nothing in the
|
||||
// test suite covers the picked source file. That's the most informative
|
||||
// mutation result there is — effectively 0%, every mutant no-coverage — so we
|
||||
// record it as a score-0 red ledger row instead of hard-failing the job. The
|
||||
// picker can then advance to the next file the following night. See DEVP-414.
|
||||
const noTestsExecuted = /no tests were executed/i.test(strykerOutput) && !existsSync(rawJsonPath);
|
||||
const outcome = classifyRun({
|
||||
exitCode: strykerExitCode,
|
||||
output: strykerOutput,
|
||||
hasReport: existsSync(rawJsonPath),
|
||||
});
|
||||
|
||||
// A non-zero exit shouldn't discard a report Stryker already wrote — salvage a
|
||||
// partial summary from it below; only die when there's genuinely nothing to keep.
|
||||
const strykerFailedWithoutReport =
|
||||
strykerExitCode !== 0 && !noTestsExecuted && !existsSync(rawJsonPath);
|
||||
if (strykerFailedWithoutReport) {
|
||||
die(
|
||||
3,
|
||||
`Stryker exited with code ${strykerExitCode} without producing ${path.relative(repoRoot, rawJsonPath)}`,
|
||||
if (outcome === 'failed') {
|
||||
process.stderr.write(
|
||||
`✗ ${packageDir}: Stryker exited ${strykerExitCode} without producing ` +
|
||||
`${path.relative(repoRoot, rawJsonPath)}\n`,
|
||||
);
|
||||
return { packageDir, summaryJsonPath, failed: true };
|
||||
}
|
||||
|
||||
if (noTestsExecuted) {
|
||||
// Best-effort mutant count from the instrument phase log line, e.g.
|
||||
// INFO Instrumenter Instrumented 1 source file(s) with 47 mutant(s)
|
||||
// Falls back to 0 if Stryker never got that far.
|
||||
if (outcome === 'no-tests') {
|
||||
const mutantMatch = strykerOutput.match(
|
||||
/Instrumented\s+\d+\s+source file\(s\)\s+with\s+(\d+)\s+mutant/i,
|
||||
);
|
||||
const noCoverage = mutantMatch ? Number(mutantMatch[1]) : 0;
|
||||
const summary = buildNoTestsSummary({
|
||||
threshold: THRESHOLD,
|
||||
target,
|
||||
noCoverage,
|
||||
noCoverage: mutantMatch ? Number(mutantMatch[1]) : 0,
|
||||
generatedAt: new Date().toISOString(),
|
||||
});
|
||||
await writeFile(summaryJsonPath, JSON.stringify(summary, null, 2));
|
||||
process.stderr.write(
|
||||
`\n=== Mutation summary ===\n` +
|
||||
`✗ ${target} 0.00% (no covering tests — recorded as score-0 red)\n` +
|
||||
`Summary written: ${summaryJsonPath}\n`,
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!existsSync(rawJsonPath)) {
|
||||
die(3, `Stryker did not produce ${rawJsonPath}`);
|
||||
return { packageDir, summaryJsonPath, summary, noTests: true };
|
||||
}
|
||||
|
||||
const raw = JSON.parse(await readFile(rawJsonPath, 'utf8'));
|
||||
@@ -410,20 +584,27 @@ async function main() {
|
||||
target,
|
||||
generatedAt: new Date().toISOString(),
|
||||
});
|
||||
|
||||
// Report present but non-zero exit → run was cut short; flag it so consumers
|
||||
// (and the gate below) know the survivor list may be incomplete.
|
||||
const partial = strykerExitCode !== 0;
|
||||
if (partial) summary.partial = true;
|
||||
if (outcome === 'partial') summary.partial = true;
|
||||
|
||||
await writeFile(summaryJsonPath, JSON.stringify(summary, null, 2));
|
||||
return { packageDir, summaryJsonPath, summary };
|
||||
}
|
||||
|
||||
if (partial) {
|
||||
function reportJob({ packageDir, summaryJsonPath, summary, noTests, failed }) {
|
||||
if (failed) return;
|
||||
if (noTests) {
|
||||
process.stderr.write(
|
||||
`\n⚠ Stryker exited with code ${strykerExitCode}; summary built from a partial raw.json — results may be incomplete.\n`,
|
||||
`✗ ${packageDir} ${summary.target} 0.00% (no covering tests — recorded as score-0 red)\n`,
|
||||
);
|
||||
process.stderr.write(` summary: ${path.relative(repoRoot, summaryJsonPath)}\n`);
|
||||
return;
|
||||
}
|
||||
if (summary.partial) {
|
||||
process.stderr.write(
|
||||
`⚠ ${packageDir}: Stryker exited non-zero; summary built from a partial raw.json — ` +
|
||||
'results may be incomplete.\n',
|
||||
);
|
||||
}
|
||||
process.stderr.write('\n=== Mutation summary ===\n');
|
||||
for (const f of summary.files) {
|
||||
const mark = f.thresholdMet ? '✓' : '✗';
|
||||
process.stderr.write(
|
||||
@@ -436,24 +617,136 @@ async function main() {
|
||||
);
|
||||
}
|
||||
for (const ig of f.ignored) {
|
||||
const reason = ig.reason ? ` — ${ig.reason}` : ' — (no reason given)';
|
||||
process.stderr.write(
|
||||
` · ${'ignored'.padEnd(10)} ${ig.mutator.padEnd(22)} ${ig.location}${reason}\n`,
|
||||
` · ${'ignored'.padEnd(10)} ${ig.mutator.padEnd(22)} ${ig.location}` +
|
||||
`${ig.reason ? ` — ${ig.reason}` : ' — (no reason given)'}\n`,
|
||||
);
|
||||
}
|
||||
}
|
||||
// A partial run never passes: mutants it never tested could be survivors, so
|
||||
// reporting PASS would mark the file done with work left undone.
|
||||
const passed = summary.overall.thresholdMet && !summary.partial;
|
||||
const gateState = passed ? 'PASS' : summary.partial ? 'FAIL (partial)' : 'FAIL';
|
||||
const unjustified = summary.overall.counts.survived + summary.overall.counts.noCoverage;
|
||||
process.stderr.write(
|
||||
`\nGate: ${gateState} • threshold: ${THRESHOLD}% • score: ${summary.overall.score.toFixed(2)}% • ` +
|
||||
`unjustified survivors: ${unjustified} • ignored (justified): ${summary.overall.counts.ignored}\n`,
|
||||
);
|
||||
process.stderr.write(`Summary written: ${summaryJsonPath}\n`);
|
||||
process.stderr.write(` summary: ${path.relative(repoRoot, summaryJsonPath)}\n`);
|
||||
}
|
||||
|
||||
process.exit(passed ? 0 : 1);
|
||||
// --- entry point ---
|
||||
|
||||
const USAGE = `Usage:
|
||||
node scripts/mutation-health/mutate.mjs <file>[:<start>-<end>] [--package-dir <dir>] [--config <path>]
|
||||
node scripts/mutation-health/mutate.mjs --diff [--base <ref>] [--config <path>]
|
||||
|
||||
# one file, whole
|
||||
node scripts/mutation-health/mutate.mjs packages/@n8n/crdt/src/utils.ts
|
||||
# one file, only lines 40-75
|
||||
node scripts/mutation-health/mutate.mjs packages/@n8n/crdt/src/utils.ts:40-75
|
||||
# package-relative target
|
||||
node scripts/mutation-health/mutate.mjs src/cron.ts --package-dir packages/workflow
|
||||
# every line this branch changed, batched one Stryker run per package
|
||||
node scripts/mutation-health/mutate.mjs --diff --base origin/master`;
|
||||
|
||||
function planFromTarget(targetArg, packageDirArg) {
|
||||
const { file, range } = splitRange(targetArg);
|
||||
|
||||
let pkgRoot;
|
||||
let rel;
|
||||
if (packageDirArg) {
|
||||
pkgRoot = path.resolve(repoRoot, packageDirArg);
|
||||
if (!existsSync(pkgRoot)) die(2, `Package dir not found: ${pkgRoot}`);
|
||||
rel = path.isAbsolute(file) ? path.relative(pkgRoot, file) : file;
|
||||
} else {
|
||||
const abs = path.resolve(repoRoot, file);
|
||||
if (!existsSync(abs)) die(2, `Target not found: ${abs}\n${USAGE}`);
|
||||
pkgRoot = findPackageRoot(abs);
|
||||
if (!pkgRoot) die(2, `Could not infer the package for ${file} — pass --package-dir.\n${USAGE}`);
|
||||
rel = path.relative(pkgRoot, abs);
|
||||
}
|
||||
|
||||
if (rel.startsWith('..') || path.isAbsolute(rel)) {
|
||||
die(2, `Target must live inside the package. Got: ${rel}`);
|
||||
}
|
||||
if (!existsSync(path.join(pkgRoot, rel))) {
|
||||
die(2, `Target not found: ${path.join(pkgRoot, rel)}`);
|
||||
}
|
||||
if (!isMutableSource(rel)) {
|
||||
die(2, `Not a mutable source file (test/declaration/config/build output): ${rel}`);
|
||||
}
|
||||
// --diff skips an ineligible package. A named target must refuse for the same
|
||||
// reason, or it starts a run that is known to crash.
|
||||
const reason = ineligibleReason(pkgRoot);
|
||||
if (reason) die(2, `Cannot mutate ${rel}: ${reason}`);
|
||||
|
||||
return {
|
||||
pkgRoot,
|
||||
packageDir: path.relative(repoRoot, pkgRoot),
|
||||
targets: [range ? `${rel}:${range}` : rel],
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const argv = process.argv.slice(2);
|
||||
let packageDirArg;
|
||||
let configArg;
|
||||
let targetArg;
|
||||
let baseArg = 'origin/master';
|
||||
let diffMode = false;
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (a === '--package-dir') packageDirArg = argv[++i];
|
||||
else if (a === '--config') configArg = argv[++i];
|
||||
else if (a === '--base') baseArg = argv[++i];
|
||||
else if (a === '--diff') diffMode = true;
|
||||
else if (!a.startsWith('--') && targetArg === undefined) targetArg = a;
|
||||
}
|
||||
|
||||
if (diffMode && targetArg) die(2, `--diff takes no positional target.\n${USAGE}`);
|
||||
if (!diffMode && !targetArg) die(2, `Missing mutate target.\n${USAGE}`);
|
||||
|
||||
let jobs;
|
||||
let skipped = [];
|
||||
if (diffMode) {
|
||||
({ jobs, skipped } = planFromDiff(baseArg));
|
||||
for (const [file, why] of skipped) process.stderr.write(` skipped ${file} — ${why}\n`);
|
||||
if (jobs.length === 0) {
|
||||
process.stderr.write(`\nNothing mutable changed vs ${baseArg}.\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
const files = jobs.reduce((n, j) => n + j.targets.length, 0);
|
||||
process.stderr.write(
|
||||
`\nMutating ${files} changed range(s) across ${jobs.length} package(s) vs ${baseArg}.\n`,
|
||||
);
|
||||
} else {
|
||||
jobs = [planFromTarget(targetArg, packageDirArg)];
|
||||
}
|
||||
|
||||
const results = [];
|
||||
for (const job of jobs) {
|
||||
results.push(await runJob(job, { configArg }));
|
||||
}
|
||||
|
||||
process.stderr.write('\n=== Mutation summary ===\n');
|
||||
for (const r of results) reportJob(r);
|
||||
|
||||
if (results.some((r) => r.failed)) {
|
||||
process.stderr.write('\nGate: ERROR — at least one Stryker run produced no report.\n');
|
||||
process.exit(3);
|
||||
}
|
||||
|
||||
// A partial run never passes. The mutants it did not test can be survivors.
|
||||
const overall = results.reduce(
|
||||
(acc, r) => {
|
||||
const c = r.summary.overall.counts;
|
||||
acc.survived += c.survived + c.noCoverage;
|
||||
acc.ignored += c.ignored;
|
||||
acc.passed &&= r.summary.overall.thresholdMet && !r.summary.partial;
|
||||
acc.partial ||= Boolean(r.summary.partial);
|
||||
return acc;
|
||||
},
|
||||
{ survived: 0, ignored: 0, passed: true, partial: false },
|
||||
);
|
||||
|
||||
const gateState = overall.passed ? 'PASS' : overall.partial ? 'FAIL (partial)' : 'FAIL';
|
||||
process.stderr.write(
|
||||
`\nGate: ${gateState} • threshold: ${THRESHOLD}% • ` +
|
||||
`unjustified survivors: ${overall.survived} • ignored (justified): ${overall.ignored}\n`,
|
||||
);
|
||||
process.exit(overall.passed ? 0 : 1);
|
||||
}
|
||||
|
||||
const isCli = import.meta.url === `file://${process.argv[1]}`;
|
||||
|
||||
@@ -2,17 +2,17 @@ import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
coverageFromCounts,
|
||||
buildSummary,
|
||||
buildNoTestsSummary,
|
||||
buildSummary,
|
||||
classifyRun,
|
||||
coverageFromCounts,
|
||||
formatMutateArg,
|
||||
isMutableSource,
|
||||
mergeRanges,
|
||||
parseHunkRanges,
|
||||
scoreFromCounts,
|
||||
splitRange,
|
||||
} from './mutate.mjs';
|
||||
import {
|
||||
buildPayload,
|
||||
coverageForLedger,
|
||||
makeChurnFor,
|
||||
makeFixDensityFor,
|
||||
} from './emit-payload.mjs';
|
||||
|
||||
// A minimal Stryker Mutation Testing Elements report for one source file. Mix
|
||||
// of statuses so coverage (anything that ran / ran + no-coverage) is a genuine
|
||||
@@ -162,218 +162,189 @@ describe('buildNoTestsSummary (no covering tests)', () => {
|
||||
});
|
||||
});
|
||||
|
||||
// PR-gate contract (DEVP-496):
|
||||
// "mutate.mjs fixture run produces ledger row with coverage field in [0,1]"
|
||||
describe('coverage writeback to the ledger row (DEVP-496 PR gate)', () => {
|
||||
it('a fixture run produces a ledger row whose coverage is in [0,1]', () => {
|
||||
const summary = buildSummary(RAW_FIXTURE, RUN_META);
|
||||
const { ledger } = buildPayload(summary, {
|
||||
pkg: 'n8n-workflow',
|
||||
sha: 'deadbeef',
|
||||
pkgRelToRepo: 'packages/workflow',
|
||||
});
|
||||
describe('classifyRun', () => {
|
||||
const DONE = 'Instrumented 1 source file(s) with 8 mutant(s)';
|
||||
const NO_TESTS = 'ERROR Stryker No tests were executed. Stryker will exit prematurely.';
|
||||
|
||||
assert.equal(ledger.length, 1);
|
||||
const row = ledger[0];
|
||||
assert.ok(Object.hasOwn(row, 'coverage'), 'ledger row carries a coverage field');
|
||||
assert.ok(isFractionInUnitInterval(row.coverage), `coverage ${row.coverage} must be in [0,1]`);
|
||||
assert.equal(row.coverage, 0.75);
|
||||
assert.equal(row.source_file_path, 'packages/workflow/src/cron.ts');
|
||||
it('is complete when the run wrote a report and exited zero', () => {
|
||||
assert.equal(classifyRun({ exitCode: 0, output: DONE, hasReport: true }), 'complete');
|
||||
});
|
||||
|
||||
it('forwards coverage onto the event row dimensions too', () => {
|
||||
const summary = buildSummary(RAW_FIXTURE, RUN_META);
|
||||
const { events } = buildPayload(summary, {
|
||||
pkg: 'n8n-workflow',
|
||||
sha: 'deadbeef',
|
||||
pkgRelToRepo: 'packages/workflow',
|
||||
});
|
||||
assert.ok(isFractionInUnitInterval(events[0].dimensions.coverage));
|
||||
it('is partial when the run wrote a report and then exited non-zero', () => {
|
||||
assert.equal(classifyRun({ exitCode: 1, output: DONE, hasReport: true }), 'partial');
|
||||
});
|
||||
|
||||
it('clamps an out-of-range summary coverage into [0,1]', () => {
|
||||
assert.equal(coverageForLedger({ coverage: 1.4, counts: {} }), 1);
|
||||
assert.equal(coverageForLedger({ coverage: -0.2, counts: {} }), 0);
|
||||
it('is no-tests when nothing covers the target', () => {
|
||||
assert.equal(classifyRun({ exitCode: 1, output: NO_TESTS, hasReport: false }), 'no-tests');
|
||||
});
|
||||
|
||||
it('falls back to deriving coverage from counts for pre-writeback summaries', () => {
|
||||
// No `coverage` field (an older summary) → derived from the mutant census.
|
||||
const row = coverageForLedger({
|
||||
counts: { killed: 3, survived: 0, timeout: 0, noCoverage: 1, runtimeError: 0 },
|
||||
});
|
||||
assert.equal(row, 0.75);
|
||||
assert.ok(isFractionInUnitInterval(row));
|
||||
it('is failed when the run produced no report', () => {
|
||||
assert.equal(classifyRun({ exitCode: 1, output: 'SIGABRT', hasReport: false }), 'failed');
|
||||
});
|
||||
|
||||
// The caller deletes the previous reports before each run. Without that, a
|
||||
// crashed run finds the earlier report and is classified `partial`, so it
|
||||
// reports the earlier target and its score instead of failing.
|
||||
it('trusts hasReport as this run only — a report plus a crash is partial, never failed', () => {
|
||||
assert.equal(classifyRun({ exitCode: 3, output: 'SIGABRT', hasReport: true }), 'partial');
|
||||
});
|
||||
|
||||
// Same trap for the no-tests path: a leftover report used to suppress it, and
|
||||
// a genuine score-0 red was reported as the earlier run's passing score.
|
||||
it('still detects no-tests when the run crashed without a report', () => {
|
||||
assert.equal(classifyRun({ exitCode: 3, output: NO_TESTS, hasReport: false }), 'no-tests');
|
||||
});
|
||||
});
|
||||
|
||||
// DEVP-546: the ledger row also carries a git-derived `churn` count so the
|
||||
// global picker can rank by the value formula's churn term.
|
||||
describe('churn writeback to the ledger row (DEVP-546)', () => {
|
||||
it('forwards the per-file churn count and fix-density onto the ledger and event rows', () => {
|
||||
const summary = buildSummary(RAW_FIXTURE, RUN_META);
|
||||
const churnFor = (p) => (p === 'packages/workflow/src/cron.ts' ? 7 : null);
|
||||
const fixDensityFor = (p) => (p === 'packages/workflow/src/cron.ts' ? 2.5 : null);
|
||||
const { ledger, events } = buildPayload(summary, {
|
||||
pkg: 'n8n-workflow',
|
||||
sha: 'deadbeef',
|
||||
pkgRelToRepo: 'packages/workflow',
|
||||
churnFor,
|
||||
fixDensityFor,
|
||||
});
|
||||
|
||||
assert.ok(Object.hasOwn(ledger[0], 'churn'), 'ledger row carries a churn field');
|
||||
assert.equal(ledger[0].churn, 7);
|
||||
assert.equal(events[0].dimensions.churn, 7);
|
||||
assert.ok(Object.hasOwn(ledger[0], 'fix_density'), 'ledger row carries a fix_density field');
|
||||
assert.equal(ledger[0].fix_density, 2.5);
|
||||
assert.equal(events[0].dimensions.fix_density, 2.5);
|
||||
describe('isMutableSource', () => {
|
||||
it('accepts product source wherever a package keeps it', () => {
|
||||
assert.ok(isMutableSource('packages/workflow/src/cron.ts'));
|
||||
// nodes-base has no src/. An allowlist drops the largest surface in the repo.
|
||||
assert.ok(isMutableSource('packages/nodes-base/nodes/Slack/Slack.node.ts'));
|
||||
assert.ok(isMutableSource('packages/nodes-base/credentials/SlackApi.credentials.ts'));
|
||||
assert.ok(isMutableSource('packages/frontend/editor-ui/src/stores/ui.store.ts'));
|
||||
// `[cm]?` in the extension test is there for the ESM/CJS variants.
|
||||
assert.ok(isMutableSource('packages/@n8n/db/src/index.mts'));
|
||||
assert.ok(isMutableSource('packages/@n8n/db/src/index.cts'));
|
||||
});
|
||||
|
||||
it('defaults churn and fix_density to null when no signal source is wired in', () => {
|
||||
const summary = buildSummary(RAW_FIXTURE, RUN_META);
|
||||
const { ledger } = buildPayload(summary, {
|
||||
pkg: 'n8n-workflow',
|
||||
sha: 'deadbeef',
|
||||
pkgRelToRepo: 'packages/workflow',
|
||||
});
|
||||
it('rejects tests, declarations, configs and build output', () => {
|
||||
assert.equal(isMutableSource('packages/workflow/src/cron.test.ts'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/src/cron.spec.ts'), false);
|
||||
// The ESM/CJS variants are accepted as source, so they have to be
|
||||
// excluded as tests too.
|
||||
assert.equal(isMutableSource('packages/workflow/src/cron.test.mts'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/src/__tests__/cron.ts'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/src/__mocks__/cron.ts'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/src/types.d.ts'), false);
|
||||
assert.equal(isMutableSource('packages/cli/vitest.config.ts'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/dist/cron.js'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/test/helper.ts'), false);
|
||||
assert.equal(isMutableSource('packages/@n8n/db/src/migrations/sqlite/x.ts'), false);
|
||||
assert.equal(isMutableSource('packages/design-system/src/Button.stories.ts'), false);
|
||||
// The extension test is anchored: `.ts` has to end the path, not merely
|
||||
// appear in it. Committed snapshots sit next to their source and would
|
||||
// otherwise be handed to Stryker as mutable TypeScript.
|
||||
assert.equal(isMutableSource('packages/cli/src/__snapshots__/foo.test.ts.snap'), false);
|
||||
});
|
||||
|
||||
assert.ok(Object.hasOwn(ledger[0], 'churn'), 'ledger row carries a churn field');
|
||||
assert.equal(ledger[0].churn, null);
|
||||
assert.ok(Object.hasOwn(ledger[0], 'fix_density'), 'ledger row carries a fix_density field');
|
||||
assert.equal(ledger[0].fix_density, null);
|
||||
// .vue stays out. Each SFC package crashed Stryker's mutate step in the
|
||||
// 2026-06 sweep, and the component layer gives little value.
|
||||
it('rejects everything that is not TypeScript', () => {
|
||||
assert.equal(isMutableSource('packages/frontend/editor-ui/src/App.vue'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/src/cron.js'), false);
|
||||
assert.equal(isMutableSource('README.md'), false);
|
||||
assert.equal(isMutableSource('packages/workflow/package.json'), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('makeChurnFor (git-derived churn)', () => {
|
||||
// Stub git: shallow probe answers `false`, every rev-list answers `count`.
|
||||
function stubGit({ shallow = 'false', count } = {}) {
|
||||
return (args) => {
|
||||
if (args.includes('--is-shallow-repository')) return `${shallow}\n`;
|
||||
if (typeof count === 'function') return count(args);
|
||||
return `${count}\n`;
|
||||
};
|
||||
}
|
||||
|
||||
it('counts commits touching a file within the window', () => {
|
||||
const churnFor = makeChurnFor({ runGit: stubGit({ count: 7 }) });
|
||||
assert.equal(churnFor('packages/workflow/src/cron.ts'), 7);
|
||||
});
|
||||
|
||||
it('passes the configured window through to git rev-list', () => {
|
||||
const seen = [];
|
||||
const runGit = (args) => {
|
||||
seen.push(args);
|
||||
return args.includes('--is-shallow-repository') ? 'false\n' : '3\n';
|
||||
};
|
||||
const churnFor = makeChurnFor({ since: '30 days', runGit });
|
||||
churnFor('a/b.ts');
|
||||
const revList = seen.find((a) => a[0] === 'rev-list');
|
||||
assert.ok(revList.includes('--since=30 days'));
|
||||
assert.ok(revList.includes('a/b.ts'));
|
||||
});
|
||||
|
||||
it('returns null on a shallow clone (truncated history would undercount)', () => {
|
||||
const churnFor = makeChurnFor({ runGit: stubGit({ shallow: 'true', count: 999 }) });
|
||||
assert.equal(churnFor('any/file.ts'), null);
|
||||
});
|
||||
|
||||
it('returns null when git fails for a file', () => {
|
||||
const runGit = (args) => {
|
||||
if (args.includes('--is-shallow-repository')) return 'false\n';
|
||||
throw new Error('git boom');
|
||||
};
|
||||
assert.equal(makeChurnFor({ runGit })('x.ts'), null);
|
||||
});
|
||||
|
||||
it('returns null when the count is not a finite number', () => {
|
||||
const churnFor = makeChurnFor({ runGit: stubGit({ count: 'not-a-number' }) });
|
||||
assert.equal(churnFor('x.ts'), null);
|
||||
});
|
||||
|
||||
it('treats a non-git directory (probe throws) as unknown churn', () => {
|
||||
const runGit = () => {
|
||||
throw new Error('not a git repository');
|
||||
};
|
||||
assert.equal(makeChurnFor({ runGit })('x.ts'), null);
|
||||
});
|
||||
});
|
||||
|
||||
describe('makeFixDensityFor (git-derived fix-density)', () => {
|
||||
const NOW = 1_700_000_000; // fixed reference time (unix seconds) for determinism
|
||||
|
||||
// Stub git: shallow probe answers `shallow`, the log read answers `log`
|
||||
// (synthetic `git log --numstat` output in signals.mjs's GIT_LOG_FORMAT).
|
||||
function stubGit({ shallow = 'false', log = '' } = {}) {
|
||||
return (args) => (args.includes('--is-shallow-repository') ? `${shallow}\n` : log);
|
||||
}
|
||||
|
||||
// A fix commit and a non-fix commit, both touching the same file at `NOW`.
|
||||
const FIX_AT_NOW = [
|
||||
`COMMIT abc123 ${NOW} fix(core): patch a bug`,
|
||||
'10\t0\tpackages/core/src/hot.ts',
|
||||
'',
|
||||
`COMMIT def456 ${NOW} feat(core): add a feature`,
|
||||
'5\t0\tpackages/core/src/hot.ts',
|
||||
].join('\n');
|
||||
|
||||
it('sums delta-weighted contributions of fix commits touching the file', () => {
|
||||
const fixDensityFor = makeFixDensityFor({ now: NOW, runGit: stubGit({ log: FIX_AT_NOW }) });
|
||||
// Only the fix commit counts; age 0 → weight 1 → 10 lines changed.
|
||||
assert.equal(fixDensityFor('packages/core/src/hot.ts'), 10);
|
||||
});
|
||||
|
||||
it('scores a file with no fix commits as 0 (known: no fixes), not null', () => {
|
||||
const fixDensityFor = makeFixDensityFor({ now: NOW, runGit: stubGit({ log: FIX_AT_NOW }) });
|
||||
assert.equal(fixDensityFor('packages/core/src/cold.ts'), 0);
|
||||
});
|
||||
|
||||
it('decays older fixes by the configured half-life', () => {
|
||||
const oneHalfLifeAgo = NOW - 90 * 86_400;
|
||||
const log = [
|
||||
`COMMIT old111 ${oneHalfLifeAgo} fix: older patch`,
|
||||
'8\t0\tpackages/core/src/hot.ts',
|
||||
describe('parseHunkRanges', () => {
|
||||
it('reads new-side ranges out of `git diff -U0` headers', () => {
|
||||
const diff = [
|
||||
'diff --git a/src/cron.ts b/src/cron.ts',
|
||||
'--- a/src/cron.ts',
|
||||
'+++ b/src/cron.ts',
|
||||
'@@ -12,0 +13,4 @@ export function tick() {',
|
||||
'+const a = 1;',
|
||||
'@@ -40,2 +44,2 @@',
|
||||
'+const b = 2;',
|
||||
// Counts run past one digit on both sides for any hunk of ten lines
|
||||
// or more, which is most of them.
|
||||
'@@ -80,12 +90,14 @@',
|
||||
'+const c = 3;',
|
||||
].join('\n');
|
||||
const fixDensityFor = makeFixDensityFor({
|
||||
now: NOW,
|
||||
halfLifeDays: 90,
|
||||
runGit: stubGit({ log }),
|
||||
});
|
||||
// One half-life old → weight 0.5 → 8 * 0.5 = 4.
|
||||
assert.equal(fixDensityFor('packages/core/src/hot.ts'), 4);
|
||||
assert.deepEqual(parseHunkRanges(diff), [
|
||||
{ start: 13, end: 16 },
|
||||
{ start: 44, end: 45 },
|
||||
{ start: 90, end: 103 },
|
||||
]);
|
||||
});
|
||||
|
||||
it('passes the configured window + pathspec through to git log', () => {
|
||||
const seen = [];
|
||||
const runGit = (args) => {
|
||||
seen.push(args);
|
||||
return args.includes('--is-shallow-repository') ? 'false\n' : '';
|
||||
};
|
||||
makeFixDensityFor({ since: '6 months', pathspec: 'packages/core', now: NOW, runGit })('x.ts');
|
||||
const logArgs = seen.find((a) => a[0] === 'log');
|
||||
assert.ok(logArgs.includes('--since=6 months'));
|
||||
assert.ok(logArgs.includes('packages/core'));
|
||||
// `git diff` of a file that itself talks about diffs (a patch fixture, this
|
||||
// very test file) carries hunk-header text inside `+`/`-` content lines.
|
||||
// Only a header at the start of a line is a header.
|
||||
it('ignores hunk-header text that appears inside a content line', () => {
|
||||
const diff = ['@@ -1,0 +5,1 @@', "+const H = '@@ -1,2 +300,4 @@';"].join('\n');
|
||||
assert.deepEqual(parseHunkRanges(diff), [{ start: 5, end: 5 }]);
|
||||
});
|
||||
|
||||
it('returns null on a shallow clone (truncated history would undercount)', () => {
|
||||
const fixDensityFor = makeFixDensityFor({
|
||||
now: NOW,
|
||||
runGit: stubGit({ shallow: 'true', log: FIX_AT_NOW }),
|
||||
});
|
||||
assert.equal(fixDensityFor('packages/core/src/hot.ts'), null);
|
||||
it('treats a header with no new-side count as a single line', () => {
|
||||
assert.deepEqual(parseHunkRanges('@@ -5 +7 @@'), [{ start: 7, end: 7 }]);
|
||||
});
|
||||
|
||||
it('returns null when git fails', () => {
|
||||
const runGit = (args) => {
|
||||
if (args.includes('--is-shallow-repository')) return 'false\n';
|
||||
throw new Error('git boom');
|
||||
};
|
||||
assert.equal(makeFixDensityFor({ now: NOW, runGit })('x.ts'), null);
|
||||
it('drops pure deletions — nothing survives there to mutate', () => {
|
||||
assert.deepEqual(parseHunkRanges('@@ -10,4 +9,0 @@'), []);
|
||||
});
|
||||
|
||||
it('treats a non-git directory (probe throws) as unknown fix-density', () => {
|
||||
const runGit = () => {
|
||||
throw new Error('not a git repository');
|
||||
};
|
||||
assert.equal(makeFixDensityFor({ now: NOW, runGit })('x.ts'), null);
|
||||
it('returns nothing for a diff with no hunks', () => {
|
||||
assert.deepEqual(parseHunkRanges(''), []);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeRanges', () => {
|
||||
it('merges overlapping ranges', () => {
|
||||
assert.deepEqual(
|
||||
mergeRanges([
|
||||
{ start: 1, end: 5 },
|
||||
{ start: 3, end: 9 },
|
||||
]),
|
||||
[{ start: 1, end: 9 }],
|
||||
);
|
||||
});
|
||||
|
||||
it('merges adjacent ranges so Stryker gets one span per region', () => {
|
||||
assert.deepEqual(
|
||||
mergeRanges([
|
||||
{ start: 1, end: 4 },
|
||||
{ start: 5, end: 8 },
|
||||
]),
|
||||
[{ start: 1, end: 8 }],
|
||||
);
|
||||
});
|
||||
|
||||
it('keeps ranges with a real gap apart, and sorts them', () => {
|
||||
assert.deepEqual(
|
||||
mergeRanges([
|
||||
{ start: 20, end: 22 },
|
||||
{ start: 1, end: 4 },
|
||||
]),
|
||||
[
|
||||
{ start: 1, end: 4 },
|
||||
{ start: 20, end: 22 },
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
it('leaves a fully-contained range absorbed', () => {
|
||||
assert.deepEqual(
|
||||
mergeRanges([
|
||||
{ start: 1, end: 20 },
|
||||
{ start: 5, end: 9 },
|
||||
]),
|
||||
[{ start: 1, end: 20 }],
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('formatMutateArg', () => {
|
||||
it('comma-joins every target into one flag value', () => {
|
||||
assert.equal(
|
||||
formatMutateArg(['src/a.ts:1-4', 'src/a.ts:20-22', 'src/b.ts']),
|
||||
'src/a.ts:1-4,src/a.ts:20-22,src/b.ts',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('splitRange', () => {
|
||||
it('splits a trailing line range off the path', () => {
|
||||
assert.deepEqual(splitRange('src/cron.ts:13-16'), { file: 'src/cron.ts', range: '13-16' });
|
||||
});
|
||||
|
||||
it('leaves a bare path alone', () => {
|
||||
assert.deepEqual(splitRange('src/cron.ts'), { file: 'src/cron.ts', range: null });
|
||||
});
|
||||
|
||||
it('does not mistake a Windows drive letter or a colon in a dirname for a range', () => {
|
||||
assert.deepEqual(splitRange('src/a:b/cron.ts'), { file: 'src/a:b/cron.ts', range: null });
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,625 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Walk a package's source tree (per-package mode) or every vitest-eligible
|
||||
* package's source tree (global mode), merge with the live BQ ledger snapshot,
|
||||
* return the next file(s) to mutate.
|
||||
*
|
||||
* Files present in src/ but absent from the live ledger are synthesised as
|
||||
* status='new'. No separate seed step needed — the ledger fills in
|
||||
* organically as files get scored.
|
||||
*
|
||||
* Stored statuses (from BQ): new | red | green
|
||||
* Effective statuses (computed at pick time): new | red | stale | green
|
||||
*
|
||||
* Picker priority: new → red → stale → skip green
|
||||
*
|
||||
* Per-package mode tiebreaks within each bucket:
|
||||
* - new: alphabetical by source_file_path
|
||||
* - red: lowest score first (focus on weakest tests)
|
||||
* - stale: oldest last_checked_at first (natural cycling)
|
||||
*
|
||||
* Global mode tiebreaks within each bucket: highest value first, where
|
||||
* value = w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage)
|
||||
* — churn and fix-density come from `signals.mjs`, coverage from an optional
|
||||
* input file. Path used as final lexical tiebreak for determinism.
|
||||
*
|
||||
* "Stale" is an in-memory promotion of green rows older than
|
||||
* STALE_AFTER_WEEKS (default 4). Not stored.
|
||||
*
|
||||
* Inputs (per-package mode):
|
||||
* --package-dir <path> Required. Repo-relative path to the package, e.g. packages/workflow
|
||||
* --ledger-file <path> Required. Live ledger JSON: { "ledger": [ ... ] }
|
||||
* --mode <baseline|coverage> Optional. Restrict the picker to one bucket.
|
||||
* --stale-after-weeks <n> Optional. Default 4.
|
||||
*
|
||||
* Inputs (global mode):
|
||||
* --global Required to enter global mode.
|
||||
* --ledger-file <path> Required. Live read-all ledger JSON (rows for every package).
|
||||
* --signals-file <path> Optional. JSON from `signals.mjs gatherSignals`.
|
||||
* --coverage-file <path> Optional. JSON map { "<repo-rel-path>": 0..1 } of line-coverage.
|
||||
* --top-n <n> Optional. Default 1. How many top-ranked rows to emit.
|
||||
* --block <list> Optional. Comma-separated package names to exclude from the walk.
|
||||
* --w-churn / --w-fix-density / --w-coverage <n> Optional. Per-signal weights for value formula.
|
||||
* --mode <baseline|coverage> Optional. Restrict picker to one bucket (same as per-package).
|
||||
* --stale-after-weeks <n> Optional. Default 4.
|
||||
*
|
||||
* Output (stdout):
|
||||
* per-package mode →
|
||||
* { picked: { source_file_path, package, prior_status, effective_status } }
|
||||
* OR { picked: null, reason: "all-green" | "empty-source-tree"
|
||||
* | "no-new-files" | "nothing-below-threshold" }
|
||||
* global mode →
|
||||
* { picked: [ { source_file_path, package, prior_status, effective_status, value }, ... ] }
|
||||
* OR { picked: [], reason: "all-green" | "empty-source-tree"
|
||||
* | "no-new-files" | "nothing-below-threshold" }
|
||||
*
|
||||
* Exit codes:
|
||||
* 0 — picked a row OR nothing to do (with picked: null / [] sentinel)
|
||||
* 2 — usage / config error
|
||||
*/
|
||||
|
||||
import { readdir, readFile } from 'node:fs/promises';
|
||||
import { existsSync, readFileSync } from 'node:fs';
|
||||
import { execFileSync } from 'node:child_process';
|
||||
import path from 'node:path';
|
||||
|
||||
import { readLedger } from './ledger.mjs';
|
||||
|
||||
function die(code, msg) {
|
||||
process.stderr.write(`${msg}\n`);
|
||||
process.exit(code);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (!a.startsWith('--')) continue;
|
||||
const key = a.slice(2);
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith('--')) {
|
||||
out[key] = true;
|
||||
} else {
|
||||
out[key] = next;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// Files with no useful mutation surface: barrels, declarations, type-only modules.
|
||||
// Matched against either the full basename (`types.ts`) or the trailing
|
||||
// dot-segment (`foo.types.ts` → `types`), so dotted-suffix declaration files
|
||||
// are caught the same as their plain-named counterparts. `methods` covers
|
||||
// NativeDoc `*.methods.ts` descriptor files; `schemas` covers pure Zod
|
||||
// declaration files; `message-event-bus` is an exact-basename entry for a
|
||||
// bulk enum + interfaces + Zod module with no function logic.
|
||||
const LOW_VALUE_BASENAMES = new Set([
|
||||
'interfaces',
|
||||
'index',
|
||||
'constants',
|
||||
'types',
|
||||
'methods',
|
||||
'schemas',
|
||||
'message-event-bus',
|
||||
]);
|
||||
|
||||
// Files with fewer than this many non-blank, non-import lines have so little
|
||||
// surface area they're not worth mutating — e.g. trivial error subclasses with
|
||||
// empty bodies or one hardcoded super() call. This catches what an exact-name
|
||||
// list can't (`error` can't be skipped wholesale because files like
|
||||
// `workflow-activation.error.ts` have real branching logic).
|
||||
const MIN_MEANINGFUL_LINES = 15;
|
||||
|
||||
// Directories whose contents are tests/fixtures/mocks, not production code.
|
||||
// Pruned in `walkSources` so we don't descend; `isMutationWorthy` re-checks as a
|
||||
// safety net for packages that co-locate tests as siblings (e.g. `foo.test.ts`).
|
||||
const NON_SOURCE_DIRS = new Set(['__tests__', '__mocks__', 'fixtures']);
|
||||
|
||||
// The vitest-eligible mutation-tracked packages. Single source of truth for
|
||||
// global mode: the picker walks every entry here unless overridden via
|
||||
// --block. The nightly workflow's `setup` job builds its matrix from this
|
||||
// export (DEVP-497), so adding a package = one-line append here, nothing else.
|
||||
export const ELIGIBLE_PACKAGES = [
|
||||
{ name: 'n8n-workflow', dir: 'packages/workflow' },
|
||||
{ name: '@n8n/crdt', dir: 'packages/@n8n/crdt' },
|
||||
{ name: '@n8n/decorators', dir: 'packages/@n8n/decorators' },
|
||||
{ name: '@n8n/expression-runtime', dir: 'packages/@n8n/expression-runtime' },
|
||||
{ name: 'n8n', dir: 'packages/cli' },
|
||||
];
|
||||
|
||||
export function isEligible(pkgName) {
|
||||
return ELIGIBLE_PACKAGES.some((p) => p.name === pkgName);
|
||||
}
|
||||
|
||||
export const DEFAULT_WEIGHTS = Object.freeze({ churn: 1, fixDensity: 1, coverage: 1 });
|
||||
|
||||
/**
|
||||
* Value formula: w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage).
|
||||
*
|
||||
* Missing signals contribute 0 (no penalty, no boost). Missing coverage is
|
||||
* treated as 0 so the (1 − coverage) term becomes 1 — unknown coverage =
|
||||
* worst case = highest urge to score, matching the picker's bias toward
|
||||
* surfacing untracked files.
|
||||
*/
|
||||
export function computeValue(
|
||||
{ churn = 0, fixDensity = 0, coverage = 0 } = {},
|
||||
weights = DEFAULT_WEIGHTS,
|
||||
) {
|
||||
const churnVal = Number.isFinite(Number(churn)) ? Number(churn) : 0;
|
||||
const fdVal = Number.isFinite(Number(fixDensity)) ? Number(fixDensity) : 0;
|
||||
const covRaw = Number(coverage);
|
||||
const covVal = Number.isFinite(covRaw) ? Math.max(0, Math.min(1, covRaw)) : 0;
|
||||
return (
|
||||
(weights.churn ?? 0) * churnVal +
|
||||
(weights.fixDensity ?? 0) * fdVal +
|
||||
(weights.coverage ?? 0) * (1 - covVal)
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* `signals` shape matches `gatherSignals`'s JSON output: churn[path] = { commits, linesChanged }
|
||||
* (or a bare number from caller transforms), fixDensity[path] = number.
|
||||
* Returns the per-row `{ churn, fixDensity, coverage }` triple used by `computeValue`.
|
||||
*/
|
||||
export function extractSignals(row, { signals = {}, coverage = {} } = {}) {
|
||||
const churnEntry = signals.churn?.[row.source_file_path];
|
||||
const churnVal =
|
||||
typeof churnEntry === 'number'
|
||||
? churnEntry
|
||||
: typeof churnEntry?.commits === 'number'
|
||||
? churnEntry.commits
|
||||
: 0;
|
||||
const fixDensity = signals.fixDensity?.[row.source_file_path] ?? 0;
|
||||
const cov = coverage[row.source_file_path];
|
||||
return {
|
||||
churn: churnVal,
|
||||
fixDensity,
|
||||
coverage: typeof cov === 'number' ? cov : 0,
|
||||
};
|
||||
}
|
||||
|
||||
function countMeaningfulLines(content) {
|
||||
let count = 0;
|
||||
for (const raw of content.split('\n')) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
if (line.startsWith('import ')) continue;
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
export function isMutationWorthy(absPath, { read = readFileSync } = {}) {
|
||||
if (absPath.endsWith('.d.ts')) return false;
|
||||
if (/\.(test|spec)\.ts$/.test(absPath)) return false;
|
||||
if (absPath.includes(`${path.sep}__tests__${path.sep}`)) return false;
|
||||
if (absPath.includes(`${path.sep}__mocks__${path.sep}`)) return false;
|
||||
const base = path.basename(absPath, '.ts');
|
||||
const lastSegment = base.split('.').at(-1);
|
||||
if (LOW_VALUE_BASENAMES.has(base) || LOW_VALUE_BASENAMES.has(lastSegment)) return false;
|
||||
if (countMeaningfulLines(read(absPath, 'utf8')) < MIN_MEANINGFUL_LINES) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function walkSources(dir) {
|
||||
const entries = await readdir(dir, { withFileTypes: true });
|
||||
const out = [];
|
||||
for (const e of entries) {
|
||||
const full = path.join(dir, e.name);
|
||||
if (e.isDirectory()) {
|
||||
if (NON_SOURCE_DIRS.has(e.name)) continue;
|
||||
out.push(...(await walkSources(full)));
|
||||
} else if (e.isFile() && e.name.endsWith('.ts')) {
|
||||
out.push(full);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
const PRIORITY = { new: 0, red: 1, stale: 2, green: 3 };
|
||||
|
||||
const MODE_BUCKETS = {
|
||||
baseline: new Set(['new']),
|
||||
coverage: new Set(['red', 'stale']),
|
||||
};
|
||||
|
||||
export function computeEffectiveStatus(row, { now, staleAfterMs }) {
|
||||
if (row.status === 'new') return 'new';
|
||||
if (row.status === 'red') return 'red';
|
||||
if (row.last_checked_at) {
|
||||
const age = now - Date.parse(row.last_checked_at);
|
||||
if (age > staleAfterMs) return 'stale';
|
||||
}
|
||||
return 'green';
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank a flat list of (already-merged) candidate rows by bucket priority,
|
||||
* then by value formula (descending) within each bucket, with the source
|
||||
* path as the final lexical tiebreak.
|
||||
*
|
||||
* Pure function — used by both the global CLI path and the test suite.
|
||||
* Excludes any row whose package is in the `blocked` set, exits the
|
||||
* `green` bucket entirely (those are "nothing to do"), and applies the
|
||||
* optional `--mode` bucket filter.
|
||||
*
|
||||
* Returns the rows in rank order, each annotated with `effective_status`
|
||||
* and `value`. Caller decides how many to keep.
|
||||
*/
|
||||
export function rankCandidates(
|
||||
rows,
|
||||
{
|
||||
now,
|
||||
staleAfterMs,
|
||||
mode,
|
||||
blocked = new Set(),
|
||||
signals = {},
|
||||
coverage = {},
|
||||
weights = DEFAULT_WEIGHTS,
|
||||
} = {},
|
||||
) {
|
||||
const annotated = rows
|
||||
.filter((r) => !blocked.has(r.package))
|
||||
.map((row) => {
|
||||
const effective_status = computeEffectiveStatus(row, { now, staleAfterMs });
|
||||
const signalTriple = extractSignals(row, { signals, coverage });
|
||||
return {
|
||||
...row,
|
||||
effective_status,
|
||||
value: computeValue(signalTriple, weights),
|
||||
};
|
||||
});
|
||||
|
||||
const filtered = annotated.filter((r) => {
|
||||
if (r.effective_status === 'green') return false;
|
||||
if (mode && !MODE_BUCKETS[mode].has(r.effective_status)) return false;
|
||||
return true;
|
||||
});
|
||||
|
||||
filtered.sort((a, b) => {
|
||||
const pa = PRIORITY[a.effective_status] ?? 99;
|
||||
const pb = PRIORITY[b.effective_status] ?? 99;
|
||||
if (pa !== pb) return pa - pb;
|
||||
if (b.value !== a.value) return b.value - a.value;
|
||||
return a.source_file_path.localeCompare(b.source_file_path);
|
||||
});
|
||||
|
||||
return filtered;
|
||||
}
|
||||
|
||||
/**
|
||||
* Synthesise a "new" row for every mutation-worthy source file that has no
|
||||
* ledger row yet, then layer the live ledger rows on top (ledger wins). Used
|
||||
* by both per-package and global walks.
|
||||
*/
|
||||
export function mergeWithLedger({ worthyPaths, pkgName, ledgerRows }) {
|
||||
const byPath = new Map();
|
||||
for (const row of ledgerRows) {
|
||||
if (row.package === pkgName) byPath.set(row.source_file_path, row);
|
||||
}
|
||||
return worthyPaths.map(
|
||||
(p) =>
|
||||
byPath.get(p) ?? {
|
||||
source_file_path: p,
|
||||
package: pkgName,
|
||||
last_score: null,
|
||||
threshold_at_run: null,
|
||||
last_checked_at: null,
|
||||
status: 'new',
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
function parseStaleAfterWeeks(staleArg) {
|
||||
const DEFAULT = 4;
|
||||
if (staleArg === undefined) return DEFAULT;
|
||||
const parsed = Number(staleArg);
|
||||
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
||||
process.stderr.write(`Invalid --stale-after-weeks=${staleArg}, falling back to ${DEFAULT}.\n`);
|
||||
return DEFAULT;
|
||||
}
|
||||
|
||||
function parseWeights(args) {
|
||||
const w = { ...DEFAULT_WEIGHTS };
|
||||
for (const [flag, key] of [
|
||||
['w-churn', 'churn'],
|
||||
['w-fix-density', 'fixDensity'],
|
||||
['w-coverage', 'coverage'],
|
||||
]) {
|
||||
if (args[flag] === undefined) continue;
|
||||
const parsed = Number(args[flag]);
|
||||
if (!Number.isFinite(parsed) || parsed < 0) {
|
||||
die(2, `Invalid --${flag}=${args[flag]} (expected non-negative number).`);
|
||||
}
|
||||
w[key] = parsed;
|
||||
}
|
||||
return w;
|
||||
}
|
||||
|
||||
function parseTopN(args) {
|
||||
if (args['top-n'] === undefined) return 1;
|
||||
const parsed = Number(args['top-n']);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
die(2, `Invalid --top-n=${args['top-n']} (expected positive integer).`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
function parseBlocked(args) {
|
||||
const raw = args.block;
|
||||
if (raw === undefined || raw === true) return new Set();
|
||||
return new Set(
|
||||
String(raw)
|
||||
.split(',')
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean),
|
||||
);
|
||||
}
|
||||
|
||||
function resolveLedgerPath(ledgerFile) {
|
||||
return path.isAbsolute(ledgerFile) ? ledgerFile : path.join(process.cwd(), ledgerFile);
|
||||
}
|
||||
|
||||
async function readJsonIfPresent(file) {
|
||||
if (!file) return null;
|
||||
const resolved = path.isAbsolute(file) ? file : path.join(process.cwd(), file);
|
||||
if (!existsSync(resolved)) die(2, `File not found: ${resolved}`);
|
||||
try {
|
||||
return JSON.parse(await readFile(resolved, 'utf8'));
|
||||
} catch (err) {
|
||||
die(2, `Failed to parse JSON at ${resolved}: ${err.message}`);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function collectPackageCandidates({ repoRoot, pkg }) {
|
||||
const srcDir = path.join(repoRoot, pkg.dir, 'src');
|
||||
if (!existsSync(srcDir)) return null;
|
||||
const allSources = (await walkSources(srcDir)).sort();
|
||||
const worthy = allSources
|
||||
.filter((p) => isMutationWorthy(p))
|
||||
.map((abs) => path.relative(repoRoot, abs));
|
||||
return { pkg, worthy };
|
||||
}
|
||||
|
||||
async function runGlobal({ args, repoRoot, now }) {
|
||||
const ledgerFile = args['ledger-file'];
|
||||
if (!ledgerFile) die(2, 'Missing required --ledger-file <path>');
|
||||
const ledgerPath = resolveLedgerPath(ledgerFile);
|
||||
if (!existsSync(ledgerPath)) die(2, `Ledger file not found: ${ledgerPath}`);
|
||||
|
||||
const mode = args.mode;
|
||||
if (mode !== undefined && mode !== true && !Object.hasOwn(MODE_BUCKETS, mode)) {
|
||||
die(2, `Invalid --mode=${mode}. Use 'baseline' or 'coverage' (omit for combined).`);
|
||||
}
|
||||
const modeArg = mode === true ? undefined : mode;
|
||||
|
||||
const staleAfterWeeks = parseStaleAfterWeeks(args['stale-after-weeks']);
|
||||
const staleAfterMs = staleAfterWeeks * 7 * 24 * 60 * 60 * 1000;
|
||||
const topN = parseTopN(args);
|
||||
const blocked = parseBlocked(args);
|
||||
const weights = parseWeights(args);
|
||||
|
||||
let liveLedger;
|
||||
try {
|
||||
({ rows: liveLedger } = await readLedger({ path: ledgerPath }));
|
||||
} catch (err) {
|
||||
die(2, err.message);
|
||||
}
|
||||
|
||||
const signals = (await readJsonIfPresent(args['signals-file'])) ?? { churn: {}, fixDensity: {} };
|
||||
const coverage = (await readJsonIfPresent(args['coverage-file'])) ?? {};
|
||||
|
||||
const eligible = ELIGIBLE_PACKAGES.filter((p) => !blocked.has(p.name));
|
||||
const merged = [];
|
||||
for (const pkg of eligible) {
|
||||
const collected = await collectPackageCandidates({ repoRoot, pkg });
|
||||
if (!collected) {
|
||||
process.stderr.write(`No src/ for ${pkg.name} at ${pkg.dir}; skipping.\n`);
|
||||
continue;
|
||||
}
|
||||
merged.push(
|
||||
...mergeWithLedger({
|
||||
worthyPaths: collected.worthy,
|
||||
pkgName: pkg.name,
|
||||
ledgerRows: liveLedger,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
if (merged.length === 0) {
|
||||
process.stderr.write('No mutation-worthy source files found across eligible packages.\n');
|
||||
process.stdout.write(JSON.stringify({ picked: [], reason: 'empty-source-tree' }) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const ranked = rankCandidates(merged, {
|
||||
now,
|
||||
staleAfterMs,
|
||||
mode: modeArg,
|
||||
blocked,
|
||||
signals,
|
||||
coverage,
|
||||
weights,
|
||||
});
|
||||
|
||||
const counts = ranked.reduce((acc, r) => {
|
||||
acc[r.effective_status] = (acc[r.effective_status] ?? 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
process.stderr.write(
|
||||
`Global walk: candidates=${merged.length} ranked=${ranked.length} • ` +
|
||||
`new=${counts.new ?? 0} red=${counts.red ?? 0} stale=${counts.stale ?? 0}\n`,
|
||||
);
|
||||
|
||||
if (ranked.length === 0) {
|
||||
const reason =
|
||||
modeArg === 'baseline'
|
||||
? 'no-new-files'
|
||||
: modeArg === 'coverage'
|
||||
? 'nothing-below-threshold'
|
||||
: 'all-green';
|
||||
process.stderr.write(`Nothing to do for mode=${modeArg ?? 'combined'} (${reason}).\n`);
|
||||
process.stdout.write(JSON.stringify({ picked: [], reason }) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const top = ranked.slice(0, topN).map((r) => ({
|
||||
source_file_path: r.source_file_path,
|
||||
package: r.package,
|
||||
prior_status: r.status,
|
||||
effective_status: r.effective_status,
|
||||
value: r.value,
|
||||
}));
|
||||
|
||||
for (const t of top) {
|
||||
process.stderr.write(
|
||||
`Picked: ${t.source_file_path} ` +
|
||||
`[${t.package}] priority=${t.effective_status} value=${t.value.toFixed(4)}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
process.stdout.write(JSON.stringify({ picked: top }) + '\n');
|
||||
}
|
||||
|
||||
async function runPerPackage({ args, repoRoot, now }) {
|
||||
const pkgDirArg = args['package-dir'];
|
||||
const ledgerFile = args['ledger-file'];
|
||||
if (!pkgDirArg) die(2, 'Missing required --package-dir <relative-path-to-package>');
|
||||
if (!ledgerFile) die(2, 'Missing required --ledger-file <path>');
|
||||
|
||||
const pkgDir = path.isAbsolute(pkgDirArg) ? pkgDirArg : path.join(repoRoot, pkgDirArg);
|
||||
if (!existsSync(pkgDir)) die(2, `Package dir not found: ${pkgDir}`);
|
||||
|
||||
const pkgJsonPath = path.join(pkgDir, 'package.json');
|
||||
if (!existsSync(pkgJsonPath)) die(2, `No package.json at ${pkgJsonPath}`);
|
||||
const pkgName = JSON.parse(await readFile(pkgJsonPath, 'utf8')).name;
|
||||
|
||||
const srcDir = path.join(pkgDir, 'src');
|
||||
if (!existsSync(srcDir)) die(2, `No src/ in ${pkgDir}`);
|
||||
|
||||
const ledgerPath = resolveLedgerPath(ledgerFile);
|
||||
if (!existsSync(ledgerPath)) die(2, `Ledger file not found: ${ledgerPath}`);
|
||||
|
||||
const staleAfterWeeks = parseStaleAfterWeeks(args['stale-after-weeks']);
|
||||
const staleAfterMs = staleAfterWeeks * 7 * 24 * 60 * 60 * 1000;
|
||||
|
||||
// One read returns every row across every package; we narrow to this
|
||||
// package's rows internally so the picker's per-package behaviour is
|
||||
// preserved whether the file holds one package or many.
|
||||
let liveLedger;
|
||||
try {
|
||||
({ rows: liveLedger } = await readLedger({ path: ledgerPath, pkg: pkgName }));
|
||||
} catch (err) {
|
||||
die(2, err.message);
|
||||
}
|
||||
|
||||
const allSources = (await walkSources(srcDir)).sort();
|
||||
const worthy = allSources
|
||||
.filter((p) => isMutationWorthy(p))
|
||||
.map((abs) => path.relative(repoRoot, abs));
|
||||
|
||||
if (worthy.length === 0) {
|
||||
process.stderr.write('No mutation-worthy source files found under src/.\n');
|
||||
process.stdout.write(JSON.stringify({ picked: null, reason: 'empty-source-tree' }) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const merged = mergeWithLedger({ worthyPaths: worthy, pkgName, ledgerRows: liveLedger });
|
||||
|
||||
const annotated = merged.map((row) => ({
|
||||
...row,
|
||||
effective_status: computeEffectiveStatus(row, { now, staleAfterMs }),
|
||||
}));
|
||||
|
||||
annotated.sort((a, b) => {
|
||||
const pa = PRIORITY[a.effective_status] ?? 99;
|
||||
const pb = PRIORITY[b.effective_status] ?? 99;
|
||||
if (pa !== pb) return pa - pb;
|
||||
|
||||
if (a.effective_status === 'new') {
|
||||
return a.source_file_path.localeCompare(b.source_file_path);
|
||||
}
|
||||
|
||||
if (a.effective_status === 'red') {
|
||||
const sa = a.last_score == null ? Infinity : Number(a.last_score);
|
||||
const sb = b.last_score == null ? Infinity : Number(b.last_score);
|
||||
if (sa !== sb) return sa - sb;
|
||||
return a.source_file_path.localeCompare(b.source_file_path);
|
||||
}
|
||||
|
||||
// stale: oldest last_checked_at first
|
||||
const ta = a.last_checked_at ? Date.parse(a.last_checked_at) : 0;
|
||||
const tb = b.last_checked_at ? Date.parse(b.last_checked_at) : 0;
|
||||
if (ta !== tb) return ta - tb;
|
||||
return a.source_file_path.localeCompare(b.source_file_path);
|
||||
});
|
||||
|
||||
const counts = annotated.reduce((acc, r) => {
|
||||
acc[r.effective_status] = (acc[r.effective_status] ?? 0) + 1;
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
process.stderr.write(
|
||||
`Source files: ${worthy.length} • ` +
|
||||
`new=${counts.new ?? 0} red=${counts.red ?? 0} stale=${counts.stale ?? 0} green=${counts.green ?? 0}\n`,
|
||||
);
|
||||
|
||||
const mode = args.mode;
|
||||
if (mode !== undefined && mode !== true && !Object.hasOwn(MODE_BUCKETS, mode)) {
|
||||
die(
|
||||
2,
|
||||
`Invalid --mode=${mode}. Use 'baseline' or 'coverage' (omit for combined new→red→stale).`,
|
||||
);
|
||||
}
|
||||
const modeArg = mode === true ? undefined : mode;
|
||||
const candidates = modeArg
|
||||
? annotated.filter((r) => MODE_BUCKETS[modeArg].has(r.effective_status))
|
||||
: annotated;
|
||||
|
||||
const top = candidates[0];
|
||||
|
||||
if (!top || (!modeArg && top.effective_status === 'green')) {
|
||||
const reason =
|
||||
modeArg === 'baseline'
|
||||
? 'no-new-files'
|
||||
: modeArg === 'coverage'
|
||||
? 'nothing-below-threshold'
|
||||
: 'all-green';
|
||||
process.stderr.write(`Nothing to do for mode=${modeArg ?? 'combined'} (${reason}).\n`);
|
||||
process.stdout.write(JSON.stringify({ picked: null, reason }) + '\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
process.stderr.write(
|
||||
`Picked: ${top.source_file_path}\n` +
|
||||
` priority=${top.effective_status} ` +
|
||||
`(was ${top.status}, last_checked_at=${top.last_checked_at ?? 'never'})\n`,
|
||||
);
|
||||
|
||||
process.stdout.write(
|
||||
JSON.stringify({
|
||||
picked: {
|
||||
source_file_path: top.source_file_path,
|
||||
package: top.package,
|
||||
prior_status: top.status,
|
||||
effective_status: top.effective_status,
|
||||
},
|
||||
}) + '\n',
|
||||
);
|
||||
}
|
||||
|
||||
const isCli = import.meta.url === `file://${process.argv[1]}`;
|
||||
if (isCli) {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const repoRoot = path.resolve(
|
||||
execFileSync('git', ['rev-parse', '--show-toplevel'], { encoding: 'utf8' }).trim(),
|
||||
);
|
||||
const now = Date.now();
|
||||
if (args.global) {
|
||||
await runGlobal({ args, repoRoot, now });
|
||||
} else {
|
||||
await runPerPackage({ args, repoRoot, now });
|
||||
}
|
||||
}
|
||||
@@ -1,412 +0,0 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
DEFAULT_WEIGHTS,
|
||||
ELIGIBLE_PACKAGES,
|
||||
computeEffectiveStatus,
|
||||
computeValue,
|
||||
extractSignals,
|
||||
isEligible,
|
||||
mergeWithLedger,
|
||||
rankCandidates,
|
||||
} from './pick-next.mjs';
|
||||
|
||||
// Fixed reference epoch (2026-06-20 00:00:00 UTC). All age-based assertions
|
||||
// derive from this so they don't drift with wall-clock.
|
||||
const NOW = Date.parse('2026-06-20T00:00:00.000Z');
|
||||
const WEEK_MS = 7 * 24 * 60 * 60 * 1000;
|
||||
const STALE_AFTER_MS = 4 * WEEK_MS;
|
||||
|
||||
// Three-package fixture ledger. Each row pins a known status (new is
|
||||
// implicit — files not in the ledger become `new` when merged) so the
|
||||
// bucketed-ordering assertions don't depend on staleness math.
|
||||
const MULTI_PKG_LEDGER = [
|
||||
// n8n-workflow — one red row (will be ordered by value)
|
||||
{
|
||||
source_file_path: 'packages/workflow/src/a.ts',
|
||||
package: 'n8n-workflow',
|
||||
status: 'red',
|
||||
last_score: 30,
|
||||
last_checked_at: '2026-05-01T00:00:00.000Z',
|
||||
},
|
||||
// @n8n/crdt — one red row
|
||||
{
|
||||
source_file_path: 'packages/@n8n/crdt/src/x.ts',
|
||||
package: '@n8n/crdt',
|
||||
status: 'red',
|
||||
last_score: 40,
|
||||
last_checked_at: '2026-05-15T00:00:00.000Z',
|
||||
},
|
||||
// @n8n/decorators — one green row that has aged past STALE_AFTER_MS
|
||||
{
|
||||
source_file_path: 'packages/@n8n/decorators/src/y.ts',
|
||||
package: '@n8n/decorators',
|
||||
status: 'green',
|
||||
last_score: 85,
|
||||
last_checked_at: '2026-04-01T00:00:00.000Z',
|
||||
},
|
||||
// excluded package — should never appear in any candidate set
|
||||
{
|
||||
source_file_path: 'packages/@n8n/expression-runtime/src/forbidden.ts',
|
||||
package: '@n8n/expression-runtime',
|
||||
status: 'red',
|
||||
last_score: 10,
|
||||
last_checked_at: '2026-06-10T00:00:00.000Z',
|
||||
},
|
||||
];
|
||||
|
||||
// Synthetic worthy-file inputs per package — these come from walking each
|
||||
// package's src/ in the real CLI. Here we feed them straight to
|
||||
// mergeWithLedger so the test has no filesystem dependency.
|
||||
const WALK_INPUTS = {
|
||||
'n8n-workflow': ['packages/workflow/src/a.ts', 'packages/workflow/src/b.ts'],
|
||||
'@n8n/crdt': ['packages/@n8n/crdt/src/x.ts', 'packages/@n8n/crdt/src/z.ts'],
|
||||
'@n8n/decorators': ['packages/@n8n/decorators/src/y.ts'],
|
||||
};
|
||||
|
||||
// Signals are tuned so the value ordering within each bucket is
|
||||
// deterministic and obvious:
|
||||
// - within `new`: crdt/z.ts (churn 100) > workflow/b.ts (churn 1)
|
||||
// - within `red`: workflow/a.ts (churn 50, fix 5) > crdt/x.ts (churn 2, fix 0)
|
||||
const SIGNALS = {
|
||||
churn: {
|
||||
'packages/workflow/src/a.ts': { commits: 50, linesChanged: 999 },
|
||||
'packages/workflow/src/b.ts': { commits: 1, linesChanged: 1 },
|
||||
'packages/@n8n/crdt/src/x.ts': { commits: 2, linesChanged: 2 },
|
||||
'packages/@n8n/crdt/src/z.ts': { commits: 100, linesChanged: 200 },
|
||||
'packages/@n8n/decorators/src/y.ts': { commits: 0, linesChanged: 0 },
|
||||
},
|
||||
fixDensity: {
|
||||
'packages/workflow/src/a.ts': 5,
|
||||
'packages/@n8n/crdt/src/x.ts': 0,
|
||||
'packages/@n8n/crdt/src/z.ts': 1,
|
||||
},
|
||||
};
|
||||
|
||||
const COVERAGE = {
|
||||
'packages/workflow/src/a.ts': 0.4,
|
||||
'packages/workflow/src/b.ts': 0.9,
|
||||
'packages/@n8n/crdt/src/x.ts': 0.95,
|
||||
'packages/@n8n/crdt/src/z.ts': 0.1,
|
||||
};
|
||||
|
||||
function buildMerged() {
|
||||
const merged = [];
|
||||
for (const [pkgName, worthy] of Object.entries(WALK_INPUTS)) {
|
||||
merged.push(...mergeWithLedger({ worthyPaths: worthy, pkgName, ledgerRows: MULTI_PKG_LEDGER }));
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
describe('isEligible (vitest allowlist unit test)', () => {
|
||||
it('admits every name in ELIGIBLE_PACKAGES', () => {
|
||||
assert.ok(ELIGIBLE_PACKAGES.length >= 2, 'allowlist must span at least two packages');
|
||||
for (const pkg of ELIGIBLE_PACKAGES) {
|
||||
assert.equal(isEligible(pkg.name), true, `expected eligible: ${pkg.name}`);
|
||||
}
|
||||
});
|
||||
|
||||
it('rejects packages outside the allowlist', () => {
|
||||
assert.equal(isEligible('@n8n/foo'), false);
|
||||
assert.equal(isEligible(''), false);
|
||||
assert.equal(isEligible(undefined), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeValue', () => {
|
||||
it('applies w_churn·churn + w_fix·fixDensity + w_cov·(1 − coverage)', () => {
|
||||
const v = computeValue(
|
||||
{ churn: 10, fixDensity: 2, coverage: 0.25 },
|
||||
{ churn: 1, fixDensity: 3, coverage: 4 },
|
||||
);
|
||||
// 1*10 + 3*2 + 4*(1 - 0.25) = 10 + 6 + 3 = 19
|
||||
assert.equal(v, 19);
|
||||
});
|
||||
|
||||
it('treats missing signals as zero (no penalty, no boost)', () => {
|
||||
// Default weights {1,1,1}. With everything missing → 1*0 + 1*0 + 1*(1-0) = 1.
|
||||
assert.equal(computeValue({}), 1);
|
||||
assert.equal(computeValue({ churn: undefined, fixDensity: null }), 1);
|
||||
});
|
||||
|
||||
it('clamps coverage to [0,1]', () => {
|
||||
assert.equal(computeValue({ coverage: 2 }, { churn: 0, fixDensity: 0, coverage: 1 }), 0);
|
||||
assert.equal(computeValue({ coverage: -1 }, { churn: 0, fixDensity: 0, coverage: 1 }), 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('extractSignals', () => {
|
||||
it('handles `{ commits, linesChanged }` shape from gatherSignals', () => {
|
||||
const triple = extractSignals(
|
||||
{ source_file_path: 'packages/workflow/src/a.ts' },
|
||||
{ signals: SIGNALS, coverage: COVERAGE },
|
||||
);
|
||||
assert.deepEqual(triple, { churn: 50, fixDensity: 5, coverage: 0.4 });
|
||||
});
|
||||
|
||||
it('also accepts a bare-number churn value', () => {
|
||||
const triple = extractSignals(
|
||||
{ source_file_path: 'foo.ts' },
|
||||
{ signals: { churn: { 'foo.ts': 7 }, fixDensity: { 'foo.ts': 3 } }, coverage: {} },
|
||||
);
|
||||
assert.deepEqual(triple, { churn: 7, fixDensity: 3, coverage: 0 });
|
||||
});
|
||||
|
||||
it('zero-fills when the file is absent from signals/coverage', () => {
|
||||
assert.deepEqual(extractSignals({ source_file_path: 'missing.ts' }), {
|
||||
churn: 0,
|
||||
fixDensity: 0,
|
||||
coverage: 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeEffectiveStatus', () => {
|
||||
it('promotes a green row older than the stale window to stale', () => {
|
||||
const row = { status: 'green', last_checked_at: '2026-04-01T00:00:00.000Z' };
|
||||
assert.equal(computeEffectiveStatus(row, { now: NOW, staleAfterMs: STALE_AFTER_MS }), 'stale');
|
||||
});
|
||||
|
||||
it('keeps a fresh green row green', () => {
|
||||
const row = { status: 'green', last_checked_at: '2026-06-19T00:00:00.000Z' };
|
||||
assert.equal(computeEffectiveStatus(row, { now: NOW, staleAfterMs: STALE_AFTER_MS }), 'green');
|
||||
});
|
||||
|
||||
it('passes new/red through unchanged', () => {
|
||||
assert.equal(
|
||||
computeEffectiveStatus({ status: 'new' }, { now: NOW, staleAfterMs: STALE_AFTER_MS }),
|
||||
'new',
|
||||
);
|
||||
assert.equal(
|
||||
computeEffectiveStatus({ status: 'red' }, { now: NOW, staleAfterMs: STALE_AFTER_MS }),
|
||||
'red',
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe('mergeWithLedger', () => {
|
||||
it('synthesises new rows for worthy paths not in the ledger', () => {
|
||||
const merged = mergeWithLedger({
|
||||
worthyPaths: ['packages/workflow/src/a.ts', 'packages/workflow/src/b.ts'],
|
||||
pkgName: 'n8n-workflow',
|
||||
ledgerRows: MULTI_PKG_LEDGER,
|
||||
});
|
||||
const a = merged.find((r) => r.source_file_path === 'packages/workflow/src/a.ts');
|
||||
const b = merged.find((r) => r.source_file_path === 'packages/workflow/src/b.ts');
|
||||
assert.equal(a.status, 'red'); // from the live ledger
|
||||
assert.equal(b.status, 'new'); // synthesised
|
||||
assert.equal(b.last_score, null);
|
||||
});
|
||||
|
||||
it("never lets another package's rows leak into the merge", () => {
|
||||
// Ledger has a row for @n8n/expression-runtime; merging for n8n-workflow
|
||||
// must ignore it entirely.
|
||||
const merged = mergeWithLedger({
|
||||
worthyPaths: ['packages/workflow/src/a.ts'],
|
||||
pkgName: 'n8n-workflow',
|
||||
ledgerRows: MULTI_PKG_LEDGER,
|
||||
});
|
||||
assert.equal(merged.length, 1);
|
||||
assert.equal(merged[0].package, 'n8n-workflow');
|
||||
});
|
||||
});
|
||||
|
||||
// PR-gate contract from DEVP-494:
|
||||
// "integration tests: fixture ledger spanning ≥2 packages returns N rows
|
||||
// ordered by value within buckets, ≥2 packages present, zero
|
||||
// excluded-package rows"
|
||||
describe('rankCandidates (DEVP-494 PR gate)', () => {
|
||||
it('orders results bucket-first (new → red → stale), then by value desc within each bucket', () => {
|
||||
const merged = buildMerged();
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
weights: DEFAULT_WEIGHTS,
|
||||
});
|
||||
|
||||
// Effective bucket sequence must be monotonic in priority.
|
||||
const priority = { new: 0, red: 1, stale: 2, green: 3 };
|
||||
for (let i = 1; i < ranked.length; i++) {
|
||||
assert.ok(
|
||||
priority[ranked[i - 1].effective_status] <= priority[ranked[i].effective_status],
|
||||
`bucket order broken at index ${i}: ${ranked[i - 1].effective_status} → ${ranked[i].effective_status}`,
|
||||
);
|
||||
}
|
||||
|
||||
// Within each bucket, value must be non-increasing.
|
||||
let prev = null;
|
||||
for (const row of ranked) {
|
||||
if (prev && prev.effective_status === row.effective_status) {
|
||||
assert.ok(
|
||||
prev.value >= row.value,
|
||||
`value descending broken in bucket=${row.effective_status}: ${prev.value} → ${row.value}`,
|
||||
);
|
||||
}
|
||||
prev = row;
|
||||
}
|
||||
|
||||
// Spot-check the new bucket: crdt/z.ts (churn 100) must outrank
|
||||
// workflow/b.ts (churn 1).
|
||||
const newBucket = ranked.filter((r) => r.effective_status === 'new');
|
||||
assert.equal(newBucket[0].source_file_path, 'packages/@n8n/crdt/src/z.ts');
|
||||
|
||||
// Spot-check the red bucket: workflow/a.ts (churn 50, fix 5) must
|
||||
// outrank crdt/x.ts (churn 2, fix 0).
|
||||
const redBucket = ranked.filter((r) => r.effective_status === 'red');
|
||||
assert.equal(redBucket[0].source_file_path, 'packages/workflow/src/a.ts');
|
||||
|
||||
// Stale bucket has just decorators/y.ts (green → stale by age).
|
||||
const staleBucket = ranked.filter((r) => r.effective_status === 'stale');
|
||||
assert.equal(staleBucket.length, 1);
|
||||
assert.equal(staleBucket[0].package, '@n8n/decorators');
|
||||
});
|
||||
|
||||
it('top-N output spans ≥2 packages on the multi-package fixture', () => {
|
||||
const merged = buildMerged();
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
});
|
||||
const topN = ranked.slice(0, 4);
|
||||
const distinctPackages = new Set(topN.map((r) => r.package));
|
||||
assert.ok(
|
||||
distinctPackages.size >= 2,
|
||||
`top-${topN.length} must span ≥2 packages; got ${[...distinctPackages].join(', ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('excludes blocked-package rows entirely (zero excluded rows)', () => {
|
||||
// Add a worthy-file row for the blocked package as if its src/ tree
|
||||
// were walked. The picker must still drop every row from that pkg.
|
||||
const blockedRows = mergeWithLedger({
|
||||
worthyPaths: ['packages/@n8n/expression-runtime/src/forbidden.ts'],
|
||||
pkgName: '@n8n/expression-runtime',
|
||||
ledgerRows: MULTI_PKG_LEDGER,
|
||||
});
|
||||
const merged = [...buildMerged(), ...blockedRows];
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
blocked: new Set(['@n8n/expression-runtime']),
|
||||
});
|
||||
const blockedHits = ranked.filter((r) => r.package === '@n8n/expression-runtime');
|
||||
assert.equal(blockedHits.length, 0, 'blocked package rows must not appear');
|
||||
});
|
||||
|
||||
it('drops every green row from the candidate set (combined mode)', () => {
|
||||
const merged = buildMerged();
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
});
|
||||
assert.equal(
|
||||
ranked.filter((r) => r.effective_status === 'green').length,
|
||||
0,
|
||||
'green rows must be filtered out of the candidate set',
|
||||
);
|
||||
});
|
||||
|
||||
it('--mode baseline restricts the candidate set to `new` only', () => {
|
||||
const merged = buildMerged();
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
mode: 'baseline',
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
});
|
||||
assert.ok(ranked.length > 0);
|
||||
assert.ok(
|
||||
ranked.every((r) => r.effective_status === 'new'),
|
||||
`baseline mode must yield only "new" rows; got ${ranked.map((r) => r.effective_status).join(', ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('--mode coverage restricts the candidate set to `red`/`stale` only', () => {
|
||||
const merged = buildMerged();
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
mode: 'coverage',
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
});
|
||||
assert.ok(ranked.length > 0);
|
||||
assert.ok(
|
||||
ranked.every((r) => r.effective_status === 'red' || r.effective_status === 'stale'),
|
||||
`coverage mode must yield only red/stale rows; got ${ranked.map((r) => r.effective_status).join(', ')}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('at equal churn/fix-density, the lower-coverage file outranks the higher-coverage one within a bucket', () => {
|
||||
// The (1 − coverage) term is load-bearing now that build-matrix feeds
|
||||
// ledger coverage into the picker. With identical churn and fix-density,
|
||||
// coverage alone decides order: the less-covered file is the more urgent
|
||||
// re-score target and must rank first.
|
||||
const merged = mergeWithLedger({
|
||||
worthyPaths: ['packages/workflow/src/low.ts', 'packages/workflow/src/high.ts'],
|
||||
pkgName: 'n8n-workflow',
|
||||
ledgerRows: [],
|
||||
});
|
||||
const signals = {
|
||||
churn: {
|
||||
'packages/workflow/src/low.ts': { commits: 10, linesChanged: 20 },
|
||||
'packages/workflow/src/high.ts': { commits: 10, linesChanged: 20 },
|
||||
},
|
||||
fixDensity: {
|
||||
'packages/workflow/src/low.ts': 3,
|
||||
'packages/workflow/src/high.ts': 3,
|
||||
},
|
||||
};
|
||||
const coverage = {
|
||||
'packages/workflow/src/low.ts': 0.1,
|
||||
'packages/workflow/src/high.ts': 0.9,
|
||||
};
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
signals,
|
||||
coverage,
|
||||
weights: DEFAULT_WEIGHTS,
|
||||
});
|
||||
assert.equal(ranked[0].source_file_path, 'packages/workflow/src/low.ts');
|
||||
assert.equal(ranked[1].source_file_path, 'packages/workflow/src/high.ts');
|
||||
assert.ok(
|
||||
ranked[0].value > ranked[1].value,
|
||||
`low-coverage value (${ranked[0].value}) must exceed high-coverage value (${ranked[1].value})`,
|
||||
);
|
||||
});
|
||||
|
||||
it('weights tune the ordering — boosting churn flips priority within a bucket', () => {
|
||||
// Two new files in the same bucket: crdt/z.ts (churn=100, fix=1) and
|
||||
// workflow/b.ts (churn=1). With default weights crdt wins. If we zero
|
||||
// out churn and only weight fix-density, the row WITHOUT a fix
|
||||
// (workflow/b.ts) falls behind crdt/z.ts (fix=1), but with churn=0
|
||||
// and fixDensity=0 the tiebreak becomes the (1−coverage) term:
|
||||
// workflow/b.ts has coverage 0.9 → value 0.1, crdt/z.ts has 0.1 →
|
||||
// value 0.9. crdt/z.ts still wins. Now zero out everything except
|
||||
// w_coverage and watch the lower-coverage file climb.
|
||||
const merged = buildMerged();
|
||||
const ranked = rankCandidates(merged, {
|
||||
now: NOW,
|
||||
staleAfterMs: STALE_AFTER_MS,
|
||||
mode: 'baseline',
|
||||
signals: SIGNALS,
|
||||
coverage: COVERAGE,
|
||||
weights: { churn: 0, fixDensity: 0, coverage: 1 },
|
||||
});
|
||||
const top = ranked[0];
|
||||
// crdt/z.ts has the lowest coverage (0.1) among new rows → highest
|
||||
// (1 − coverage) value.
|
||||
assert.equal(top.source_file_path, 'packages/@n8n/crdt/src/z.ts');
|
||||
});
|
||||
});
|
||||
@@ -1,233 +0,0 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Per-file risk signals derived from git history.
|
||||
*
|
||||
* Two signals, both keyed by repo-relative file path:
|
||||
*
|
||||
* churn — how often the file changes (commits + lines touched)
|
||||
* fixDensity — how "buggy" the file looks: a sum over fix commits
|
||||
* touching the file, where each contribution is the lines
|
||||
* changed in that commit, weighted by a half-life decay on
|
||||
* the commit's age. Recent fixes dominate; ancient fixes
|
||||
* fade out smoothly.
|
||||
*
|
||||
* Fix detection is conventional-commit shaped: the commit subject must match
|
||||
* `fix:` or `fix(scope): ` (case-insensitive; `!` breaking marker allowed).
|
||||
*
|
||||
* The module is split into pure helpers so unit tests can feed synthetic
|
||||
* `git log` output without touching the filesystem or invoking git:
|
||||
*
|
||||
* parseGitLog(text) → Commit[]
|
||||
* computeChurn(commits) → Map<file, { commits, linesChanged }>
|
||||
* computeFixDensity(commits, { halfLifeDays, now }) → Map<file, number>
|
||||
*
|
||||
* A high-level `gatherSignals` shells out to git in the current repo for
|
||||
* pipeline use. The unit-test contract (cold-file < hot-file at identical
|
||||
* mutation status) is proved against `parseGitLog` + the two `compute*`
|
||||
* helpers; the git wrapper is the same pure pipeline with a real log on the
|
||||
* front.
|
||||
*/
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
|
||||
const execFileP = promisify(execFile);
|
||||
|
||||
const COMMIT_MARKER = 'COMMIT';
|
||||
// `--pretty=format:'COMMIT %H %ct %s'` — fields are space-separated; subjects
|
||||
// may contain spaces so we only split on the first two.
|
||||
const FIX_SUBJECT_RE = /^fix(\([^)]+\))?!?:\s/i;
|
||||
const SECONDS_PER_DAY = 86_400;
|
||||
|
||||
export const DEFAULT_HALF_LIFE_DAYS = 90;
|
||||
export const GIT_LOG_FORMAT = `${COMMIT_MARKER} %H %ct %s`;
|
||||
|
||||
/**
|
||||
* Detect whether a commit subject is a conventional fix.
|
||||
* Exported so callers can override fix detection if they need to.
|
||||
*/
|
||||
export function isFixSubject(subject) {
|
||||
return typeof subject === 'string' && FIX_SUBJECT_RE.test(subject);
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse the output of
|
||||
* git log --no-merges --pretty=format:'COMMIT %H %ct %s' --numstat
|
||||
* into an array of commit records.
|
||||
*
|
||||
* Numstat emits added/removed counts per file, with `-` for binary files —
|
||||
* those are normalised to 0. Blank lines between commits and trailing
|
||||
* whitespace are tolerated; unknown lines are skipped silently so callers can
|
||||
* pre-process or post-process without breaking the parser.
|
||||
*/
|
||||
export function parseGitLog(input) {
|
||||
if (typeof input !== 'string') return [];
|
||||
const commits = [];
|
||||
let current = null;
|
||||
for (const raw of input.split('\n')) {
|
||||
const line = raw.replace(/\r$/, '');
|
||||
if (line.startsWith(`${COMMIT_MARKER} `)) {
|
||||
if (current) commits.push(current);
|
||||
const rest = line.slice(COMMIT_MARKER.length + 1);
|
||||
const firstSpace = rest.indexOf(' ');
|
||||
if (firstSpace === -1) {
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
const sha = rest.slice(0, firstSpace);
|
||||
const afterSha = rest.slice(firstSpace + 1);
|
||||
const secondSpace = afterSha.indexOf(' ');
|
||||
if (secondSpace === -1) {
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
const timestamp = Number(afterSha.slice(0, secondSpace));
|
||||
const subject = afterSha.slice(secondSpace + 1);
|
||||
if (!Number.isFinite(timestamp)) {
|
||||
current = null;
|
||||
continue;
|
||||
}
|
||||
current = {
|
||||
sha,
|
||||
timestamp,
|
||||
subject,
|
||||
isFix: isFixSubject(subject),
|
||||
files: [],
|
||||
};
|
||||
continue;
|
||||
}
|
||||
if (!current || !line.trim()) continue;
|
||||
// numstat row: "<added>\t<removed>\t<path>"; `-` means binary.
|
||||
const parts = line.split('\t');
|
||||
if (parts.length < 3) continue;
|
||||
const added = parts[0] === '-' ? 0 : Number(parts[0]);
|
||||
const removed = parts[1] === '-' ? 0 : Number(parts[1]);
|
||||
if (!Number.isFinite(added) || !Number.isFinite(removed)) continue;
|
||||
// Renames surface as `old => new` or `{old => new}/file`; keep the
|
||||
// path verbatim for now — the picker side will normalise once renames
|
||||
// are in scope (deferred to C5+).
|
||||
current.files.push({ path: parts.slice(2).join('\t'), added, removed });
|
||||
}
|
||||
if (current) commits.push(current);
|
||||
return commits;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-file churn: commit count + total lines added+removed within the window.
|
||||
* `since`/`until` are unix-seconds, both inclusive; omit to include all.
|
||||
*/
|
||||
export function computeChurn(commits, { since, until } = {}) {
|
||||
const out = new Map();
|
||||
for (const c of commits) {
|
||||
if (since !== undefined && c.timestamp < since) continue;
|
||||
if (until !== undefined && c.timestamp > until) continue;
|
||||
for (const f of c.files) {
|
||||
let entry = out.get(f.path);
|
||||
if (!entry) {
|
||||
entry = { commits: 0, linesChanged: 0 };
|
||||
out.set(f.path, entry);
|
||||
}
|
||||
entry.commits += 1;
|
||||
entry.linesChanged += f.added + f.removed;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-file time-decayed, delta-weighted fix-density.
|
||||
*
|
||||
* density(file) = Σ over fix commits c touching file:
|
||||
* delta(c, file) * 0.5 ^ (age_days(c) / halfLifeDays)
|
||||
*
|
||||
* `now` is required (unix-seconds) so the result is fully deterministic — the
|
||||
* caller decides the reference time so tests aren't wall-clock dependent.
|
||||
*/
|
||||
export function computeFixDensity(
|
||||
commits,
|
||||
{ halfLifeDays = DEFAULT_HALF_LIFE_DAYS, now } = {},
|
||||
) {
|
||||
if (!Number.isFinite(halfLifeDays) || halfLifeDays <= 0) {
|
||||
throw new RangeError(
|
||||
`halfLifeDays must be a positive finite number; got ${halfLifeDays}`,
|
||||
);
|
||||
}
|
||||
if (typeof now !== 'number' || !Number.isFinite(now)) {
|
||||
throw new TypeError(
|
||||
'computeFixDensity requires explicit `now` (unix-seconds number) for determinism',
|
||||
);
|
||||
}
|
||||
const halfLifeSeconds = halfLifeDays * SECONDS_PER_DAY;
|
||||
const out = new Map();
|
||||
for (const c of commits) {
|
||||
if (!c.isFix) continue;
|
||||
const ageSeconds = Math.max(0, now - c.timestamp);
|
||||
const weight = Math.pow(0.5, ageSeconds / halfLifeSeconds);
|
||||
for (const f of c.files) {
|
||||
const delta = f.added + f.removed;
|
||||
if (delta === 0) continue;
|
||||
out.set(f.path, (out.get(f.path) ?? 0) + weight * delta);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read git log from `cwd` and return both signals.
|
||||
*
|
||||
* Returned shape is plain objects (not Maps) so it round-trips through JSON
|
||||
* for the eventual writer payload; callers that want a Map can rebuild via
|
||||
* `new Map(Object.entries(...))`.
|
||||
*/
|
||||
export async function gatherSignals({
|
||||
cwd = process.cwd(),
|
||||
since,
|
||||
halfLifeDays = DEFAULT_HALF_LIFE_DAYS,
|
||||
now = Math.floor(Date.now() / 1000),
|
||||
} = {}) {
|
||||
const args = ['log', '--no-merges', `--pretty=format:${GIT_LOG_FORMAT}`, '--numstat'];
|
||||
if (since) args.push(`--since=${since}`);
|
||||
// maxBuffer: huge histories blow the default 1MB cap; 256MB is plenty for
|
||||
// year-scale windows on packages/ subdirectories.
|
||||
const { stdout } = await execFileP('git', args, { cwd, maxBuffer: 256 * 1024 * 1024 });
|
||||
const commits = parseGitLog(stdout);
|
||||
const churn = computeChurn(commits);
|
||||
const density = computeFixDensity(commits, { halfLifeDays, now });
|
||||
return {
|
||||
halfLifeDays,
|
||||
now,
|
||||
churn: Object.fromEntries(churn),
|
||||
fixDensity: Object.fromEntries(density),
|
||||
};
|
||||
}
|
||||
|
||||
const isCli = import.meta.url === `file://${process.argv[1]}`;
|
||||
if (isCli) {
|
||||
const argv = process.argv.slice(2);
|
||||
const opts = {};
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
const a = argv[i];
|
||||
if (!a.startsWith('--')) continue;
|
||||
const key = a.slice(2);
|
||||
const next = argv[i + 1];
|
||||
if (next === undefined || next.startsWith('--')) {
|
||||
opts[key] = true;
|
||||
} else {
|
||||
opts[key] = next;
|
||||
i++;
|
||||
}
|
||||
}
|
||||
const halfLifeDays = opts['half-life-days'] ? Number(opts['half-life-days']) : undefined;
|
||||
gatherSignals({
|
||||
cwd: opts.cwd ?? process.cwd(),
|
||||
since: opts.since,
|
||||
halfLifeDays,
|
||||
})
|
||||
.then((result) => {
|
||||
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`);
|
||||
})
|
||||
.catch((err) => {
|
||||
process.stderr.write(`${err.message}\n`);
|
||||
process.exit(1);
|
||||
});
|
||||
}
|
||||
@@ -1,211 +0,0 @@
|
||||
import { describe, it } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
|
||||
import {
|
||||
DEFAULT_HALF_LIFE_DAYS,
|
||||
computeChurn,
|
||||
computeFixDensity,
|
||||
isFixSubject,
|
||||
parseGitLog,
|
||||
} from './signals.mjs';
|
||||
|
||||
const DAY = 86_400;
|
||||
// Fixed reference epoch (2026-06-20 00:00:00 UTC) — tests must not depend on
|
||||
// wall-clock; the signal API requires explicit `now` for exactly this reason.
|
||||
const NOW = 1_750_377_600;
|
||||
|
||||
const commit = (sha, ageDays, subject, files) => {
|
||||
const lines = [`COMMIT ${sha} ${NOW - ageDays * DAY} ${subject}`];
|
||||
for (const [path, added, removed] of files) {
|
||||
lines.push(`${added}\t${removed}\t${path}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
};
|
||||
|
||||
const fixture = (...blocks) => blocks.join('\n\n');
|
||||
|
||||
describe('isFixSubject', () => {
|
||||
it('matches conventional fix prefixes', () => {
|
||||
assert.equal(isFixSubject('fix: oops'), true);
|
||||
assert.equal(isFixSubject('fix(workflow): oops'), true);
|
||||
assert.equal(isFixSubject('fix(core)!: breaking fix'), true);
|
||||
assert.equal(isFixSubject('FIX: case-insensitive'), true);
|
||||
});
|
||||
|
||||
it('rejects non-fix subjects', () => {
|
||||
assert.equal(isFixSubject('feat: new thing'), false);
|
||||
assert.equal(isFixSubject('chore: rename'), false);
|
||||
// "prefix" detection — we intentionally don't accept loose "fixes a bug"
|
||||
// in commit bodies; the convention here is `fix:` at the start.
|
||||
assert.equal(isFixSubject('refactor: this fixes a bug indirectly'), false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('parseGitLog', () => {
|
||||
it('parses sha, timestamp, subject, and files', () => {
|
||||
const log = commit('aaa1', 1, 'fix(core): null pointer', [['src/a.ts', 40, 10]]);
|
||||
const commits = parseGitLog(log);
|
||||
assert.equal(commits.length, 1);
|
||||
assert.equal(commits[0].sha, 'aaa1');
|
||||
assert.equal(commits[0].timestamp, NOW - DAY);
|
||||
assert.equal(commits[0].subject, 'fix(core): null pointer');
|
||||
assert.equal(commits[0].isFix, true);
|
||||
assert.deepEqual(commits[0].files, [{ path: 'src/a.ts', added: 40, removed: 10 }]);
|
||||
});
|
||||
|
||||
it('treats binary numstat (- -) as zero-change', () => {
|
||||
const log = commit('bin1', 1, 'fix: refresh sprite', [
|
||||
['assets/foo.png', '-', '-'],
|
||||
]).replace('-\t-', '-\t-'); // sanity: still a real tab
|
||||
const commits = parseGitLog(log);
|
||||
assert.equal(commits[0].files[0].added, 0);
|
||||
assert.equal(commits[0].files[0].removed, 0);
|
||||
});
|
||||
|
||||
it('returns [] on non-string input and tolerates noise lines', () => {
|
||||
assert.deepEqual(parseGitLog(undefined), []);
|
||||
const noisy = ['', 'totally not a commit line', '5\t5\torphan.ts', ''].join('\n');
|
||||
assert.deepEqual(parseGitLog(noisy), []);
|
||||
});
|
||||
|
||||
it('marks non-fix commits as isFix=false', () => {
|
||||
const log = commit('c1', 1, 'chore: rename var', [['src/a.ts', 2, 2]]);
|
||||
assert.equal(parseGitLog(log)[0].isFix, false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeChurn', () => {
|
||||
it('aggregates commit count and total lines per file', () => {
|
||||
const log = fixture(
|
||||
commit('a', 1, 'fix: x', [['src/hot.ts', 40, 10]]),
|
||||
commit('b', 10, 'chore: y', [['src/hot.ts', 5, 5]]),
|
||||
commit('c', 30, 'fix: z', [
|
||||
['src/hot.ts', 20, 8],
|
||||
['src/other.ts', 1, 0],
|
||||
]),
|
||||
);
|
||||
const churn = computeChurn(parseGitLog(log));
|
||||
assert.deepEqual(churn.get('src/hot.ts'), { commits: 3, linesChanged: 88 });
|
||||
assert.deepEqual(churn.get('src/other.ts'), { commits: 1, linesChanged: 1 });
|
||||
});
|
||||
|
||||
it('honours since/until windows', () => {
|
||||
const log = fixture(
|
||||
commit('recent', 1, 'fix: x', [['src/a.ts', 10, 0]]),
|
||||
commit('old', 365, 'fix: y', [['src/a.ts', 10, 0]]),
|
||||
);
|
||||
const commits = parseGitLog(log);
|
||||
const recent = computeChurn(commits, { since: NOW - 30 * DAY });
|
||||
assert.equal(recent.get('src/a.ts').commits, 1);
|
||||
});
|
||||
});
|
||||
|
||||
describe('computeFixDensity', () => {
|
||||
it('refuses to run without an explicit `now`', () => {
|
||||
assert.throws(() => computeFixDensity([], {}), /requires explicit `now`/);
|
||||
assert.throws(
|
||||
() => computeFixDensity([], { now: NOW, halfLifeDays: 0 }),
|
||||
/halfLifeDays/,
|
||||
);
|
||||
});
|
||||
|
||||
it('weighs recent fixes more than ancient fixes (same delta, same file)', () => {
|
||||
const log = fixture(
|
||||
commit('recent', 1, 'fix: x', [['src/a.ts', 10, 0]]),
|
||||
commit('ancient', 365, 'fix: y', [['src/b.ts', 10, 0]]),
|
||||
);
|
||||
const density = computeFixDensity(parseGitLog(log), {
|
||||
halfLifeDays: 90,
|
||||
now: NOW,
|
||||
});
|
||||
assert.ok(
|
||||
density.get('src/a.ts') > density.get('src/b.ts'),
|
||||
`recent ${density.get('src/a.ts')} should exceed ancient ${density.get('src/b.ts')}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('decays by exactly the half-life ratio', () => {
|
||||
const log = fixture(
|
||||
commit('t0', 0, 'fix: x', [['src/a.ts', 100, 0]]),
|
||||
commit('t1', DEFAULT_HALF_LIFE_DAYS, 'fix: y', [['src/b.ts', 100, 0]]),
|
||||
);
|
||||
const density = computeFixDensity(parseGitLog(log), { now: NOW });
|
||||
const ratio = density.get('src/b.ts') / density.get('src/a.ts');
|
||||
assert.ok(Math.abs(ratio - 0.5) < 1e-9, `expected half-life ratio, got ${ratio}`);
|
||||
});
|
||||
|
||||
it('weighs bigger deltas more (same age, same file)', () => {
|
||||
const log = fixture(
|
||||
commit('big', 1, 'fix: x', [['src/a.ts', 100, 0]]),
|
||||
commit('small', 1, 'fix: y', [['src/b.ts', 5, 0]]),
|
||||
);
|
||||
const density = computeFixDensity(parseGitLog(log), {
|
||||
halfLifeDays: 90,
|
||||
now: NOW,
|
||||
});
|
||||
assert.ok(density.get('src/a.ts') > density.get('src/b.ts'));
|
||||
});
|
||||
|
||||
it('ignores non-fix commits entirely', () => {
|
||||
const log = commit('chore', 1, 'chore: refactor', [['src/a.ts', 100, 0]]);
|
||||
const density = computeFixDensity(parseGitLog(log), {
|
||||
halfLifeDays: 90,
|
||||
now: NOW,
|
||||
});
|
||||
assert.equal(density.get('src/a.ts'), undefined);
|
||||
});
|
||||
|
||||
it('treats future-dated commits (skew/clock-drift) as age 0, not negative', () => {
|
||||
// Floor at 0 guards against decay > 1 if a clock-skewed commit lands
|
||||
// "in the future" relative to `now`. Behaviour: weight === 1, not >1.
|
||||
const log = commit('future', -7, 'fix: x', [['src/a.ts', 10, 0]]);
|
||||
const density = computeFixDensity(parseGitLog(log), {
|
||||
halfLifeDays: 90,
|
||||
now: NOW,
|
||||
});
|
||||
assert.equal(density.get('src/a.ts'), 10);
|
||||
});
|
||||
});
|
||||
|
||||
// PR-gate contract from DEVP-492:
|
||||
// "cold-file value < hot-file value at identical mutation status
|
||||
// using fixture git logs"
|
||||
//
|
||||
// Build two files that share the same hypothetical mutation status — what
|
||||
// differs is purely their history. `hot.ts` has three recent fix commits with
|
||||
// meaningful deltas; `cold.ts` has one ancient fix and one recent chore. The
|
||||
// risk signal MUST rank hot above cold on both churn and fix-density,
|
||||
// regardless of what their mutation score happens to be.
|
||||
describe('cold-vs-hot contract (DEVP-492 PR gate)', () => {
|
||||
const log = fixture(
|
||||
commit('h1', 1, 'fix(core): null pointer in hot path', [['src/hot.ts', 40, 10]]),
|
||||
commit('h2', 10, 'fix: edge case', [['src/hot.ts', 30, 5]]),
|
||||
commit('h3', 30, 'fix(workflow): off-by-one', [['src/hot.ts', 20, 8]]),
|
||||
commit('c1', 600, 'fix: ancient bug', [['src/cold.ts', 40, 10]]),
|
||||
commit('c2', 5, 'chore: rename variable', [['src/cold.ts', 2, 2]]),
|
||||
);
|
||||
const commits = parseGitLog(log);
|
||||
|
||||
it('hot file has higher churn (commit count and lines changed)', () => {
|
||||
const churn = computeChurn(commits);
|
||||
const hot = churn.get('src/hot.ts');
|
||||
const cold = churn.get('src/cold.ts');
|
||||
assert.ok(hot.commits > cold.commits, `hot.commits=${hot.commits} cold=${cold.commits}`);
|
||||
assert.ok(
|
||||
hot.linesChanged > cold.linesChanged,
|
||||
`hot.linesChanged=${hot.linesChanged} cold=${cold.linesChanged}`,
|
||||
);
|
||||
});
|
||||
|
||||
it('hot file has higher fix-density across realistic half-lives', () => {
|
||||
for (const halfLifeDays of [30, 90, 180]) {
|
||||
const density = computeFixDensity(commits, { halfLifeDays, now: NOW });
|
||||
const hot = density.get('src/hot.ts');
|
||||
const cold = density.get('src/cold.ts') ?? 0;
|
||||
assert.ok(
|
||||
hot > cold,
|
||||
`half-life=${halfLifeDays}d: hot=${hot} must exceed cold=${cold}`,
|
||||
);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user