diff --git a/packages/testing/README.md b/packages/testing/README.md new file mode 100644 index 00000000000..cc391eaf167 --- /dev/null +++ b/packages/testing/README.md @@ -0,0 +1,49 @@ +# packages/testing + +n8n's **test platform** — the infrastructure that decides what to test, runs it, and measures it. Each package is one concern; they compose rather than overlap. + +## Packages + +| Package | Name | Purpose | Entry point | Consumed by | +|---|---|---|---|---| +| **rules-engine** | `@n8n/rules-engine` | Generic, typed rules engine for static-analysis tools (register → run → report). The shared substrate. | library | janitor, code-health | +| **test-impact** | `@n8n/test-impact` | Test Impact Analysis: build the coverage→impact map, select impacted specs, distribute them across shards. Framework-agnostic. | library | janitor (CLI), playwright | +| **janitor** | `@n8n/playwright-janitor` | Static analysis + architecture enforcement for the Playwright suite; also hosts the impact/orchestrate CLI used by CI. | `janitor` CLI | playwright, CI | +| **code-health** | `@n8n/code-health` | Static analysis for monorepo dependency hygiene. | `code-health` CLI | CI | +| **containers** | `n8n-containers` | Composable Docker stack for tests (sqlite / postgres / queue / multi-main / observability / kafka …). | `stack:*` scripts | playwright, local dev | +| **playwright** | `n8n-playwright` | The E2E harness — page objects, composables, fixtures, and the shard distributor that drives CI. | `test:*` scripts | CI, local dev | +| **performance** | `@n8n/performance` | Microbenchmarks for critical code paths (`bench`, baseline, compare). | `bench:*` scripts | CI (nightly), local | + +## How they fit together + +```mermaid +graph TD + rules[rules-engine
rules substrate] --> janitor[janitor
PW lint + impact CLI] + rules --> codehealth[code-health
monorepo deps] + impact[test-impact
select + distribute] --> janitor + impact --> playwright[playwright
E2E harness] + janitor --> playwright + containers[containers
Docker stacks] --> playwright + performance[performance
benchmarks] + + classDef substrate fill:#eef,stroke:#88a; + class rules,impact substrate; +``` + +- **Static-analysis lane:** `rules-engine` (substrate) → `janitor` (Playwright architecture) and `code-health` (monorepo deps). +- **Selection lane:** `test-impact` (coverage map → select → distribute) feeds the janitor CLI and the playwright shard distributor. +- **Execution lane:** `playwright` runs E2E against `containers`-provided stacks. +- **Performance lane:** `performance` is standalone (benchmarks). + +## Where to look (quick nav for humans + agents) + +| I want to… | Go to | +|---|---| +| change which E2E specs a PR runs (impact map, selection, sharding) | `test-impact/` (+ `janitor` CLI `select` / `distribute`) | +| add/modify an architecture-lint rule for tests | `janitor/src/rules/` (engine: `rules-engine/`) | +| write or fix an E2E test, page object, or fixture | `playwright/` (see its `AGENTS.md`) | +| spin a DB/queue/multi-main stack for a test | `containers/` (`stack:*`) | +| add a microbenchmark | `performance/` | +| enforce a monorepo dependency rule | `code-health/` | + +Per-package detail lives in each package's own `README.md` (and `playwright/AGENTS.md`). diff --git a/packages/testing/janitor/package.json b/packages/testing/janitor/package.json index c3bcf7098ed..adc88b24e60 100644 --- a/packages/testing/janitor/package.json +++ b/packages/testing/janitor/package.json @@ -37,6 +37,7 @@ "license": "MIT", "dependencies": { "@n8n/rules-engine": "workspace:*", + "@n8n/test-impact": "workspace:*", "glob": "^10.0.0", "yaml": "catalog:" }, diff --git a/packages/testing/janitor/src/cli.ts b/packages/testing/janitor/src/cli.ts index 798e0a04f65..3f80bb876f1 100644 --- a/packages/testing/janitor/src/cli.ts +++ b/packages/testing/janitor/src/cli.ts @@ -16,6 +16,7 @@ * invoked from any package via `pnpm exec janitor ...`. */ +import { encodeImpactMap, buildImpactMap, distributeShards, selectTests } from '@n8n/test-impact'; import * as fs from 'node:fs'; import * as path from 'node:path'; @@ -45,7 +46,6 @@ import { formatBaselineInfo, getBaselinePath, } from './core/baseline.js'; -import { encodeImpactMap, mergeCoverage } from './core/coverage-map.js'; import { extractDiffs } from './core/extract-diffs.js'; import { ImpactAnalyzer, @@ -71,12 +71,10 @@ import { formatMethodUsageIndexConsole, formatMethodUsageIndexJSON, } from './core/method-usage-analyzer.js'; -import { orchestrate } from './core/orchestrator.js'; import { createProject } from './core/project-loader.js'; import { toJSON, toConsole } from './core/reporter.js'; import { filterToFailedSpecs } from './core/retry-filter.js'; import { computeScope, formatScope } from './core/scope-analyzer.js'; -import { selectE2e } from './core/select-e2e.js'; import { TcrExecutor, formatTcrResultConsole, formatTcrResultJSON } from './core/tcr-executor.js'; import { TestDiscoveryAnalyzer } from './core/test-discovery-analyzer.js'; import { runTestScoped } from './core/test-scoped-runner.js'; @@ -486,7 +484,7 @@ async function runFilterShard(options: CliOptions): Promise { } } -async function runOrchestrate(options: CliOptions): Promise { +async function runDistribute(options: CliOptions): Promise { const config = getConfig(); if (!options.shards || options.shards < 1) { @@ -534,7 +532,7 @@ async function runOrchestrate(options: CliOptions): Promise { } // Composable allowlist filter. distribute-tests.mjs pre-computes the union - // of AST + V8 selection and writes it here; orchestrate then balances shards + // of AST + V8 selection and writes it here; distributeShards then balances shards // against that subset instead of the full discovered set. if (options.includeSpecsFile) { const includeRaw = fs.readFileSync(options.includeSpecsFile, 'utf-8'); @@ -577,7 +575,7 @@ async function runOrchestrate(options: CliOptions): Promise { } } - const result = orchestrate(specs, options.shards, metrics, config.orchestration); + const result = distributeShards(specs, options.shards, metrics, config.orchestration); if (options.shardIndex !== undefined) { if (Number.isNaN(options.shardIndex) || options.shardIndex < 0) { @@ -671,7 +669,7 @@ function runMergeCoverage(options: CliOptions): void { const files = fs.existsSync(options.inputsDir) ? findLcovFiles(options.inputsDir) : []; // spec attribution comes from each lcov's TN:; the path is only a fallback. const inputs = files.map((f) => ({ text: fs.readFileSync(f, 'utf8'), spec: f })); - const result = mergeCoverage(inputs); + const result = buildImpactMap(inputs); fs.writeFileSync(options.outLcov, result.lcov); // Interned on-disk form — spec paths once, referenced by index (~10x smaller). fs.writeFileSync(options.outMap, JSON.stringify(encodeImpactMap(result.impactMap))); @@ -681,10 +679,10 @@ function runMergeCoverage(options: CliOptions): void { ); } -/** select-e2e: changed files + impact map → spec list (JSON). I/O wrapper - * around {@link selectE2e}, where the fail-open safety contract lives. */ -function runSelectE2e(options: CliOptions): void { - const result = selectE2e({ +/** select: changed files + impact map → spec list (JSON). I/O wrapper + * around {@link selectTests}, where the fail-open safety contract lives. */ +function runSelect(options: CliOptions): void { + const result = selectTests({ changedFiles: readChangedFiles(options) ?? [], mapFile: options.mapFile, allSpecsFile: options.allSpecsFile, @@ -719,7 +717,7 @@ async function main(): Promise { case 'discover': showDiscoverHelp(); break; - case 'orchestrate': + case 'distribute': showOrchestrateHelp(); break; case 'affected-packages': @@ -754,8 +752,8 @@ async function main(): Promise { runMergeCoverage(options); return; } - if (options.command === 'select-e2e') { - runSelectE2e(options); + if (options.command === 'select') { + runSelect(options); return; } @@ -796,8 +794,8 @@ async function main(): Promise { case 'discover': runDiscover(); break; - case 'orchestrate': - await runOrchestrate(options); + case 'distribute': + await runDistribute(options); break; case 'filter-shard': await runFilterShard(options); diff --git a/packages/testing/janitor/src/cli/arg-parser.ts b/packages/testing/janitor/src/cli/arg-parser.ts index a667fe9dc29..8dc2e7dfdf0 100644 --- a/packages/testing/janitor/src/cli/arg-parser.ts +++ b/packages/testing/janitor/src/cli/arg-parser.ts @@ -14,13 +14,13 @@ export type Command = | 'baseline' | 'rules' | 'discover' - | 'orchestrate' + | 'distribute' | 'affected-packages' | 'scope' | 'test-scoped' | 'filter-shard' | 'merge-coverage' - | 'select-e2e'; + | 'select'; export interface CliOptions { command: Command; @@ -63,13 +63,13 @@ export interface CliOptions { passthroughArgs: string[]; // filter-shard-specific options url?: string; - // coverage map options (merge-coverage / select-e2e) + // coverage map options (merge-coverage / select) inputsDir?: string; outLcov?: string; outMap?: string; mapFile?: string; allSpecsFile?: string; - /** Path to a newline-separated allowlist of spec paths (orchestrate). */ + /** Path to a newline-separated allowlist of spec paths (distribute). */ includeSpecsFile?: string; } @@ -81,13 +81,13 @@ const SUBCOMMANDS: Record = { baseline: 'baseline', rules: 'rules', discover: 'discover', - orchestrate: 'orchestrate', + distribute: 'distribute', 'affected-packages': 'affected-packages', scope: 'scope', 'test-scoped': 'test-scoped', 'filter-shard': 'filter-shard', 'merge-coverage': 'merge-coverage', - 'select-e2e': 'select-e2e', + select: 'select', }; interface FlagHandler { diff --git a/packages/testing/janitor/src/cli/help.ts b/packages/testing/janitor/src/cli/help.ts index 7bf51e31edb..61e5035b35f 100644 --- a/packages/testing/janitor/src/cli/help.ts +++ b/packages/testing/janitor/src/cli/help.ts @@ -20,7 +20,7 @@ Commands: method-impact Find tests that use a specific method (e.g., CanvasPage.addNode) tcr Run TCR (Test && Commit || Revert) workflow discover Discover test specs and capabilities (for orchestration) - orchestrate Distribute specs across shards using capability-aware bin-packing + distribute Distribute specs across shards using capability-aware bin-packing Analysis Options: --config= Path to janitor.config.js (default: ./janitor.config.js) @@ -48,7 +48,7 @@ For command-specific help: playwright-janitor method-impact --help playwright-janitor tcr --help playwright-janitor discover --help - playwright-janitor orchestrate --help + playwright-janitor distribute --help `); } @@ -183,8 +183,8 @@ Groups tests by capability to minimize fixture overhead, then uses greedy bin-packing to balance test time across shards. Outputs JSON to stdout. Usage: - playwright-janitor orchestrate --shards= # Full result as JSON - playwright-janitor orchestrate --shards= --shard-index= # Specs for one shard + playwright-janitor distribute --shards= # Full result as JSON + playwright-janitor distribute --shards= --shard-index= # Specs for one shard Options: --shards= Number of shards (required) @@ -201,9 +201,9 @@ Output: { shards: [{ shard, specs, testTime, capabilities, fixtureCount }], totalTestTime } Examples: - playwright-janitor orchestrate --shards=14 | jq '.shards[0].specs' - playwright-janitor orchestrate --shards=8 --impact - playwright-janitor orchestrate --shards=4 --impact --file=pages/CanvasPage.ts + playwright-janitor distribute --shards=14 | jq '.shards[0].specs' + playwright-janitor distribute --shards=8 --impact + playwright-janitor distribute --shards=4 --impact --file=pages/CanvasPage.ts `); } diff --git a/packages/testing/janitor/src/core/test-discovery-analyzer.ts b/packages/testing/janitor/src/core/test-discovery-analyzer.ts index 295fe92ea6c..94ee9cc4530 100644 --- a/packages/testing/janitor/src/core/test-discovery-analyzer.ts +++ b/packages/testing/janitor/src/core/test-discovery-analyzer.ts @@ -5,18 +5,18 @@ * Replaces the Playwright `--list` + regex approach used by distribute-tests.mjs. */ +import type { DiscoveredSpec } from '@n8n/test-impact'; import { SyntaxKind, type Project, type SourceFile, type CallExpression } from 'ts-morph'; import { getConfig } from '../config.js'; import { getSourceFiles } from './project-loader.js'; import { getRelativePath } from '../utils/paths.js'; -export interface DiscoveredSpec { - /** Spec file path relative to rootDir */ - path: string; - /** Capabilities extracted from tags matching capabilityPrefix */ - capabilities: string[]; -} +// DiscoveredSpec is owned by @n8n/test-impact (the framework-free orchestrator +// consumes it); re-exported here so this module's DiscoveryReport + existing +// importers keep their API. + +export type { DiscoveredSpec }; export interface DiscoveryReport { /** Active specs (specs with all tests skipped are excluded) */ diff --git a/packages/testing/playwright/package.json b/packages/testing/playwright/package.json index cb09245280c..3d896465852 100644 --- a/packages/testing/playwright/package.json +++ b/packages/testing/playwright/package.json @@ -59,6 +59,7 @@ "@n8n/instance-ai": "workspace:*", "@n8n/permissions": "workspace:*", "@n8n/playwright-janitor": "workspace:*", + "@n8n/test-impact": "workspace:*", "@n8n/workflow-sdk": "workspace:*", "@playwright/cli": "catalog:e2e", "@playwright/test": "catalog:e2e", diff --git a/packages/testing/playwright/scripts/backend-coverage-resolver.ts b/packages/testing/playwright/scripts/backend-coverage-resolver.ts index 9cb6235fea3..5c73cd34798 100644 --- a/packages/testing/playwright/scripts/backend-coverage-resolver.ts +++ b/packages/testing/playwright/scripts/backend-coverage-resolver.ts @@ -146,12 +146,6 @@ export function formatBackendStats(stats: BackendResolveStats): string { ); } -/** - * Force every `TN:` record in an lcov to `spec`, so the merge attributes the - * coverage to exactly this spec (the impact map keys on TN). Prepends a `TN:` - * if the lcov has none. Mirrors the frontend per-spec emitter. - */ -export function forceSpecTn(lcov: string, spec: string): string { - const tagged = lcov.replace(/^TN:.*$/gm, `TN:${spec}`); - return tagged.startsWith('TN:') ? tagged : `TN:${spec}\n${tagged}`; -} +// forceSpecTn is owned by @n8n/test-impact's map-build kernel; re-exported here +// so existing importers (and the test) keep their path. +export { forceSpecTn } from '@n8n/test-impact'; diff --git a/packages/testing/playwright/scripts/distribute-tests.mjs b/packages/testing/playwright/scripts/distribute-tests.mjs index cbdb8622342..3215f781cb1 100644 --- a/packages/testing/playwright/scripts/distribute-tests.mjs +++ b/packages/testing/playwright/scripts/distribute-tests.mjs @@ -4,7 +4,7 @@ /** * n8n CI Adapter for Test Distribution * - * Thin wrapper that calls `janitor orchestrate` for generic shard distribution, + * Thin wrapper that calls `janitor distribute` for generic shard distribution, * then maps capabilities to n8n-specific Docker images for the CI matrix. * * Impact scoping is a domain-partitioned UNION of two analyzers (DEVP-364): @@ -174,7 +174,7 @@ function logSelectionDecision(decision) { } function getOrchestration(numShards, options = {}) { - const cliArgs = ['orchestrate', `--shards=${numShards}`]; + const cliArgs = ['distribute', `--shards=${numShards}`]; const includeFile = options.includeSpecsFile; if (includeFile) cliArgs.push(`--include-specs-file=${includeFile}`); const output = execFileSync('node', [JANITOR_CLI, ...cliArgs], { diff --git a/packages/testing/playwright/scripts/emit-spec-backend-lcovs.ts b/packages/testing/playwright/scripts/emit-spec-backend-lcovs.ts index a8972be7e7c..3352d02aec6 100644 --- a/packages/testing/playwright/scripts/emit-spec-backend-lcovs.ts +++ b/packages/testing/playwright/scripts/emit-spec-backend-lcovs.ts @@ -7,22 +7,22 @@ * resolves each entry's dist URL to the checkout's `packages//src/*.ts` (the * same resolution the shard emitter uses), and emits one lcov per spec tagged * with the spec id in `TN:`. These join the frontend per-spec lcovs in - * `coverage/by-spec/` and let the impact map attribute backend source files to - * the specs that exercise them — so a backend change selects E2E specs instead - * of running the whole suite. + * `coverage/by-spec/` so the impact map attributes backend source files to the + * specs that exercise them. * - * Best-effort: a spec with no resolvable backend coverage is simply skipped - * (fail-open — no backend rows, never a failure). + * Thin wrapper over the generic @n8n/test-impact build kernel; the n8n-specific + * dist→source resolution is injected via `feedRaws`. Best-effort: a spec with no + * resolvable backend coverage is skipped (fail-open). */ -import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { CoverageReport } from 'monocart-coverage-reports'; +import { emitPerSpecLcovs } from '@n8n/test-impact'; +import type { CoverageReport } from 'monocart-coverage-reports'; import { buildPackageMap, createBackendResolveStats, - forceSpecTn, formatBackendStats, resolveBackendEntries, } from './backend-coverage-resolver'; @@ -31,57 +31,35 @@ import { BACKEND_BY_SPEC_DIR, coverageOptions } from '../coverage-options'; const OUT_DIR = join(coverageOptions.outputDir ?? './coverage', 'by-spec'); async function main() { - if (!existsSync(BACKEND_BY_SPEC_DIR)) { - console.log(`emit-spec-backend-lcovs: ${BACKEND_BY_SPEC_DIR} absent — no per-spec backend raw`); - return; - } - mkdirSync(OUT_DIR, { recursive: true }); const pkgMap = buildPackageMap(); const stats = createBackendResolveStats(); - const dirs = readdirSync(BACKEND_BY_SPEC_DIR).filter((d) => - statSync(join(BACKEND_BY_SPEC_DIR, d)).isDirectory(), - ); - let emitted = 0; - for (const slug of dirs) { - const dir = join(BACKEND_BY_SPEC_DIR, slug); - const specMarker = join(dir, '.spec'); - if (!existsSync(specMarker)) continue; - const spec = readFileSync(specMarker, 'utf8').trim(); - const rawFiles = readdirSync(dir).filter((f) => f.startsWith('raw-') && f.endsWith('.json')); - if (!rawFiles.length) continue; - const report = new CoverageReport({ - ...coverageOptions, - name: spec, - outputDir: dir, - reports: ['lcovonly'], - }); - let added = 0; - for (const rf of rawFiles) { - let parsed: { result?: Array<{ url: string }> }; - try { - parsed = JSON.parse(readFileSync(join(dir, rf), 'utf8')); - } catch { - continue; + const emit = await emitPerSpecLcovs({ + bySpecDir: BACKEND_BY_SPEC_DIR, + outDir: OUT_DIR, + coverageOptions, + suffix: '-backend', + feedRaws: async (report: CoverageReport, rawFiles, dir) => { + let added = 0; + for (const rf of rawFiles) { + let parsed: { result?: Array<{ url: string }> }; + try { + parsed = JSON.parse(readFileSync(join(dir, rf), 'utf8')); + } catch { + continue; + } + const entries = resolveBackendEntries(parsed, pkgMap, stats); + if (entries.length) { + await report.add(entries as never); + added += entries.length; + } } - const entries = resolveBackendEntries(parsed, pkgMap, stats); - if (entries.length) { - await report.add(entries as never); - added += entries.length; - } - } - if (!added) continue; + return added > 0; + }, + }); - const result = await report.generate(); - const lcovPath = join(dir, 'lcov.info'); - if (!result || !result.files?.length || !existsSync(lcovPath)) continue; - // Force every record's TN to the real spec id so the merge attributes it. - const lcov = forceSpecTn(readFileSync(lcovPath, 'utf8'), spec); - writeFileSync(join(OUT_DIR, `${slug}-backend.lcov`), lcov); - emitted++; - } console.log( - `emit-spec-backend-lcovs: ${dirs.length} dirs → ${emitted} per-spec backend lcov(s) in ${OUT_DIR} ` + + `emit-spec-backend-lcovs: ${emit.dirs} dirs → ${emit.emitted} per-spec backend lcov(s) in ${OUT_DIR} ` + `(${formatBackendStats(stats)})`, ); } diff --git a/packages/testing/playwright/scripts/emit-spec-lcovs.ts b/packages/testing/playwright/scripts/emit-spec-lcovs.ts index 4a35c97c672..895b385adf6 100644 --- a/packages/testing/playwright/scripts/emit-spec-lcovs.ts +++ b/packages/testing/playwright/scripts/emit-spec-lcovs.ts @@ -4,62 +4,42 @@ * `TN:`. These feed the impact map, letting a git diff select the E2E specs that * exercise the touched frontend code. * + * Thin wrapper: the generic build kernel lives in @n8n/test-impact; this script + * supplies the n8n-specific input dir + monocart coverage options, and feeds the + * raw page.coverage directly into the report. + * * Frontend only: backend coverage is a shared worker-scoped process with no * per-test boundary, so it stays at report granularity. See DEVP-205. */ -import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { readFileSync } from 'node:fs'; import { join } from 'node:path'; -import { CoverageReport } from 'monocart-coverage-reports'; +import { emitPerSpecLcovs } from '@n8n/test-impact'; +import type { CoverageReport } from 'monocart-coverage-reports'; import { BY_SPEC_DIR, coverageOptions } from '../coverage-options'; const OUT_DIR = join(coverageOptions.outputDir ?? './coverage', 'by-spec'); async function main() { - if (!existsSync(BY_SPEC_DIR)) { - console.log(`emit-spec-lcovs: ${BY_SPEC_DIR} absent — no per-spec coverage collected`); - return; - } - mkdirSync(OUT_DIR, { recursive: true }); - const dirs = readdirSync(BY_SPEC_DIR).filter((d) => statSync(join(BY_SPEC_DIR, d)).isDirectory()); - let withMarker = 0; - let withRaw = 0; - let emitted = 0; - for (const slug of dirs) { - const dir = join(BY_SPEC_DIR, slug); - const specMarker = join(dir, '.spec'); - if (!existsSync(specMarker)) continue; - withMarker++; - const spec = readFileSync(specMarker, 'utf8').trim(); - const rawFiles = readdirSync(dir).filter((f) => f.startsWith('raw-') && f.endsWith('.json')); - if (!rawFiles.length) continue; - withRaw++; - const report = new CoverageReport({ - ...coverageOptions, - name: spec, - outputDir: dir, - reports: ['lcovonly'], - }); - for (const rf of rawFiles) { - try { - await report.add(JSON.parse(readFileSync(join(dir, rf), 'utf8'))); - } catch (error) { - console.warn(` ⚠ ${slug}/${rf}: ${String(error)}`); + const stats = await emitPerSpecLcovs({ + bySpecDir: BY_SPEC_DIR, + outDir: OUT_DIR, + coverageOptions, + feedRaws: async (report: CoverageReport, rawFiles, dir) => { + for (const rf of rawFiles) { + try { + await report.add(JSON.parse(readFileSync(join(dir, rf), 'utf8'))); + } catch (error) { + console.warn(` ⚠ ${rf}: ${String(error)}`); + } } - } - const result = await report.generate(); - const lcovPath = join(dir, 'lcov.info'); - if (!result || !result.files?.length || !existsSync(lcovPath)) continue; - // Force every record's TN to the real spec id so the merge attributes it. - let lcov = readFileSync(lcovPath, 'utf8').replace(/^TN:.*$/gm, `TN:${spec}`); - if (!lcov.startsWith('TN:')) lcov = `TN:${spec}\n${lcov}`; - writeFileSync(join(OUT_DIR, `${slug}.lcov`), lcov); - emitted++; - } + return true; + }, + }); console.log( - `emit-spec-lcovs: ${dirs.length} dirs, ${withMarker} with .spec, ${withRaw} with raw → ` + - `${emitted} per-spec lcov(s) in ${OUT_DIR}`, + `emit-spec-lcovs: ${stats.dirs} dirs, ${stats.withMarker} with .spec, ${stats.withRaw} with raw → ` + + `${stats.emitted} per-spec lcov(s) in ${OUT_DIR}`, ); } diff --git a/packages/testing/playwright/scripts/select-affected-e2e.mjs b/packages/testing/playwright/scripts/select-affected-e2e.mjs index d485f409849..8819bc0790e 100644 --- a/packages/testing/playwright/scripts/select-affected-e2e.mjs +++ b/packages/testing/playwright/scripts/select-affected-e2e.mjs @@ -3,7 +3,7 @@ /** * Resolve changed files → the E2E specs that must run, using the coverage - * impact map. Prints the janitor select-e2e JSON ({ specs, unmapped, mode }). + * impact map. Prints the janitor select JSON ({ specs, unmapped, mode }). * * node select-affected-e2e.mjs [ ...] * CHANGED_FILES=a.ts,b.vue node select-affected-e2e.mjs @@ -57,7 +57,7 @@ export function resolveMapPath(opts = {}) { * @returns {string[]} */ export function buildArgs({ changedFiles, mapPath, allSpecs }) { - const args = ['select-e2e', `--changed-files=${changedFiles}`]; + const args = ['select', `--changed-files=${changedFiles}`]; if (mapPath) args.push(`--map=${mapPath}`); if (allSpecs) args.push(`--all-specs=${allSpecs}`); return args; diff --git a/packages/testing/playwright/scripts/select-affected-e2e.test.ts b/packages/testing/playwright/scripts/select-affected-e2e.test.ts index cab66a03ed1..34769f8c47e 100644 --- a/packages/testing/playwright/scripts/select-affected-e2e.test.ts +++ b/packages/testing/playwright/scripts/select-affected-e2e.test.ts @@ -8,7 +8,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { buildArgs, resolveMapPath } from './select-affected-e2e.mjs'; // The wrapper is the bridge between CI (raw changed-files list) and janitor -// select-e2e. Its only job is to keep selection FAIL-OPEN: any failure in +// select. Its only job is to keep selection FAIL-OPEN: any failure in // locating the map must degrade to broad — never throw, never hide selection. describe('select-affected-e2e wrapper — fail-open contract', () => { @@ -54,19 +54,14 @@ describe('select-affected-e2e wrapper — fail-open contract', () => { // the janitor CLI — assert by absence, not by a placeholder value. it('omits --map when mapPath is null (fail-open broad)', () => { const args = buildArgs({ changedFiles: 'a.ts,b.ts', mapPath: null }); - expect(args).toEqual(['select-e2e', '--changed-files=a.ts,b.ts']); + expect(args).toEqual(['select', '--changed-files=a.ts,b.ts']); expect(args.some((a: string) => a.startsWith('--map='))).toBe(false); }); it('passes --map and --all-specs through when provided', () => { expect( buildArgs({ changedFiles: 'a.ts', mapPath: '/tmp/m.json', allSpecs: '/tmp/s.txt' }), - ).toEqual([ - 'select-e2e', - '--changed-files=a.ts', - '--map=/tmp/m.json', - '--all-specs=/tmp/s.txt', - ]); + ).toEqual(['select', '--changed-files=a.ts', '--map=/tmp/m.json', '--all-specs=/tmp/s.txt']); }); }); }); diff --git a/packages/testing/test-impact/eslint.config.mjs b/packages/testing/test-impact/eslint.config.mjs new file mode 100644 index 00000000000..29dae474b1a --- /dev/null +++ b/packages/testing/test-impact/eslint.config.mjs @@ -0,0 +1,14 @@ +import { defineConfig } from 'eslint/config'; +import { baseConfig } from '@n8n/eslint-config/base'; + +export default defineConfig( + baseConfig, + { ignores: ['dist/**', 'coverage/**'] }, + { + // Test fixtures use file paths, spec names and line numbers as object keys + // (the impact-map shape is `file → { line → specs }`) — these are data, not + // identifiers, so the naming-convention rule doesn't apply. + files: ['**/*.test.ts'], + rules: { '@typescript-eslint/naming-convention': 'off' }, + }, +); diff --git a/packages/testing/test-impact/package.json b/packages/testing/test-impact/package.json new file mode 100644 index 00000000000..f70cc2a5e8c --- /dev/null +++ b/packages/testing/test-impact/package.json @@ -0,0 +1,50 @@ +{ + "name": "@n8n/test-impact", + "private": true, + "version": "0.1.0", + "description": "Test Impact Analysis: coverage-map selection + shard orchestration for n8n CI. Framework-agnostic core.", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "exports": { + ".": { + "types": "./dist/index.d.ts", + "default": "./dist/index.js" + } + }, + "files": [ + "dist" + ], + "scripts": { + "clean": "rimraf dist .turbo", + "build": "tsc", + "build:unchecked": "tsc --noCheck", + "dev": "tsc --watch", + "test": "vitest run", + "test:unit": "vitest run", + "test:watch": "vitest", + "test:coverage": "vitest run --coverage", + "format": "biome format --write .", + "format:check": "biome ci .", + "lint": "eslint . --quiet", + "lint:fix": "eslint . --fix", + "typecheck": "tsc --noEmit" + }, + "keywords": [ + "testing", + "test-impact-analysis", + "coverage", + "ci", + "sharding" + ], + "license": "MIT", + "dependencies": { + "monocart-coverage-reports": "^2.12.0" + }, + "devDependencies": { + "@types/node": "catalog:", + "@vitest/coverage-v8": "catalog:", + "fast-check": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + } +} diff --git a/packages/testing/janitor/src/core/coverage-map.test.ts b/packages/testing/test-impact/src/impact-map.test.ts similarity index 93% rename from packages/testing/janitor/src/core/coverage-map.test.ts rename to packages/testing/test-impact/src/impact-map.test.ts index c198526a61b..6a6b2c109cc 100644 --- a/packages/testing/janitor/src/core/coverage-map.test.ts +++ b/packages/testing/test-impact/src/impact-map.test.ts @@ -7,10 +7,10 @@ import { type LcovInput, decodeImpactMap, encodeImpactMap, - mergeCoverage, + buildImpactMap, parseLcov, resolveImpact, -} from './coverage-map.js'; +} from './impact-map.js'; // --------------------------------------------------------------------------- // Model + generators @@ -93,9 +93,9 @@ describe('parseLcov', () => { }); }); -describe('mergeCoverage', () => { +describe('buildImpactMap', () => { it('attributes a function to every spec that executed it (hits > 0)', () => { - const { impactMap } = mergeCoverage([ + const { impactMap } = buildImpactMap([ { spec: 'A', text: 'TN:A\nSF:f.ts\nFN:10,fn\nFNDA:2,fn\nend_of_record\n' }, { spec: 'B', text: 'TN:B\nSF:f.ts\nFN:10,fn\nFNDA:1,fn\nend_of_record\n' }, ]); @@ -103,7 +103,7 @@ describe('mergeCoverage', () => { }); it('excludes load-only functions (hits === 0) from the map', () => { - const { impactMap } = mergeCoverage([ + const { impactMap } = buildImpactMap([ { spec: 'A', text: 'TN:A\nSF:f.ts\nFN:10,hit\nFN:20,loaded\nFNDA:4,hit\nFNDA:0,loaded\nend_of_record\n', @@ -114,7 +114,7 @@ describe('mergeCoverage', () => { }); it('sums hit counts across specs in the unified lcov', () => { - const { lcov } = mergeCoverage([ + const { lcov } = buildImpactMap([ { spec: 'A', text: 'TN:A\nSF:f.ts\nFN:10,fn\nFNDA:2,fn\nDA:10,2\nend_of_record\n' }, { spec: 'B', text: 'TN:B\nSF:f.ts\nFN:10,fn\nFNDA:3,fn\nDA:10,3\nend_of_record\n' }, ]); @@ -229,11 +229,11 @@ describe('resolveImpact — sibling fallback', () => { // Property + metamorphic tests — the soundness guarantee // =========================================================================== -describe('mergeCoverage — properties', () => { +describe('buildImpactMap — properties', () => { it('SOUNDNESS: every executed (spec, function) appears in the map', () => { fc.assert( fc.property(arbExecs, (execs) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); for (const e of execs) { if (e.hits > 0) { expect(impactMap[e.file]?.[String(e.fnLine)] ?? []).toContain(e.spec); @@ -246,7 +246,7 @@ describe('mergeCoverage — properties', () => { it('NO PHANTOMS: a spec is not attributed to a function it only loaded (hits 0)', () => { fc.assert( fc.property(arbExecs, (execs) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); for (const e of execs) { if (e.hits === 0) { // (spec,file,fnLine) is unique per execs, so no hits>0 record revives it. @@ -261,8 +261,8 @@ describe('mergeCoverage — properties', () => { fc.assert( fc.property(arbExecs, fc.array(fc.nat(), { maxLength: 40 }), (execs, keys) => { const inputs = buildInputs(execs); - const a = mergeCoverage(inputs); - const b = mergeCoverage(shuffle(inputs, keys)); + const a = buildImpactMap(inputs); + const b = buildImpactMap(shuffle(inputs, keys)); expect(b.impactMap).toEqual(a.impactMap); expect(b.lcov).toEqual(a.lcov); }), @@ -272,8 +272,8 @@ describe('mergeCoverage — properties', () => { it('MONOTONIC: adding coverage never removes a map entry', () => { fc.assert( fc.property(arbExecs, arbExecs, (a, b) => { - const mapA = mergeCoverage(buildInputs(a)).impactMap; - const mapAB = mergeCoverage([...buildInputs(a), ...buildInputs(b)]).impactMap; + const mapA = buildImpactMap(buildInputs(a)).impactMap; + const mapAB = buildImpactMap([...buildInputs(a), ...buildInputs(b)]).impactMap; for (const file of Object.keys(mapA)) { for (const line of Object.keys(mapA[file])) { for (const spec of mapA[file][line]) { @@ -289,8 +289,8 @@ describe('mergeCoverage — properties', () => { fc.assert( fc.property(arbExecs, (execs) => { const inputs = buildInputs(execs); - expect(mergeCoverage([...inputs, ...inputs]).impactMap).toEqual( - mergeCoverage(inputs).impactMap, + expect(buildImpactMap([...inputs, ...inputs]).impactMap).toEqual( + buildImpactMap(inputs).impactMap, ); }), ); @@ -301,7 +301,7 @@ describe('resolveImpact — properties', () => { it('SOUNDNESS: a change to an executed function selects every spec that ran it', () => { fc.assert( fc.property(arbExecs, (execs) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); for (const e of execs) { if (e.hits > 0) { const r = resolveImpact([{ file: e.file, lines: [e.fnLine] }], impactMap); @@ -315,7 +315,7 @@ describe('resolveImpact — properties', () => { it('LINE-PRECISE ⊆ FILE-LEVEL: pinning a line never selects more than the whole file', () => { fc.assert( fc.property(arbExecs, fc.integer({ min: 1, max: 60 }), (execs, line) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); const files = Object.keys(impactMap); if (files.length === 0) return; const file = files[0]; @@ -329,7 +329,7 @@ describe('resolveImpact — properties', () => { it('MONOTONIC: resolving more changed files never selects fewer specs', () => { fc.assert( fc.property(arbExecs, (execs) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); const files = Object.keys(impactMap); if (files.length < 2) return; const a: ChangedFile[] = [{ file: files[0] }]; @@ -347,7 +347,7 @@ describe('resolveImpact — properties', () => { arbExecs, fc.array(fc.string(), { minLength: 1, maxLength: 5 }), (execs, allSpecs) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); const r = resolveImpact([{ file: 'packages/never/covered.ts' }], impactMap, { allSpecs }); expect(r.mode).toBe('broad'); for (const s of allSpecs) expect(r.specs).toContain(s); @@ -365,11 +365,11 @@ describe('resolveImpact — properties', () => { // =========================================================================== // Serializer + parser hardening — the serializer is exercised by every merge -// but the algebra properties use mergeCoverage as their own oracle, so a +// but the algebra properties use buildImpactMap as their own oracle, so a // wrong-but-deterministic serialization slips through. These pin it externally. // =========================================================================== -describe('serializeLcov (via mergeCoverage.lcov)', () => { +describe('serializeLcov (via buildImpactMap.lcov)', () => { it('SERIALIZE-CORRECT: parse∘serialize round-trips FN/FNDA/DA hits', () => { fc.assert( fc.property(arbExecs, (execs) => { @@ -380,7 +380,7 @@ describe('serializeLcov (via mergeCoverage.lcov)', () => { expected.set(e.file, f); f.set(e.fnLine, (f.get(e.fnLine) ?? 0) + e.hits); } - const { lcov } = mergeCoverage(buildInputs(execs)); + const { lcov } = buildImpactMap(buildInputs(execs)); for (const rec of parseLcov(lcov)) { const exp = expected.get(rec.file); if (!exp) continue; @@ -392,7 +392,7 @@ describe('serializeLcov (via mergeCoverage.lcov)', () => { }); it('emits FN/DA sorted by line and correct FNF/FNH/LF/LH', () => { - const { lcov } = mergeCoverage([ + const { lcov } = buildImpactMap([ { spec: 'A', text: 'TN:A\nSF:f.ts\nFN:30,c\nFN:10,a\nFN:20,b\nFNDA:0,c\nFNDA:5,a\nFNDA:2,b\nDA:30,0\nDA:10,5\nDA:20,2\nend_of_record\n', @@ -414,14 +414,14 @@ describe('encode/decode impact map (interned on-disk form)', () => { it('LOSSLESS: decode∘encode round-trips any impact map', () => { fc.assert( fc.property(arbExecs, (execs) => { - const { impactMap } = mergeCoverage(buildInputs(execs)); + const { impactMap } = buildImpactMap(buildInputs(execs)); expect(decodeImpactMap(encodeImpactMap(impactMap))).toEqual(impactMap); }), ); }); it('interns each spec path exactly once', () => { - const { impactMap } = mergeCoverage([ + const { impactMap } = buildImpactMap([ { spec: 'A', text: 'TN:A\nSF:f.ts\nFN:10,a\nFN:20,b\nFNDA:1,a\nFNDA:1,b\nend_of_record\n' }, ]); const enc = encodeImpactMap(impactMap); diff --git a/packages/testing/janitor/src/core/coverage-map.ts b/packages/testing/test-impact/src/impact-map.ts similarity index 97% rename from packages/testing/janitor/src/core/coverage-map.ts rename to packages/testing/test-impact/src/impact-map.ts index 8a14a4b558c..85c05806209 100644 --- a/packages/testing/janitor/src/core/coverage-map.ts +++ b/packages/testing/test-impact/src/impact-map.ts @@ -3,7 +3,7 @@ * * Pure functions (no I/O) so they can be exhaustively unit/property tested: * parseLcov — lcov text → per-spec coverage records - * mergeCoverage — per-spec lcovs → unified lcov + bidirectional impact map + * buildImpactMap — per-spec lcovs → unified lcov + bidirectional impact map * resolveImpact — changed files (± lines) → the E2E specs that must run * * SOUNDNESS is the contract everything is tested against: the selected set must @@ -134,7 +134,7 @@ export function parseLcov(text: string, fallbackSpec = ''): LcovRecord[] { * the map records, per function, the SET of specs that executed it (hits > 0). * Output is deterministic (sorted) so the merge is order-independent. */ -export function mergeCoverage(inputs: LcovInput[]): MergeResult { +export function buildImpactMap(inputs: LcovInput[]): MergeResult { const files = new Map(); const funcToSpecs = new Map>(); const allSpecs = new Set(); @@ -168,7 +168,7 @@ export function mergeCoverage(inputs: LcovInput[]): MergeResult { return { lcov: serializeLcov(files), - impactMap: buildImpactMap(funcToSpecs), + impactMap: assembleImpactMap(funcToSpecs), stats: { files: files.size, functions: [...files.values()].reduce((n, f) => n + f.fns.size, 0), @@ -204,7 +204,7 @@ function serializeLcov(files: Map): string { return out.join('\n') + (out.length ? '\n' : ''); } -function buildImpactMap(funcToSpecs: Map>): ImpactMap { +function assembleImpactMap(funcToSpecs: Map>): ImpactMap { const map: ImpactMap = {}; for (const [key, specs] of funcToSpecs) { const hash = key.lastIndexOf('#'); @@ -249,7 +249,7 @@ export function encodeImpactMap(map: ImpactMap): InternedImpactMap { * Expand an {@link InternedImpactMap} back to a full {@link ImpactMap}. Handles * both entry forms (index list or `"b:…"` bitmask) and older maps that predate * the bitmask form (all index lists). Spec lists come back sorted, matching - * {@link mergeCoverage}'s output, so decode∘encode round-trips exactly. + * {@link buildImpactMap}'s output, so decode∘encode round-trips exactly. */ export function decodeImpactMap(interned: InternedImpactMap): ImpactMap { const { specs, files } = interned; diff --git a/packages/testing/test-impact/src/index.ts b/packages/testing/test-impact/src/index.ts new file mode 100644 index 00000000000..a12b8030f59 --- /dev/null +++ b/packages/testing/test-impact/src/index.ts @@ -0,0 +1,17 @@ +/** + * `@n8n/test-impact` — framework-agnostic Test Impact Analysis core. + * + * Phase 1 (ts-morph-free): coverage-map (build + resolve), orchestrator + * (shard bin-packing), and the V8 selection path. AST-based selection and + * the Playwright fixture subpath land in later phases. + */ +export * from './impact-map.js'; +export * from './shard-distributor.js'; +export * from './select.js'; +export * from './map-build.js'; +export type { DiscoveredSpec } from './types.js'; + +// Strategy + Pipeline selection layer. +export type { SelectionStrategy } from './select/strategy.js'; +export { CoverageMapStrategy } from './select/coverage-map-strategy.js'; +export { selectImpactedTests } from './select/pipeline.js'; diff --git a/packages/testing/test-impact/src/map-build.test.ts b/packages/testing/test-impact/src/map-build.test.ts new file mode 100644 index 00000000000..29022d02bee --- /dev/null +++ b/packages/testing/test-impact/src/map-build.test.ts @@ -0,0 +1,59 @@ +import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { forceSpecTn, emitPerSpecLcovs, type EmitPerSpecLcovsOptions } from './map-build.js'; + +describe('forceSpecTn', () => { + it('rewrites every record TN to the spec id', () => { + const lcov = 'TN:wrong\nSF:a.ts\nend_of_record\nTN:also\nSF:b.ts\nend_of_record'; + expect(forceSpecTn(lcov, 'tests/e2e/x.spec.ts').match(/^TN:.*$/gm)).toEqual([ + 'TN:tests/e2e/x.spec.ts', + 'TN:tests/e2e/x.spec.ts', + ]); + }); + + it('prepends TN when the lcov has none', () => { + expect(forceSpecTn('SF:a.ts\nend_of_record', 'spec')).toBe('TN:spec\nSF:a.ts\nend_of_record'); + }); +}); + +describe('emitPerSpecLcovs (scan/skip logic)', () => { + let root: string; + beforeEach(() => { + root = mkdtempSync(join(tmpdir(), 'mapbuild-')); + }); + afterEach(() => { + rmSync(root, { recursive: true, force: true }); + }); + const opts = (feedRaws: EmitPerSpecLcovsOptions['feedRaws']): EmitPerSpecLcovsOptions => ({ + bySpecDir: join(root, 'by-spec'), + outDir: join(root, 'out'), + coverageOptions: {}, + feedRaws, + }); + + it('returns empty stats when bySpecDir is absent', async () => { + expect(await emitPerSpecLcovs(opts(() => false))).toEqual({ + dirs: 0, + withMarker: 0, + withRaw: 0, + emitted: 0, + }); + }); + + it('counts dirs / markers / raws and skips when feedRaws returns false', async () => { + const bs = join(root, 'by-spec'); + mkdirSync(join(bs, 'spec_a'), { recursive: true }); + writeFileSync(join(bs, 'spec_a', '.spec'), 'tests/e2e/a.spec.ts'); + writeFileSync(join(bs, 'spec_a', 'raw-1.json'), '{}'); + mkdirSync(join(bs, 'no_marker'), { recursive: true }); + writeFileSync(join(bs, 'no_marker', 'raw-1.json'), '{}'); + mkdirSync(join(bs, 'no_raw'), { recursive: true }); + writeFileSync(join(bs, 'no_raw', '.spec'), 'x'); + + const stats = await emitPerSpecLcovs(opts(() => false)); + expect(stats).toEqual({ dirs: 3, withMarker: 2, withRaw: 1, emitted: 0 }); + }); +}); diff --git a/packages/testing/test-impact/src/map-build.ts b/packages/testing/test-impact/src/map-build.ts new file mode 100644 index 00000000000..8519209f625 --- /dev/null +++ b/packages/testing/test-impact/src/map-build.ts @@ -0,0 +1,88 @@ +import { CoverageReport, type CoverageReportOptions } from 'monocart-coverage-reports'; +import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, writeFileSync } from 'node:fs'; +import { join } from 'node:path'; + +/** + * Force every lcov record's `TN:` to the real spec id, so the impact-map merge + * attributes the file→spec edges to that spec (monocart stamps its own TN). + */ +export function forceSpecTn(lcov: string, spec: string): string { + let out = lcov.replace(/^TN:.*$/gm, `TN:${spec}`); + if (!out.startsWith('TN:')) out = `TN:${spec}\n${out}`; + return out; +} + +export interface EmitPerSpecLcovsOptions { + /** Dir of per-spec raw-coverage folders, each with a `.spec` marker + `raw-*.json`. */ + bySpecDir: string; + /** Where to write `.lcov`. */ + outDir: string; + /** monocart report options (the bundle-specific entry/source filters live here). */ + coverageOptions: CoverageReportOptions; + /** Output filename suffix — `''` frontend, `'-backend'` backend. */ + suffix?: string; + /** + * Feed a spec's raw files into the report. Frontend adds raws directly; + * backend resolves dist→source first. Return `false` to skip the spec. + * Keeps this kernel framework-agnostic — the runner-specific raw handling + * (and any n8n dist→src resolution) is injected by the caller. + */ + feedRaws: ( + report: CoverageReport, + rawFiles: string[], + dir: string, + spec: string, + ) => boolean | Promise; +} + +export interface EmitStats { + dirs: number; + withMarker: number; + withRaw: number; + emitted: number; +} + +/** + * Turn per-spec raw coverage folders into one lcov per spec (TN-tagged with the + * spec id) under `outDir`. These per-spec lcovs are the impact-map inputs — a + * git diff then selects the specs that executed the touched code. Generic over + * how raws are fed in (see {@link EmitPerSpecLcovsOptions.feedRaws}). + */ +export async function emitPerSpecLcovs(opts: EmitPerSpecLcovsOptions): Promise { + const { bySpecDir, outDir, coverageOptions, suffix = '', feedRaws } = opts; + const stats: EmitStats = { dirs: 0, withMarker: 0, withRaw: 0, emitted: 0 }; + if (!existsSync(bySpecDir)) return stats; + + mkdirSync(outDir, { recursive: true }); + const dirs = readdirSync(bySpecDir).filter((d) => statSync(join(bySpecDir, d)).isDirectory()); + stats.dirs = dirs.length; + + for (const slug of dirs) { + const dir = join(bySpecDir, slug); + const specMarker = join(dir, '.spec'); + if (!existsSync(specMarker)) continue; + stats.withMarker++; + const spec = readFileSync(specMarker, 'utf8').trim(); + const rawFiles = readdirSync(dir).filter((f) => f.startsWith('raw-') && f.endsWith('.json')); + if (!rawFiles.length) continue; + stats.withRaw++; + + const report = new CoverageReport({ + ...coverageOptions, + name: spec, + outputDir: dir, + reports: ['lcovonly'], + }); + if (!(await feedRaws(report, rawFiles, dir, spec))) continue; + + const result = await report.generate(); + const lcovPath = join(dir, 'lcov.info'); + if (!result?.files?.length || !existsSync(lcovPath)) continue; + writeFileSync( + join(outDir, `${slug}${suffix}.lcov`), + forceSpecTn(readFileSync(lcovPath, 'utf8'), spec), + ); + stats.emitted++; + } + return stats; +} diff --git a/packages/testing/janitor/src/core/select-e2e.test.ts b/packages/testing/test-impact/src/select.test.ts similarity index 93% rename from packages/testing/janitor/src/core/select-e2e.test.ts rename to packages/testing/test-impact/src/select.test.ts index 76d347b9916..5d0f09634b1 100644 --- a/packages/testing/janitor/src/core/select-e2e.test.ts +++ b/packages/testing/test-impact/src/select.test.ts @@ -3,19 +3,19 @@ import * as os from 'node:os'; import * as path from 'node:path'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; -import { type ImpactMap, type InternedImpactMap, encodeImpactMap } from './coverage-map.js'; -import { selectE2e } from './select-e2e.js'; +import { type ImpactMap, type InternedImpactMap, encodeImpactMap } from './impact-map.js'; +import { selectTests } from './select.js'; // The handler's fail-open contract is its safety guarantee: every failure mode // of the map source must degrade to mode 'broad', never to an empty spec set // (which would silently skip real tests). These cases pin that contract so a // future refactor of resolveImpact / loadMap can't quietly invert it. -describe('selectE2e — fail-open contract', () => { +describe('selectTests — fail-open contract', () => { let tempDir: string; beforeEach(() => { - tempDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'select-e2e-'))); + tempDir = fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), 'select-'))); }); afterEach(() => { @@ -31,7 +31,7 @@ describe('selectE2e — fail-open contract', () => { }; it('no --map provided → broad with failOpen reason', () => { - const result = selectE2e({ + const result = selectTests({ changedFiles: ['packages/cli/src/x.ts'], allSpecsFile: writeAllSpecs(ALL_SPECS.join('\n')), }); @@ -42,7 +42,7 @@ describe('selectE2e — fail-open contract', () => { it("map path that doesn't exist → broad with failOpen reason", () => { const missing = path.join(tempDir, 'never-existed.json'); - const result = selectE2e({ + const result = selectTests({ changedFiles: ['packages/cli/src/x.ts'], mapFile: missing, allSpecsFile: writeAllSpecs(ALL_SPECS.join('\n')), @@ -55,7 +55,7 @@ describe('selectE2e — fail-open contract', () => { it('corrupt / non-JSON map → broad with failOpen reason', () => { const mapPath = path.join(tempDir, 'corrupt.json'); fs.writeFileSync(mapPath, '{not valid json,,,'); - const result = selectE2e({ + const result = selectTests({ changedFiles: ['packages/cli/src/x.ts'], mapFile: mapPath, allSpecsFile: writeAllSpecs(ALL_SPECS.join('\n')), @@ -71,7 +71,7 @@ describe('selectE2e — fail-open contract', () => { it('empty map {} → broad (the fail-open collapse)', () => { const mapPath = path.join(tempDir, 'empty.json'); fs.writeFileSync(mapPath, '{}'); - const result = selectE2e({ + const result = selectTests({ changedFiles: ['packages/cli/src/x.ts'], mapFile: mapPath, allSpecsFile: writeAllSpecs(ALL_SPECS.join('\n')), @@ -92,7 +92,7 @@ describe('selectE2e — fail-open contract', () => { const mapPath = path.join(tempDir, 'interned.json'); fs.writeFileSync(mapPath, JSON.stringify(interned)); - const result = selectE2e({ + const result = selectTests({ changedFiles: ['packages/cli/src/x.ts'], mapFile: mapPath, }); @@ -103,7 +103,7 @@ describe('selectE2e — fail-open contract', () => { describe('--all-specs parsing', () => { const triggerBroad = (allSpecsFile: string) => - selectE2e({ + selectTests({ changedFiles: ['packages/never/covered.ts'], allSpecsFile, }); diff --git a/packages/testing/janitor/src/core/select-e2e.ts b/packages/testing/test-impact/src/select.ts similarity index 73% rename from packages/testing/janitor/src/core/select-e2e.ts rename to packages/testing/test-impact/src/select.ts index 3ace55be234..959eca32394 100644 --- a/packages/testing/janitor/src/core/select-e2e.ts +++ b/packages/testing/test-impact/src/select.ts @@ -1,12 +1,12 @@ /** - * `select-e2e` handler: changed files + impact map → spec list. + * `select` handler: changed files + impact map → spec list. * * The file-system-aware wrapper around the pure {@link resolveImpact} resolver. * This is where the FAIL-OPEN safety contract lives — every failure mode * (missing map, unreadable map, corrupt JSON, empty map) must degrade to * `mode: 'broad'` so the caller runs the full suite, never an empty one. * - * Extracted from {@link runSelectE2e} in `cli.ts` so the contract can be + * Extracted from {@link runSelect} in `cli.ts` so the contract can be * exhaustively unit-tested without spawning a subprocess. */ @@ -17,10 +17,11 @@ import { type InternedImpactMap, type ResolveResult, decodeImpactMap, - resolveImpact, -} from './coverage-map.js'; +} from './impact-map.js'; +import { CoverageMapStrategy } from './select/coverage-map-strategy.js'; +import { selectImpactedTests } from './select/pipeline.js'; -export interface SelectE2eInput { +export interface SelectTestsInput { /** Changed files (file paths). */ changedFiles: string[]; /** Path to the impact map JSON. Missing/unreadable → fail-open broad. */ @@ -29,7 +30,7 @@ export interface SelectE2eInput { allSpecsFile?: string; } -export interface SelectE2eResult extends ResolveResult { +export interface SelectTestsResult extends ResolveResult { /** Set when the map could not be loaded; the result is broad as a safety. */ failOpen?: string; } @@ -61,14 +62,17 @@ function loadMap(mapFile: string | undefined): { map: ImpactMap; failOpen?: stri * empty/missing map every changed file is "unmapped" → {@link resolveImpact} * returns `mode: 'broad'`, so fail-open falls out of the same code path. */ -export function selectE2e(input: SelectE2eInput): SelectE2eResult { +export function selectTests(input: SelectTestsInput): SelectTestsResult { const allSpecs = input.allSpecsFile ? parseSpecList(fs.readFileSync(input.allSpecsFile, 'utf8')) : undefined; const { map, failOpen } = loadMap(input.mapFile); const changed = input.changedFiles.map((file) => ({ file })); - // Sibling fallback on: a new/unmapped file scopes to its nearest covered - // directory's specs rather than forcing the whole suite (see resolveImpact). - const result = resolveImpact(changed, map, { allSpecs, siblingFallback: true }); + // The live V8 selection runs through the pipeline as the single selection + // mechanism; the AST / dep-graph selectors join this array later. Sibling + // fallback scopes a new/unmapped file to its nearest covered directory. + const result = selectImpactedTests(changed, [ + new CoverageMapStrategy(map, { allSpecs, siblingFallback: true }), + ]); return { ...result, failOpen }; } diff --git a/packages/testing/test-impact/src/select/coverage-map-strategy.ts b/packages/testing/test-impact/src/select/coverage-map-strategy.ts new file mode 100644 index 00000000000..263e01ce845 --- /dev/null +++ b/packages/testing/test-impact/src/select/coverage-map-strategy.ts @@ -0,0 +1,22 @@ +import type { ChangedFile, ImpactMap, ResolveResult } from '../impact-map.js'; +import { resolveImpact } from '../impact-map.js'; +import type { SelectionStrategy } from './strategy.js'; + +/** + * Selection via the runtime V8 coverage map: a changed source file resolves to + * the specs that executed it (function/line-level when `ChangedFile.lines` is + * supplied, whole-file otherwise). Thin Strategy wrapper around the pure + * {@link resolveImpact}; the fail-open-to-broad contract lives there. + */ +export class CoverageMapStrategy implements SelectionStrategy { + readonly name = 'coverage-map'; + + constructor( + private readonly map: ImpactMap, + private readonly opts: { allSpecs?: string[]; siblingFallback?: boolean } = {}, + ) {} + + resolve(changed: ChangedFile[]): ResolveResult { + return resolveImpact(changed, this.map, this.opts); + } +} diff --git a/packages/testing/test-impact/src/select/pipeline.test.ts b/packages/testing/test-impact/src/select/pipeline.test.ts new file mode 100644 index 00000000000..7a3e48985ed --- /dev/null +++ b/packages/testing/test-impact/src/select/pipeline.test.ts @@ -0,0 +1,64 @@ +import { describe, it, expect } from 'vitest'; + +import type { ChangedFile, ImpactMap, ResolveResult } from '../impact-map.js'; +import { resolveImpact } from '../impact-map.js'; +import { CoverageMapStrategy } from './coverage-map-strategy.js'; +import { selectImpactedTests } from './pipeline.js'; +import type { SelectionStrategy } from './strategy.js'; + +const MAP: ImpactMap = { + 'packages/cli/src/a.ts': { '1': ['tests/e2e/a.spec.ts'] }, + 'packages/cli/src/b.ts': { '1': ['tests/e2e/b.spec.ts', 'tests/e2e/a.spec.ts'] }, +}; +const ALL = ['tests/e2e/a.spec.ts', 'tests/e2e/b.spec.ts', 'tests/e2e/c.spec.ts']; + +const stub = (name: string, result: ResolveResult): SelectionStrategy => ({ + name, + resolve: () => result, +}); + +describe('CoverageMapStrategy', () => { + it('is a thin wrapper — resolve() equals resolveImpact()', () => { + const changed: ChangedFile[] = [{ file: 'packages/cli/src/b.ts' }]; + const opts = { allSpecs: ALL, siblingFallback: true }; + const selector = new CoverageMapStrategy(MAP, opts); + expect(selector.resolve(changed)).toEqual(resolveImpact(changed, MAP, opts)); + }); + + it('exposes a stable name', () => { + expect(new CoverageMapStrategy(MAP).name).toBe('coverage-map'); + }); +}); + +describe('selectImpactedTests (pipeline)', () => { + it('unions scoped spec sets across selectors, sorted', () => { + const a = stub('a', { specs: ['tests/e2e/b.spec.ts'], unmapped: [], mode: 'scoped' }); + const b = stub('b', { specs: ['tests/e2e/a.spec.ts'], unmapped: [], mode: 'scoped' }); + const result = selectImpactedTests([{ file: 'x' }], [a, b]); + expect(result.mode).toBe('scoped'); + expect(result.specs).toEqual(['tests/e2e/a.spec.ts', 'tests/e2e/b.spec.ts']); + }); + + it('broad wins — any broad selector makes the whole result broad', () => { + const scoped = stub('scoped', { specs: ['tests/e2e/a.spec.ts'], unmapped: [], mode: 'scoped' }); + const broad = stub('broad', { specs: ALL, unmapped: ['pnpm-lock.yaml'], mode: 'broad' }); + const result = selectImpactedTests([{ file: 'x' }], [scoped, broad]); + expect(result.mode).toBe('broad'); + }); + + it('merges unmapped + viaSibling from scoped selectors', () => { + const a = stub('a', { specs: ['s1'], unmapped: ['u1'], mode: 'scoped', viaSibling: ['v1'] }); + const b = stub('b', { specs: ['s2'], unmapped: ['u2'], mode: 'scoped' }); + const result = selectImpactedTests([{ file: 'x' }], [a, b]); + expect(result.specs).toEqual(['s1', 's2']); + expect(result.unmapped.sort()).toEqual(['u1', 'u2']); + expect(result.viaSibling).toEqual(['v1']); + }); + + it('composes with CoverageMapStrategy as the only strategy', () => { + const changed: ChangedFile[] = [{ file: 'packages/cli/src/a.ts' }]; + const result = selectImpactedTests(changed, [new CoverageMapStrategy(MAP, { allSpecs: ALL })]); + expect(result.mode).toBe('scoped'); + expect(result.specs).toEqual(['tests/e2e/a.spec.ts']); + }); +}); diff --git a/packages/testing/test-impact/src/select/pipeline.ts b/packages/testing/test-impact/src/select/pipeline.ts new file mode 100644 index 00000000000..666cd51b8d1 --- /dev/null +++ b/packages/testing/test-impact/src/select/pipeline.ts @@ -0,0 +1,37 @@ +import type { ChangedFile, ResolveResult } from '../impact-map.js'; +import type { SelectionStrategy } from './strategy.js'; + +/** + * Run every selector over the changed files and combine, biased to OVER-select: + * + * - if ANY selector returns `mode: 'broad'`, the whole result is broad (broad + * wins — never skip a test on uncertainty); + * - otherwise union the scoped spec sets. + * + * This is the single place the fail-open contract lives — it formalises the + * `V8 ∪ AST` union that `distribute-tests` previously did inline, so adding a + * dep-graph or AST selector needs no change here. + */ +export function selectImpactedTests( + changed: ChangedFile[], + selectors: SelectionStrategy[], +): ResolveResult { + const specs = new Set(); + const unmapped = new Set(); + const viaSibling = new Set(); + + for (const selector of selectors) { + const result = selector.resolve(changed); + if (result.mode === 'broad') return result; // broad wins, short-circuit + for (const spec of result.specs) specs.add(spec); + for (const file of result.unmapped) unmapped.add(file); + for (const file of result.viaSibling ?? []) viaSibling.add(file); + } + + return { + specs: [...specs].sort(), + unmapped: [...unmapped], + mode: 'scoped', + ...(viaSibling.size ? { viaSibling: [...viaSibling] } : {}), + }; +} diff --git a/packages/testing/test-impact/src/select/strategy.ts b/packages/testing/test-impact/src/select/strategy.ts new file mode 100644 index 00000000000..d33a2755a88 --- /dev/null +++ b/packages/testing/test-impact/src/select/strategy.ts @@ -0,0 +1,16 @@ +import type { ChangedFile, ResolveResult } from '../impact-map.js'; + +/** + * A selection strategy: given changed files, decide which specs must run. + * + * Each strategy resolves *independently* and may return `mode: 'broad'` to mean + * "I can't scope this — run everything." The {@link selectImpactedTests} pipeline + * unions scoped results and lets broad win (fail-open). Implementations: + * `CoverageMapStrategy` (runtime coverage), and — later — an AST selector for + * test-internal changes and a dep-graph selector for dependency changes. + */ +export interface SelectionStrategy { + /** Stable id for logging / telemetry (e.g. 'coverage-map', 'ast', 'dep-graph'). */ + readonly name: string; + resolve(changed: ChangedFile[]): ResolveResult; +} diff --git a/packages/testing/janitor/src/core/orchestrator.test.ts b/packages/testing/test-impact/src/shard-distributor.test.ts similarity index 77% rename from packages/testing/janitor/src/core/orchestrator.test.ts rename to packages/testing/test-impact/src/shard-distributor.test.ts index f57f1db0f15..625e090a926 100644 --- a/packages/testing/janitor/src/core/orchestrator.test.ts +++ b/packages/testing/test-impact/src/shard-distributor.test.ts @@ -1,7 +1,7 @@ import { describe, it, expect } from 'vitest'; -import { orchestrate } from './orchestrator.js'; -import type { DiscoveredSpec } from './test-discovery-analyzer.js'; +import { distributeShards } from './shard-distributor.js'; +import type { DiscoveredSpec } from './types.js'; const DEFAULT_CONFIG = { defaultDuration: 60_000, maxGroupDuration: 300_000 }; @@ -9,16 +9,16 @@ function spec(path: string, capabilities: string[] = []): DiscoveredSpec { return { path, capabilities }; } -describe('orchestrate', () => { +describe('distributeShards', () => { it('returns 0 shards when no specs provided', () => { - const result = orchestrate([], 3, {}, DEFAULT_CONFIG); + const result = distributeShards([], 3, {}, DEFAULT_CONFIG); expect(result.shards).toHaveLength(0); expect(result.totalTestTime).toBe(0); }); it('assigns single spec to single shard', () => { - const result = orchestrate([spec('test.spec.ts')], 1, {}, DEFAULT_CONFIG); + const result = distributeShards([spec('test.spec.ts')], 1, {}, DEFAULT_CONFIG); expect(result.shards).toHaveLength(1); expect(result.shards[0].specs).toEqual(['test.spec.ts']); @@ -26,21 +26,21 @@ describe('orchestrate', () => { }); it('strips empty shards when more shards than specs', () => { - const result = orchestrate([spec('a.spec.ts'), spec('b.spec.ts')], 5, {}, DEFAULT_CONFIG); + const result = distributeShards([spec('a.spec.ts'), spec('b.spec.ts')], 5, {}, DEFAULT_CONFIG); expect(result.shards).toHaveLength(2); expect(result.shards.every((s) => s.specs.length > 0)).toBe(true); }); it('re-numbers shards sequentially after stripping empty ones', () => { - const result = orchestrate([spec('a.spec.ts'), spec('b.spec.ts')], 5, {}, DEFAULT_CONFIG); + const result = distributeShards([spec('a.spec.ts'), spec('b.spec.ts')], 5, {}, DEFAULT_CONFIG); expect(result.shards.map((s) => s.shard)).toEqual([1, 2]); }); it('uses defaultDuration when metrics are missing', () => { const config = { defaultDuration: 30_000, maxGroupDuration: 300_000 }; - const result = orchestrate([spec('a.spec.ts')], 1, {}, config); + const result = distributeShards([spec('a.spec.ts')], 1, {}, config); expect(result.shards[0].testTime).toBe(30_000); expect(result.totalTestTime).toBe(30_000); @@ -48,7 +48,7 @@ describe('orchestrate', () => { it('uses metric duration when available', () => { const metrics = { 'a.spec.ts': 120_000 }; - const result = orchestrate([spec('a.spec.ts')], 1, metrics, DEFAULT_CONFIG); + const result = distributeShards([spec('a.spec.ts')], 1, metrics, DEFAULT_CONFIG); expect(result.shards[0].testTime).toBe(120_000); expect(result.totalTestTime).toBe(120_000); @@ -60,7 +60,7 @@ describe('orchestrate', () => { spec('email2.spec.ts', ['email']), spec('standard.spec.ts'), ]; - const result = orchestrate(specs, 3, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 3, {}, DEFAULT_CONFIG); const emailShard = result.shards.find((s) => s.capabilities.includes('email')); expect(emailShard).toBeDefined(); @@ -70,7 +70,7 @@ describe('orchestrate', () => { it('places different capabilities on separate shards when space allows', () => { const specs = [spec('email.spec.ts', ['email']), spec('proxy.spec.ts', ['proxy'])]; - const result = orchestrate(specs, 2, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 2, {}, DEFAULT_CONFIG); const emailShard = result.shards.find((s) => s.capabilities.includes('email')); const proxyShard = result.shards.find((s) => s.capabilities.includes('proxy')); @@ -87,7 +87,7 @@ describe('orchestrate', () => { const config = { defaultDuration: 60_000, maxGroupDuration: 200_000 }; const specs = [spec('email1.spec.ts', ['email']), spec('email2.spec.ts', ['email'])]; - const result = orchestrate(specs, 2, metrics, config); + const result = distributeShards(specs, 2, metrics, config); const shardsWithEmail = result.shards.filter((s) => s.capabilities.includes('email')); expect(shardsWithEmail.length).toBeGreaterThanOrEqual(2); @@ -107,7 +107,7 @@ describe('orchestrate', () => { spec('light2.spec.ts'), ]; - const result = orchestrate(specs, 2, metrics, DEFAULT_CONFIG); + const result = distributeShards(specs, 2, metrics, DEFAULT_CONFIG); const times = result.shards.map((s) => s.testTime).sort((a, b) => a - b); expect(times[0]).toBe(100_000); @@ -121,7 +121,7 @@ describe('orchestrate', () => { spec('standard.spec.ts'), ]; - const result = orchestrate(specs, 1, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 1, {}, DEFAULT_CONFIG); const shard = result.shards[0]; // 2 capabilities + standard specs = 3 @@ -130,28 +130,28 @@ describe('orchestrate', () => { it('fixtureCount is 1 for shard with only standard specs', () => { const specs = [spec('a.spec.ts'), spec('b.spec.ts')]; - const result = orchestrate(specs, 1, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 1, {}, DEFAULT_CONFIG); expect(result.shards[0].fixtureCount).toBe(1); }); it('fixtureCount is 1 for shard with only one capability and no standard specs', () => { const specs = [spec('email.spec.ts', ['email'])]; - const result = orchestrate(specs, 1, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 1, {}, DEFAULT_CONFIG); expect(result.shards[0].fixtureCount).toBe(1); }); it('sorts capabilities alphabetically', () => { const specs = [spec('proxy.spec.ts', ['proxy']), spec('email.spec.ts', ['email'])]; - const result = orchestrate(specs, 1, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 1, {}, DEFAULT_CONFIG); expect(result.shards[0].capabilities).toEqual(['email', 'proxy']); }); it('uses 1-indexed shard numbers', () => { const specs = [spec('a.spec.ts'), spec('b.spec.ts'), spec('c.spec.ts')]; - const result = orchestrate(specs, 3, {}, DEFAULT_CONFIG); + const result = distributeShards(specs, 3, {}, DEFAULT_CONFIG); expect(result.shards.map((s) => s.shard)).toEqual([1, 2, 3]); }); @@ -160,7 +160,7 @@ describe('orchestrate', () => { const metrics = { 'a.spec.ts': 100_000, 'b.spec.ts': 200_000 }; const specs = [spec('a.spec.ts'), spec('b.spec.ts'), spec('c.spec.ts')]; - const result = orchestrate(specs, 2, metrics, DEFAULT_CONFIG); + const result = distributeShards(specs, 2, metrics, DEFAULT_CONFIG); expect(result.totalTestTime).toBe(100_000 + 200_000 + 60_000); }); diff --git a/packages/testing/janitor/src/core/orchestrator.ts b/packages/testing/test-impact/src/shard-distributor.ts similarity index 95% rename from packages/testing/janitor/src/core/orchestrator.ts rename to packages/testing/test-impact/src/shard-distributor.ts index 866c2e3edad..3e6ffa2c664 100644 --- a/packages/testing/janitor/src/core/orchestrator.ts +++ b/packages/testing/test-impact/src/shard-distributor.ts @@ -6,7 +6,7 @@ * 4. Greedy bin-packing: assign heaviest items to lightest shard */ -import type { DiscoveredSpec } from './test-discovery-analyzer.js'; +import type { DiscoveredSpec } from './types.js'; export interface ShardAssignment { shard: number; @@ -16,12 +16,12 @@ export interface ShardAssignment { fixtureCount: number; } -export interface OrchestrationResult { +export interface ShardDistribution { shards: ShardAssignment[]; totalTestTime: number; } -interface OrchestrationConfig { +interface DistributeConfig { defaultDuration: number; maxGroupDuration: number; } @@ -150,12 +150,12 @@ function assignToShards(items: PackingItem[], numShards: number): Bucket[] { return buckets; } -export function orchestrate( +export function distributeShards( specs: DiscoveredSpec[], numShards: number, metrics: Record, - config: OrchestrationConfig, -): OrchestrationResult { + config: DistributeConfig, +): ShardDistribution { const enriched = enrichWithDuration(specs, metrics, config.defaultDuration); const { groups, standard } = groupByCapability(enriched); diff --git a/packages/testing/test-impact/src/types.ts b/packages/testing/test-impact/src/types.ts new file mode 100644 index 00000000000..4282ad9c5f5 --- /dev/null +++ b/packages/testing/test-impact/src/types.ts @@ -0,0 +1,13 @@ +/** + * A test spec discovered by a runner adapter, with the capabilities (container + * tags) it needs. Owned here — not by any runner-specific discovery — so the + * framework-free orchestrator can bin-pack without depending on the ts-morph + * discovery that produces these. Janitor's `test-discovery-analyzer` re-exports + * this type for its own `DiscoveryReport`. + */ +export interface DiscoveredSpec { + /** Spec file path relative to rootDir. */ + path: string; + /** Capabilities extracted from tags matching the capability prefix. */ + capabilities: string[]; +} diff --git a/packages/testing/test-impact/tsconfig.json b/packages/testing/test-impact/tsconfig.json new file mode 100644 index 00000000000..0d590513e5a --- /dev/null +++ b/packages/testing/test-impact/tsconfig.json @@ -0,0 +1,21 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "outDir": "dist", + "rootDir": "src", + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "target": "ES2022", + "types": ["node"], + "strict": true, + "esModuleInterop": true, + "skipLibCheck": true, + "forceConsistentCasingInFileNames": true + }, + "include": ["src/**/*.ts"], + "exclude": ["dist", "node_modules", "**/__tests__/**"] +} diff --git a/packages/testing/test-impact/vitest.config.ts b/packages/testing/test-impact/vitest.config.ts new file mode 100644 index 00000000000..5a7d8bcc36f --- /dev/null +++ b/packages/testing/test-impact/vitest.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from 'vitest/config'; + +export default defineConfig({ + test: { + globals: true, + environment: 'node', + include: ['src/**/*.test.ts'], + coverage: { + provider: 'v8', + reporter: ['text', 'json', 'html'], + include: ['src/**/*.ts'], + exclude: ['src/**/*.test.ts', 'src/index.ts'], + }, + }, +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a44a1890d69..2391fb0cf67 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -5103,6 +5103,9 @@ importers: '@n8n/rules-engine': specifier: workspace:* version: link:../rules-engine + '@n8n/test-impact': + specifier: workspace:* + version: link:../test-impact glob: specifier: 10.5.0 version: 10.5.0 @@ -5179,6 +5182,9 @@ importers: '@n8n/playwright-janitor': specifier: workspace:* version: link:../janitor + '@n8n/test-impact': + specifier: workspace:* + version: link:../test-impact '@n8n/workflow-sdk': specifier: workspace:* version: link:../../@n8n/workflow-sdk @@ -5277,6 +5283,28 @@ importers: specifier: 'catalog:' version: 4.1.1(@opentelemetry/api@1.9.0)(@types/node@20.19.41)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@20.19.41)(esbuild@0.25.10)(jiti@2.6.1)(sass-embedded@1.98.0)(sass@1.98.0)(terser@5.16.1)(tsx@4.19.3)(yaml@2.8.3)) + packages/testing/test-impact: + dependencies: + monocart-coverage-reports: + specifier: ^2.12.0 + version: 2.12.12 + devDependencies: + '@types/node': + specifier: ^20.17.50 + version: 20.19.41 + '@vitest/coverage-v8': + specifier: 'catalog:' + version: 4.1.1(vitest@4.1.1(@opentelemetry/api@1.9.0)(@types/node@20.19.41)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@20.19.41)(esbuild@0.25.10)(jiti@2.6.1)(sass-embedded@1.98.0)(sass@1.98.0)(terser@5.16.1)(tsx@4.19.3)(yaml@2.8.3))) + fast-check: + specifier: 'catalog:' + version: 3.23.2 + typescript: + specifier: 6.0.2 + version: 6.0.2 + vitest: + specifier: 'catalog:' + version: 4.1.1(@opentelemetry/api@1.9.0)(@types/node@20.19.41)(jsdom@23.0.1(bufferutil@4.0.9)(utf-8-validate@5.0.10))(vite@8.0.2(@types/node@20.19.41)(esbuild@0.25.10)(jiti@2.6.1)(sass-embedded@1.98.0)(sass@1.98.0)(terser@5.16.1)(tsx@4.19.3)(yaml@2.8.3)) + packages/workflow: dependencies: '@n8n/errors': @@ -25427,14 +25455,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 20.19.21 + '@types/node': 20.19.41 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@20.19.21)(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.21)(typescript@6.0.2)) + jest-config: 29.7.0(@types/node@20.19.41)(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.21)(typescript@6.0.2)) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -25462,14 +25490,14 @@ snapshots: '@jest/test-result': 29.7.0 '@jest/transform': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 20.19.21 + '@types/node': 20.19.41 ansi-escapes: 4.3.2 chalk: 4.1.2 ci-info: 3.9.0 exit: 0.1.2 graceful-fs: 4.2.11 jest-changed-files: 29.7.0 - jest-config: 29.7.0(@types/node@20.19.21)(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.41)(typescript@6.0.2)) + jest-config: 29.7.0(@types/node@20.19.41)(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.41)(typescript@6.0.2)) jest-haste-map: 29.7.0 jest-message-util: 29.7.0 jest-regex-util: 29.6.3 @@ -25494,7 +25522,7 @@ snapshots: dependencies: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 - '@types/node': 20.19.21 + '@types/node': 20.19.41 jest-mock: 29.7.0 '@jest/expect-utils@29.7.0': @@ -25512,7 +25540,7 @@ snapshots: dependencies: '@jest/types': 29.6.3 '@sinonjs/fake-timers': 10.3.0 - '@types/node': 20.19.21 + '@types/node': 20.19.41 jest-message-util: 29.7.0 jest-mock: 29.7.0 jest-util: 29.7.0 @@ -25604,7 +25632,7 @@ snapshots: '@jest/schemas': 29.6.3 '@types/istanbul-lib-coverage': 2.0.6 '@types/istanbul-reports': 3.0.4 - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/yargs': 17.0.33 chalk: 4.1.2 @@ -29107,11 +29135,11 @@ snapshots: '@types/adm-zip@0.5.7': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/amqplib@0.10.1': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/argparse@1.0.38': {} @@ -29119,15 +29147,15 @@ snapshots: '@types/asn1@0.2.0': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/autocannon@7.12.7': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/aws4@1.11.2': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/babel__core@7.20.5': dependencies: @@ -29266,7 +29294,7 @@ snapshots: '@types/formidable@3.4.5': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/fs-extra@9.0.13': dependencies: @@ -29274,16 +29302,16 @@ snapshots: '@types/ftp@0.3.33': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/glob@8.0.0': dependencies: '@types/minimatch': 5.1.2 - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/gm@1.25.0': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/graceful-fs@4.1.9': dependencies: @@ -29297,7 +29325,7 @@ snapshots: '@types/http-proxy@1.17.16': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/humanize-duration@3.27.1': {} @@ -29337,7 +29365,7 @@ snapshots: '@types/jsdom@20.0.1': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/tough-cookie': 4.0.5 parse5: 7.1.2 @@ -29374,7 +29402,7 @@ snapshots: '@types/mailparser@3.4.4': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 iconv-lite: 0.6.3 '@types/markdown-it-emoji@2.0.5': @@ -29416,7 +29444,7 @@ snapshots: '@types/mssql@9.1.5': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/tedious': 4.0.14 tarn: 3.0.2 @@ -29440,13 +29468,13 @@ snapshots: '@types/nodemailer@7.0.3': dependencies: '@aws-sdk/client-sesv2': 3.918.0 - '@types/node': 20.19.21 + '@types/node': 20.19.41 transitivePeerDependencies: - aws-crt '@types/oracledb@6.10.3': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/pg@8.15.6': dependencies: @@ -29468,14 +29496,14 @@ snapshots: dependencies: '@types/bluebird': 3.5.37 '@types/ftp': 0.3.33 - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/promise-ftp-common': 1.1.0 '@types/prop-types@15.7.15': {} '@types/proxy-from-env@1.0.4': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/psl@1.1.3': {} @@ -29507,7 +29535,7 @@ snapshots: '@types/resolve@1.17.1': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/responselike@1.0.3': dependencies: @@ -29538,7 +29566,7 @@ snapshots: '@types/shelljs@0.8.11': dependencies: '@types/glob': 8.0.0 - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/showdown@1.9.4': {} @@ -29557,18 +29585,18 @@ snapshots: '@types/ssh2@1.11.6': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/sshpk@1.17.4': dependencies: '@types/asn1': 0.2.0 - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/stack-utils@2.0.3': {} '@types/stream-chain@2.1.0': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/stream-json@1.7.8': dependencies: @@ -29581,7 +29609,7 @@ snapshots: dependencies: '@types/cookiejar': 2.1.5 '@types/methods': 1.1.4 - '@types/node': 20.19.21 + '@types/node': 20.19.41 form-data: 4.0.4 '@types/supertest@6.0.3': @@ -29596,11 +29624,11 @@ snapshots: '@types/syslog-client@1.1.2': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/tedious@4.0.14': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/temp@0.9.4': dependencies: @@ -29608,7 +29636,7 @@ snapshots: '@types/through@0.0.30': dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/tough-cookie@4.0.5': {} @@ -34177,7 +34205,7 @@ snapshots: groq-sdk@0.19.0(encoding@0.1.13): dependencies: - '@types/node': 20.19.21 + '@types/node': 20.19.41 '@types/node-fetch': 2.6.13 abort-controller: 3.0.0 agentkeepalive: 4.6.0 @@ -35049,7 +35077,7 @@ snapshots: - babel-plugin-macros - supports-color - jest-config@29.7.0(@types/node@20.19.21)(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.41)(typescript@6.0.2)): + jest-config@29.7.0(@types/node@20.19.41)(ts-node@10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.21)(typescript@6.0.2)): dependencies: '@babel/core': 7.29.0 '@jest/test-sequencer': 29.7.0 @@ -35074,8 +35102,8 @@ snapshots: slash: 3.0.0 strip-json-comments: 3.1.1 optionalDependencies: - '@types/node': 20.19.21 - ts-node: 10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.41)(typescript@6.0.2) + '@types/node': 20.19.41 + ts-node: 10.9.2(@swc/core@1.15.8(@swc/helpers@0.5.17))(@types/node@20.19.21)(typescript@6.0.2) transitivePeerDependencies: - babel-plugin-macros - supports-color @@ -35136,7 +35164,7 @@ snapshots: '@jest/fake-timers': 29.7.0 '@jest/types': 29.6.3 '@types/jsdom': 20.0.1 - '@types/node': 20.19.21 + '@types/node': 20.19.41 jest-mock: 29.7.0 jest-util: 29.7.0 jsdom: 20.0.2(bufferutil@4.0.9)(utf-8-validate@5.0.10) @@ -35220,7 +35248,7 @@ snapshots: jest-mock@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 20.19.21 + '@types/node': 20.19.41 jest-util: 29.7.0 jest-pnp-resolver@1.2.3(jest-resolve@29.7.0): @@ -35329,7 +35357,7 @@ snapshots: jest-util@29.7.0: dependencies: '@jest/types': 29.6.3 - '@types/node': 20.19.21 + '@types/node': 20.19.41 chalk: 4.1.2 ci-info: 3.9.0 graceful-fs: 4.2.11 @@ -38514,7 +38542,7 @@ snapshots: '@protobufjs/path': 1.1.2 '@protobufjs/pool': 1.1.0 '@protobufjs/utf8': 1.1.1 - '@types/node': 20.19.21 + '@types/node': 20.19.41 long: 5.3.2 proxy-addr@2.0.7: