ci: Unify QA metrics pipeline to single webhook, format, and BigQuery table (no-changelog) (#27111)

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-09-24 23:22:38 +08:00 · 2026-03-17 05:50:10 +00:00
parent dcd306bc5c
commit 7c7c70f142
36 changed files with 739 additions and 749 deletions
@@ -1,104 +1,67 @@
 #!/usr/bin/env node
 /**
- * Sends Turbo build summary JSON to a webhook
+ * Sends Turbo build stats to the unified QA metrics webhook.
+ *
+ * Reads the Turbo run summary from .turbo/runs/ and emits per-package
+ * build-duration metrics with {package, cache, task} dimensions, plus
+ * a run-level build-total-duration summary.
 *
 * Usage: node send-build-stats.mjs
 *
- * Auto-detects summary from .turbo/runs/ directory.
- *
 * Environment variables:
- *   BUILD_STATS_WEBHOOK_URL - Webhook URL (required to send)
- *   BUILD_STATS_WEBHOOK_USER - Basic auth username (required if URL set)
- *   BUILD_STATS_WEBHOOK_PASSWORD - Basic auth password (required if URL set)
+ *   QA_METRICS_WEBHOOK_URL      - Webhook URL (required to send)
+ *   QA_METRICS_WEBHOOK_USER     - Basic auth username
+ *   QA_METRICS_WEBHOOK_PASSWORD - Basic auth password
 */

 import { existsSync, readFileSync, readdirSync } from 'node:fs';
-import * as os from 'node:os';
 import { join } from 'node:path';

+import { sendMetrics, metric } from './send-metrics.mjs';
+
 const runsDir = '.turbo/runs';
 if (!existsSync(runsDir)) {
 	console.log('No .turbo/runs directory found (turbo --summarize not used), skipping.');
 	process.exit(0);
 }
-const files = readdirSync(runsDir).filter((f) => f.endsWith('.json'));
-const summaryPath = files.length > 0 ? join(runsDir, files[0]) : null;

-const webhookUrl = process.env.BUILD_STATS_WEBHOOK_URL;
-const webhookUser = process.env.BUILD_STATS_WEBHOOK_USER;
-const webhookPassword = process.env.BUILD_STATS_WEBHOOK_PASSWORD;
-
-if (!summaryPath) {
+const files = readdirSync(runsDir)
+	.filter((f) => f.endsWith('.json'))
+	.sort();
+if (files.length === 0) {
 	console.error('No summary file found in .turbo/runs/');
 	process.exit(1);
 }

-if (!webhookUrl) {
-	console.log('BUILD_STATS_WEBHOOK_URL not set, skipping.');
-	process.exit(0);
+const summary = JSON.parse(readFileSync(join(runsDir, files.at(-1)), 'utf-8'));
+
+const metrics = [];
+
+for (const task of summary.tasks ?? []) {
+	if (task.execution?.exitCode !== 0) continue;
+	const durationMs = task.execution.durationMs ?? 0;
+	const cacheHit = task.cache?.status === 'HIT';
+	// taskId format: "package-name#task-name"
+	const [pkg, taskName] = task.taskId?.split('#') ?? [task.package, task.task];
+
+	metrics.push(
+		metric('build-duration', durationMs / 1000, 's', {
+			package: pkg ?? 'unknown',
+			task: taskName ?? 'build',
+			cache: cacheHit ? 'hit' : 'miss',
+		}),
+	);
 }

-if (!webhookUser || !webhookPassword) {
-	console.error('BUILD_STATS_WEBHOOK_USER and BUILD_STATS_WEBHOOK_PASSWORD are required');
-	process.exit(1);
-}
+const totalMs = summary.durationMs ?? 0;
+const totalTasks = summary.tasks?.length ?? 0;
+const cachedTasks = summary.tasks?.filter((t) => t.cache?.status === 'HIT').length ?? 0;

-const basicAuth = Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64');
+metrics.push(
+	metric('build-total-duration', totalMs / 1000, 's', {
+		total_tasks: totalTasks,
+		cached_tasks: cachedTasks,
+	}),
+);

-const summary = JSON.parse(readFileSync(summaryPath, 'utf-8'));
-
-// Extract PR number from GITHUB_REF (refs/pull/123/merge)
-const ref = process.env.GITHUB_REF ?? '';
-const prMatch = ref.match(/refs\/pull\/(\d+)/);
-
-// Detect runner provider (matches packages/testing/containers/telemetry.ts)
-function getRunnerProvider() {
-	if (!process.env.CI) return 'local';
-	if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') return 'github';
-	return 'blacksmith';
-}
-
-// Add git context (aligned with container telemetry)
-summary.git = {
-	sha: process.env.GITHUB_SHA?.slice(0, 8) || null,
-	branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null,
-	pr: prMatch ? parseInt(prMatch[1], 10) : null,
-};
-
-// Add CI context
-summary.ci = {
-	runId: process.env.GITHUB_RUN_ID || null,
-	runUrl: process.env.GITHUB_RUN_ID
-		? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
-		: null,
-	job: process.env.GITHUB_JOB || null,
-	workflow: process.env.GITHUB_WORKFLOW || null,
-	attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : null,
-};
-
-// Add runner info
-summary.runner = {
-	provider: getRunnerProvider(),
-	cpuCores: os.cpus().length,
-	memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10,
-};
-
-const headers = {
-	'Content-Type': 'application/json',
-	'Authorization': `Basic ${basicAuth}`,
-};
-
-const response = await fetch(webhookUrl, {
-	method: 'POST',
-	headers,
-	body: JSON.stringify(summary),
-});
-
-if (!response.ok) {
-	console.error(`Webhook failed: ${response.status} ${response.statusText}`);
-	const body = await response.text();
-	if (body) console.error(`Response: ${body}`);
-	process.exit(1);
-}
-
-console.log(`Build stats sent: ${response.status}`);
+await sendMetrics(metrics, 'build-stats');
@@ -1,18 +1,22 @@
 #!/usr/bin/env node
 /**
- * Sends Docker build stats to a webhook for BigQuery ingestion.
+ * Sends Docker build stats to the unified QA metrics webhook.
 *
- * Reads manifests produced by build-n8n.mjs and dockerize-n8n.mjs,
- * enriches with git/CI/runner context, and POSTs to a webhook.
+ * Reads manifests produced by build-n8n.mjs and dockerize-n8n.mjs and emits
+ * per-image docker-image-size metrics and build duration metrics with
+ * {image, platform} dimensions.
 *
 * Usage: node send-docker-stats.mjs
 *
 * Environment variables:
- *   DOCKER_STATS_WEBHOOK_URL - Webhook URL (required to send)
+ *   QA_METRICS_WEBHOOK_URL      - Webhook URL (required to send)
+ *   QA_METRICS_WEBHOOK_USER     - Basic auth username
+ *   QA_METRICS_WEBHOOK_PASSWORD - Basic auth password
 */

 import { existsSync, readFileSync } from 'node:fs';
-import * as os from 'node:os';
+
+import { sendMetrics, metric } from './send-metrics.mjs';

 const buildManifestPath = 'compiled/build-manifest.json';
 const dockerManifestPath = 'docker-build-manifest.json';
@@ -22,13 +26,6 @@ if (!existsSync(buildManifestPath) && !existsSync(dockerManifestPath)) {
 	process.exit(0);
 }

-const webhookUrl = process.env.DOCKER_STATS_WEBHOOK_URL;
-
-if (!webhookUrl) {
-	console.log('DOCKER_STATS_WEBHOOK_URL not set, skipping.');
-	process.exit(0);
-}
-
 const buildManifest = existsSync(buildManifestPath)
 	? JSON.parse(readFileSync(buildManifestPath, 'utf-8'))
 	: null;
@@ -37,68 +34,41 @@ const dockerManifest = existsSync(dockerManifestPath)
 	? JSON.parse(readFileSync(dockerManifestPath, 'utf-8'))
 	: null;

-// Extract PR number from GITHUB_REF (refs/pull/123/merge)
-const ref = process.env.GITHUB_REF ?? '';
-const prMatch = ref.match(/refs\/pull\/(\d+)/);
+const metrics = [];

-// Detect runner provider (matches packages/testing/containers/telemetry.ts)
-function getRunnerProvider() {
-	if (!process.env.CI) return 'local';
-	if (process.env.RUNNER_ENVIRONMENT === 'github-hosted') return 'github';
-	return 'blacksmith';
+if (buildManifest) {
+	if (buildManifest.artifactSize != null) {
+		metrics.push(metric('artifact-size', buildManifest.artifactSize, 'bytes', { artifact: 'compiled' }));
+	}
+	if (buildManifest.buildDuration != null) {
+		metrics.push(metric('build-duration', buildManifest.buildDuration / 1000, 's', { artifact: 'compiled' }));
+	}
 }

-const payload = {
-	build: buildManifest
-		? {
-				artifactSize: buildManifest.artifactSize,
-				buildDuration: buildManifest.buildDuration,
-			}
-		: null,
+if (dockerManifest) {
+	const platform = dockerManifest.platform ?? 'unknown';

-	docker: dockerManifest
-		? {
-				platform: dockerManifest.platform,
-				images: dockerManifest.images,
-			}
-		: null,
+	for (const image of dockerManifest.images ?? []) {
+		if (image.sizeBytes != null) {
+			metrics.push(
+				metric('docker-image-size', image.sizeBytes, 'bytes', {
+					image: image.name ?? 'unknown',
+					platform,
+				}),
+			);
+		}
+	}

-	git: {
-		sha: process.env.GITHUB_SHA?.slice(0, 8) || null,
-		branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null,
-		pr: prMatch ? parseInt(prMatch[1], 10) : null,
-	},
-
-	ci: {
-		runId: process.env.GITHUB_RUN_ID || null,
-		runUrl: process.env.GITHUB_RUN_ID
-			? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`
-			: null,
-		job: process.env.GITHUB_JOB || null,
-		workflow: process.env.GITHUB_WORKFLOW || null,
-		attempt: process.env.GITHUB_RUN_ATTEMPT ? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) : null,
-	},
-
-	runner: {
-		provider: getRunnerProvider(),
-		cpuCores: os.cpus().length,
-		memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10,
-	},
-};
-
-const response = await fetch(webhookUrl, {
-	method: 'POST',
-	headers: {
-		'Content-Type': 'application/json',
-	},
-	body: JSON.stringify(payload),
-});
-
-if (!response.ok) {
-	console.error(`Webhook failed: ${response.status} ${response.statusText}`);
-	const body = await response.text();
-	if (body) console.error(`Response: ${body}`);
-	process.exit(1);
+	if (dockerManifest.buildDurationMs != null) {
+		metrics.push(
+			metric('docker-build-duration', dockerManifest.buildDurationMs / 1000, 's', { platform }),
+		);
+	}
 }

-console.log(`Docker build stats sent: ${response.status}`);
+if (metrics.length === 0) {
+	console.log('No metrics to send.');
+	process.exit(0);
+}
+
+await sendMetrics(metrics, 'docker-stats');
@@ -0,0 +1,94 @@
+#!/usr/bin/env node
+/**
+ * Shared metrics sender for CI scripts.
+ * See .github/CI-TELEMETRY.md for payload shape and BigQuery schema.
+ *
+ * Usage:
+ *   import { sendMetrics, metric } from './send-metrics.mjs';
+ *   await sendMetrics([metric('build-duration', 45.2, 's', { package: '@n8n/cli' })]);
+ *
+ * Env: QA_METRICS_WEBHOOK_URL, QA_METRICS_WEBHOOK_USER, QA_METRICS_WEBHOOK_PASSWORD
+ */
+
+import * as os from 'node:os';
+
+/** Build a single metric object. */
+export function metric(name, value, unit, dimensions = {}) {
+	return { metric_name: name, value, unit, dimensions };
+}
+
+/** Build git/ci/runner context from environment variables. */
+export function buildContext(benchmarkName = null) {
+	const ref = process.env.GITHUB_REF ?? '';
+	const prMatch = ref.match(/refs\/pull\/(\d+)/);
+	const runId = process.env.GITHUB_RUN_ID ?? null;
+
+	return {
+		timestamp: new Date().toISOString(),
+		benchmark_name: benchmarkName,
+		git: {
+			sha: process.env.GITHUB_SHA?.slice(0, 8) ?? null,
+			branch: process.env.GITHUB_HEAD_REF ?? process.env.GITHUB_REF_NAME ?? null,
+			pr: prMatch ? parseInt(prMatch[1], 10) : null,
+		},
+		ci: {
+			runId,
+			runUrl:
+				runId && process.env.GITHUB_REPOSITORY
+					? `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${runId}`
+					: null,
+			job: process.env.GITHUB_JOB ?? null,
+			workflow: process.env.GITHUB_WORKFLOW ?? null,
+			attempt: process.env.GITHUB_RUN_ATTEMPT
+				? parseInt(process.env.GITHUB_RUN_ATTEMPT, 10)
+				: null,
+		},
+		runner: {
+			provider: !process.env.CI
+				? 'local'
+				: process.env.RUNNER_ENVIRONMENT === 'github-hosted'
+					? 'github'
+					: 'blacksmith',
+			cpuCores: os.cpus().length,
+			memoryGb: Math.round((os.totalmem() / (1024 * 1024 * 1024)) * 10) / 10,
+		},
+	};
+}
+
+export async function sendMetrics(metrics, benchmarkName = null) {
+	const webhookUrl = process.env.QA_METRICS_WEBHOOK_URL;
+	const webhookUser = process.env.QA_METRICS_WEBHOOK_USER;
+	const webhookPassword = process.env.QA_METRICS_WEBHOOK_PASSWORD;
+
+	if (!webhookUrl) {
+		console.log('QA_METRICS_WEBHOOK_URL not set, skipping.');
+		return;
+	}
+
+	if (!webhookUser || !webhookPassword) {
+		console.log('QA_METRICS_WEBHOOK_USER/PASSWORD not set, skipping.');
+		return;
+	}
+
+	const payload = { ...buildContext(benchmarkName), metrics };
+	const basicAuth = Buffer.from(`${webhookUser}:${webhookPassword}`).toString('base64');
+
+	const response = await fetch(webhookUrl, {
+		method: 'POST',
+		headers: {
+			'Content-Type': 'application/json',
+			Authorization: `Basic ${basicAuth}`,
+		},
+		body: JSON.stringify(payload),
+		signal: AbortSignal.timeout(30_000),
+	});
+
+	if (!response.ok) {
+		const body = await response.text().catch(() => '');
+		throw new Error(
+			`Webhook failed: ${response.status} ${response.statusText}${body ? `\n${body}` : ''}`,
+		);
+	}
+
+	console.log(`Sent ${metrics.length} metric(s): ${response.status}`);
+}