build: Package @confluentinc/kafka-javascript native dependency (no-changelog) (#35321)

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

.github/WORKFLOWS.md

@@ -507,6 +507,7 @@ Scripts in `.github/scripts/`:
 |-------------------------|-------------------|------------------------|
 | `docker/docker-config.mjs`| Build context   | `docker-build-push.yml`|
 | `docker/docker-tags.mjs`  | Image tags      | `docker-build-push.yml`|
+| `docker/kafka-native-smoke-check.mjs`| Verify librdkafka binary loads in built image | `docker-build-push.yml`|
 
 ### Validation Scripts
 

.github/scripts/docker/docker-config.mjs

@@ -7,7 +7,7 @@ class BuildContext {
 		this.githubOutput = process.env.GITHUB_OUTPUT || null;
 	}
 
-	determine({ event, pr, branch, version, releaseType, pushEnabled }) {
+	determine({ event, pr, branch, version, releaseType, pushEnabled, includeArm64 }) {
 		let context = {
 			version: '',
 			release_type: '',
@@ -37,7 +37,9 @@ class BuildContext {
 				case 'workflow_dispatch':
 					context.version = `branch-${this.sanitizeBranch(branch)}`;
 					context.release_type = 'branch';
-					context.platforms = ['linux/amd64'];
+					// Manual pre-merge proof (e.g. a native-binary check) can opt into arm64;
+					// defaults to amd64-only like other non-release branch builds.
+					context.platforms = includeArm64 ? ['linux/amd64', 'linux/arm64'] : ['linux/amd64'];
 					break;
 
 				case 'push':
@@ -152,6 +154,7 @@ if (import.meta.url === `file://${process.argv[1]}`) {
 			releaseType: getArg('release-type'),
 			pushEnabled:
 				pushEnabledArg === 'true' ? true : pushEnabledArg === 'false' ? false : undefined,
+			includeArm64: getArg('include-arm64') === 'true',
 		});
 
 		const matrix = context.buildMatrix(result.platforms);

.github/scripts/docker/kafka-native-smoke-check.mjs

@@ -0,0 +1,42 @@
+#!/usr/bin/env node
+
+// Verifies the @confluentinc/kafka-javascript native binding (librdkafka) loads
+// correctly inside a built n8n image. Resolves the module the same way n8n's
+// runtime would - from within n8n-nodes-base, not via a hardcoded pnpm store path,
+// since that path's hash suffix depends on the exact dependency graph.
+
+import { createRequire } from 'node:module';
+import { realpathSync } from 'node:fs';
+import path from 'node:path';
+
+// The features our existing Kafka credential depends on (TLS + SASL SCRAM auth).
+const REQUIRED_FEATURES = ['ssl', 'sasl_scram'];
+const COMPRESSION_CODECS = ['gzip', 'snappy', 'lz4', 'zstd'];
+
+const n8nInstallDir = process.env.N8N_INSTALL_DIR || '/usr/local/lib/node_modules/n8n';
+const nodesBasePackageJson = path.join(n8nInstallDir, 'node_modules/n8n-nodes-base/package.json');
+// n8n-nodes-base is a pnpm symlink; resolve it so require() walks up from its real
+// location in the pnpm virtual store, where its dependencies actually live.
+const require = createRequire(realpathSync(nodesBasePackageJson));
+
+const kafka = require('@confluentinc/kafka-javascript');
+
+// Construct a client object (no broker connection attempted) to prove the native
+// binding is fully usable, not just importable.
+new kafka.KafkaJS.Kafka({
+	kafkaJS: { brokers: ['localhost:9092'], clientId: 'ent-216-smoke-check' },
+});
+
+console.log(`librdkafka version: ${kafka.librdkafkaVersion}`);
+console.log(`Reported features: ${kafka.features.join(', ')}`);
+
+const missingFeatures = REQUIRED_FEATURES.filter((feature) => !kafka.features.includes(feature));
+if (missingFeatures.length > 0) {
+	console.error(`Missing required librdkafka features: ${missingFeatures.join(', ')}`);
+	process.exit(1);
+}
+
+const supportedCodecs = COMPRESSION_CODECS.filter((codec) => kafka.features.includes(codec));
+console.log(`Supported compression codecs: ${supportedCodecs.join(', ') || 'none'}`);
+
+console.log('Kafka native smoke check passed.');

.github/workflows/docker-build-push.yml

@@ -47,6 +47,11 @@ on:
         required: false
         type: boolean
         default: true
+      include_arm64:
+        description: 'Also build linux/arm64 (default is amd64-only for manual/branch builds)'
+        required: false
+        type: boolean
+        default: false
       success_url:
         description: 'URL to call after the build is successful'
         required: false
@@ -81,6 +86,7 @@ jobs:
           N8N_VERSION: ${{ inputs.n8n_version }}
           RELEASE_TYPE: ${{ inputs.release_type }}
           PUSH_ENABLED: ${{ inputs.push_enabled }}
+          INCLUDE_ARM64: ${{ inputs.include_arm64 }}
           GITHUB_REF: ${{ github.ref_name }}
         run: |
           node .github/scripts/docker/docker-config.mjs \
@@ -89,7 +95,8 @@ jobs:
             --branch "$GITHUB_REF" \
             --version "$N8N_VERSION" \
             --release-type "$RELEASE_TYPE" \
-            --push-enabled "$PUSH_ENABLED"
+            --push-enabled "$PUSH_ENABLED" \
+            --include-arm64 "$INCLUDE_ARM64"
 
   build-and-push-docker:
     name: Build App, then Build and Push Docker Image (${{ matrix.platform }})
@@ -174,6 +181,22 @@ jobs:
           push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }}
           tags: ${{ steps.determine-tags.outputs.n8n_tags }}
 
+      - name: Kafka native binding smoke check
+        # `load: true` isn't an option here: with sbom:true, buildx produces a manifest
+        # list even for a single platform, and the docker exporter (what --load uses)
+        # can't materialize manifest lists locally. Pull the pushed image back instead.
+        if: needs.determine-build-context.outputs.push_enabled == 'true'
+        env:
+          N8N_TAGS: ${{ steps.determine-tags.outputs.n8n_tags }}
+        run: |
+          IMAGE_TAG=$(echo "$N8N_TAGS" | cut -d',' -f1)
+          docker pull "$IMAGE_TAG"
+          docker run --rm \
+            --entrypoint node \
+            -v "${{ github.workspace }}/.github/scripts/docker/kafka-native-smoke-check.mjs:/tmp/kafka-native-smoke-check.mjs:ro" \
+            "$IMAGE_TAG" \
+            /tmp/kafka-native-smoke-check.mjs
+
       - name: Build and push task runners Docker image (Alpine)
         id: build-runners
         uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2

package.json

@@ -103,6 +103,7 @@
   },
   "pnpm": {
     "onlyBuiltDependencies": [
+      "@confluentinc/kafka-javascript",
       "@vscode/ripgrep",
       "isolated-vm",
       "sqlite3"

packages/nodes-base/package.json

@@ -936,6 +936,7 @@
     "@smithy/signature-v4": "5.3.5",
     "@smithy/types": "4.13.1",
     "@n8n/backend-network": "workspace:*",
+    "@confluentinc/kafka-javascript": "catalog:",
     "@kafkajs/confluent-schema-registry": "3.8.0",
     "@langchain/core": "catalog:",
     "@mozilla/readability": "catalog:",

pnpm-lock.yaml

@@ -96,6 +96,9 @@ catalogs:
     '@codemirror/view':
       specifier: 6.39.8
       version: 6.39.8
+    '@confluentinc/kafka-javascript':
+      specifier: 1.9.1
+      version: 1.9.1
     '@daytona/sdk':
       specifier: 0.187.0
       version: 0.187.0
@@ -6267,6 +6270,9 @@ importers:
       '@aws-sdk/credential-providers':
         specifier: 3.808.0
         version: 3.808.0
+      '@confluentinc/kafka-javascript':
+        specifier: 'catalog:'
+        version: 1.9.1(encoding@0.1.13)
       '@e965/xlsx':
         specifier: 'catalog:'
         version: 0.20.3
@@ -8410,6 +8416,10 @@ packages:
     peerDependencies:
       commander: ~13.1.0
 
+  '@confluentinc/kafka-javascript@1.9.1':
+    resolution: {integrity: sha512-Qc7IZGSiWb9PVcfGwhMDmVcn0fj4V2jZjJ7r1ObNdnu1JSFY1yMZvINS4QY9dAI00S7Ng6EE2zIaEQ7UrVDrPw==}
+    engines: {node: '>=18.0.0'}
+
   '@cspotcode/source-map-support@0.8.1':
     resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==}
     engines: {node: '>=12'}
@@ -9854,6 +9864,11 @@ packages:
     resolution: {integrity: sha512-9QOtNffcOF/c1seMCDnjckb3R9WHcG34tky+FHpNKKCW0wc/scYLwMtO+ptyGUfMW0/b/n4qRiALlaFHc9Oj7Q==}
     engines: {node: '>= 10.0.0'}
 
+  '@mapbox/node-pre-gyp@2.0.3':
+    resolution: {integrity: sha512-uwPAhccfFJlsfCxMYTwOdVfOz3xqyj8xYL3zJj8f0pb30tLohnnFPhLuqp4/qoEz8sNxe4SESZedcBojRefIzg==}
+    engines: {node: '>=18'}
+    hasBin: true
+
   '@marijn/find-cluster-break@1.0.2':
     resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==}
 
@@ -13738,6 +13753,10 @@ packages:
   abbrev@1.1.1:
     resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==}
 
+  abbrev@3.0.1:
+    resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==}
+    engines: {node: ^18.17.0 || >=20.5.0}
+
   abbrev@4.0.0:
     resolution: {integrity: sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA==}
     engines: {node: ^20.17.0 || >=22.9.0}
@@ -14911,6 +14930,10 @@ packages:
   config-chain@1.1.13:
     resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==}
 
+  consola@3.4.2:
+    resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==}
+    engines: {node: ^14.18.0 || >=16.10.0}
+
   console-browserify@1.2.0:
     resolution: {integrity: sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==}
 
@@ -19157,6 +19180,11 @@ packages:
     engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0}
     hasBin: true
 
+  nopt@8.1.0:
+    resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==}
+    engines: {node: ^18.17.0 || >=20.5.0}
+    hasBin: true
+
   nopt@9.0.0:
     resolution: {integrity: sha512-Zhq3a+yFKrYwSBluL4H9XP3m3y5uvQkB/09CwDruCiRmR/UJYnn9W4R48ry0uGC70aeTPKLynBtscP9efFFcPw==}
     engines: {node: ^20.17.0 || >=22.9.0}
@@ -25570,6 +25598,15 @@ snapshots:
     dependencies:
       commander: 13.1.0
 
+  '@confluentinc/kafka-javascript@1.9.1(encoding@0.1.13)':
+    dependencies:
+      '@mapbox/node-pre-gyp': 2.0.3(encoding@0.1.13)
+      bindings: 1.5.0
+      nan: 2.26.2
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
+
   '@cspotcode/source-map-support@0.8.1':
     dependencies:
       '@jridgewell/trace-mapping': 0.3.9
@@ -27029,6 +27066,19 @@ snapshots:
     transitivePeerDependencies:
       - supports-color
 
+  '@mapbox/node-pre-gyp@2.0.3(encoding@0.1.13)':
+    dependencies:
+      consola: 3.4.2
+      detect-libc: 2.1.2
+      https-proxy-agent: 7.0.6
+      node-fetch: 2.7.0(encoding@0.1.13)
+      nopt: 8.1.0
+      semver: 7.7.3
+      tar: 7.5.19
+    transitivePeerDependencies:
+      - encoding
+      - supports-color
+
   '@marijn/find-cluster-break@1.0.2': {}
 
   '@mdx-js/react@3.0.1(@types/react@18.0.27)(react@18.2.0)':
@@ -31482,6 +31532,8 @@ snapshots:
 
   abbrev@1.1.1: {}
 
+  abbrev@3.0.1: {}
+
   abbrev@4.0.0: {}
 
   abort-controller-x@0.4.3: {}
@@ -32807,6 +32859,8 @@ snapshots:
       ini: 1.3.8
       proto-list: 1.2.4
 
+  consola@3.4.2: {}
+
   console-browserify@1.2.0: {}
 
   console-control-strings@1.1.0:
@@ -38015,6 +38069,10 @@ snapshots:
     dependencies:
       abbrev: 1.1.1
 
+  nopt@8.1.0:
+    dependencies:
+      abbrev: 3.0.1
+
   nopt@9.0.0:
     dependencies:
       abbrev: 4.0.0

pnpm-workspace.yaml

@@ -37,6 +37,7 @@ catalog:
   '@codemirror/search': 6.5.11
   '@codemirror/state': 6.5.3
   '@codemirror/view': 6.39.8
+  '@confluentinc/kafka-javascript': 1.9.1
   '@daytona/sdk': 0.187.0
   '@iconify-json/lucide': ^1.2.112
   '@iconify-json/mdi': ^1.1.63

scripts/build-n8n.mjs

@@ -218,6 +218,14 @@ for (const pattern of phantomDirs) {
 }
 echo(chalk.green('✅ Phantom dirs stripped'));
 
+// @confluentinc/kafka-javascript vendors librdkafka's full C source tree for its
+// build-from-source fallback (~11MB), but the prebuilt binary - librdkafka statically
+// linked in, no .so/.a shipped - is what actually loads at runtime on Alpine. The
+// source is dead weight in the shipped image.
+echo(chalk.yellow('INFO: Stripping unused librdkafka source tree...'));
+await $`find ${config.compiledAppDir}/node_modules/.pnpm -type d -path "*/@confluentinc/kafka-javascript/deps" -exec rm -rf {} + 2>/dev/null || true`;
+echo(chalk.green('✅ librdkafka source tree stripped'));
+
 // Strip TypeScript declaration artifacts to cut the image's file count, which
 // dominates layer extraction time on constrained hosts. Only these two explicit
 // patterns are safe to remove by extension: several features read other