diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index 21cb7aa118a..7643fab6fce 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -507,6 +507,7 @@ Scripts in `.github/scripts/`: |-------------------------|-------------------|------------------------| | `docker/docker-config.mjs`| Build context | `docker-build-push.yml`| | `docker/docker-tags.mjs` | Image tags | `docker-build-push.yml`| +| `docker/kafka-native-smoke-check.mjs`| Verify librdkafka binary loads in built image | `docker-build-push.yml`| ### Validation Scripts diff --git a/.github/scripts/docker/docker-config.mjs b/.github/scripts/docker/docker-config.mjs index 3099024ce01..00dd1b8e035 100644 --- a/.github/scripts/docker/docker-config.mjs +++ b/.github/scripts/docker/docker-config.mjs @@ -7,7 +7,7 @@ class BuildContext { this.githubOutput = process.env.GITHUB_OUTPUT || null; } - determine({ event, pr, branch, version, releaseType, pushEnabled }) { + determine({ event, pr, branch, version, releaseType, pushEnabled, includeArm64 }) { let context = { version: '', release_type: '', @@ -37,7 +37,9 @@ class BuildContext { case 'workflow_dispatch': context.version = `branch-${this.sanitizeBranch(branch)}`; context.release_type = 'branch'; - context.platforms = ['linux/amd64']; + // Manual pre-merge proof (e.g. a native-binary check) can opt into arm64; + // defaults to amd64-only like other non-release branch builds. + context.platforms = includeArm64 ? ['linux/amd64', 'linux/arm64'] : ['linux/amd64']; break; case 'push': @@ -152,6 +154,7 @@ if (import.meta.url === `file://${process.argv[1]}`) { releaseType: getArg('release-type'), pushEnabled: pushEnabledArg === 'true' ? true : pushEnabledArg === 'false' ? false : undefined, + includeArm64: getArg('include-arm64') === 'true', }); const matrix = context.buildMatrix(result.platforms); diff --git a/.github/scripts/docker/kafka-native-smoke-check.mjs b/.github/scripts/docker/kafka-native-smoke-check.mjs new file mode 100644 index 00000000000..c711143a153 --- /dev/null +++ b/.github/scripts/docker/kafka-native-smoke-check.mjs @@ -0,0 +1,42 @@ +#!/usr/bin/env node + +// Verifies the @confluentinc/kafka-javascript native binding (librdkafka) loads +// correctly inside a built n8n image. Resolves the module the same way n8n's +// runtime would - from within n8n-nodes-base, not via a hardcoded pnpm store path, +// since that path's hash suffix depends on the exact dependency graph. + +import { createRequire } from 'node:module'; +import { realpathSync } from 'node:fs'; +import path from 'node:path'; + +// The features our existing Kafka credential depends on (TLS + SASL SCRAM auth). +const REQUIRED_FEATURES = ['ssl', 'sasl_scram']; +const COMPRESSION_CODECS = ['gzip', 'snappy', 'lz4', 'zstd']; + +const n8nInstallDir = process.env.N8N_INSTALL_DIR || '/usr/local/lib/node_modules/n8n'; +const nodesBasePackageJson = path.join(n8nInstallDir, 'node_modules/n8n-nodes-base/package.json'); +// n8n-nodes-base is a pnpm symlink; resolve it so require() walks up from its real +// location in the pnpm virtual store, where its dependencies actually live. +const require = createRequire(realpathSync(nodesBasePackageJson)); + +const kafka = require('@confluentinc/kafka-javascript'); + +// Construct a client object (no broker connection attempted) to prove the native +// binding is fully usable, not just importable. +new kafka.KafkaJS.Kafka({ + kafkaJS: { brokers: ['localhost:9092'], clientId: 'ent-216-smoke-check' }, +}); + +console.log(`librdkafka version: ${kafka.librdkafkaVersion}`); +console.log(`Reported features: ${kafka.features.join(', ')}`); + +const missingFeatures = REQUIRED_FEATURES.filter((feature) => !kafka.features.includes(feature)); +if (missingFeatures.length > 0) { + console.error(`Missing required librdkafka features: ${missingFeatures.join(', ')}`); + process.exit(1); +} + +const supportedCodecs = COMPRESSION_CODECS.filter((codec) => kafka.features.includes(codec)); +console.log(`Supported compression codecs: ${supportedCodecs.join(', ') || 'none'}`); + +console.log('Kafka native smoke check passed.'); diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index 487ce21b6a4..bd1e284c689 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -47,6 +47,11 @@ on: required: false type: boolean default: true + include_arm64: + description: 'Also build linux/arm64 (default is amd64-only for manual/branch builds)' + required: false + type: boolean + default: false success_url: description: 'URL to call after the build is successful' required: false @@ -81,6 +86,7 @@ jobs: N8N_VERSION: ${{ inputs.n8n_version }} RELEASE_TYPE: ${{ inputs.release_type }} PUSH_ENABLED: ${{ inputs.push_enabled }} + INCLUDE_ARM64: ${{ inputs.include_arm64 }} GITHUB_REF: ${{ github.ref_name }} run: | node .github/scripts/docker/docker-config.mjs \ @@ -89,7 +95,8 @@ jobs: --branch "$GITHUB_REF" \ --version "$N8N_VERSION" \ --release-type "$RELEASE_TYPE" \ - --push-enabled "$PUSH_ENABLED" + --push-enabled "$PUSH_ENABLED" \ + --include-arm64 "$INCLUDE_ARM64" build-and-push-docker: name: Build App, then Build and Push Docker Image (${{ matrix.platform }}) @@ -174,6 +181,22 @@ jobs: push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }} tags: ${{ steps.determine-tags.outputs.n8n_tags }} + - name: Kafka native binding smoke check + # `load: true` isn't an option here: with sbom:true, buildx produces a manifest + # list even for a single platform, and the docker exporter (what --load uses) + # can't materialize manifest lists locally. Pull the pushed image back instead. + if: needs.determine-build-context.outputs.push_enabled == 'true' + env: + N8N_TAGS: ${{ steps.determine-tags.outputs.n8n_tags }} + run: | + IMAGE_TAG=$(echo "$N8N_TAGS" | cut -d',' -f1) + docker pull "$IMAGE_TAG" + docker run --rm \ + --entrypoint node \ + -v "${{ github.workspace }}/.github/scripts/docker/kafka-native-smoke-check.mjs:/tmp/kafka-native-smoke-check.mjs:ro" \ + "$IMAGE_TAG" \ + /tmp/kafka-native-smoke-check.mjs + - name: Build and push task runners Docker image (Alpine) id: build-runners uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2 diff --git a/package.json b/package.json index 842afc1b134..bb102bc791b 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,7 @@ }, "pnpm": { "onlyBuiltDependencies": [ + "@confluentinc/kafka-javascript", "@vscode/ripgrep", "isolated-vm", "sqlite3" diff --git a/packages/nodes-base/package.json b/packages/nodes-base/package.json index be831b48352..e58baa86916 100644 --- a/packages/nodes-base/package.json +++ b/packages/nodes-base/package.json @@ -936,6 +936,7 @@ "@smithy/signature-v4": "5.3.5", "@smithy/types": "4.13.1", "@n8n/backend-network": "workspace:*", + "@confluentinc/kafka-javascript": "catalog:", "@kafkajs/confluent-schema-registry": "3.8.0", "@langchain/core": "catalog:", "@mozilla/readability": "catalog:", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 505ac414726..d09a903456d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -96,6 +96,9 @@ catalogs: '@codemirror/view': specifier: 6.39.8 version: 6.39.8 + '@confluentinc/kafka-javascript': + specifier: 1.9.1 + version: 1.9.1 '@daytona/sdk': specifier: 0.187.0 version: 0.187.0 @@ -6267,6 +6270,9 @@ importers: '@aws-sdk/credential-providers': specifier: 3.808.0 version: 3.808.0 + '@confluentinc/kafka-javascript': + specifier: 'catalog:' + version: 1.9.1(encoding@0.1.13) '@e965/xlsx': specifier: 'catalog:' version: 0.20.3 @@ -8410,6 +8416,10 @@ packages: peerDependencies: commander: ~13.1.0 + '@confluentinc/kafka-javascript@1.9.1': + resolution: {integrity: sha512-Qc7IZGSiWb9PVcfGwhMDmVcn0fj4V2jZjJ7r1ObNdnu1JSFY1yMZvINS4QY9dAI00S7Ng6EE2zIaEQ7UrVDrPw==} + engines: {node: '>=18.0.0'} + '@cspotcode/source-map-support@0.8.1': resolution: {integrity: sha512-IchNf6dN4tHoMFIn/7OE8LWZ19Y6q/67Bmf6vnGREv8RSbBVb9LPJxEcnwrcwX6ixSvaiGoomAUvu4YSxXrVgw==} engines: {node: '>=12'} @@ -9854,6 +9864,11 @@ packages: resolution: {integrity: sha512-9QOtNffcOF/c1seMCDnjckb3R9WHcG34tky+FHpNKKCW0wc/scYLwMtO+ptyGUfMW0/b/n4qRiALlaFHc9Oj7Q==} engines: {node: '>= 10.0.0'} + '@mapbox/node-pre-gyp@2.0.3': + resolution: {integrity: sha512-uwPAhccfFJlsfCxMYTwOdVfOz3xqyj8xYL3zJj8f0pb30tLohnnFPhLuqp4/qoEz8sNxe4SESZedcBojRefIzg==} + engines: {node: '>=18'} + hasBin: true + '@marijn/find-cluster-break@1.0.2': resolution: {integrity: sha512-l0h88YhZFyKdXIFNfSWpyjStDjGHwZ/U7iobcK1cQQD8sejsONdQtTVU+1wVN1PBw40PiiHB1vA5S7VTfQiP9g==} @@ -13738,6 +13753,10 @@ packages: abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} + abbrev@3.0.1: + resolution: {integrity: sha512-AO2ac6pjRB3SJmGJo+v5/aK6Omggp6fsLrs6wN9bd35ulu4cCwaAU9+7ZhXjeqHVkaHThLuzH0nZr0YpCDhygg==} + engines: {node: ^18.17.0 || >=20.5.0} + abbrev@4.0.0: resolution: {integrity: sha512-a1wflyaL0tHtJSmLSOVybYhy22vRih4eduhhrkcjgrWGnRfrZtovJ2FRjxuTtkkj47O/baf0R86QU5OuYpz8fA==} engines: {node: ^20.17.0 || >=22.9.0} @@ -14911,6 +14930,10 @@ packages: config-chain@1.1.13: resolution: {integrity: sha512-qj+f8APARXHrM0hraqXYb2/bOVSV4PvJQlNZ/DVj0QrmNM2q2euizkeuVckQ57J+W0mRH6Hvi+k50M4Jul2VRQ==} + consola@3.4.2: + resolution: {integrity: sha512-5IKcdX0nnYavi6G7TtOhwkYzyjfJlatbjMjuLSfE2kYT5pMDOilZ4OvMhi637CcDICTmz3wARPoyhqyX1Y+XvA==} + engines: {node: ^14.18.0 || >=16.10.0} + console-browserify@1.2.0: resolution: {integrity: sha512-ZMkYO/LkF17QvCPqM0gxw8yUzigAOZOSWSHg91FH6orS7vcEj5dVZTidN2fQ14yBSdg97RqhSNwLUXInd52OTA==} @@ -19157,6 +19180,11 @@ packages: engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} hasBin: true + nopt@8.1.0: + resolution: {integrity: sha512-ieGu42u/Qsa4TFktmaKEwM6MQH0pOWnaB3htzh0JRtx84+Mebc0cbZYN5bC+6WTZ4+77xrL9Pn5m7CV6VIkV7A==} + engines: {node: ^18.17.0 || >=20.5.0} + hasBin: true + nopt@9.0.0: resolution: {integrity: sha512-Zhq3a+yFKrYwSBluL4H9XP3m3y5uvQkB/09CwDruCiRmR/UJYnn9W4R48ry0uGC70aeTPKLynBtscP9efFFcPw==} engines: {node: ^20.17.0 || >=22.9.0} @@ -25570,6 +25598,15 @@ snapshots: dependencies: commander: 13.1.0 + '@confluentinc/kafka-javascript@1.9.1(encoding@0.1.13)': + dependencies: + '@mapbox/node-pre-gyp': 2.0.3(encoding@0.1.13) + bindings: 1.5.0 + nan: 2.26.2 + transitivePeerDependencies: + - encoding + - supports-color + '@cspotcode/source-map-support@0.8.1': dependencies: '@jridgewell/trace-mapping': 0.3.9 @@ -27029,6 +27066,19 @@ snapshots: transitivePeerDependencies: - supports-color + '@mapbox/node-pre-gyp@2.0.3(encoding@0.1.13)': + dependencies: + consola: 3.4.2 + detect-libc: 2.1.2 + https-proxy-agent: 7.0.6 + node-fetch: 2.7.0(encoding@0.1.13) + nopt: 8.1.0 + semver: 7.7.3 + tar: 7.5.19 + transitivePeerDependencies: + - encoding + - supports-color + '@marijn/find-cluster-break@1.0.2': {} '@mdx-js/react@3.0.1(@types/react@18.0.27)(react@18.2.0)': @@ -31482,6 +31532,8 @@ snapshots: abbrev@1.1.1: {} + abbrev@3.0.1: {} + abbrev@4.0.0: {} abort-controller-x@0.4.3: {} @@ -32807,6 +32859,8 @@ snapshots: ini: 1.3.8 proto-list: 1.2.4 + consola@3.4.2: {} + console-browserify@1.2.0: {} console-control-strings@1.1.0: @@ -38015,6 +38069,10 @@ snapshots: dependencies: abbrev: 1.1.1 + nopt@8.1.0: + dependencies: + abbrev: 3.0.1 + nopt@9.0.0: dependencies: abbrev: 4.0.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 68dbc212ec4..9a6345e9ae6 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -37,6 +37,7 @@ catalog: '@codemirror/search': 6.5.11 '@codemirror/state': 6.5.3 '@codemirror/view': 6.39.8 + '@confluentinc/kafka-javascript': 1.9.1 '@daytona/sdk': 0.187.0 '@iconify-json/lucide': ^1.2.112 '@iconify-json/mdi': ^1.1.63 diff --git a/scripts/build-n8n.mjs b/scripts/build-n8n.mjs index 8c9cfa7a674..e21d731582d 100755 --- a/scripts/build-n8n.mjs +++ b/scripts/build-n8n.mjs @@ -218,6 +218,14 @@ for (const pattern of phantomDirs) { } echo(chalk.green('✅ Phantom dirs stripped')); +// @confluentinc/kafka-javascript vendors librdkafka's full C source tree for its +// build-from-source fallback (~11MB), but the prebuilt binary - librdkafka statically +// linked in, no .so/.a shipped - is what actually loads at runtime on Alpine. The +// source is dead weight in the shipped image. +echo(chalk.yellow('INFO: Stripping unused librdkafka source tree...')); +await $`find ${config.compiledAppDir}/node_modules/.pnpm -type d -path "*/@confluentinc/kafka-javascript/deps" -exec rm -rf {} + 2>/dev/null || true`; +echo(chalk.green('✅ librdkafka source tree stripped')); + // Strip TypeScript declaration artifacts to cut the image's file count, which // dominates layer extraction time on constrained hosts. Only these two explicit // patterns are safe to remove by extension: several features read other