diff --git a/.github/WORKFLOWS.md b/.github/WORKFLOWS.md index 13c59c18b3b..06ad00d7697 100644 --- a/.github/WORKFLOWS.md +++ b/.github/WORKFLOWS.md @@ -480,9 +480,18 @@ Composite actions in `.github/actions/`: inputs: node-version: # default: '24.18.1' enable-docker-cache: # default: 'false' (Blacksmith Buildx) + docker-cache-key: # required when enable-docker-cache is true build-command: # default: 'pnpm build' ``` +The Blacksmith layer cache lives on a sticky disk identified by +`docker-cache-key`, and commits are last-writer-wins. Splitting the key per +image would avoid that, but Blacksmith currently never populates a +newly created sticky disk - it stays at 0 bytes however many runs commit to it, +while the build reports a successful commit. Every job therefore shares the +`n8n-io/n8n` key, which is the only disk that actually retains layers. Revisit +once new-disk retention works. + ### docker-registry-login ```yaml diff --git a/.github/actions/build-n8n-docker/action.yml b/.github/actions/build-n8n-docker/action.yml index 37cf04a8d95..d24fe913ed1 100644 --- a/.github/actions/build-n8n-docker/action.yml +++ b/.github/actions/build-n8n-docker/action.yml @@ -23,8 +23,8 @@ runs: id: cache-check uses: actions/cache/restore@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3 with: - key: n8n-docker-image-${{ inputs.build-variant }}-${{ github.sha }} - path: /tmp/n8n-image.tar.zst + key: n8n-docker-image-v2-${{ inputs.build-variant }}-${{ github.sha }} + path: /tmp/n8n-images lookup-only: true - name: Build Docker image @@ -33,16 +33,17 @@ runs: with: build-command: ${{ inputs.build-variant == 'coverage' && 'pnpm build:docker:coverage' || 'pnpm build:docker' }} enable-docker-cache: true + docker-cache-key: n8n-io/n8n env: INCLUDE_TEST_CONTROLLER: 'true' - # Keep the pc pins in sync with the -pc build in docker-build-push.yml. - BUILDER_IMAGE: ${{ inputs.build-variant == 'pc' && 'n8nio/node-pc:26.7.0-dev@sha256:c64642dcb9464e50bd08aef8504affafeb795d01aed2f7ab158976c334a36ef6' || '' }} - RUNTIME_IMAGE: ${{ inputs.build-variant == 'pc' && 'n8nio/node-pc:26.7.0@sha256:6577d0742ad043baaa39db7dda2ce8aeb73f32c84eb241c49bf716f170c53297' || '' }} + DOCKER_BUILD_TARBALL_DIR: /tmp/n8n-images + DOCKER_BUILD_PC: ${{ inputs.build-variant == 'pc' }} - name: Assert pointer compression if: steps.cache-check.outputs.cache-hit != 'true' && inputs.build-variant == 'pc' shell: bash run: | + docker load -i /tmp/n8n-images/n8n-pc.tar docker run --rm --entrypoint node n8nio/n8n:local -e " if (!process.config.variables.v8_enable_pointer_compression) { throw new Error('pc variant image is not pointer-compressed'); @@ -50,16 +51,14 @@ runs: console.log(process.version, 'pointer compression: on'); " - - name: Save image tarball + - name: Report tarball sizes if: steps.cache-check.outputs.cache-hit != 'true' shell: bash - run: | - docker save n8nio/n8n:local n8nio/runners:local | zstd -T0 -3 -o /tmp/n8n-image.tar.zst - ls -lh /tmp/n8n-image.tar.zst + run: ls -lh /tmp/n8n-images - name: Publish image tarball to cache if: steps.cache-check.outputs.cache-hit != 'true' uses: actions/cache/save@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3 with: - key: n8n-docker-image-${{ inputs.build-variant }}-${{ github.sha }} - path: /tmp/n8n-image.tar.zst + key: n8n-docker-image-v2-${{ inputs.build-variant }}-${{ github.sha }} + path: /tmp/n8n-images diff --git a/.github/actions/load-n8n-docker/action.yml b/.github/actions/load-n8n-docker/action.yml index 1534fd00ed6..939a230cccd 100644 --- a/.github/actions/load-n8n-docker/action.yml +++ b/.github/actions/load-n8n-docker/action.yml @@ -11,7 +11,7 @@ # the tarball to cache as a side effect. name: 'Load n8n Docker images from cache' -description: 'Restores the zstd-compressed n8n + runners image tarball from the variant+SHA-keyed GHA cache and loads both images into the local docker daemon. Falls back to rebuilding on cache miss.' +description: 'Restores the n8n + runners image tarballs from the variant+SHA-keyed GHA cache and loads both images into the local docker daemon. Falls back to rebuilding on cache miss.' inputs: build-variant: @@ -30,13 +30,24 @@ runs: id: restore uses: actions/cache/restore@640a1c2554105b57832a23eea0b4672fc7a790d5 # v4.2.3 with: - key: n8n-docker-image-${{ inputs.build-variant }}-${{ inputs.cache-sha || github.sha }} - path: /tmp/n8n-image.tar.zst + key: n8n-docker-image-v2-${{ inputs.build-variant }}-${{ inputs.cache-sha || github.sha }} + path: /tmp/n8n-images - name: Load n8n and runners images into docker if: steps.restore.outputs.cache-hit == 'true' shell: bash - run: zstd -d -c /tmp/n8n-image.tar.zst | docker load + # A restore reports a hit whatever it wrote, and writes to the paths + # recorded at save time. Fail loudly rather than letting an unexpanded + # glob reach `docker load`. + run: | + set -euo pipefail + shopt -s nullglob + files=(/tmp/n8n-images/*.tar) + if [ ${#files[@]} -eq 0 ]; then + echo "::error::cache hit but /tmp/n8n-images holds no archives" + exit 1 + fi + for f in "${files[@]}"; do docker load -i "$f"; done - name: Warn on cache miss if: steps.restore.outputs.cache-hit != 'true' @@ -45,10 +56,25 @@ runs: BUILD_VARIANT: ${{ inputs.build-variant }} GITHUB_SHA: ${{ inputs.cache-sha || github.sha }} run: | - echo "::warning::Cache miss for n8n-docker-image-$BUILD_VARIANT-$GITHUB_SHA (SHA $GITHUB_SHA); falling back to rebuild via build-n8n-docker." + echo "::warning::Cache miss for n8n-docker-image-v2-$BUILD_VARIANT-$GITHUB_SHA (SHA $GITHUB_SHA); falling back to rebuild via build-n8n-docker." - name: Rebuild image on cache miss if: steps.restore.outputs.cache-hit != 'true' uses: ./.github/actions/build-n8n-docker with: build-variant: ${{ inputs.build-variant }} + + # build-n8n-docker writes archives rather than loading them, so the fallback + # has to load them here or downstream steps find no images. + - name: Load rebuilt images into docker + if: steps.restore.outputs.cache-hit != 'true' + shell: bash + run: | + set -euo pipefail + shopt -s nullglob + files=(/tmp/n8n-images/*.tar) + if [ ${#files[@]} -eq 0 ]; then + echo "::error::rebuild produced no archives in /tmp/n8n-images" + exit 1 + fi + for f in "${files[@]}"; do docker load -i "$f"; done diff --git a/.github/actions/setup-nodejs/action.yml b/.github/actions/setup-nodejs/action.yml index b31f3906267..d3bb69309db 100644 --- a/.github/actions/setup-nodejs/action.yml +++ b/.github/actions/setup-nodejs/action.yml @@ -14,6 +14,10 @@ inputs: description: 'Whether to set up Blacksmith Buildx for Docker layer caching (Blacksmith runners only).' required: false default: 'false' + docker-cache-key: + description: 'Sticky-disk identity for the Docker layer cache. Required when enable-docker-cache is true. Use n8n-io/n8n: Blacksmith only retains disks that already exist, and a newly created key stays empty across runs, so a per-image key silently disables caching entirely.' + required: false + default: '' build-command: description: 'Command to execute for building the project or an optional command. Leave empty to skip build step.' required: false @@ -30,6 +34,15 @@ inputs: runs: using: 'composite' steps: + # An empty key makes the builder use a local, uncached one. The job stays + # green and loses the cache. Fail here instead. + - name: Verify Docker cache key is set + if: ${{ inputs.enable-docker-cache == 'true' && inputs.docker-cache-key == '' }} + shell: bash + run: | + echo "::error::enable-docker-cache is true but docker-cache-key is empty" + exit 1 + - name: Setup pnpm uses: pnpm/setup@84cb39b217b10273981911c288cd62326dc7c6d2 # v2.0.2 with: @@ -277,7 +290,9 @@ runs: - name: Setup Docker Builder for Docker Cache (Blacksmith) if: ${{ inputs.enable-docker-cache == 'true' && contains(runner.name, 'blacksmith') }} - uses: useblacksmith/setup-docker-builder@ef12d5b165b596e3aa44ea8198d8fde563eab402 # v1.4.0 + uses: useblacksmith/setup-docker-builder@a5256a73e30f09e37e3eceb8ca36043d17621d24 # v2.1.0 + with: + cache-key: ${{ inputs.docker-cache-key }} - name: Setup Docker Builder (GitHub fallback) if: ${{ inputs.enable-docker-cache == 'true' && !contains(runner.name, 'blacksmith') }} diff --git a/.github/scripts/docker/assert-manifest-format.mjs b/.github/scripts/docker/assert-manifest-format.mjs new file mode 100644 index 00000000000..d6f953de569 --- /dev/null +++ b/.github/scripts/docker/assert-manifest-format.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +/** + * Asserts that a pushed image is an OCI image index with only real platform + * manifests. + * + * n8n 2.26.0 shipped as a Docker manifest list, not an OCI index. Older + * containerd on AKS then read the attestation manifests as image manifests, and + * every pull failed (#31997). This check verifies the format directly. + * + * Usage: node assert-manifest-format.mjs [--expect-platforms n] + */ + +import { execFileSync } from 'node:child_process'; +import { pathToFileURL } from 'node:url'; + +export const OCI_INDEX = 'application/vnd.oci.image.index.v1+json'; + +/** + * Partitions an index's manifests and collects format failures. + * + * Pure so it can be unit-tested without a registry. `platform` is optional in + * the OCI spec, so a descriptor without one is a failure rather than a crash. + * + * @param {object} manifest parsed image index + * @param {number|null} expectPlatforms distinct platform count to require + */ +export function checkManifestFormat(manifest, expectPlatforms = null) { + const failures = []; + + if (manifest.mediaType !== OCI_INDEX) { + failures.push(`mediaType is ${manifest.mediaType}, expected ${OCI_INDEX}`); + } + + const entries = Array.isArray(manifest.manifests) ? manifest.manifests : []; + const unplaced = entries.filter((m) => !m.platform); + if (unplaced.length > 0) { + failures.push(`${unplaced.length} manifest(s) carry no platform descriptor`); + } + + const placed = entries.filter((m) => m.platform); + const platforms = placed.filter((m) => m.platform.architecture !== 'unknown'); + const attestations = placed.filter((m) => m.platform.architecture === 'unknown'); + + const distinct = new Set( + platforms.map((m) => + [m.platform.os, m.platform.architecture, m.platform.variant ?? ''].join('/'), + ), + ); + // Distinct, not length: a duplicated platform entry would otherwise pass. + if (expectPlatforms !== null && distinct.size !== expectPlatforms) { + failures.push( + `${distinct.size} distinct platforms (${[...distinct].join(', ')}), expected ${expectPlatforms}`, + ); + } + if (platforms.length !== distinct.size) { + failures.push(`${platforms.length} platform manifests but only ${distinct.size} distinct`); + } + + return { failures, platforms, attestations, distinct }; +} + +function main() { + const [ref, ...rest] = process.argv.slice(2); + if (!ref) { + console.error('usage: assert-manifest-format.mjs [--expect-platforms n]'); + process.exit(2); + } + + const expectIdx = rest.indexOf('--expect-platforms'); + const expectPlatforms = expectIdx === -1 ? null : Number(rest[expectIdx + 1]); + + const raw = execFileSync('docker', ['buildx', 'imagetools', 'inspect', '--raw', ref], { + encoding: 'utf-8', + }); + const manifest = JSON.parse(raw); + + const { failures, platforms, attestations } = checkManifestFormat(manifest, expectPlatforms); + + console.log(`ref: ${ref}`); + console.log(`mediaType: ${manifest.mediaType}`); + for (const m of platforms) { + console.log(` platform ${m.platform.os}/${m.platform.architecture}`); + } + for (const _ of attestations) { + console.log(' attestation (unknown/unknown)'); + } + + // Not fatal. `--sbom=true` adds these, and they caused the 2.26.0 pull failure. + // Report the count and let the caller decide. + console.log(`\nplatforms: ${platforms.length}, attestations: ${attestations.length}`); + + if (failures.length > 0) { + for (const f of failures) console.error(`::error::${f}`); + process.exit(1); + } + console.log('OK: manifest is an OCI image index'); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/.github/scripts/docker/assert-manifest-format.test.mjs b/.github/scripts/docker/assert-manifest-format.test.mjs new file mode 100644 index 00000000000..37d62530cd3 --- /dev/null +++ b/.github/scripts/docker/assert-manifest-format.test.mjs @@ -0,0 +1,96 @@ +import { describe, it } from 'node:test'; +import assert from 'node:assert/strict'; +import { checkManifestFormat, OCI_INDEX } from './assert-manifest-format.mjs'; + +const DOCKER_LIST = 'application/vnd.docker.distribution.manifest.list.v2+json'; + +const platform = (os, architecture, variant) => ({ platform: { os, architecture, variant } }); +const attestation = () => ({ platform: { os: 'unknown', architecture: 'unknown' } }); + +describe('checkManifestFormat', () => { + it('passes a two-platform OCI index', () => { + const { failures, platforms, attestations } = checkManifestFormat( + { mediaType: OCI_INDEX, manifests: [platform('linux', 'amd64'), platform('linux', 'arm64')] }, + 2, + ); + assert.deepEqual(failures, []); + assert.equal(platforms.length, 2); + assert.equal(attestations.length, 0); + }); + + it('rejects a Docker manifest list', () => { + const { failures } = checkManifestFormat({ + mediaType: DOCKER_LIST, + manifests: [platform('linux', 'amd64')], + }); + assert.equal(failures.length, 1); + assert.match(failures[0], /expected application\/vnd\.oci\.image\.index/); + }); + + it('counts attestations separately from platforms', () => { + const { platforms, attestations, failures } = checkManifestFormat( + { + mediaType: OCI_INDEX, + manifests: [platform('linux', 'amd64'), platform('linux', 'arm64'), attestation()], + }, + 2, + ); + assert.deepEqual(failures, []); + assert.equal(platforms.length, 2); + assert.equal(attestations.length, 1); + }); + + it('fails when the distinct platform count does not match', () => { + const { failures } = checkManifestFormat( + { mediaType: OCI_INDEX, manifests: [platform('linux', 'amd64')] }, + 2, + ); + assert.equal(failures.length, 1); + assert.match(failures[0], /1 distinct platforms/); + }); + + it('does not let a duplicated platform satisfy the expected count', () => { + const { failures } = checkManifestFormat( + { mediaType: OCI_INDEX, manifests: [platform('linux', 'amd64'), platform('linux', 'amd64')] }, + 2, + ); + assert.ok(failures.some((f) => /1 distinct platforms/.test(f))); + assert.ok(failures.some((f) => /2 platform manifests but only 1 distinct/.test(f))); + }); + + it('treats a descriptor with no platform as a failure, not a crash', () => { + const { failures } = checkManifestFormat( + { mediaType: OCI_INDEX, manifests: [platform('linux', 'amd64'), { digest: 'sha256:x' }] }, + 1, + ); + assert.ok(failures.some((f) => /carry no platform descriptor/.test(f))); + }); + + it('treats a missing or non-array manifests field as empty', () => { + assert.equal(checkManifestFormat({ mediaType: OCI_INDEX }).platforms.length, 0); + assert.equal( + checkManifestFormat({ mediaType: OCI_INDEX, manifests: {} }).platforms.length, + 0, + ); + }); + + it('distinguishes platforms that differ only by variant', () => { + const { platforms, failures } = checkManifestFormat( + { + mediaType: OCI_INDEX, + manifests: [platform('linux', 'arm', 'v6'), platform('linux', 'arm', 'v7')], + }, + 2, + ); + assert.deepEqual(failures, []); + assert.equal(platforms.length, 2); + }); + + it('skips the platform-count check when no expectation is given', () => { + const { failures } = checkManifestFormat({ + mediaType: OCI_INDEX, + manifests: [platform('linux', 'amd64')], + }); + assert.deepEqual(failures, []); + }); +}); diff --git a/.github/scripts/docker/docker-config.mjs b/.github/scripts/docker/docker-config.mjs index 00dd1b8e035..fce22011995 100644 --- a/.github/scripts/docker/docker-config.mjs +++ b/.github/scripts/docker/docker-config.mjs @@ -90,7 +90,7 @@ class BuildContext { buildMatrix(platforms) { const runners = { - 'linux/amd64': 'blacksmith-4vcpu-ubuntu-2204', + 'linux/amd64': 'blacksmith-8vcpu-ubuntu-2204', 'linux/arm64': 'blacksmith-8vcpu-ubuntu-2204-arm', }; diff --git a/.github/scripts/package.json b/.github/scripts/package.json index 51c62541e91..1d60aad2a3c 100644 --- a/.github/scripts/package.json +++ b/.github/scripts/package.json @@ -6,7 +6,7 @@ }, "packageManager": "pnpm@11.22.0", "scripts": { - "test": "node --test --experimental-test-module-mocks ./*.test.mjs ./quality/*.test.mjs ./slack/*.test.mjs ./stale/*.test.mjs ../../scripts/licenses/*.test.mjs ../../scripts/mutation-health/*.test.mjs", + "test": "node --test --experimental-test-module-mocks ./*.test.mjs ./docker/*.test.mjs ./quality/*.test.mjs ./slack/*.test.mjs ./stale/*.test.mjs ../../scripts/licenses/*.test.mjs ../../scripts/mutation-health/*.test.mjs", "generate-sbom": "FETCH_LICENSE=true cdxgen -t pnpm --no-install-deps --profile license-compliance --spec-version 1.6 -o ../../sbom-source.cdx.json ../../compiled/", "enrich-sbom": "node ../../scripts/licenses/enrich-sbom.mjs ../../sbom-source.cdx.json", "render-licenses-md": "node ../../scripts/licenses/render-licenses-md.mjs ../../sbom-source.cdx.json ../../packages/cli/THIRD_PARTY_LICENSES.md ../../compiled/node_modules", diff --git a/.github/workflows/docker-build-push.yml b/.github/workflows/docker-build-push.yml index b75e6699883..7a7ef5ccf76 100644 --- a/.github/workflows/docker-build-push.yml +++ b/.github/workflows/docker-build-push.yml @@ -156,6 +156,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} build-command: pnpm build:n8n enable-docker-cache: 'true' + docker-cache-key: n8n-io/n8n env: RELEASE: ${{ needs.determine-build-context.outputs.n8n_version }} SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }} @@ -193,105 +194,36 @@ jobs: dockerhub-username: ${{ secrets.DOCKER_USERNAME }} dockerhub-password: ${{ secrets.DOCKER_PASSWORD }} - - name: Build and push n8n Docker image - id: build-n8n - uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2 - with: - context: . - file: ./docker/images/n8n/Dockerfile - build-args: | - NODE_VERSION=${{ env.NODE_VERSION }} - N8N_VERSION=${{ needs.determine-build-context.outputs.n8n_version }} - N8N_RELEASE_TYPE=${{ needs.determine-build-context.outputs.release_type }} - platforms: ${{ matrix.docker_platform }} - provenance: false # Disabled - using SLSA L3 generator for isolated provenance - sbom: true - push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }} - tags: ${{ steps.determine-tags.outputs.n8n_tags }} - - - name: Build and push n8n -pc Docker image - id: build-n8n-pc - if: needs.determine-build-context.outputs.push_enabled == 'true' && steps.determine-tags.outputs.n8n_pc_tags != '' - uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2 - with: - context: . - file: ./docker/images/n8n/Dockerfile - build-args: | - NODE_VERSION=${{ env.NODE_VERSION }} - N8N_VERSION=${{ needs.determine-build-context.outputs.n8n_version }} - N8N_RELEASE_TYPE=${{ needs.determine-build-context.outputs.release_type }} - BUILDER_IMAGE=n8nio/node-pc:26.7.0-dev@sha256:c64642dcb9464e50bd08aef8504affafeb795d01aed2f7ab158976c334a36ef6 - RUNTIME_IMAGE=n8nio/node-pc:26.7.0@sha256:6577d0742ad043baaa39db7dda2ce8aeb73f32c84eb241c49bf716f170c53297 - IMAGE_DESCRIPTION=Workflow Automation Tool (pointer-compressed variant, internal to n8n Cloud, no support or stability guarantees) - platforms: ${{ matrix.docker_platform }} - provenance: false # Disabled - using SLSA L3 generator for isolated provenance - sbom: true - push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }} - tags: ${{ steps.determine-tags.outputs.n8n_pc_tags }} - - - name: Kafka native binding smoke check - # `load: true` isn't an option here: with sbom:true, buildx produces a manifest - # list even for a single platform, and the docker exporter (what --load uses) - # can't materialize manifest lists locally. Pull the pushed image back instead. - if: needs.determine-build-context.outputs.push_enabled == 'true' + - name: Build and push Docker images env: + N8N_VERSION: ${{ needs.determine-build-context.outputs.n8n_version }} + N8N_RELEASE_TYPE: ${{ needs.determine-build-context.outputs.release_type }} + PLATFORMS: ${{ matrix.docker_platform }} N8N_TAGS: ${{ steps.determine-tags.outputs.n8n_tags }} - run: | - IMAGE_TAG=$(echo "$N8N_TAGS" | cut -d',' -f1) - docker pull "$IMAGE_TAG" - docker run --rm \ - --entrypoint node \ - -v "${{ github.workspace }}/.github/scripts/docker/kafka-native-smoke-check.mjs:/tmp/kafka-native-smoke-check.mjs:ro" \ - "$IMAGE_TAG" \ - /tmp/kafka-native-smoke-check.mjs - - - name: Build and push task runners Docker image (Alpine) - id: build-runners - uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2 - with: - context: . - file: ./docker/images/runners/Dockerfile - build-args: | - NODE_VERSION=${{ env.NODE_VERSION }} - N8N_VERSION=${{ needs.determine-build-context.outputs.n8n_version }} - N8N_RELEASE_TYPE=${{ needs.determine-build-context.outputs.release_type }} - platforms: ${{ matrix.docker_platform }} - provenance: false # Disabled - using SLSA L3 generator for isolated provenance - sbom: true - push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }} - tags: ${{ steps.determine-tags.outputs.runners_tags }} - - - name: Build and push task runners Docker image (distroless) - id: build-runners-distroless - uses: useblacksmith/build-push-action@30c71162f16ea2c27c3e21523255d209b8b538c1 # v2 - with: - context: . - file: ./docker/images/runners/Dockerfile.distroless - build-args: | - NODE_VERSION=${{ env.NODE_VERSION }} - N8N_VERSION=${{ needs.determine-build-context.outputs.n8n_version }} - N8N_RELEASE_TYPE=${{ needs.determine-build-context.outputs.release_type }} - platforms: ${{ matrix.docker_platform }} - provenance: false # Disabled - using SLSA L3 generator for isolated provenance - sbom: true - push: ${{ needs.determine-build-context.outputs.push_enabled == 'true' }} - tags: ${{ steps.determine-tags.outputs.runners_distroless_tags }} - - - name: Runners interpreters smoke check - # The runtime stages assemble node/python by copying binaries across images, - # so a missing shared library only surfaces at exec time. Runs the exact - # interpreter paths the launcher config (n8n-task-runners.json) uses. - if: needs.determine-build-context.outputs.push_enabled == 'true' - env: + N8N_PC_TAGS: ${{ steps.determine-tags.outputs.n8n_pc_tags }} RUNNERS_TAGS: ${{ steps.determine-tags.outputs.runners_tags }} RUNNERS_DISTROLESS_TAGS: ${{ steps.determine-tags.outputs.runners_distroless_tags }} + PUSH_ENABLED: ${{ needs.determine-build-context.outputs.push_enabled }} run: | - for TAGS in "$RUNNERS_TAGS" "$RUNNERS_DISTROLESS_TAGS"; do - IMAGE_TAG=$(echo "$TAGS" | cut -d',' -f1) - docker pull "$IMAGE_TAG" - docker run --rm --entrypoint /usr/local/bin/node "$IMAGE_TAG" --version - docker run --rm --entrypoint /opt/runners/task-runner-python/.venv/bin/python "$IMAGE_TAG" --version - done + TARGETS=(n8n runners runners-distroless) + if [ -n "$N8N_PC_TAGS" ] && [ "$PUSH_ENABLED" = 'true' ]; then + TARGETS+=(n8n-pc) + fi + + # oci-mediatypes keeps the merged manifest an OCI image index. Older + # containerd cannot pull a Docker manifest list that has attestation + # manifests (#31997). create_multi_arch_manifest asserts the merged + # format before the release proceeds. + OUTPUT=() + if [ "$PUSH_ENABLED" = 'true' ]; then + OUTPUT=(--set '*.output=type=image,oci-mediatypes=true,push=true') + fi + + docker buildx bake -f docker/docker-bake.hcl "${TARGETS[@]}" \ + --provenance=false \ + --sbom=false \ + --metadata-file /tmp/bake-metadata.json \ + "${OUTPUT[@]}" create_multi_arch_manifest: name: Create Multi-Arch Manifest @@ -378,6 +310,28 @@ jobs: create_manifest "runners (date)" "$RUNNERS_DATE_MANIFEST_TAG" create_manifest "runners-distroless (date)" "$RUNNERS_DISTROLESS_DATE_MANIFEST_TAG" + # Gates the release on the format itself. 2.26.0 shipped as a Docker + # manifest list and every pull failed on older containerd (#31997). The + # exporter flags that keep this an OCI index live in the build job, but + # the merge happens here, so this is the only place the published shape + # can be asserted. + - name: Assert merged manifests are OCI image indexes + env: + RELEASE_TYPE: ${{ needs.determine-build-context.outputs.release_type }} + N8N_TAG: ${{ needs.build-and-push-docker.outputs.primary_ghcr_manifest_tag }} + N8N_PC_TAG: ${{ needs.build-and-push-docker.outputs.n8n_pc_primary_ghcr_manifest_tag }} + RUNNERS_TAG: ${{ needs.build-and-push-docker.outputs.runners_primary_ghcr_manifest_tag }} + RUNNERS_DISTROLESS_TAG: ${{ needs.build-and-push-docker.outputs.runners_distroless_primary_ghcr_manifest_tag }} + run: | + set -euo pipefail + # Branch builds are amd64-only; everything else is amd64 + arm64. + if [[ "$RELEASE_TYPE" == "branch" ]]; then EXPECT=1; else EXPECT=2; fi + + for TAG in "$N8N_TAG" "$N8N_PC_TAG" "$RUNNERS_TAG" "$RUNNERS_DISTROLESS_TAG"; do + [[ -n "$TAG" ]] || continue + node .github/scripts/docker/assert-manifest-format.mjs "$TAG" --expect-platforms "$EXPECT" + done + - name: Create Docker Hub manifests if: needs.determine-build-context.outputs.push_to_docker == 'true' env: @@ -453,6 +407,43 @@ jobs: DISTROLESS_TAG: ${{ needs.build-and-push-docker.outputs.runners_distroless_primary_ghcr_manifest_tag }} run: node .github/scripts/docker/get-manifest-digests.mjs + # Execs the artifact that actually shipped. docker-build-smoke.yml runs the + # same checks, but against its own no-cache build of the branch tip - a + # different image, and its PR trigger deliberately skips pnpm-workspace.yaml, + # which is where the native deps are pinned. So a catalog bump of + # @confluentinc/kafka-javascript or isolated-vm reaches a published tag + # unexercised unless something pulls the pushed image back and runs it. + verify-pushed-images: + name: Verify Pushed Images + needs: [determine-build-context, create_multi_arch_manifest] + runs-on: ubuntu-latest + if: needs.create_multi_arch_manifest.result == 'success' + steps: + - name: Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + + # Installs only, no build: the script needs zx and yaml, not a compiled + # workspace. The image under test is pulled from the registry. + - name: Setup Node.js + uses: ./.github/actions/setup-nodejs + with: + node-version: '26.7.0' + build-command: '' + + - name: Login to GHCR + uses: ./.github/actions/docker-registry-login + with: + login-ghcr: true + + - name: Exec native bindings and runner interpreters + env: + SMOKE_IMAGE: ${{ needs.create_multi_arch_manifest.outputs.n8n_image }}@${{ needs.create_multi_arch_manifest.outputs.n8n_digest }} + SMOKE_RUNNERS_IMAGES: ${{ needs.create_multi_arch_manifest.outputs.runners_image }}@${{ needs.create_multi_arch_manifest.outputs.runners_digest }},${{ needs.create_multi_arch_manifest.outputs.runners_distroless_image }}@${{ needs.create_multi_arch_manifest.outputs.runners_distroless_digest }} + # The cloud-chart invocations need a token this job does not carry; + # the binding and interpreter execs are what guard the release. + SMOKE_SKIP_CLOUD: 'true' + run: node scripts/smoke-n8n-image.mjs + call-success-url: name: Call Success URL needs: [create_multi_arch_manifest] diff --git a/.github/workflows/docker-build-smoke.yml b/.github/workflows/docker-build-smoke.yml index 48ed9743e8d..41381657468 100644 --- a/.github/workflows/docker-build-smoke.yml +++ b/.github/workflows/docker-build-smoke.yml @@ -24,12 +24,24 @@ on: - cron: '0 3 * * *' # 3:00 AM UTC, after the nightly Docker build at midnight pull_request: paths: + - 'docker/docker-bake.hcl' - 'docker/images/n8n/**' - 'docker/images/n8n-base/**' - 'docker/images/runners/**' - 'scripts/build-n8n.mjs' - 'scripts/dockerize-n8n.mjs' - 'scripts/smoke-n8n-image.mjs' + - '.github/scripts/docker/kafka-native-smoke-check.mjs' + # These drive the build and the image distribution, so a change here can + # break the chain without touching a Dockerfile. + - '.github/actions/build-n8n-docker/action.yml' + - '.github/actions/load-n8n-docker/action.yml' + - '.github/workflows/docker-build-push.yml' + # Patched native dependencies change here. pnpm-workspace.yaml is + # excluded: it changes too often for a two-arch no-cache build. A native + # catalog bump that slips past this is still caught before it ships, by + # verify-pushed-images in docker-build-push.yml. + - 'patches/**' workflow_dispatch: concurrency: @@ -70,6 +82,9 @@ jobs: node-version: ${{ env.NODE_VERSION }} build-command: 'pnpm build:docker:clean' enable-docker-cache: true + # Use a separate key. This job builds with --no-cache. A shared key + # would write an empty snapshot over the release layers. + docker-cache-key: docker-smoke - uses: azure/setup-helm@b9e51907a09c216f16ebe8536097933489208112 # v4.3.0 diff --git a/.github/workflows/docker-sbom-probe.yml b/.github/workflows/docker-sbom-probe.yml new file mode 100644 index 00000000000..631011d5c8e --- /dev/null +++ b/.github/workflows/docker-sbom-probe.yml @@ -0,0 +1,225 @@ +name: 'Docker: SBOM manifest-format probe' +run-name: "Docker: SBOM probe (sbom=${{ inputs.sbom }})" + +# Asserts that the merged manifest is an OCI image index. 2.26.0 shipped as a +# Docker manifest list and did not pull on AKS (#31997). Run this after a +# buildx, BuildKit or base image change. It pushes only to a separate +# -format-probe package, so the published image repo is untouched. +# +# The release itself is gated by create_multi_arch_manifest in +# docker-build-push.yml; this probe is for investigating exporter behaviour. + +on: + workflow_dispatch: + inputs: + sbom: + description: 'Build with SBOM attestations (true reproduces the current release path)' + required: false + type: boolean + default: false + +env: + NODE_VERSION: '26.7.0' + SCRATCH_TAG: sbom-probe-${{ github.run_id }} + # A separate package. These are throwaway fixtures (one is a two-layer + # alpine), and ghcr.io/ is where the real release tags live - they + # would show up in the public n8n tag listing and never get cleaned up. + SCRATCH_IMAGE: ghcr.io/${{ github.repository }}-format-probe + +permissions: + contents: read + +jobs: + build: + name: Build (${{ matrix.platform }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 35 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + - platform: amd64 + runner: blacksmith-4vcpu-ubuntu-2204 + - platform: arm64 + runner: blacksmith-8vcpu-ubuntu-2204-arm + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Setup and Build + uses: ./.github/actions/setup-nodejs + with: + node-version: ${{ env.NODE_VERSION }} + build-command: pnpm build:n8n + enable-docker-cache: 'true' + docker-cache-key: n8n-io/n8n + + - name: Login to GHCR + uses: ./.github/actions/docker-registry-login + with: + login-ghcr: 'true' + + - name: Build and push scratch image + env: + PLATFORMS: linux/${{ matrix.platform }} + N8N_TAGS: ${{ env.SCRATCH_IMAGE }}:${{ env.SCRATCH_TAG }}-${{ matrix.platform }} + SBOM: ${{ inputs.sbom }} + run: | + if [ "$SBOM" = 'true' ]; then + OUT=(--push) + else + OUT=(--set '*.output=type=image,oci-mediatypes=true,push=true') + fi + docker buildx bake -f docker/docker-bake.hcl n8n \ + --provenance=false \ + --sbom="$SBOM" \ + "${OUT[@]}" + + verify: + name: Merge and verify manifest format + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 + + - name: Login to GHCR + uses: ./.github/actions/docker-registry-login + with: + login-ghcr: 'true' + + - name: Merge per-arch manifests + env: + IMAGE: ${{ env.SCRATCH_IMAGE }} + run: | + docker buildx imagetools create \ + --tag "${IMAGE}:${SCRATCH_TAG}" \ + "${IMAGE}:${SCRATCH_TAG}-amd64" \ + "${IMAGE}:${SCRATCH_TAG}-arm64" + + - name: Assert OCI image index + env: + IMAGE: ${{ env.SCRATCH_IMAGE }} + run: | + node .github/scripts/docker/assert-manifest-format.mjs \ + "${IMAGE}:${SCRATCH_TAG}" --expect-platforms 2 + + - name: Report scratch tag + env: + IMAGE: ${{ env.SCRATCH_IMAGE }} + run: | + { + echo "### SBOM probe (sbom=${{ inputs.sbom }})" + echo + echo 'Pull for a preview deploy:' + echo + echo '```' + echo "docker pull ${IMAGE}:${SCRATCH_TAG}" + echo '```' + echo + echo 'Media types match only proves the producer side. The 2.26.0' + echo 'failure was a containerd bug, so verify a real pull too.' + } >> "$GITHUB_STEP_SUMMARY" + + format-matrix: + name: Exporter format matrix (${{ matrix.name }}) + runs-on: blacksmith-4vcpu-ubuntu-2204 + permissions: + contents: read + packages: write + strategy: + fail-fast: false + matrix: + include: + # Negative control: proves the check can still detect the 2.26.0 format. + - name: sbom-off + sbom: 'false' + oci_mediatypes: 'false' + expect_fail: 'true' + - name: sbom-off-oci-mediatypes + sbom: 'false' + oci_mediatypes: 'true' + - name: sbom-on + sbom: 'true' + oci_mediatypes: 'false' + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + # Its own key: this builds a throwaway fixture, and commits to the shared + # disk are last-writer-wins. + - uses: useblacksmith/setup-docker-builder@a5256a73e30f09e37e3eceb8ca36043d17621d24 # v2.1.0 + with: + cache-key: docker-format-probe + + - name: Login to GHCR + uses: ./.github/actions/docker-registry-login + with: + login-ghcr: 'true' + + - name: Build both arches and merge + env: + IMAGE: ${{ env.SCRATCH_IMAGE }} + TAG: fmt-${{ github.run_id }}-${{ matrix.name }} + SBOM: ${{ matrix.sbom }} + OCI_MEDIATYPES: ${{ matrix.oci_mediatypes }} + run: | + mkdir -p /tmp/fmt && printf 'FROM alpine:3.22\nRUN echo hi > /hi\n' > /tmp/fmt/Dockerfile + for arch in amd64 arm64; do + REF="${IMAGE}:${TAG}-${arch}" + # oci-mediatypes is an exporter option, so this variant needs + # --output instead of -t and --push. + if [ "$OCI_MEDIATYPES" = 'true' ]; then + OUT=(--output "type=image,name=${REF},oci-mediatypes=true,push=true") + else + OUT=(-t "${REF}" --push) + fi + docker buildx build /tmp/fmt \ + --platform "linux/${arch}" --provenance=false "--sbom=${SBOM}" \ + "${OUT[@]}" + done + docker buildx imagetools create --tag "${IMAGE}:${TAG}" \ + "${IMAGE}:${TAG}-amd64" "${IMAGE}:${TAG}-arm64" + + - name: Report merged format + env: + IMAGE: ${{ env.SCRATCH_IMAGE }} + TAG: fmt-${{ github.run_id }}-${{ matrix.name }} + EXPECT_FAIL: ${{ matrix.expect_fail }} + run: | + set -o pipefail + rc=0 + node .github/scripts/docker/assert-manifest-format.mjs \ + "${IMAGE}:${TAG}" --expect-platforms 2 | tee /tmp/out.txt || rc=$? + if [ "$EXPECT_FAIL" = 'true' ]; then + if [ "$rc" -eq 0 ]; then + echo "::error::negative control passed - the check no longer detects a Docker manifest list" + exit 1 + fi + # A nonzero exit is not proof. A registry error also gives one. + if ! grep -q 'docker.distribution.manifest.list' /tmp/out.txt; then + echo "::error::negative control failed for the wrong reason - expected a Docker manifest list" + cat /tmp/out.txt + exit 1 + fi + echo "control produced a Docker manifest list, as expected" + elif [ "$rc" -ne 0 ]; then + exit "$rc" + fi + { + echo "### ${{ matrix.name }}" + echo '```' + cat /tmp/out.txt + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/test-e2e-helm.yml b/.github/workflows/test-e2e-helm.yml index 61bf480aae1..b1602070b7b 100644 --- a/.github/workflows/test-e2e-helm.yml +++ b/.github/workflows/test-e2e-helm.yml @@ -56,6 +56,7 @@ jobs: with: build-command: 'pnpm build:docker' enable-docker-cache: true + docker-cache-key: n8n-io/n8n env: INCLUDE_TEST_CONTROLLER: 'true' IMAGE_BASE_NAME: ghcr.io/${{ github.repository }} diff --git a/docker/docker-bake.hcl b/docker/docker-bake.hcl new file mode 100644 index 00000000000..362aefb583e --- /dev/null +++ b/docker/docker-bake.hcl @@ -0,0 +1,99 @@ +# Every n8n image build uses this file. `pnpm build:docker` and CI both drive +# these targets, so a pin changed here changes both. + +variable "NODE_VERSION" { default = "26.7.0" } +variable "N8N_VERSION" { default = "snapshot" } +variable "N8N_RELEASE_TYPE" { default = "dev" } + +variable "IMAGE_BASE_NAME" { default = "n8nio/n8n" } +variable "IMAGE_TAG" { default = "local" } +variable "RUNNERS_IMAGE_BASE_NAME" { default = "n8nio/runners" } + +# An empty value keeps the Dockerfile ARG default, so `docker build` still +# works. The n8n-pc target always sets these. +variable "BUILDER_IMAGE" { default = "" } +variable "RUNTIME_IMAGE" { default = "" } + +variable "PC_BUILDER_IMAGE" { + default = "n8nio/node-pc:26.7.0-dev@sha256:c64642dcb9464e50bd08aef8504affafeb795d01aed2f7ab158976c334a36ef6" +} +variable "PC_RUNTIME_IMAGE" { + default = "n8nio/node-pc:26.7.0@sha256:6577d0742ad043baaa39db7dda2ce8aeb73f32c84eb241c49bf716f170c53297" +} + +variable "DHI_REF" { + default = "dhi.io/node:26.7.0-alpine3.24-dev@sha256:4b494d89fb26c950ce97865acf45b480dc7a6868fdc2b81c2d66599702eeac3f" +} + +variable "N8N_TAGS" { default = "" } +variable "N8N_PC_TAGS" { default = "" } +variable "RUNNERS_TAGS" { default = "" } +variable "RUNNERS_DISTROLESS_TAGS" { default = "" } +variable "BASE_TAGS" { default = "" } + +variable "PLATFORMS" { default = "" } + +function "tags" { + params = [override, fallback] + result = override != "" ? split(",", override) : [fallback] +} + +target "_context" { + context = "." + # BAKE_LOCAL_PLATFORM gives the host platform. On macOS it reads + # `darwin/arm64/v8`. These images are linux, so use the architecture only. + platforms = PLATFORMS != "" ? split(",", PLATFORMS) : ["linux/${split("/", BAKE_LOCAL_PLATFORM)[1]}"] +} + +target "_app" { + inherits = ["_context"] + args = { + NODE_VERSION = NODE_VERSION + N8N_VERSION = N8N_VERSION + N8N_RELEASE_TYPE = N8N_RELEASE_TYPE + } +} + +target "n8n" { + inherits = ["_app"] + dockerfile = "docker/images/n8n/Dockerfile" + tags = tags(N8N_TAGS, "${IMAGE_BASE_NAME}:${IMAGE_TAG}") + args = merge( + BUILDER_IMAGE != "" ? { BUILDER_IMAGE = BUILDER_IMAGE } : {}, + RUNTIME_IMAGE != "" ? { RUNTIME_IMAGE = RUNTIME_IMAGE } : {}, + ) +} + +target "n8n-pc" { + inherits = ["n8n"] + tags = tags(N8N_PC_TAGS, "${IMAGE_BASE_NAME}:${IMAGE_TAG}-pc") + args = { + BUILDER_IMAGE = PC_BUILDER_IMAGE + RUNTIME_IMAGE = PC_RUNTIME_IMAGE + IMAGE_DESCRIPTION = "Workflow Automation Tool (pointer-compressed variant, internal to n8n Cloud, no support or stability guarantees)" + } +} + +target "runners" { + inherits = ["_app"] + dockerfile = "docker/images/runners/Dockerfile" + tags = tags(RUNNERS_TAGS, "${RUNNERS_IMAGE_BASE_NAME}:${IMAGE_TAG}") +} + +target "runners-distroless" { + inherits = ["_app"] + dockerfile = "docker/images/runners/Dockerfile.distroless" + tags = tags(RUNNERS_DISTROLESS_TAGS, "${RUNNERS_IMAGE_BASE_NAME}:${IMAGE_TAG}-distroless") +} + +target "base" { + inherits = ["_context"] + dockerfile = "docker/images/n8n-base/Dockerfile" + args = { DHI_REF = DHI_REF } + tags = tags(BASE_TAGS, "n8nio/base:${NODE_VERSION}") +} + +group "default" { targets = ["n8n", "runners"] } +group "distroless" { targets = ["n8n", "runners", "runners-distroless"] } +group "all" { targets = ["base", "n8n", "runners", "runners-distroless"] } +group "release" { targets = ["n8n", "n8n-pc", "runners", "runners-distroless"] } diff --git a/docker/images/n8n/Dockerfile b/docker/images/n8n/Dockerfile index d1c4326e49c..f72e1a3f3aa 100644 --- a/docker/images/n8n/Dockerfile +++ b/docker/images/n8n/Dockerfile @@ -8,43 +8,44 @@ ARG RUNTIME_IMAGE=n8nio/base:26.7.0@sha256:33687300c4e94dc00f42ec79ae15082ae0733 # The runtime base has no compiler, so this stage supplies one. FROM ${BUILDER_IMAGE} AS toolchain RUN apk add --no-cache python3 make g++ +ENV NODE_GYP=/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js -# Build isolated-vm in a separate stage. The only cache input is the module source. -# Below `COPY ./compiled`, each application change starts a new compile. +# Compile the native modules above `COPY ./compiled`. An application change +# then does not start a new compile. FROM toolchain AS native-builder -# `node_modules/isolated-vm` is a pnpm symlink, so copy the target directory. -# The wildcard tracks the version in the catalog. + +# These paths are pnpm symlinks. Copying through them uses the resolved version. COPY ./compiled/node_modules/.pnpm/isolated-vm@*/node_modules/isolated-vm /build/isolated-vm WORKDIR /build/isolated-vm -# node-gyp-build reads /etc/alpine-release to detect musl. DHI Alpine has no such file, -# so the loader takes the glibc prebuild and crashes. Build from source instead. -# Call node-gyp directly: `npm rebuild` starts two builds that conflict, and -# `npx node-gyp` downloads a different version at build time. -RUN rm -rf prebuilds && \ - node /usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js rebuild --release -j max +# node-gyp-build reads /etc/alpine-release to find musl. DHI Alpine does not +# have that file, so the loader uses the glibc prebuild and crashes. +RUN rm -rf prebuilds && node "$NODE_GYP" rebuild --release -j max -FROM toolchain AS builder -COPY ./compiled /usr/local/lib/node_modules/n8n -# Call node-gyp directly rather than `npm rebuild sqlite3`: sqlite3 is reachable through -# several symlinks in the pnpm tree, and npm runs one install script per link -# concurrently in the same store directory, where they collide on build/node_gyp_bins -# and on each other's make output. npm's bundled node-gyp also fixes the toolchain -# version to the pinned image digest instead of fetching it at build time. -RUN cd /usr/local/lib/node_modules/n8n/node_modules/sqlite3 && \ - node /usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js rebuild --release -# The workspace install cannot build this binding, because CI installs on a Node version -# Confluent ships no prebuild for, so `./compiled` arrives without one. Compile it here -# and link against the runtime base's librdkafka. See: https://github.com/confluentinc/confluent-kafka-javascript/issues/397 -RUN apk add --no-cache librdkafka-dev && \ - cd /usr/local/lib/node_modules/n8n/node_modules/.pnpm/@confluentinc+kafka-javascript@*/node_modules/@confluentinc/kafka-javascript && \ - BUILD_LIBRDKAFKA=0 node /usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js rebuild --release -COPY --from=native-builder /build/isolated-vm/build/Release/isolated_vm.node \ - /usr/local/lib/node_modules/n8n/node_modules/isolated-vm/build/Release/isolated_vm.node -# node-gyp-build finds build/Release first. Delete the prebuilds to remove the glibc fallback. -RUN rm -rf /usr/local/lib/node_modules/n8n/node_modules/isolated-vm/prebuilds +# sqlite3 needs node-addon-api for the headers, and `tar` because a gyp action +# unpacks the sqlite source. pnpm gives each package its own directory, so the +# dependencies of `tar` must come too. +COPY ./compiled/node_modules/.pnpm/sqlite3@*/node_modules/node-addon-api /build/node_modules/node-addon-api +COPY ./compiled/node_modules/.pnpm/sqlite3@*/node_modules/tar /build/node_modules/tar +COPY ./compiled/node_modules/.pnpm/tar@*/node_modules/@isaacs/fs-minipass /build/node_modules/@isaacs/fs-minipass +COPY ./compiled/node_modules/.pnpm/tar@*/node_modules/chownr /build/node_modules/chownr +COPY ./compiled/node_modules/.pnpm/tar@*/node_modules/minipass /build/node_modules/minipass +COPY ./compiled/node_modules/.pnpm/tar@*/node_modules/minizlib /build/node_modules/minizlib +COPY ./compiled/node_modules/.pnpm/tar@*/node_modules/yallist /build/node_modules/yallist +COPY ./compiled/node_modules/.pnpm/sqlite3@*/node_modules/sqlite3 /build/node_modules/sqlite3 +WORKDIR /build/node_modules/sqlite3 +RUN node "$NODE_GYP" rebuild --release -# A separate workflow builds the base image, so a new base reaches this image only -# when you change the reference here (see BUILDER_IMAGE/RUNTIME_IMAGE above). +# Confluent has no prebuild for the Node version that CI installs, so +# `./compiled` has none. Compile it here against the base image librdkafka. +# See https://github.com/confluentinc/confluent-kafka-javascript/issues/397 +RUN apk add --no-cache librdkafka-dev +COPY ./compiled/node_modules/.pnpm/@confluentinc+kafka-javascript@*/node_modules/nan /build/node_modules/nan +COPY ./compiled/node_modules/.pnpm/@confluentinc+kafka-javascript@*/node_modules/@confluentinc/kafka-javascript /build/node_modules/@confluentinc/kafka-javascript +WORKDIR /build/node_modules/@confluentinc/kafka-javascript +RUN BUILD_LIBRDKAFKA=0 node "$NODE_GYP" rebuild --release + +# A separate workflow builds the base image. A new base reaches this image only +# when you change RUNTIME_IMAGE above. FROM ${RUNTIME_IMAGE} ARG N8N_VERSION @@ -56,14 +57,30 @@ ENV SHELL=/bin/sh WORKDIR /home/node -COPY --from=builder /usr/local/lib/node_modules/n8n /usr/local/lib/node_modules/n8n -COPY docker/images/n8n/docker-entrypoint.sh / +COPY --link ./compiled /usr/local/lib/node_modules/n8n +COPY --link docker/images/n8n/docker-entrypoint.sh / -# The base keeps node in /usr/bin and has no /usr/local/bin. Make that directory for the symlink. -RUN mkdir -p /usr/local/bin && \ - ln -s /usr/local/lib/node_modules/n8n/bin/n8n /usr/local/bin/n8n && \ - mkdir -p /home/node/.n8n && \ - chown -R node:node /home/node && \ +# Do not use --link here. These destinations are pnpm symlinks. A --link copy +# resolves the destination against an empty root. It replaces the symlink with +# a directory, and the module does not load. The build still succeeds. +COPY --from=native-builder /build/node_modules/sqlite3/build/Release/node_sqlite3.node \ + /usr/local/lib/node_modules/n8n/node_modules/sqlite3/build/Release/node_sqlite3.node +COPY --from=native-builder /build/isolated-vm/build/Release/isolated_vm.node \ + /usr/local/lib/node_modules/n8n/node_modules/isolated-vm/build/Release/isolated_vm.node +# kafka-javascript has no top-level symlink, and its pnpm directory name holds +# a patch hash. The shell must expand this destination. +COPY --from=native-builder /build/node_modules/@confluentinc/kafka-javascript/build/Release/confluent-kafka-javascript.node /tmp/ + +# The base image keeps node in /usr/bin and has no /usr/local/bin. +RUN set -e; \ + N8N=/usr/local/lib/node_modules/n8n; \ + kafka=$(echo "$N8N"/node_modules/.pnpm/@confluentinc+kafka-javascript@*/node_modules/@confluentinc/kafka-javascript); \ + install -D /tmp/confluent-kafka-javascript.node "$kafka/build/Release/confluent-kafka-javascript.node"; \ + rm -rf "$N8N"/node_modules/.pnpm/isolated-vm@*/node_modules/isolated-vm/prebuilds; \ + mkdir -p /usr/local/bin; \ + ln -s "$N8N/bin/n8n" /usr/local/bin/n8n; \ + mkdir -p /home/node/.n8n; \ + chown -R node:node /home/node; \ rm -rf /root/.npm /tmp/* EXPOSE 5678/tcp diff --git a/scripts/build-n8n.mjs b/scripts/build-n8n.mjs index aff6ddf2970..04c31c883f9 100755 --- a/scripts/build-n8n.mjs +++ b/scripts/build-n8n.mjs @@ -184,6 +184,13 @@ echo(chalk.green('✅ Phantom dirs stripped')); // build-from-source fallback (~11MB), but the prebuilt binary - librdkafka statically // linked in, no .so/.a shipped - is what actually loads at runtime on Alpine. The // source is dead weight in the shipped image. +// isolated-vm ships prebuilds for darwin, win32 and linux. The image compiles +// the binding from source, so these are unused. Removing them also keeps 15MB +// out of the build context. +echo(chalk.yellow('INFO: Stripping isolated-vm prebuilds...')); +await $`find ${config.compiledAppDir}/node_modules/.pnpm -type d -path "*/isolated-vm/prebuilds" -exec rm -rf {} + 2>/dev/null || true`; +echo(chalk.green('✅ isolated-vm prebuilds stripped')); + echo(chalk.yellow('INFO: Stripping unused librdkafka source tree...')); await $`find ${config.compiledAppDir}/node_modules/.pnpm -type d -path "*/@confluentinc/kafka-javascript/deps" -exec rm -rf {} + 2>/dev/null || true`; echo(chalk.green('✅ librdkafka source tree stripped')); @@ -243,7 +250,8 @@ const verifySingleInstance = async (label, dir) => { echo(chalk.yellow(`INFO: Verifying single-instance dependency integrity in ${label}...`)); // `--dir` rather than `--filter`: a filter that matches nothing exits 0, so a renamed or moved // package would report a passing check having run no verifier at all. - const verifyProcess = $`cd ${config.rootDir} && pnpm --dir packages/testing/code-health exec tsx src/cli.ts verify-closure ${dir}`.nothrow(); + const verifyProcess = + $`cd ${config.rootDir} && pnpm --dir packages/testing/code-health exec tsx src/cli.ts verify-closure ${dir}`.nothrow(); verifyProcess.pipe(process.stdout); const { exitCode } = await verifyProcess; // 0 and 3 are the only codes the verifier itself produces; everything else (tsx failing to load, @@ -299,7 +307,9 @@ if (generateLicenses) { echo(chalk.yellow('⚠️ Warning: continuing local build (CI=true would have failed)')); } } else { - echo(chalk.gray('INFO: Skipping SBOM/license generation (set N8N_GENERATE_LICENSES=true to enable)')); + echo( + chalk.gray('INFO: Skipping SBOM/license generation (set N8N_GENERATE_LICENSES=true to enable)'), + ); } // Restore package.json files diff --git a/scripts/dockerize-n8n.mjs b/scripts/dockerize-n8n.mjs index 0b0537ee826..8e2cefbf217 100755 --- a/scripts/dockerize-n8n.mjs +++ b/scripts/dockerize-n8n.mjs @@ -1,73 +1,72 @@ #!/usr/bin/env node /** - * Build n8n and runners Docker images locally + * Build the n8n and runners Docker images. * - * This script simulates the CI build process for local testing. - * Default output: 'n8nio/n8n:local' and 'n8nio/runners:local' - * Override with IMAGE_BASE_NAME and IMAGE_TAG environment variables. + * Targets, tags and build args live in docker/docker-bake.hcl. CI drives the + * same file. This script selects the targets, sets the output, and records the + * image sizes for the metrics pipeline. + * + * Default output: 'n8nio/n8n:local' and 'n8nio/runners:local'. + * + * Environment: + * IMAGE_BASE_NAME, IMAGE_TAG, RUNNERS_IMAGE_BASE_NAME - image naming + * NODE_VERSION, BUILDER_IMAGE, RUNTIME_IMAGE - read by bake directly + * DOCKER_PLATFORM - cross-platform builds + * DOCKER_BUILD_NO_CACHE, DOCKER_BUILD_BASE_IMAGE, DOCKER_BUILD_DISTROLESS + * DOCKER_BUILD_TARBALL_DIR - write per-target docker-archives here instead of + * loading into the daemon (CI image distribution) + * CONTAINER_ENGINE - force 'docker' or 'podman' */ import { $, echo, fs, chalk, os } from 'zx'; import { fileURLToPath } from 'url'; import path from 'path'; -// Disable verbose mode for cleaner output $.verbose = false; process.env.FORCE_COLOR = '1'; -// #region ===== Helper Functions ===== +const __dirname = path.dirname(fileURLToPath(import.meta.url)); +const rootDir = path.basename(__dirname) === 'scripts' ? path.join(__dirname, '..') : __dirname; + +const BAKE_FILE = path.join(rootDir, 'docker/docker-bake.hcl'); + +const noCache = process.env.DOCKER_BUILD_NO_CACHE === 'true'; +const withBaseImage = process.env.DOCKER_BUILD_BASE_IMAGE === 'true'; +// Opt-in: only cloud deploys the distroless runners image, so local builds skip it. +const withDistroless = process.env.DOCKER_BUILD_DISTROLESS === 'true'; +// Build n8n on the pointer-compressed bases. The pins live in the bake file. +const pointerCompressed = process.env.DOCKER_BUILD_PC === 'true'; + +const imageBaseName = process.env.IMAGE_BASE_NAME || 'n8nio/n8n'; +const runnersImageBaseName = process.env.RUNNERS_IMAGE_BASE_NAME || 'n8nio/runners'; +const imageTag = process.env.IMAGE_TAG || 'local'; + +// Push directly when the name has a registry host. This avoids the slow +// --load export and import. +const hasRegistryHost = (name) => name.split('/').length > 2; +const shouldPush = hasRegistryHost(imageBaseName); +// CI needs a tarball, not images in the daemon. BuildKit writes it directly. +// This removes the dockerd import and the `docker save` that follows it. +const tarballDir = process.env.DOCKER_BUILD_TARBALL_DIR; + +const compiledAppDir = path.join(rootDir, 'compiled'); +const compiledTaskRunnerDir = path.join(rootDir, 'dist', 'task-runner-javascript'); /** - * Get Docker platform string based on host architecture or environment override - * @returns {string} Platform string (e.g., 'linux/amd64') + * Which bake targets to build. n8n and runners are always built; the base image + * and the distroless runners are opt-in. + * @returns {string[]} */ -function getDockerPlatform() { - // Allow environment variable override for cross-platform builds - if (process.env.DOCKER_PLATFORM) { - return process.env.DOCKER_PLATFORM; - } - - const arch = os.arch(); - const dockerArch = { - x64: 'amd64', - arm64: 'arm64', - }[arch]; - - if (!dockerArch) { - throw new Error(`Unsupported architecture: ${arch}. Only x64 and arm64 are supported.`); - } - - return `linux/${dockerArch}`; +function selectTargets() { + // The pc target only differs by its base images, so it keeps the plain name - + // downstream jobs load `n8nio/n8n:local` either way. + const targets = [pointerCompressed ? 'n8n-pc' : 'n8n', 'runners']; + if (withDistroless) targets.push('runners-distroless'); + if (withBaseImage) targets.unshift('base'); + return targets; } -/** - * Format duration in seconds - * @param {number} ms - Duration in milliseconds - * @returns {string} Formatted duration - */ -function formatDuration(ms) { - return `${Math.floor(ms / 1000)}s`; -} - -/** - * Get Docker image size - * @param {string} imageName - Full image name with tag - * @returns {Promise} Image size or 'Unknown' - */ -async function getImageSize(imageName) { - try { - const { stdout } = await $`docker images ${imageName} --format "{{.Size}}"`; - return stdout.trim(); - } catch { - return 'Unknown'; - } -} - -/** - * Check if a command exists - * @param {string} command - Command to check - * @returns {Promise} True if command exists - */ +/** @returns {Promise} */ async function commandExists(command) { try { await $`command -v ${command}`; @@ -77,317 +76,224 @@ async function commandExists(command) { } } -const SupportedContainerEngines = /** @type {const} */ (['docker', 'podman']); - -/** - * Detect if the local `docker` CLI is actually Podman via the docker shim. - * @returns {Promise} - */ -async function isDockerPodmanShim() { +/** @returns {Promise} */ +async function hasBake() { try { - const { stdout } = await $`docker version`; - return stdout.toLowerCase().includes('podman'); + await $`docker buildx bake --help`; + return true; } catch { return false; } } /** - * Get the driver of the currently selected buildx builder ('docker', 'docker-container', etc). - * Colima defaults to the 'docker' driver, which doesn't support buildkit-container flags - * like `--load` or `--provenance=false`. + * Buildx driver of the selected builder. Colima defaults to the 'docker' + * driver, which rejects `--load` and `--provenance=false`. * @returns {Promise} */ -async function getBuildxDriver() { +async function buildxDriver() { try { const { stdout } = await $`docker buildx inspect`; - const match = stdout.match(/Driver:\s+(\S+)/); - return match ? match[1] : null; + return stdout.match(/Driver:\s+(\S+)/)?.[1] ?? null; } catch { return null; } } -/** - * @returns {Promise<(typeof SupportedContainerEngines[number])>} - */ -async function getContainerEngine() { - // Allow explicit override via env var - const override = process.env.CONTAINER_ENGINE?.toLowerCase(); - if (override && /** @type {readonly string[]} */ (SupportedContainerEngines).includes(override)) { - return /** @type {typeof SupportedContainerEngines[number]} */ (override); + +/** @returns {string} */ +function hostPlatform() { + if (process.env.DOCKER_PLATFORM) return process.env.DOCKER_PLATFORM; + const dockerArch = { x64: 'amd64', arm64: 'arm64' }[os.arch()]; + if (!dockerArch) { + throw new Error(`Unsupported architecture: ${os.arch()}. Only x64 and arm64 are supported.`); } - - const hasDocker = await commandExists('docker'); - const hasPodman = await commandExists('podman'); - - if (hasDocker) { - // If docker is actually a Podman shim, use podman path to avoid unsupported flags like --load - if (hasPodman && (await isDockerPodmanShim())) { - return 'podman'; - } - return 'docker'; - } - - if (hasPodman) return 'podman'; - - throw new Error('No supported container engine found. Please install Docker or Podman.'); + return `linux/${dockerArch}`; } -// #endregion ===== Helper Functions ===== +/** Resolved bake plan, so tags and platform are read back rather than re-derived. */ +async function bakePlan(targets) { + const { stdout } = await $`docker buildx bake -f ${BAKE_FILE} ${targets} --print`; + return JSON.parse(stdout); +} -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); -const isInScriptsDir = path.basename(__dirname) === 'scripts'; -const rootDir = isInScriptsDir ? path.join(__dirname, '..') : __dirname; - -const noCache = process.env.DOCKER_BUILD_NO_CACHE === 'true'; -const withBaseImage = process.env.DOCKER_BUILD_BASE_IMAGE === 'true'; -// Opt-in: only cloud deploys the distroless runners image, so local builds skip it. -const withDistroless = process.env.DOCKER_BUILD_DISTROLESS === 'true'; -// The pc CI variant pins the n8n base images; runners have no such variant. -const baseImageArgs = [ - process.env.BUILDER_IMAGE && `BUILDER_IMAGE=${process.env.BUILDER_IMAGE}`, - process.env.RUNTIME_IMAGE && `RUNTIME_IMAGE=${process.env.RUNTIME_IMAGE}`, -].filter(Boolean); -// Keep in sync with NODE_VERSION in .github/workflows/docker-build-push.yml, -// which is what the published images are actually built with. -const nodeVersion = process.env.NODE_VERSION || '26.7.0'; - -const config = { - base: { - dockerfilePath: path.join(rootDir, 'docker/images/n8n-base/Dockerfile'), - get fullImageName() { - return `n8nio/base:${nodeVersion}`; - }, - }, - n8n: { - dockerfilePath: path.join(rootDir, 'docker/images/n8n/Dockerfile'), - imageBaseName: process.env.IMAGE_BASE_NAME || 'n8nio/n8n', - imageTag: process.env.IMAGE_TAG || 'local', - get fullImageName() { - return `${this.imageBaseName}:${this.imageTag}`; - }, - }, - runners: { - dockerfilePath: path.join(rootDir, 'docker/images/runners/Dockerfile'), - imageBaseName: process.env.RUNNERS_IMAGE_BASE_NAME || 'n8nio/runners', - get imageTag() { - // Runners use the same tag as n8n for consistency - return config.n8n.imageTag; - }, - get fullImageName() { - return `${this.imageBaseName}:${this.imageTag}`; - }, - }, - runnersDistroless: { - dockerfilePath: path.join(rootDir, 'docker/images/runners/Dockerfile.distroless'), - get fullImageName() { - return `${config.runners.fullImageName}-distroless`; - }, - }, - buildContext: rootDir, - compiledAppDir: path.join(rootDir, 'compiled'), - compiledTaskRunnerDir: path.join(rootDir, 'dist', 'task-runner-javascript'), -}; - -// #region ===== Main Build Process ===== - -const platform = getDockerPlatform(); - -async function main() { - echo(chalk.blue.bold('===== Docker Build for n8n & Runners =====')); - echo(`INFO: n8n Image: ${config.n8n.fullImageName}`); - echo(`INFO: Runners Image: ${config.runners.fullImageName}`); - echo(`INFO: Platform: ${platform}`); - if (noCache) echo(chalk.yellow('INFO: Docker layer cache disabled (DOCKER_BUILD_NO_CACHE=true)')); - if (withBaseImage) echo(chalk.yellow('INFO: Building base image first (DOCKER_BUILD_BASE_IMAGE=true)')); - if (baseImageArgs.length > 0) - echo(chalk.yellow(`INFO: Base image overrides: ${baseImageArgs.join(', ')}`)); - echo(chalk.gray('-'.repeat(47))); - - await checkPrerequisites(); - - if (withBaseImage) { - await buildDockerImage({ - name: 'base', - dockerfilePath: config.base.dockerfilePath, - fullImageName: config.base.fullImageName, - buildArgs: [`NODE_VERSION=${nodeVersion}`], - }); +/** @returns {Promise} */ +async function getImageSize(imageName) { + try { + const { stdout } = await $`docker images ${imageName} --format {{.Size}}`; + return stdout.trim() || 'Unknown'; + } catch { + return 'Unknown'; } - - const nodeVersionArgs = withBaseImage ? [`NODE_VERSION=${nodeVersion}`] : []; - - const n8nBuildTime = await buildDockerImage({ - name: 'n8n', - dockerfilePath: config.n8n.dockerfilePath, - fullImageName: config.n8n.fullImageName, - buildArgs: [...nodeVersionArgs, ...baseImageArgs], - }); - - const runnersBuildTime = await buildDockerImage({ - name: 'runners', - dockerfilePath: config.runners.dockerfilePath, - fullImageName: config.runners.fullImageName, - buildArgs: nodeVersionArgs, - }); - - // Get image details - const n8nImageSize = await getImageSize(config.n8n.fullImageName); - const runnersImageSize = await getImageSize(config.runners.fullImageName); - - const imageStats = [ - { - imageName: config.n8n.fullImageName, - platform, - size: n8nImageSize, - buildTime: n8nBuildTime, - }, - { - imageName: config.runners.fullImageName, - platform, - size: runnersImageSize, - buildTime: runnersBuildTime, - }, - ]; - - if (withDistroless) { - const buildTime = await buildDockerImage({ - name: 'runners-distroless', - dockerfilePath: config.runnersDistroless.dockerfilePath, - fullImageName: config.runnersDistroless.fullImageName, - buildArgs: nodeVersionArgs, - }); - imageStats.push({ - imageName: config.runnersDistroless.fullImageName, - platform, - size: await getImageSize(config.runnersDistroless.fullImageName), - buildTime, - }); - } - - // Write docker build manifest for telemetry collection - const dockerManifest = { - buildTime: new Date().toISOString(), - platform, - images: imageStats.map(({ imageName, size, buildTime }) => ({ - imageName, - size, - buildTime, - })), - }; - await fs.writeJson(path.join(config.buildContext, 'docker-build-manifest.json'), dockerManifest, { - spaces: 2, - }); - - // Display summary - displaySummary(imageStats); } async function checkPrerequisites() { - if (!(await fs.pathExists(config.compiledAppDir))) { - echo(chalk.red(`Error: Compiled app directory not found at ${config.compiledAppDir}`)); + if (!(await fs.pathExists(compiledAppDir))) { + echo(chalk.red(`Error: Compiled app directory not found at ${compiledAppDir}`)); echo(chalk.yellow('Please run build-n8n.mjs first!')); process.exit(1); } - if (!(await fs.pathExists(config.compiledTaskRunnerDir))) { - echo(chalk.red(`Error: Task runner directory not found at ${config.compiledTaskRunnerDir}`)); + if (!(await fs.pathExists(compiledTaskRunnerDir))) { + echo(chalk.red(`Error: Task runner directory not found at ${compiledTaskRunnerDir}`)); echo(chalk.yellow('Please run build-n8n.mjs first!')); process.exit(1); } - - // Ensure at least one supported container engine is available - if (!(await commandExists('docker')) && !(await commandExists('podman'))) { - echo(chalk.red('Error: Neither Docker nor Podman is installed or in PATH')); - process.exit(1); - } } -async function buildDockerImage({ name, dockerfilePath, fullImageName, buildArgs = [] }) { - const startTime = Date.now(); - const containerEngine = await getContainerEngine(); - // Push directly if image name contains a registry (e.g., ghcr.io/...) - // This avoids the slow --load step (export/import tarball) when pushing to a registry - const shouldPush = fullImageName.includes('/') && fullImageName.split('/').length > 2; +async function buildWithBake(targets) { + const driver = await buildxDriver(); + const isContainerDriver = driver !== 'docker'; - const extraFlags = [ - ...buildArgs.flatMap((arg) => ['--build-arg', arg]), + const tarballOutputs = targets.flatMap((t) => [ + '--set', + `${t}.output=type=docker,dest=${path.join(tarballDir ?? '', `${t}.tar`)},compression=zstd,compression-level=3`, + ]); + + if (tarballDir && !isContainerDriver) { + throw new Error( + "DOCKER_BUILD_TARBALL_DIR needs a container-driver builder. The 'docker' driver builds " + + 'into the daemon and cannot write an archive.', + ); + } + + const flags = [ ...(noCache ? ['--no-cache'] : []), + // The 'docker' driver builds into the daemon and rejects both flags. + ...(isContainerDriver + ? [ + '--provenance=false', + ...(tarballDir ? tarballOutputs : [shouldPush ? '--push' : '--load']), + ] + : []), ]; - echo(chalk.yellow(`INFO: Building ${name} Docker image using ${containerEngine}...`)); - if (shouldPush) { - echo(chalk.yellow(`INFO: Registry detected - pushing directly to ${fullImageName}`)); - } + echo(chalk.yellow(`INFO: Building ${targets.join(', ')} with docker buildx bake...`)); + if (tarballDir) echo(chalk.yellow(`INFO: Writing image tarballs to ${tarballDir}`)); + if (shouldPush) echo(chalk.yellow(`INFO: Registry detected - pushing directly`)); - const buildxDriver = containerEngine === 'docker' ? await getBuildxDriver() : null; - const useLegacyDockerBuild = containerEngine === 'docker' && buildxDriver === 'docker'; - - try { - if (containerEngine === 'podman') { - const { stdout } = await $`podman build \ - --platform ${platform} \ - --build-arg TARGETPLATFORM=${platform} \ - ${extraFlags} \ - -t ${fullImageName} \ - -f ${dockerfilePath} \ - ${config.buildContext}`; - echo(stdout); - } else if (useLegacyDockerBuild) { - // Buildx 'docker' driver (colima default) doesn't support `--load` or - // `--provenance=false`. Use plain `docker build` instead. - const { stdout } = await $`docker build \ - --platform ${platform} \ - --build-arg TARGETPLATFORM=${platform} \ - ${extraFlags} \ - -t ${fullImageName} \ - -f ${dockerfilePath} \ - ${config.buildContext}`; - echo(stdout); - } else { - // Use docker buildx build to leverage Blacksmith's layer caching when running in CI. - // The setup-docker-builder action creates a buildx builder with sticky disk cache. - // In CI, push directly to registry to avoid slow --load (export/import tarball). - // Locally, use --load to make image available in local daemon. - const outputFlag = shouldPush ? '--push' : '--load'; - const { stdout } = await $`docker buildx build \ - --platform ${platform} \ - --build-arg TARGETPLATFORM=${platform} \ - ${extraFlags} \ - -t ${fullImageName} \ - -f ${dockerfilePath} \ - --provenance=false \ - ${outputFlag} \ - ${config.buildContext}`; - echo(stdout); - } - - return formatDuration(Date.now() - startTime); - } catch (error) { - echo(chalk.red(`ERROR: ${name} Docker build failed: ${error.stderr || error.message}`)); - process.exit(1); - } + await $({ verbose: true })`docker buildx bake -f ${BAKE_FILE} ${targets} ${flags}`; } -function displaySummary(images) { +/** + * Podman has no bake. A podman-only host also has no buildx to resolve the bake + * plan, so this list repeats the targets. It sends no build args, because this + * path has always used the Dockerfile defaults. + */ +async function buildWithPodman(platform) { + const podmanTargets = [ + { dockerfile: 'docker/images/n8n/Dockerfile', tag: `${imageBaseName}:${imageTag}` }, + { + dockerfile: 'docker/images/runners/Dockerfile', + tag: `${runnersImageBaseName}:${imageTag}`, + }, + ]; + if (withDistroless) { + podmanTargets.push({ + dockerfile: 'docker/images/runners/Dockerfile.distroless', + tag: `${runnersImageBaseName}:${imageTag}-distroless`, + }); + } + + echo(chalk.yellow('INFO: docker buildx bake unavailable - building with podman...')); + + for (const { dockerfile, tag } of podmanTargets) { + await $({ + verbose: true, + })`podman build --platform ${platform} --build-arg TARGETPLATFORM=${platform} ${noCache ? ['--no-cache'] : []} -t ${tag} -f ${path.join(rootDir, dockerfile)} ${rootDir}`; + } + + return podmanTargets.map(({ tag }) => tag); +} + +async function main() { + echo(chalk.blue.bold('===== Docker Build for n8n & Runners =====')); + + await checkPrerequisites(); + + const engineOverride = process.env.CONTAINER_ENGINE?.toLowerCase(); + const usePodman = + engineOverride === 'podman' || (engineOverride !== 'docker' && !(await hasBake())); + + if (usePodman && !(await commandExists('podman'))) { + echo(chalk.red('Error: neither `docker buildx bake` nor `podman` is available')); + process.exit(1); + } + + // The podman list is fixed, so it cannot honour these. Fail instead of + // building something different from what was asked for. + if (usePodman) { + const unsupported = [ + withBaseImage && 'DOCKER_BUILD_BASE_IMAGE', + pointerCompressed && 'DOCKER_BUILD_PC', + tarballDir && 'DOCKER_BUILD_TARBALL_DIR', + ].filter(Boolean); + if (unsupported.length > 0) { + echo(chalk.red(`Error: the podman path does not support ${unsupported.join(', ')}`)); + process.exit(1); + } + } + + // --push applies to every target in the bake call. If only the n8n name + // carries a registry, the runners target would push to its Docker Hub + // default instead - a 401 at best, a tag in the official repo at worst. + if (shouldPush && !hasRegistryHost(runnersImageBaseName)) { + echo( + chalk.red( + `Error: IMAGE_BASE_NAME (${imageBaseName}) has a registry host but ` + + `RUNNERS_IMAGE_BASE_NAME (${runnersImageBaseName}) does not. ` + + 'Set both, or neither.', + ), + ); + process.exit(1); + } + + const targets = selectTargets(); + const startTime = Date.now(); + let platform; + let imageNames; + + if (tarballDir) await fs.ensureDir(tarballDir); + + if (usePodman) { + platform = hostPlatform(); + imageNames = await buildWithPodman(platform); + } else { + if (process.env.DOCKER_PLATFORM) process.env.PLATFORMS = process.env.DOCKER_PLATFORM; + if (pointerCompressed) process.env.N8N_PC_TAGS = `${imageBaseName}:${imageTag}`; + const plan = await bakePlan(targets); + platform = plan.target[targets[0]].platforms.join(','); + imageNames = targets.map((name) => plan.target[name].tags[0]); + await buildWithBake(targets); + } + + const buildDurationMs = Date.now() - startTime; + + const images = []; + for (const imageName of imageNames) { + // Tarball mode loads nothing into the daemon, so there is no size to read. + // The archive size would change the meaning of the docker-image-size metric. + images.push({ imageName, size: tarballDir ? 'Unknown' : await getImageSize(imageName) }); + } + + await fs.writeJson( + path.join(rootDir, 'docker-build-manifest.json'), + { buildTime: new Date().toISOString(), platform, buildDurationMs, images }, + { spaces: 2 }, + ); + echo(''); echo(chalk.green.bold('═'.repeat(54))); echo(chalk.green.bold(' DOCKER BUILD COMPLETE')); echo(chalk.green.bold('═'.repeat(54))); - for (const { imageName, platform, size, buildTime } of images) { - echo(chalk.green(`✅ Image built: ${imageName}`)); - echo(` Platform: ${platform}`); - echo(` Size: ${size}`); - echo(` Build time: ${buildTime}`); - echo(''); + echo(` Platform: ${platform}`); + echo(` Build time: ${Math.floor(buildDurationMs / 1000)}s`); + for (const { imageName, size } of images) { + echo(chalk.green(`✅ ${imageName} (${size})`)); } echo(chalk.green.bold('═'.repeat(54))); } -// #endregion ===== Main Build Process ===== - main().catch((error) => { - echo(chalk.red(`Unexpected error: ${error.message}`)); + echo(chalk.red(`ERROR: Docker build failed: ${error.stderr || error.message}`)); process.exit(1); }); diff --git a/scripts/smoke-n8n-image.mjs b/scripts/smoke-n8n-image.mjs index 436cd2d2a2d..6888c0bc77b 100644 --- a/scripts/smoke-n8n-image.mjs +++ b/scripts/smoke-n8n-image.mjs @@ -5,6 +5,7 @@ import { $, echo, chalk, fs, tmpdir } from 'zx'; import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { parseAllDocuments } from 'yaml'; $.verbose = false; @@ -12,11 +13,18 @@ process.env.FORCE_COLOR = '1'; const IMAGE = process.env.SMOKE_IMAGE || 'n8nio/n8n:local'; // Runners images to exec-check. Tracks DOCKER_BUILD_DISTROLESS so the same -// flag drives both build and check. -const RUNNERS_IMAGES = [ - 'n8nio/runners:local', - ...(process.env.DOCKER_BUILD_DISTROLESS === 'true' ? ['n8nio/runners:local-distroless'] : []), -]; +// flag drives both build and check. SMOKE_RUNNERS_IMAGES (comma-separated) +// overrides both, so the release job can point this at the pushed tags. +const RUNNERS_IMAGES = process.env.SMOKE_RUNNERS_IMAGES + ? process.env.SMOKE_RUNNERS_IMAGES.split(',') + .map((s) => s.trim()) + .filter(Boolean) + : [ + 'n8nio/runners:local', + ...(process.env.DOCKER_BUILD_DISTROLESS === 'true' + ? ['n8nio/runners:local-distroless'] + : []), + ]; const TIMEOUT = '45s'; // Matches an n8n runtime image ref (e.g. `n8nio/n8n:2.4.4`, `ghcr.io/n8n-io/n8n@sha256:…`) // but not sidecars like `n8nio/runners:…` or controller images that happen to contain "n8n". @@ -126,6 +134,26 @@ async function runWorkspaceDedupCheck() { } } +// The image compiles this binding from source, so a bad build shows only at +// require() time. This is the same script the release build uses. +const KAFKA_CHECK = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '../.github/scripts/docker/kafka-native-smoke-check.mjs', +); + +async function runKafkaBindingCheck() { + const name = 'kafka native binding loads in image'; + try { + await $({ + timeout: TIMEOUT, + })`docker run --rm --entrypoint node -v ${`${KAFKA_CHECK}:/tmp/kafka-check.mjs:ro`} ${IMAGE} /tmp/kafka-check.mjs`; + echo(chalk.green(`✓ ${name}`)); + return true; + } catch (err) { + return reportFailure(name, err); + } +} + // Interpreter paths as launched by docker/images/runners/n8n-task-runners.json. // The runners images assemble node/python by copying binaries across images, so a // missing shared library only surfaces at exec time. @@ -183,6 +211,7 @@ const ok = ( await Promise.all([ ...invocations.map(run), runWorkspaceDedupCheck(), + runKafkaBindingCheck(), ...RUNNERS_IMAGES.map(runRunnersInterpreterCheck), ]) ).every(Boolean);