chore(ci): add CI_PROVIDER toggle between Blacksmith and GitHub-hosted runners (#5808)

* chore(ci): add CI_PROVIDER toggle between Blacksmith and GitHub-hosted runners

* chore(ci): fail unrecognized CI_PROVIDER values over to GitHub-hosted runners

.github/actions/cache-mount/action.yml

@@ -0,0 +1,40 @@
+name: Cache Mount
+description: Mount a build cache directory using Blacksmith sticky disks, or the GitHub Actions cache when running on GitHub-hosted runners.
+
+inputs:
+  provider:
+    description: Empty or 'blacksmith' selects sticky disks. Any other value selects actions/cache. Must stay byte-identical to the runs-on predicate in the calling workflow.
+    required: false
+    default: ''
+  key:
+    description: Cache key. Shared by both backends, so callers keep one key regardless of provider.
+    required: true
+  path:
+    description: Filesystem path the cache is mounted at.
+    required: true
+
+# The two conditions are exact complements of the runs-on expression the caller
+# uses. They must be inverted together: a job placed on ubuntu-latest that then
+# took the sticky-disk branch would hard-fail, since stickydisk hot-mounts an
+# ext4 volume from Blacksmith's storage agents and cannot run anywhere else.
+runs:
+  using: composite
+  steps:
+    - name: Mount sticky disk
+      if: inputs.provider == '' || inputs.provider == 'blacksmith'
+      uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      with:
+        key: ${{ inputs.key }}
+        path: ${{ inputs.path }}
+
+    # Sticky disks always re-commit the mounted path, so the cache rolls forward
+    # every run. actions/cache skips its save step on an exact key hit, which
+    # would freeze the cache at whatever the first run wrote — the run-scoped
+    # suffix plus a prefix restore-key reproduces the rolling behaviour.
+    - name: Restore and save cache
+      if: inputs.provider != '' && inputs.provider != 'blacksmith'
+      uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5
+      with:
+        key: ${{ inputs.key }}-${{ github.run_id }}
+        restore-keys: ${{ inputs.key }}-
+        path: ${{ inputs.path }}

.github/actions/docker-build/action.yml

@@ -0,0 +1,64 @@
+name: Docker Build and Push
+description: Set up a buildx builder and build/push an image, using Blacksmith's builder or the upstream Docker actions on GitHub-hosted runners.
+
+inputs:
+  provider:
+    description: Empty or 'blacksmith' selects Blacksmith's builder. Any other value selects the upstream docker/* actions. Must stay byte-identical to the runs-on predicate in the calling workflow.
+    required: false
+    default: ''
+  context:
+    description: Build context.
+    required: false
+    default: .
+  file:
+    description: Path to the Dockerfile.
+    required: true
+  platforms:
+    description: Target platforms, e.g. linux/amd64.
+    required: true
+  tags:
+    description: Comma-separated list of tags to push.
+    required: true
+
+# Registry logins must already have run in the calling job — both backends read
+# the same ~/.docker/config.json. provenance/sbom stay off everywhere: the
+# attestation manifests they add break `imagetools create` retagging in the
+# promote-images and create-ghcr-manifests jobs.
+runs:
+  using: composite
+  steps:
+    - name: Set up Blacksmith builder
+      if: inputs.provider == '' || inputs.provider == 'blacksmith'
+      uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
+
+    - name: Build and push (Blacksmith)
+      if: inputs.provider == '' || inputs.provider == 'blacksmith'
+      uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        push: true
+        tags: ${{ inputs.tags }}
+        provenance: false
+        sbom: false
+
+    - name: Set up Docker Buildx
+      if: inputs.provider != '' && inputs.provider != 'blacksmith'
+      uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4
+
+    # No layer cache here. Blacksmith caches layers on the builder itself;
+    # the GitHub equivalent (cache-to: type=gha) shares the same 10 GB repo
+    # cache quota as the bun/node_modules/turbo mounts, and four images of
+    # layers would evict them. Fallback builds are cold by design.
+    - name: Build and push (GitHub)
+      if: inputs.provider != '' && inputs.provider != 'blacksmith'
+      uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7
+      with:
+        context: ${{ inputs.context }}
+        file: ${{ inputs.file }}
+        platforms: ${{ inputs.platforms }}
+        push: true
+        tags: ${{ inputs.tags }}
+        provenance: false
+        sbom: false

.github/workflows/ci.yml

@@ -1,5 +1,33 @@
 name: CI
 
+# Runner provider toggle. Every job here and in the reusable workflows reads the
+# CI_PROVIDER repository variable:
+#
+#   gh variable set CI_PROVIDER --body github   # fall back to GitHub-hosted
+#   gh variable delete CI_PROVIDER              # back to Blacksmith (default)
+#
+# It is a repo variable rather than a committed value on purpose: during a
+# Blacksmith outage there is no working CI to merge a switchover commit through,
+# and the variable takes effect on the next run with nothing to land.
+#
+# Only unset or the exact value 'blacksmith' selects Blacksmith; everything else
+# selects GitHub-hosted. The unrecognized case resolves toward GitHub on purpose,
+# because that is the provider that is always available: a typo'd or whitespace-
+# padded break-glass value then yields slow-but-working CI instead of jobs queued
+# forever against the provider you were trying to escape. `==` is case-insensitive
+# in Actions expressions, so 'GitHub' and 'Blacksmith' both work. Every runs-on
+# expression and both composite actions use this same predicate — they must be
+# changed together, since a job on ubuntu-latest that took a Blacksmith-only
+# branch would hard-fail.
+#
+# GitHub mode is a break-glass fallback, not a peer. Public-repo ubuntu-latest is
+# 4 vCPU with no free 8-core tier, the 10 GB repo cache quota will thrash across
+# the bun/node_modules/turbo mounts, and image builds start from cold layers — so
+# expect slower runs, not different results. Blacksmith-only actions are reached
+# exclusively through .github/actions/cache-mount and .github/actions/docker-build,
+# which branch internally; a bare useblacksmith/* step added anywhere else will
+# hard-fail in GitHub mode rather than silently degrade.
+
 on:
   push:
     branches: [main, staging, dev]
@@ -28,7 +56,7 @@ jobs:
   # Detect if this is a version release commit (e.g., "v0.5.24: ...")
   detect-version:
     name: Detect Version
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     if: github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging' || github.ref == 'refs/heads/dev')
     outputs:
@@ -81,7 +109,7 @@ jobs:
     name: Build Dev ECR
     needs: [detect-version, migrate-dev]
     if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 30
     permissions:
       contents: read
@@ -118,9 +146,6 @@ jobs:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
-
       - name: Resolve ECR repo name
         id: ecr-repo
         run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
@@ -128,15 +153,12 @@ jobs:
           ECR_REPO: ${{ matrix.ecr_repo_secret == 'ECR_APP' && secrets.ECR_APP || matrix.ecr_repo_secret == 'ECR_MIGRATIONS' && secrets.ECR_MIGRATIONS || matrix.ecr_repo_secret == 'ECR_REALTIME' && secrets.ECR_REALTIME || matrix.ecr_repo_secret == 'ECR_PII' && secrets.ECR_PII || '' }}
 
       - name: Build and push
-        uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
+        uses: ./.github/actions/docker-build
         with:
-          context: .
+          provider: ${{ vars.CI_PROVIDER }}
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
-          push: true
           tags: ${{ steps.login-ecr.outputs.registry }}/${{ steps.ecr-repo.outputs.name }}:dev
-          provenance: false
-          sbom: false
 
   # Dev: deploy Trigger.dev background tasks to the preview "dev-sim" branch.
   # Gated after migrate-dev for the same reason as build-dev — the new task
@@ -145,7 +167,7 @@ jobs:
     name: Deploy Trigger.dev (Dev)
     needs: [migrate-dev]
     if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
     steps:
       - name: Checkout code
@@ -192,7 +214,7 @@ jobs:
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 30
     permissions:
       contents: read
@@ -242,9 +264,6 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
-
       - name: Resolve ECR repo name
         id: ecr-repo
         run: echo "name=$ECR_REPO" >> $GITHUB_OUTPUT
@@ -270,15 +289,12 @@ jobs:
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
 
       - name: Build and push images
-        uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
+        uses: ./.github/actions/docker-build
         with:
-          context: .
+          provider: ${{ vars.CI_PROVIDER }}
           file: ${{ matrix.dockerfile }}
           platforms: linux/amd64
-          push: true
           tags: ${{ steps.meta.outputs.tags }}
-          provenance: false
-          sbom: false
 
   # Promote the sha-tagged ECR images to the deploy tags once tests and
   # migrations pass. Pushing the ECR latest/staging tag is what triggers
@@ -292,7 +308,7 @@ jobs:
     if: >-
       github.event_name == 'push' &&
       (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/staging')
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 10
     permissions:
       contents: read
@@ -362,7 +378,7 @@ jobs:
   # never moves a documented tag.
   build-ghcr-arm64:
     name: Build ARM64 (GHCR Only)
-    runs-on: blacksmith-8vcpu-ubuntu-2404-arm
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404-arm' || 'ubuntu-24.04-arm' }}
     timeout-minutes: 30
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     permissions:
@@ -392,26 +408,20 @@ jobs:
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
-      - name: Set up Docker Buildx
-        uses: useblacksmith/setup-docker-builder@ab5c1da94f53f5cd75c1038092aa276dddfccbba # v1
-
       - name: Build and push ARM64 to GHCR
-        uses: useblacksmith/build-push-action@fb9e3e6a9299c78462bfadd0d93352c316adc9b8 # v2
+        uses: ./.github/actions/docker-build
         with:
-          context: .
+          provider: ${{ vars.CI_PROVIDER }}
           file: ${{ matrix.dockerfile }}
           platforms: linux/arm64
-          push: true
           tags: ${{ matrix.image }}:${{ github.sha }}-arm64
-          provenance: false
-          sbom: false
 
   # Publish all mutable GHCR tags (latest, latest-amd64/arm64, version tags)
   # and the multi-arch manifests from the immutable sha tags — only on main,
   # after the deploy gate (promote-images) and the ARM64 build both pass.
   create-ghcr-manifests:
     name: Create GHCR Manifests
-    runs-on: blacksmith-2vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-2vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 10
     needs: [promote-images, build-ghcr-arm64, detect-version]
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
@@ -477,7 +487,7 @@ jobs:
   # Check if docs changed
   check-docs-changes:
     name: Check Docs Changes
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 5
     if: github.event_name == 'push' && github.ref == 'refs/heads/main'
     outputs:
@@ -506,7 +516,7 @@ jobs:
   # Create GitHub Release (only for version commits on main, after all builds complete)
   create-release:
     name: Create GitHub Release
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 10
     needs: [create-ghcr-manifests, detect-version]
     if: needs.detect-version.outputs.is_release == 'true'

.github/workflows/docs-embeddings.yml

@@ -10,7 +10,7 @@ permissions:
 jobs:
   process-docs-embeddings:
     name: Process Documentation Embeddings
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 30
     if: github.ref == 'refs/heads/main'
 

.github/workflows/migrations.yml

@@ -24,7 +24,7 @@ permissions:
 jobs:
   migrate:
     name: Apply Database Migrations
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 45
 
     steps:

.github/workflows/publish-cli.yml

@@ -11,7 +11,7 @@ permissions:
 
 jobs:
   publish-npm:
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
     steps:
       - name: Checkout repository

.github/workflows/publish-python-sdk.yml

@@ -11,7 +11,7 @@ permissions:
 
 jobs:
   publish-pypi:
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
     steps:
       - name: Checkout repository

.github/workflows/publish-ts-sdk.yml

@@ -11,7 +11,7 @@ permissions:
 
 jobs:
   publish-npm:
-    runs-on: blacksmith-4vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
     steps:
       - name: Checkout repository

.github/workflows/test-build.yml

@@ -10,7 +10,7 @@ permissions:
 jobs:
   test-build:
     name: Lint and Test
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
 
     steps:
@@ -27,25 +27,28 @@ jobs:
         with:
           node-version: 22
 
-      # Sticky-disk keys are scoped by event name, and fork PRs get their own
-      # namespace on top: untrusted fork runs must never share a disk with
-      # push runs (whose disks feed production image builds) or with trusted
-      # internal-PR runs.
-      - name: Mount Bun cache (Sticky Disk)
-        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      # Cache keys are scoped by event name, and fork PRs get their own
+      # namespace on top: untrusted fork runs must never share a cache with
+      # push runs (whose caches feed production image builds) or with trusted
+      # internal-PR runs. The scoping holds on both backends.
+      - name: Mount Bun cache
+        uses: ./.github/actions/cache-mount
         with:
+          provider: ${{ vars.CI_PROVIDER }}
           key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
           path: ~/.bun/install/cache
 
-      - name: Mount node_modules (Sticky Disk)
-        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      - name: Mount node_modules
+        uses: ./.github/actions/cache-mount
         with:
+          provider: ${{ vars.CI_PROVIDER }}
           key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
           path: ./node_modules
 
-      - name: Mount Turbo cache (Sticky Disk)
-        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      - name: Mount Turbo cache
+        uses: ./.github/actions/cache-mount
         with:
+          provider: ${{ vars.CI_PROVIDER }}
           key: ${{ github.repository }}-turbo-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
           path: ./.turbo
 
@@ -185,10 +188,11 @@ jobs:
   # with test-build (identical content from the same lockfile — LWW loss is
   # harmless), but the Turbo cache gets its own key: with a shared key, only
   # the last committer's new entries survive each run, so the test and build
-  # Turbo entries would evict each other nondeterministically.
+  # Turbo entries would evict each other nondeterministically. The same split
+  # is correct on actions/cache, which is likewise last-writer-wins per key.
   build:
     name: Build App
-    runs-on: blacksmith-8vcpu-ubuntu-2404
+    runs-on: ${{ (vars.CI_PROVIDER == '' || vars.CI_PROVIDER == 'blacksmith') && 'blacksmith-8vcpu-ubuntu-2404' || 'ubuntu-latest' }}
     timeout-minutes: 15
 
     steps:
@@ -205,21 +209,24 @@ jobs:
         with:
           node-version: 22
 
-      - name: Mount Bun cache (Sticky Disk)
-        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      - name: Mount Bun cache
+        uses: ./.github/actions/cache-mount
         with:
+          provider: ${{ vars.CI_PROVIDER }}
           key: ${{ github.repository }}-bun-cache-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
           path: ~/.bun/install/cache
 
-      - name: Mount node_modules (Sticky Disk)
-        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      - name: Mount node_modules
+        uses: ./.github/actions/cache-mount
         with:
+          provider: ${{ vars.CI_PROVIDER }}
           key: ${{ github.repository }}-node-modules-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
           path: ./node_modules
 
-      - name: Mount Turbo cache (Sticky Disk)
-        uses: useblacksmith/stickydisk@4c034ba57b706cf0e3b4b0ce098c2a3b1071580c # v1
+      - name: Mount Turbo cache
+        uses: ./.github/actions/cache-mount
         with:
+          provider: ${{ vars.CI_PROVIDER }}
           key: ${{ github.repository }}-turbo-cache-build-${{ github.event_name }}${{ github.event.pull_request.head.repo.fork && '-fork' || '' }}
           path: ./.turbo