From 5089df372cdbd3c847480016e5e4a9b87f431fe8 Mon Sep 17 00:00:00 2001 From: Jax Liu Date: Fri, 4 Sep 2026 21:15:25 +0800 Subject: [PATCH] feat(genbi): distribute verified context loader --- .github/workflows/genbi-ci.yml | 31 +++++- .github/workflows/publish-context-loader.yml | 98 +++++++++++++++++++ apps/context-loader/artifacts.json | 6 ++ .../bin/wren-context-loader.mjs | 13 +++ apps/context-loader/lib/verified.mjs | 70 +++++++++++++ apps/context-loader/package.json | 24 +++++ .../scripts/generate-release-manifest.mjs | 38 +++++++ apps/context-loader/scripts/installer.mjs | 92 +++++++++++++++++ apps/context-loader/scripts/postinstall.mjs | 11 +++ apps/context-loader/test/installer.test.mjs | 78 +++++++++++++++ apps/genbi/INSTALL.md | 45 ++++----- apps/genbi/MAINTAINING.md | 28 +++--- apps/genbi/RUNNING.md | 19 ++-- apps/genbi/harness/compile/context-loader.ts | 84 ++++++++++++++-- apps/genbi/harness/compile/errors.ts | 2 +- apps/genbi/harness/compile/pipeline.ts | 14 +-- apps/genbi/harness/compile/types.ts | 3 +- apps/genbi/harness/npm-package-resolve.ts | 19 +++- apps/genbi/package.json | 1 + .../scripts/installed-package-acceptance.mjs | 69 ++++++++++++- .../genbi/test/compile-context-loader.test.ts | 58 ++++++++++- apps/genbi/test/packaging-contract.test.ts | 9 ++ pnpm-lock.yaml | 8 ++ pnpm-workspace.yaml | 3 + 24 files changed, 755 insertions(+), 68 deletions(-) create mode 100644 .github/workflows/publish-context-loader.yml create mode 100644 apps/context-loader/artifacts.json create mode 100755 apps/context-loader/bin/wren-context-loader.mjs create mode 100644 apps/context-loader/lib/verified.mjs create mode 100644 apps/context-loader/package.json create mode 100644 apps/context-loader/scripts/generate-release-manifest.mjs create mode 100644 apps/context-loader/scripts/installer.mjs create mode 100644 apps/context-loader/scripts/postinstall.mjs create mode 100644 apps/context-loader/test/installer.test.mjs diff --git a/.github/workflows/genbi-ci.yml b/.github/workflows/genbi-ci.yml index e24c8527e..91cbea73a 100644 --- a/.github/workflows/genbi-ci.yml +++ b/.github/workflows/genbi-ci.yml @@ -12,6 +12,7 @@ on: pull_request: paths: - 'apps/genbi/**' + - 'apps/context-loader/**' - 'package.json' - 'pnpm-workspace.yaml' - 'pnpm-lock.yaml' @@ -22,6 +23,7 @@ on: - feat/genbi-app paths: - 'apps/genbi/**' + - 'apps/context-loader/**' - 'package.json' - 'pnpm-workspace.yaml' - 'pnpm-lock.yaml' @@ -87,10 +89,33 @@ jobs: - name: Test run: pnpm run test + installed-package-macos: + name: macOS packed-install context-loader acceptance + runs-on: macos-14 + defaults: + run: + working-directory: apps/genbi + steps: + - uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 24 + + - name: Enable corepack + run: corepack enable + + - name: Install workspace dependencies without source-package lifecycle scripts + working-directory: . + run: pnpm install --frozen-lockfile + + - name: Build packaged application + run: pnpm run build + - name: Packed-install acceptance - # This packs the actual artifact, installs it below a fresh temporary - # project, and invokes `npx --no-install genbi` with no checkout PATH. - # It is deliberately separate from source-tree Vitest coverage. + # This packs both real npm artifacts, installs them below a fresh temporary + # project, and invokes npx with no checkout PATH or Rust toolchain. run: pnpm run check:installed-package vendor-contracts-macos: diff --git a/.github/workflows/publish-context-loader.yml b/.github/workflows/publish-context-loader.yml new file mode 100644 index 000000000..11effeea9 --- /dev/null +++ b/.github/workflows/publish-context-loader.yml @@ -0,0 +1,98 @@ +name: Publish context loader + +on: + push: + tags: + - 'context-loader-v*' + +permissions: + contents: write + id-token: write + +jobs: + publish-darwin-arm64: + name: Build and publish darwin-arm64 context loader + runs-on: macos-14 + environment: npm + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ github.ref_name }} + fetch-depth: 0 + + - name: Verify explicit tag version + id: version + shell: bash + run: | + version="${GITHUB_REF_NAME#context-loader-v}" + test -n "$version" + test "$(node -p "require('./apps/context-loader/package.json').version")" = "$version" + printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT" + + - name: Set up locked Rust toolchain + uses: dtolnay/rust-toolchain@1.89.0 + with: + targets: aarch64-apple-darwin + + - name: Set up Node + uses: actions/setup-node@v4 + with: + node-version: 24 + registry-url: https://registry.npmjs.org + + - name: Ensure npm supports trusted publishing + run: npm install -g npm@latest + + - name: Test generator source + working-directory: core/wren-context-loader + run: | + cargo fmt --all -- --check + cargo clippy --locked --all-targets -- -D warnings + cargo test --locked + + - name: Build target archive + id: archive + shell: bash + run: | + cargo build --locked --release --target aarch64-apple-darwin --manifest-path core/wren-context-loader/Cargo.toml + name="wren-context-loader-${{ steps.version.outputs.version }}-darwin-arm64.tar.gz" + mkdir -p release + tar -C core/wren-context-loader/target/aarch64-apple-darwin/release -czf "release/$name" wren-context-loader + printf 'name=%s\n' "$name" >> "$GITHUB_OUTPUT" + + - name: Create immutable release asset + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + generate_release_notes: true + files: release/${{ steps.archive.outputs.name }} + + - name: Generate embedded manifest + shell: bash + run: | + url="https://github.com/${GITHUB_REPOSITORY}/releases/download/${GITHUB_REF_NAME}/${{ steps.archive.outputs.name }}" + node apps/context-loader/scripts/generate-release-manifest.mjs apps/context-loader "${{ steps.version.outputs.version }}" "$GITHUB_SHA" "release/${{ steps.archive.outputs.name }}" "$url" > release/context-loader-manifest.json + cp release/context-loader-manifest.json "release/${{ steps.archive.outputs.name }}.manifest.json" + + - name: Validate installer contract + working-directory: apps/context-loader + run: node --test test/*.test.mjs + + - name: Run fresh GenBI package-binding contract + run: | + corepack enable + pnpm install --frozen-lockfile + pnpm --filter @wrenai/genbi run build + pnpm --filter @wrenai/genbi run check:installed-package + + - name: Pack and publish npm package with provenance + working-directory: apps/context-loader + run: | + npm pack + npm publish --provenance --access public + + - name: Upload machine-readable manifest + uses: softprops/action-gh-release@v2 + with: + tag_name: ${{ github.ref_name }} + files: release/${{ steps.archive.outputs.name }}.manifest.json diff --git a/apps/context-loader/artifacts.json b/apps/context-loader/artifacts.json new file mode 100644 index 000000000..de8465687 --- /dev/null +++ b/apps/context-loader/artifacts.json @@ -0,0 +1,6 @@ +{ + "schema": 1, + "package": "@wrenai/context-loader", + "version": "0.1.0", + "artifacts": {} +} diff --git a/apps/context-loader/bin/wren-context-loader.mjs b/apps/context-loader/bin/wren-context-loader.mjs new file mode 100755 index 000000000..3db9cea62 --- /dev/null +++ b/apps/context-loader/bin/wren-context-loader.mjs @@ -0,0 +1,13 @@ +#!/usr/bin/env node +import { spawnSync } from "node:child_process"; +import { resolveVerifiedBinary } from "../lib/verified.mjs"; + +try { + const binary = resolveVerifiedBinary(); + const child = spawnSync(binary, process.argv.slice(2), { stdio: "inherit" }); + if (child.error) throw child.error; + process.exit(child.status ?? 1); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/apps/context-loader/lib/verified.mjs b/apps/context-loader/lib/verified.mjs new file mode 100644 index 000000000..845e14105 --- /dev/null +++ b/apps/context-loader/lib/verified.mjs @@ -0,0 +1,70 @@ +import { createHash } from "node:crypto"; +import { existsSync, readFileSync } from "node:fs"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +export const PACKAGE_NAME = "@wrenai/context-loader"; +export const STATE_FILE = "install-state.json"; +export const CANONICAL_BINARY = path.join("bin", "wren-context-loader"); + +export class ContextLoaderPackageError extends Error { + constructor(code, detail) { + super(`@wrenai/context-loader ${code}: ${detail}`); + this.name = "ContextLoaderPackageError"; + this.code = code; + } +} + +export function sha256File(file) { + return createHash("sha256").update(readFileSync(file)).digest("hex"); +} + +export function targetFor(platform = process.platform, arch = process.arch) { + return `${platform}-${arch}`; +} + +export function packageRootFrom(moduleUrl = import.meta.url) { + return path.resolve(path.dirname(fileURLToPath(moduleUrl)), ".."); +} + +export function readVerifiedState(root = packageRootFrom()) { + const packageJson = readJson(path.join(root, "package.json"), "package.json"); + if (packageJson.name !== PACKAGE_NAME || typeof packageJson.version !== "string") { + throw new ContextLoaderPackageError("invalid-package", "package.json does not identify this package"); + } + const state = readJson(path.join(root, STATE_FILE), STATE_FILE); + const target = targetFor(); + if (state.package !== PACKAGE_NAME || state.version !== packageJson.version || state.target !== target) { + throw new ContextLoaderPackageError("stale-state", "verification record does not match package version or platform"); + } + if (!isSha256(state.binarySha256)) { + throw new ContextLoaderPackageError("invalid-state", "verification record has no binary SHA-256"); + } + if (state.binaryPath !== CANONICAL_BINARY) { + throw new ContextLoaderPackageError("invalid-state", "verification record does not name the canonical package-local binary"); + } + const binary = path.resolve(root, state.binaryPath); + if (!binary.startsWith(`${root}${path.sep}`) || !existsSync(binary)) { + throw new ContextLoaderPackageError("missing-binary", "verified package-local binary is absent"); + } + if (sha256File(binary) !== state.binarySha256) { + throw new ContextLoaderPackageError("binary-digest-mismatch", "verified package-local binary was modified"); + } + return { binary, identity: `package:${PACKAGE_NAME}@${packageJson.version}:${target}:${state.binarySha256}`, state }; +} + +export function resolveVerifiedBinary(root = packageRootFrom()) { + return readVerifiedState(root).binary; +} + +export function readJson(file, label) { + try { + return JSON.parse(readFileSync(file, "utf8")); + } catch { + throw new ContextLoaderPackageError("invalid-state", `${label} is missing or invalid JSON`); + } +} + +export function isSha256(value) { + return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value); +} diff --git a/apps/context-loader/package.json b/apps/context-loader/package.json new file mode 100644 index 000000000..67e2008c7 --- /dev/null +++ b/apps/context-loader/package.json @@ -0,0 +1,24 @@ +{ + "name": "@wrenai/context-loader", + "version": "0.1.0", + "description": "Verified native context-loader binary for @wrenai/genbi", + "license": "Apache-2.0", + "type": "module", + "files": [ + "artifacts.json", + "bin", + "install-state.json", + "lib", + "scripts" + ], + "scripts": { + "postinstall": "node scripts/postinstall.mjs", + "test": "node --test test/*.test.mjs" + }, + "bin": { + "wren-context-loader": "bin/wren-context-loader.mjs" + }, + "publishConfig": { + "access": "public" + } +} diff --git a/apps/context-loader/scripts/generate-release-manifest.mjs b/apps/context-loader/scripts/generate-release-manifest.mjs new file mode 100644 index 000000000..6d96eb191 --- /dev/null +++ b/apps/context-loader/scripts/generate-release-manifest.mjs @@ -0,0 +1,38 @@ +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const [packageDir, version, sourceCommit, archivePath, assetUrl] = process.argv.slice(2); +if (![packageDir, version, sourceCommit, archivePath, assetUrl].every(Boolean)) { + throw new Error("usage: generate-release-manifest.mjs "); +} +const archive = await readFile(archivePath); +const binaryPath = "wren-context-loader"; +const binary = await readBinaryFromTarGz(archive, binaryPath); +const sha256 = (data) => createHash("sha256").update(data).digest("hex"); +const manifest = { + schema: 1, + package: "@wrenai/context-loader", + version, + sourceCommit, + artifacts: { + "darwin-arm64": { url: assetUrl, archiveSha256: sha256(archive), binarySha256: sha256(binary), binaryPath }, + }, +}; +await writeFile(path.join(packageDir, "artifacts.json"), `${JSON.stringify(manifest, null, 2)}\n`); +process.stdout.write(`${JSON.stringify(manifest)}\n`); + +async function readBinaryFromTarGz(archive, expectedPath) { + const { gunzipSync } = await import("node:zlib"); + const tar = gunzipSync(archive); + for (let offset = 0; offset + 512 <= tar.length;) { + const header = tar.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) break; + const name = header.subarray(0, 100).toString("utf8").replace(/\0.*$/u, ""); + const size = Number.parseInt(header.subarray(124, 136).toString("utf8").replace(/\0.*$/u, "").trim() || "0", 8); + if (!Number.isSafeInteger(size) || size < 0 || offset + 512 + size > tar.length) throw new Error("invalid release archive layout"); + if (name === expectedPath) return tar.subarray(offset + 512, offset + 512 + size); + offset += 512 + Math.ceil(size / 512) * 512; + } + throw new Error(`release archive lacks ${expectedPath}`); +} diff --git a/apps/context-loader/scripts/installer.mjs b/apps/context-loader/scripts/installer.mjs new file mode 100644 index 000000000..329479b3c --- /dev/null +++ b/apps/context-loader/scripts/installer.mjs @@ -0,0 +1,92 @@ +import { createHash } from "node:crypto"; +import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { gunzipSync } from "node:zlib"; +import { CANONICAL_BINARY, ContextLoaderPackageError, isSha256, readJson, sha256File, targetFor } from "../lib/verified.mjs"; + +/** Downloads, verifies, and atomically installs the exact manifest row for this package. */ +export async function installContextLoader({ packageRoot, fetchImpl = fetch, platform = process.platform, arch = process.arch } = {}) { + if (typeof packageRoot !== "string") throw new TypeError("packageRoot is required"); + const packageJson = readJson(path.join(packageRoot, "package.json"), "package.json"); + const manifest = readJson(path.join(packageRoot, "artifacts.json"), "artifacts.json"); + const target = targetFor(platform, arch); + const artifact = manifest.artifacts?.[target]; + validateManifest(packageJson, manifest, artifact, target); + + const binary = path.join(packageRoot, CANONICAL_BINARY); + const statePath = path.join(packageRoot, "install-state.json"); + const reusable = await existingVerified(binary, statePath, packageJson.version, target, artifact.binarySha256); + if (reusable) return { binary, reused: true }; + + const staging = `${binary}.tmp-${process.pid}-${Date.now()}`; + try { + const response = await fetchImpl(artifact.url); + if (!response?.ok) throw new ContextLoaderPackageError("download-failed", `could not download the ${target} artifact`); + const archive = Buffer.from(await response.arrayBuffer()); + if (sha256(archive) !== artifact.archiveSha256) { + throw new ContextLoaderPackageError("archive-digest-mismatch", `downloaded ${target} artifact did not match its SHA-256`); + } + const content = extractSingleTarGz(archive, artifact.binaryPath); + if (sha256(content) !== artifact.binarySha256) { + throw new ContextLoaderPackageError("binary-digest-mismatch", `extracted ${target} binary did not match its SHA-256`); + } + await mkdir(path.dirname(binary), { recursive: true }); + await writeFile(staging, content, { mode: 0o755 }); + await chmod(staging, 0o755); + await rename(staging, binary); + const state = { package: packageJson.name, version: packageJson.version, target, archiveSha256: artifact.archiveSha256, binarySha256: artifact.binarySha256, binaryPath: CANONICAL_BINARY }; + const stateTmp = `${statePath}.tmp-${process.pid}-${Date.now()}`; + await writeFile(stateTmp, `${JSON.stringify(state)}\n`, { mode: 0o600 }); + await rename(stateTmp, statePath); + return { binary, reused: false }; + } catch (error) { + await rm(staging, { force: true }); + throw error instanceof ContextLoaderPackageError ? error : new ContextLoaderPackageError("download-failed", "could not download the required artifact (offline cache miss)"); + } +} + +function validateManifest(packageJson, manifest, artifact, target) { + if (manifest.schema !== 1 || manifest.package !== packageJson.name || manifest.version !== packageJson.version) { + throw new ContextLoaderPackageError("invalid-manifest", "manifest does not match package identity"); + } + if (!artifact) throw new ContextLoaderPackageError("unsupported-platform", `${target} has no certified artifact row`); + if (typeof artifact.url !== "string" || !artifact.url.startsWith("https://") || !isSha256(artifact.archiveSha256) || !isSha256(artifact.binarySha256) || typeof artifact.binaryPath !== "string" || artifact.binaryPath.includes("..") || path.isAbsolute(artifact.binaryPath)) { + throw new ContextLoaderPackageError("invalid-manifest", `${target} row is incomplete or unsafe`); + } +} + +async function existingVerified(binary, statePath, version, target, digest) { + try { + const state = JSON.parse(await readFile(statePath, "utf8")); + return state.version === version && state.target === target && state.binaryPath === CANONICAL_BINARY && state.binarySha256 === digest && sha256File(binary) === digest; + } catch { + return false; + } +} + +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function extractSingleTarGz(archive, expectedPath) { + let tar; + try { tar = gunzipSync(archive); } catch { throw new ContextLoaderPackageError("invalid-archive", "artifact is not a gzip tar archive"); } + let offset = 0; + let found; + while (offset + 512 <= tar.length) { + const header = tar.subarray(offset, offset + 512); + if (header.every((byte) => byte === 0)) break; + const name = header.subarray(0, 100).toString("utf8").replace(/\0.*$/u, ""); + const sizeText = header.subarray(124, 136).toString("utf8").replace(/\0.*$/u, "").trim(); + const size = Number.parseInt(sizeText || "0", 8); + const type = String.fromCharCode(header[156] || 48); + if (!Number.isSafeInteger(size) || size < 0 || offset + 512 + size > tar.length) throw new ContextLoaderPackageError("invalid-archive", "artifact has an invalid tar layout"); + if (name === expectedPath) { + if (type !== "0" || found !== undefined) throw new ContextLoaderPackageError("invalid-archive", "artifact has an unexpected binary layout"); + found = Buffer.from(tar.subarray(offset + 512, offset + 512 + size)); + } + offset += 512 + Math.ceil(size / 512) * 512; + } + if (found === undefined) throw new ContextLoaderPackageError("invalid-archive", "artifact does not contain the declared binary path"); + return found; +} diff --git a/apps/context-loader/scripts/postinstall.mjs b/apps/context-loader/scripts/postinstall.mjs new file mode 100644 index 000000000..cb697bdf3 --- /dev/null +++ b/apps/context-loader/scripts/postinstall.mjs @@ -0,0 +1,11 @@ +import path from "node:path"; +import { fileURLToPath } from "node:url"; +import { installContextLoader } from "./installer.mjs"; + +const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +try { + await installContextLoader({ packageRoot }); +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(1); +} diff --git a/apps/context-loader/test/installer.test.mjs b/apps/context-loader/test/installer.test.mjs new file mode 100644 index 000000000..2ae67dc4c --- /dev/null +++ b/apps/context-loader/test/installer.test.mjs @@ -0,0 +1,78 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { gzipSync } from "node:zlib"; +import { installContextLoader } from "../scripts/installer.mjs"; +import { readVerifiedState } from "../lib/verified.mjs"; + +const sha256 = (content) => createHash("sha256").update(content).digest("hex"); + +function tarEntry(name, content) { + const header = Buffer.alloc(512); + header.write(name); + header.write(content.length.toString(8).padStart(11, "0"), 124); + header[135] = 0; + header[156] = "0".charCodeAt(0); + header.write("ustar", 257); + return Buffer.concat([header, content, Buffer.alloc((512 - (content.length % 512)) % 512), Buffer.alloc(1024)]); +} + +async function fixture({ artifact = Buffer.from("#!/bin/sh\necho loader\n"), target = "darwin-arm64", row = {} } = {}) { + const root = await mkdtemp(path.join(os.tmpdir(), "context-loader-package-")); + const archive = gzipSync(tarEntry("wren-context-loader", artifact)); + const manifest = { + schema: 1, + package: "@wrenai/context-loader", + version: "0.1.0", + artifacts: { + [target]: { + url: "https://example.invalid/context-loader.tar.gz", + archiveSha256: sha256(archive), + binarySha256: sha256(artifact), + binaryPath: "wren-context-loader", + ...row, + }, + }, + }; + await writeFile(path.join(root, "package.json"), JSON.stringify({ name: manifest.package, version: manifest.version })); + await writeFile(path.join(root, "artifacts.json"), JSON.stringify(manifest)); + return { root, archive, artifact }; +} + +const fetchArchive = (archive) => async () => ({ ok: true, arrayBuffer: async () => archive }); + +test("installs a digest-verified darwin-arm64 binary atomically and records canonical state", async () => { + const { root, archive, artifact } = await fixture(); + const first = await installContextLoader({ packageRoot: root, fetchImpl: fetchArchive(archive), platform: "darwin", arch: "arm64" }); + assert.equal(first.reused, false); + assert.deepEqual(await readFile(first.binary), artifact); + const state = readVerifiedState(root); + assert.equal(state.binary, first.binary); + assert.match(state.identity, /package:@wrenai\/context-loader@0\.1\.0:darwin-arm64:[a-f0-9]{64}/); + const second = await installContextLoader({ packageRoot: root, fetchImpl: async () => { throw new Error("offline"); }, platform: "darwin", arch: "arm64" }); + assert.equal(second.reused, true); +}); + +test("fails closed for unsupported targets, archive tampering, and binary digest tampering", async () => { + const supported = await fixture(); + await assert.rejects( + installContextLoader({ packageRoot: supported.root, fetchImpl: fetchArchive(supported.archive), platform: "linux", arch: "x64" }), + /unsupported-platform/, + ); + const archiveTampered = await fixture({ row: { archiveSha256: "0".repeat(64) } }); + await assert.rejects(installContextLoader({ packageRoot: archiveTampered.root, fetchImpl: fetchArchive(archiveTampered.archive), platform: "darwin", arch: "arm64" }), /archive-digest-mismatch/); + const binaryTampered = await fixture({ row: { binarySha256: "0".repeat(64) } }); + await assert.rejects(installContextLoader({ packageRoot: binaryTampered.root, fetchImpl: fetchArchive(binaryTampered.archive), platform: "darwin", arch: "arm64" }), /binary-digest-mismatch/); +}); + +test("rejects stale or altered installed state at runtime", async () => { + const { root, archive } = await fixture(); + await installContextLoader({ packageRoot: root, fetchImpl: fetchArchive(archive), platform: "darwin", arch: "arm64" }); + const statePath = path.join(root, "install-state.json"); + const state = JSON.parse(await readFile(statePath, "utf8")); + await writeFile(statePath, JSON.stringify({ ...state, version: "0.1.1" })); + assert.throws(() => readVerifiedState(root), /stale-state/); +}); diff --git a/apps/genbi/INSTALL.md b/apps/genbi/INSTALL.md index ac674b04a..523cffd5c 100644 --- a/apps/genbi/INSTALL.md +++ b/apps/genbi/INSTALL.md @@ -17,20 +17,18 @@ answers (tables, charts, KPIs, narrative text). ## Supported platforms -`@wrenai/genbi` depends on `@warble/cli`, a Rust binary fetched from GitHub -Releases during install. That binary is published for exactly these four -platforms: +`@wrenai/genbi` has one additional native dependency for user-project binding: +`@wrenai/context-loader`. Its initial certified target is macOS Apple Silicon, +so the GenBI package is currently supported only on that platform. This limit is +intentional; the Warble package supports more platforms, but GenBI must not +claim a target until its context-loader artifact has been built and verified. | OS | Architecture | | --- | --- | -| macOS | Apple Silicon (`aarch64-apple-darwin`) | -| macOS | Intel (`x86_64-apple-darwin`) | -| Linux (glibc ≥ 2.31) | x86_64 (`x86_64-unknown-linux-gnu`) | -| Linux (glibc ≥ 2.31) | aarch64 (`aarch64-unknown-linux-gnu`) | +| macOS | Apple Silicon (`darwin-arm64`) | -**Windows is not supported.** **musl-based Linux (e.g. Alpine) is not -supported**, nor is glibc older than 2.31 — install fails outright on these; -see [Troubleshooting](#troubleshooting). +**Intel macOS, Linux, and Windows are not supported.** Installation fails +before GenBI starts; see [Troubleshooting](#troubleshooting). ## Prerequisites @@ -42,8 +40,9 @@ see [Troubleshooting](#troubleshooting). `@wrenai/genbi` shells out to it for live questions and context inspection; it does not install or configure `wren` for you. See the [Wren project](https://github.com/Canner/WrenAI). -- **Network access to `github.com`** during install (to fetch the Warble - binary) and on first use (see [Network](#network) below). Fully offline / +- **Network access to `github.com`** during install (to fetch the Warble and + exact context-loader binaries) and on first use (see [Network](#network) + below). Fully offline / air-gapped installs are not supported and are out of scope for this guide. ## Install @@ -60,10 +59,11 @@ Or install it once, if you would rather not re-fetch: npm install -g @wrenai/genbi ``` -Either way, `npm` runs `@warble/cli`'s postinstall script, which downloads the -Warble binary for your platform (about 12 MB) from GitHub Releases and verifies -its checksum against the release manifest. That is where an unsupported platform -fails, before any of the app's own code runs. +Either way, `npm` runs the binary-package postinstall scripts. The context +loader downloads its exact manifest row, verifies archive and binary SHA-256, +then writes the package-local executable atomically. Unsupported targets, +offline first installs, altered downloads, and incomplete extraction fail +before any app code runs. ## Run @@ -118,7 +118,10 @@ here because they aren't needed for a first install and run. 1. **At install time** — `@warble/cli`'s postinstall script downloads the Warble executable for your platform (~12 MB) from a GitHub Release and checksum-verifies it. -2. **At runtime, on first use** — the first time the app compiles a +2. **At install time** — `@wrenai/context-loader` downloads the exact + darwin-arm64 generator artifact and verifies its archive and binary + digests. Runtime never downloads it. +3. **At runtime, on first use** — the first time the app compiles a profile, it downloads a Hub component archive from GitHub. After that first download it's cached locally and no further network access to GitHub is needed for subsequent compiles. @@ -127,11 +130,9 @@ Offline or air-gapped environments are not supported by this guide. ## Troubleshooting -**"Platform ... is not supported by @warble/cli"** (during `npm install`) — -your OS/architecture isn't one of the four supported platforms listed -above. This includes Windows and musl-based Linux distributions (e.g. -Alpine), where the installer additionally reports a libc mismatch before -failing. There is no workaround; a matching platform is required. +**"@wrenai/context-loader unsupported-platform"** (during `npm install`) — +your OS/architecture is not the certified macOS Apple Silicon target. There is +no fallback to a nearby architecture or an ambient binary. **Node version errors, or unexpected syntax/runtime errors on startup** — confirm your Node version is 20 or later (`node --version`). Older diff --git a/apps/genbi/MAINTAINING.md b/apps/genbi/MAINTAINING.md index c4017c35a..61e2738a0 100644 --- a/apps/genbi/MAINTAINING.md +++ b/apps/genbi/MAINTAINING.md @@ -128,18 +128,24 @@ Verify a regeneration rather than trusting it: recompiling an unchanged profile did not intend it to means something else moved — the pin, the Hub, or the generator — and the diff should be understood before it is committed. -### Open question: distributing the generator +### Distributing the context-loader generator -`wren-context-loader` is resolved from an **in-repo Rust build** (or an explicit -`WREN_HARNESS_CONTEXT_LOADER_BIN` override), and resolution loud-fails when neither is present. -That is sufficient for development in this monorepo and for CI, but it is **not** a distribution -story: an installed `@wrenai/genbi` has no Rust toolchain and no `core/` tree, so a -user-project-bound compile cannot resolve the generator at all. Warble faced the same problem with -its own compiler and solved it by publishing a pinned `@warble/cli` npm package whose postinstall -downloads a prebuilt platform binary. The analogue for this generator — publish a -platform-binary npm package, vendor prebuilt binaries, statically link it into an existing -distributed artifact, or drop the separate binary and expose the projection some other way — is a -real packaging decision with release-process consequences, and is deliberately **not** settled here. +`@wrenai/genbi` pins `@wrenai/context-loader` to one exact version. The package postinstall +downloads only the immutable manifest row for the current target, verifies both archive and binary +SHA-256 values, writes the canonical package-local executable atomically, and records the verified +package/version/target/digest tuple. Runtime re-verifies that record before execution; it never +downloads a binary on first bind or uses PATH. + +The first certified row is `darwin-arm64`. Treat every other target as unsupported until it has a +reviewed artifact row and release evidence. The context-loader version is independent of both the +GenBI application and managed Wren runtime: release it only from an explicit `context-loader-v*` +tag using `.github/workflows/publish-context-loader.yml`, then separately bump GenBI's exact pin. + +Development order remains explicit `WREN_HARNESS_CONTEXT_LOADER_BIN`, then an in-repo Rust build, +then the verified package. Neither a missing/altered package record nor a failed development +override may fall through to a legacy binding. The compile cache identity includes that resolver +kind and the package tuple or development binary digest, so changing the shipped generator moves +the key. ### Why `@warble/ir-spec` stays behind diff --git a/apps/genbi/RUNNING.md b/apps/genbi/RUNNING.md index a20bb7887..10c77308b 100644 --- a/apps/genbi/RUNNING.md +++ b/apps/genbi/RUNNING.md @@ -22,20 +22,23 @@ Beyond the prerequisites in [README.md](./README.md#prerequisites): local and gateway runtimes are not accepted. Claude is by far the better exercised of the two — see [Codex status](#codex-status). - The Wren CLI on `PATH`: -- **The `wren-context-loader` generator, built from this repo.** Binding a user's project - compiles a prepared-context document first, and the generator is a Rust binary rather than an - npm dependency, so `pnpm install` does not supply it. Build it once with - `cargo build --release --manifest-path ../../core/wren-context-loader/Cargo.toml`, or point - `WREN_HARNESS_CONTEXT_LOADER_BIN` at an existing build. Without either, the first user-project - bind fails loudly — by design, since a silent fallback would leave no way to tell which path - produced an IR. Restart the BFF after rebuilding the binary: its identity is memoized for the - process lifetime. ```bash uv tool install wrenai wren --version ``` +- For an in-repo user-project bind, build the `wren-context-loader` generator once: + + ```bash + cargo build --release --manifest-path ../../core/wren-context-loader/Cargo.toml + ``` + + An explicit `WREN_HARNESS_CONTEXT_LOADER_BIN` overrides that development build. Installed + packages instead use their exact pinned, digest-verified `@wrenai/context-loader` dependency. + There is no PATH or legacy-binding fallback. Restart the BFF after rebuilding the development + binary because its identity is memoized for the process lifetime. + ## Where Warble comes from diff --git a/apps/genbi/harness/compile/context-loader.ts b/apps/genbi/harness/compile/context-loader.ts index 1202ee98e..e52ec4c81 100644 --- a/apps/genbi/harness/compile/context-loader.ts +++ b/apps/genbi/harness/compile/context-loader.ts @@ -1,8 +1,10 @@ import { execFile } from "node:child_process"; -import { existsSync } from "node:fs"; +import { existsSync, readFileSync } from "node:fs"; +import { createHash } from "node:crypto"; import { fileURLToPath } from "node:url"; import path from "node:path"; import { ContextLoaderFailedError, ContextLoaderNotFoundError } from "./errors.js"; +import { resolveInstalledPackageRoot } from "../npm-package-resolve.js"; /** Environment override for {@link resolveContextLoaderBinary}, named like the other `WREN_HARNESS_*` binary overrides. */ const CONTEXT_LOADER_BIN_ENV = "WREN_HARNESS_CONTEXT_LOADER_BIN"; @@ -18,18 +20,30 @@ const IN_REPO_BUILD_PATHS = [ * document — guards against a hang, not meant to ever fire in normal operation. */ const CONTEXT_LOADER_TIMEOUT_MS = 60 * 1000; +const CONTEXT_LOADER_PACKAGE = "@wrenai/context-loader"; +const CANONICAL_PACKAGE_BINARY = path.join("bin", "wren-context-loader"); + +export interface ResolvedContextLoader { + readonly bin: string; + readonly kind: "explicit" | "in-repo" | "package"; + /** Package resolution is already digest-verified. Development modes use the binary hash below. */ + readonly verifiedIdentity?: string; +} /** * Resolves the `wren-context-loader` binary — the generator that renders a wren project as the * prepared-context document a `kind: prepared` binding reads. * - * Two tiers, then a loud failure: + * Three deterministic tiers, then a loud failure: * * 1. `explicit`, else the `WREN_HARNESS_CONTEXT_LOADER_BIN` environment variable. Either must * exist on disk; neither falls through to a search. * 2. This repo's own build of the generator crate, found by walking up from this package looking * for `core/wren-context-loader/target/{release,debug}/wren-context-loader`. `release` wins * when both exist, so a stale `debug` build never shadows a deliberate release build. + * 3. The verified package-local binary from the exact `@wrenai/context-loader` dependency. Its + * install record must match package version and target, name the canonical binary, and pass a + * fresh SHA-256 check. Resolution never downloads at runtime. * * There is deliberately **no** `PATH` tier and — more importantly — **no fallback to the old * `wren_project` binding**. A fallback would make the extraction unverifiable: a compiled IR is @@ -39,27 +53,85 @@ const CONTEXT_LOADER_TIMEOUT_MS = 60 * 1000; * * Throws {@link ContextLoaderNotFoundError} with the accumulated attempts when neither tier works. */ -export function resolveContextLoaderBinary(explicit?: string): string { +export function resolveContextLoader(explicit?: string): ResolvedContextLoader { const override = explicit ?? process.env[CONTEXT_LOADER_BIN_ENV]; if (override !== undefined && override.length > 0) { if (!existsSync(override)) { const source = explicit !== undefined ? `explicit contextLoaderBin` : CONTEXT_LOADER_BIN_ENV; throw new ContextLoaderNotFoundError([`${source} "${override}" does not exist`]); } - return override; + return { bin: override, kind: "explicit" }; } const inRepo = findInRepoBuild(); - if (inRepo !== undefined) return inRepo; + if (inRepo !== undefined) return { bin: inRepo, kind: "in-repo" }; + + const packaged = resolveVerifiedPackageBinary(); + if (packaged !== undefined) return packaged; throw new ContextLoaderNotFoundError([ `${CONTEXT_LOADER_BIN_ENV} is not set`, `no in-repo build found (searched ancestors of this package for ` + IN_REPO_BUILD_PATHS.map((segments) => `"${path.join(...segments)}"`).join(" and ") + - `) — build it with "cargo build --release --manifest-path core/wren-context-loader/Cargo.toml"`, + `) — ${CONTEXT_LOADER_PACKAGE} is not installed with a verified ${targetForCurrentPlatform()} binary`, ]); } +/** Backward-compatible path-only API for callers that do not need cache provenance. */ +export function resolveContextLoaderBinary(explicit?: string): string { + return resolveContextLoader(explicit).bin; +} + +/** Stable cache identity: resolver kind plus either a verified package tuple or the dev binary's bytes. */ +export function getContextLoaderIdentity(resolved: ResolvedContextLoader): string { + if (resolved.verifiedIdentity !== undefined) return resolved.verifiedIdentity; + return `${resolved.kind}:${createHash("sha256").update(readFileSync(resolved.bin)).digest("hex")}`; +} + +function resolveVerifiedPackageBinary(): ResolvedContextLoader | undefined { + const packageRoot = resolveInstalledPackageRoot(CONTEXT_LOADER_PACKAGE); + if (packageRoot === undefined) return undefined; + return readVerifiedContextLoaderPackage(packageRoot); +} + +/** Validates a package-local installation record without resolving it from the host environment. */ +export function readVerifiedContextLoaderPackage(packageRoot: string, target = targetForCurrentPlatform()): ResolvedContextLoader | undefined { + try { + const packageJson = readJson(path.join(packageRoot, "package.json")); + const state = readJson(path.join(packageRoot, "install-state.json")); + const bin = path.resolve(packageRoot, CANONICAL_PACKAGE_BINARY); + if ( + packageJson.name !== CONTEXT_LOADER_PACKAGE || + typeof packageJson.version !== "string" || + state.package !== CONTEXT_LOADER_PACKAGE || + state.version !== packageJson.version || + state.target !== target || + state.binaryPath !== CANONICAL_PACKAGE_BINARY || + !isSha256(state.binarySha256) || + !bin.startsWith(`${packageRoot}${path.sep}`) || + !existsSync(bin) || + createHash("sha256").update(readFileSync(bin)).digest("hex") !== state.binarySha256 + ) { + return undefined; + } + return { bin, kind: "package", verifiedIdentity: `package:${CONTEXT_LOADER_PACKAGE}@${packageJson.version}:${target}:${state.binarySha256}` }; + } catch { + return undefined; + } +} + +function readJson(file: string): Record { + return JSON.parse(readFileSync(file, "utf8")) as Record; +} + +function isSha256(value: unknown): value is string { + return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value); +} + +function targetForCurrentPlatform(): string { + return `${process.platform}-${process.arch}`; +} + /** * Walks up from this package's own directory (`harness/compile/` in source, `dist/compile/` once * built — either way two levels below the package root), returning the first existing generator diff --git a/apps/genbi/harness/compile/errors.ts b/apps/genbi/harness/compile/errors.ts index 910d09c5d..347ac3e52 100644 --- a/apps/genbi/harness/compile/errors.ts +++ b/apps/genbi/harness/compile/errors.ts @@ -44,7 +44,7 @@ export class ContextLoaderNotFoundError extends Error { super( `could not resolve the "wren-context-loader" binary:\n` + attempts.map((attempt) => ` - ${attempt}`).join("\n") + - `\nfix: build it in-repo, from the repository root ` + + `\nfix: install the exact @wrenai/context-loader package, build it in-repo from the repository root ` + `("cargo build --release --manifest-path core/wren-context-loader/Cargo.toml") ` + `or point WREN_HARNESS_CONTEXT_LOADER_BIN at an existing build.`, ); diff --git a/apps/genbi/harness/compile/pipeline.ts b/apps/genbi/harness/compile/pipeline.ts index a7cff071b..4390d8884 100644 --- a/apps/genbi/harness/compile/pipeline.ts +++ b/apps/genbi/harness/compile/pipeline.ts @@ -5,7 +5,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { createFileSystemCompileCache } from "./cache.js"; import { composeUserProfile } from "./compose-profile.js"; -import { resolveContextLoaderBinary } from "./context-loader.js"; +import { getContextLoaderIdentity, resolveContextLoader } from "./context-loader.js"; import { WarbleCommandFailedError } from "./errors.js"; import { hashDirectory, hashFiles } from "./fingerprint.js"; import { resolveHubDir, resolveWarbleBinary } from "./resolve-binary.js"; @@ -91,21 +91,21 @@ export async function compileProfile(options: CompileProfileOptions): Promise { - resolvedLoaderBin ??= resolveContextLoaderBinary(options.contextLoaderBin); - return resolvedLoaderBin; + let resolvedLoader: ReturnType | undefined; + const ensureContextLoader = (): ReturnType => { + resolvedLoader ??= resolveContextLoader(options.contextLoaderBin); + return resolvedLoader; }; return compileProfileSource( options, await hashDirectory(path.resolve(options.userProject)), - async () => options.contextLoaderIdentity ?? (await getBinaryIdentity(ensureContextLoaderBin())), + async () => options.contextLoaderIdentity ?? getContextLoaderIdentity(ensureContextLoader()), (workDir) => composeUserProfile({ profileSource: options.profileSource, userProject: options.userProject, destDir: workDir, - contextLoaderBin: ensureContextLoaderBin(), + contextLoaderBin: ensureContextLoader().bin, }), ); } diff --git a/apps/genbi/harness/compile/types.ts b/apps/genbi/harness/compile/types.ts index 014f60db0..dbf684db1 100644 --- a/apps/genbi/harness/compile/types.ts +++ b/apps/genbi/harness/compile/types.ts @@ -96,7 +96,8 @@ export interface CompileProfileOptions { /** * Explicit path to the `wren-context-loader` binary. See `resolveContextLoaderBinary` for the * resolution order when omitted (explicit → `WREN_HARNESS_CONTEXT_LOADER_BIN` → in-repo build → - * loud failure; there is no fallback to Warble's built-in `wren_project` adapter). + * verified exact `@wrenai/context-loader` package → loud failure; there is no fallback to + * Warble's built-in `wren_project` adapter). */ readonly contextLoaderBin?: string; /** diff --git a/apps/genbi/harness/npm-package-resolve.ts b/apps/genbi/harness/npm-package-resolve.ts index 7c863d644..3c60443eb 100644 --- a/apps/genbi/harness/npm-package-resolve.ts +++ b/apps/genbi/harness/npm-package-resolve.ts @@ -22,18 +22,29 @@ import path from "node:path"; */ export function resolveInstalledPackageBin(packageName: string, binName: string): string | undefined { try { - const require = createRequire(import.meta.url); - const pkgJsonPath = require.resolve(`${packageName}/package.json`); - const pkg = JSON.parse(readFileSync(pkgJsonPath, "utf8")) as { bin?: string | Record }; + const pkgJsonPath = resolveInstalledPackageRoot(packageName); + if (pkgJsonPath === undefined) return undefined; + const manifestPath = path.join(pkgJsonPath, "package.json"); + const pkg = JSON.parse(readFileSync(manifestPath, "utf8")) as { bin?: string | Record }; const binField = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.[binName]; if (binField === undefined) return undefined; - const resolved = path.resolve(path.dirname(pkgJsonPath), binField); + const resolved = path.resolve(pkgJsonPath, binField); return existsSync(resolved) ? resolved : undefined; } catch { return undefined; } } +/** Returns an installed package's canonical root through Node resolution, never by walking PATH. */ +export function resolveInstalledPackageRoot(packageName: string): string | undefined { + try { + const require = createRequire(import.meta.url); + return path.dirname(require.resolve(`${packageName}/package.json`)); + } catch { + return undefined; + } +} + /** * Named, explicit opt-in that re-enables the legacy "sibling `warble` git checkout" * development-mode resolution tier in `resolveWarbleBinary`, `resolveAgentSdkCli`, and diff --git a/apps/genbi/package.json b/apps/genbi/package.json index bd7a1ae57..ea499b850 100644 --- a/apps/genbi/package.json +++ b/apps/genbi/package.json @@ -48,6 +48,7 @@ "@ai-sdk/provider": "~4.0.3", "@ant-design/icons": "^6.3.2", "@hono/node-server": "^2.0.10", + "@wrenai/context-loader": "0.1.0", "@warble/claude-agent-sdk": "0.10.0", "@warble/cli": "0.10.0", "@warble/codex-local": "0.10.0", diff --git a/apps/genbi/scripts/installed-package-acceptance.mjs b/apps/genbi/scripts/installed-package-acceptance.mjs index b41d30fcc..037033cc7 100644 --- a/apps/genbi/scripts/installed-package-acceptance.mjs +++ b/apps/genbi/scripts/installed-package-acceptance.mjs @@ -9,18 +9,23 @@ * minimal PATH and a require hook that turns any source-checkout read into a * hard failure, so a checkout-only fallback cannot make this pass. */ +import { createHash } from "node:crypto"; import { existsSync } from "node:fs"; -import { mkdtemp, mkdir, readdir, rm, writeFile } from "node:fs/promises"; +import { chmod, cp, mkdtemp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises"; import { createServer as createHttpServer } from "node:http"; import { createServer } from "node:net"; import os from "node:os"; import path from "node:path"; +import { pathToFileURL } from "node:url"; import { closeServerBounded, runBounded, spawnProcessGroup, stopProcessTree } from "./process-cleanup.mjs"; import { readSseFrames } from "./sse-frames.mjs"; const packageRoot = path.resolve(process.cwd()); +const contextLoaderSource = path.resolve(packageRoot, "..", "context-loader"); const tempRoot = await mkdtemp(path.join(os.tmpdir(), "genbi-installed-package-")); const packDirectory = path.join(tempRoot, "pack"); +const contextLoaderPackDirectory = path.join(tempRoot, "context-loader-pack"); +const contextLoaderStage = path.join(tempRoot, "context-loader-stage"); const installRoot = path.join(tempRoot, "fresh-install"); const workspaceRoot = path.join(tempRoot, "workspace"); const sourceAuditHook = path.join(tempRoot, "block-checkout-access.cjs"); @@ -31,21 +36,31 @@ let phase = "initialize"; try { markPhase("pack"); - await Promise.all([mkdir(packDirectory), mkdir(installRoot), mkdir(workspaceRoot)]); + await Promise.all([mkdir(packDirectory), mkdir(contextLoaderPackDirectory), mkdir(installRoot), mkdir(workspaceRoot)]); await run("pnpm", ["pack", "--pack-destination", packDirectory], { cwd: packageRoot }); + const contextLoaderTarball = await packVerifiedContextLoaderFixture(contextLoaderSource, contextLoaderStage, contextLoaderPackDirectory); const packageTarball = await onlyTarball(packDirectory); const packedFiles = await tarFiles(packageTarball); assertPublishedFiles(packedFiles); + const packedManifest = await tarJson(packageTarball, "package/package.json"); + if (packedManifest.dependencies?.["@wrenai/context-loader"] !== "0.1.0") { + throw new Error("packed @wrenai/genbi does not retain an exact @wrenai/context-loader version"); + } markPhase("install"); await run("npm", ["init", "--yes"], { cwd: installRoot }); - await run("npm", ["install", "--no-audit", "--no-fund", packageTarball], { cwd: installRoot }); + await run("npm", ["install", "--no-audit", "--no-fund", packageTarball, contextLoaderTarball], { cwd: installRoot }); const installedPackageRoot = path.join(installRoot, "node_modules", "@wrenai", "genbi"); if (!existsSync(path.join(installedPackageRoot, "package.json"))) { throw new Error("fresh install did not contain @wrenai/genbi"); } + const installedContextLoaderRoot = path.join(installRoot, "node_modules", "@wrenai", "context-loader"); + const installedContextLoaderBin = path.join(installedContextLoaderRoot, "bin", "wren-context-loader"); + if (!existsSync(path.join(installedContextLoaderRoot, "install-state.json")) || !existsSync(installedContextLoaderBin)) { + throw new Error("fresh install did not preserve the verified context-loader package record"); + } for (const forbidden of ["scripts", "test", "examples", path.join("node_modules", "examples")]) { if (existsSync(path.join(installedPackageRoot, forbidden))) { throw new Error(`published package unexpectedly contains ${forbidden}`); @@ -65,6 +80,7 @@ try { markPhase("setup-connect"); await waitForServer(port, output); await verifyFirstRunSetup(port, workspaceRoot, fixtureProvider); + await verifyInstalledProjectBind({ installRoot, workspaceRoot, sourceAuditHook, installedPackageRoot, installedContextLoaderBin, port, fixtureEndpoint: fixtureProvider.endpoint }); await stopProcessTree(serverProcess); serverProcess = undefined; @@ -75,6 +91,7 @@ try { { name: "tarball excludes checkout-only scripts, tests, fixtures, and examples", ok: true }, { name: "fresh install launches through npx with package-manager PATH", ok: true }, { name: "first-run Setup connect terminal flow works without checkout access or development escapes", ok: true }, + { name: "fresh install binds through a verified package-local context loader with no Rust toolchain or checkout access", ok: true }, ], }, null, 2)}\n`); } finally { @@ -153,6 +170,47 @@ function controlledEnvironment({ installRoot, workspaceRoot, port: selectedPort, return environment; } +async function packVerifiedContextLoaderFixture(source, stage, destination) { + await cp(source, stage, { recursive: true }); + const binary = path.join(stage, "bin", "wren-context-loader"); + const content = Buffer.from("#!/bin/sh\nprintf '{\"context_version\":1,\"parseable\":true}' > \"$3\"\n"); + const binarySha256 = createHash("sha256").update(content).digest("hex"); + await writeFile(binary, content, { mode: 0o755 }); + await chmod(binary, 0o755); + const packageJson = JSON.parse(await readFile(path.join(stage, "package.json"), "utf8")); + await writeFile(path.join(stage, "artifacts.json"), JSON.stringify({ schema: 1, package: packageJson.name, version: packageJson.version, artifacts: { "darwin-arm64": { url: "https://example.invalid/context-loader.tar.gz", archiveSha256: "0".repeat(64), binarySha256, binaryPath: "wren-context-loader" } } })); + await writeFile(path.join(stage, "install-state.json"), JSON.stringify({ package: packageJson.name, version: packageJson.version, target: "darwin-arm64", archiveSha256: "0".repeat(64), binarySha256, binaryPath: path.join("bin", "wren-context-loader") })); + await run("npm", ["pack", "--ignore-scripts", "--pack-destination", destination], { cwd: stage }); + return onlyTarball(destination); +} + +async function verifyInstalledProjectBind({ installRoot, workspaceRoot, sourceAuditHook, installedPackageRoot, installedContextLoaderBin, port: selectedPort, fixtureEndpoint }) { + const project = path.join(workspaceRoot, "bound-project"); + const warble = path.join(installRoot, "warble-fixture"); + await mkdir(project); + await writeFile(path.join(project, "wren_project.yml"), "name: bound-project\n"); + await writeFile(warble, "#!/bin/sh\nwhile [ $# -gt 0 ]; do if [ \"$1\" = \"-o\" ]; then shift; printf '{}' > \"$1\"; fi; shift; done\n", { mode: 0o755 }); + await chmod(warble, 0o755); + const pipeline = pathToFileURL(path.join(installedPackageRoot, "dist-server", "harness", "compile", "pipeline.js")).href; + const profile = path.join(installedPackageRoot, "profiles", "genbi-default"); + const bindEnv = controlledEnvironment({ installRoot, workspaceRoot, port: selectedPort, sourceAuditHook, fixtureEndpoint }); + bindEnv.XDG_CACHE_HOME = path.join(installRoot, "package-cache"); + const runBind = async () => run(process.execPath, ["--input-type=module", "--eval", `import { compileProfile } from ${JSON.stringify(pipeline)}; const result = await compileProfile({ profileSource: ${JSON.stringify(profile)}, userProject: ${JSON.stringify(project)}, mode: "native", warbleBin: ${JSON.stringify(warble)}, hubDir: "/fixture/hub" }); console.log(JSON.stringify({ cacheHit: result.cacheHit }));`], { + cwd: installRoot, + env: bindEnv, + }); + const bound = await runBind(); + if (!/"cacheHit":false/.test(bound.stdout)) throw new Error(`installed package did not bind a user project through its context loader: ${bound.stdout}${bound.stderr}`); + await writeFile(installedContextLoaderBin, "tampered", { mode: 0o755 }); + let failed = false; + try { + await runBind(); + } catch (error) { + failed = /could not resolve the "wren-context-loader" binary/.test(String(error)); + } + if (!failed) throw new Error("tampered package-local context loader did not fail closed"); +} + function createSourceAuditHook() { return [ 'const fs = require("node:fs");', @@ -314,6 +372,11 @@ async function tarFiles(tarball) { return result.stdout.split("\n").filter(Boolean); } +async function tarJson(tarball, entry) { + const result = await run("tar", ["-xOzf", tarball, entry]); + return JSON.parse(result.stdout); +} + function assertPublishedFiles(files) { if (files.length === 0) throw new Error("package tarball is empty"); const forbidden = /(^|\/)(scripts|test|tests|fixtures|examples|\.git)(\/|$)|(^|\/)node_modules(\/|$)/; diff --git a/apps/genbi/test/compile-context-loader.test.ts b/apps/genbi/test/compile-context-loader.test.ts index fcc4f2a81..c202bafae 100644 --- a/apps/genbi/test/compile-context-loader.test.ts +++ b/apps/genbi/test/compile-context-loader.test.ts @@ -1,9 +1,10 @@ -import { mkdtemp, writeFile } from "node:fs/promises"; +import { mkdir, mkdtemp, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { createInMemoryCompileCache } from "../harness/compile/cache.js"; -import { resolveContextLoaderBinary } from "../harness/compile/context-loader.js"; +import { getContextLoaderIdentity, readVerifiedContextLoaderPackage, resolveContextLoaderBinary } from "../harness/compile/context-loader.js"; import { ContextLoaderNotFoundError } from "../harness/compile/errors.js"; import { compileProfile } from "../harness/compile/pipeline.js"; import type { CompileCache, CompileCacheKey } from "../harness/compile/types.js"; @@ -41,6 +42,18 @@ async function makeWarbleStub(): Promise { return bin; } +async function makeVerifiedPackage(marker: string, version = "0.1.0"): Promise { + const root = await mkdtemp(path.join(os.tmpdir(), "wren-harness-loader-package-")); + const binary = path.join(root, "bin", "wren-context-loader"); + const content = Buffer.from(`#!/bin/sh\n# ${marker}\n`); + const digest = createHash("sha256").update(content).digest("hex"); + await mkdir(path.dirname(binary), { recursive: true }); + await writeFile(binary, content, { mode: 0o755 }); + await writeFile(path.join(root, "package.json"), JSON.stringify({ name: "@wrenai/context-loader", version })); + await writeFile(path.join(root, "install-state.json"), JSON.stringify({ package: "@wrenai/context-loader", version, target: `${process.platform}-${process.arch}`, binaryPath: path.join("bin", "wren-context-loader"), binarySha256: digest })); + return root; +} + describe("resolveContextLoaderBinary", () => { it("returns an explicit override that exists, without searching further", async () => { const bin = await makeLoaderStub("explicit"); @@ -93,6 +106,29 @@ describe("getBinaryIdentity over the generator", () => { }); }); +describe("verified package context loader", () => { + it("accepts only a canonical package-local binary and folds package/version/digest into identity", async () => { + const aRoot = await makeVerifiedPackage("a", "0.1.0"); + const bRoot = await makeVerifiedPackage("b", "0.1.1"); + const a = readVerifiedContextLoaderPackage(aRoot); + const b = readVerifiedContextLoaderPackage(bRoot); + expect(a?.kind).toBe("package"); + expect(a?.bin).toBe(path.join(aRoot, "bin", "wren-context-loader")); + expect(a).toBeDefined(); + expect(b).toBeDefined(); + expect(getContextLoaderIdentity(a!)).not.toBe(getContextLoaderIdentity(b!)); + }); + + it("rejects a tampered binary or stale package record instead of selecting an ambient fallback", async () => { + const root = await makeVerifiedPackage("before"); + await writeFile(path.join(root, "bin", "wren-context-loader"), "altered", { mode: 0o755 }); + expect(readVerifiedContextLoaderPackage(root)).toBeUndefined(); + const stale = await makeVerifiedPackage("stale"); + await writeFile(path.join(stale, "install-state.json"), JSON.stringify({ package: "@wrenai/context-loader", version: "0.0.0", target: `${process.platform}-${process.arch}`, binaryPath: path.join("bin", "wren-context-loader"), binarySha256: "0".repeat(64) })); + expect(readVerifiedContextLoaderPackage(stale)).toBeUndefined(); + }); +}); + /** * Criterion: a generator change must invalidate the compile cache **with the user project held * constant**. `contextFingerprint` already content-hashes the project, so it cannot carry this — @@ -168,4 +204,22 @@ describe("compile cache key vs a generator change (project held constant)", () = expect(keyA?.warbleIdentity).toBe(keyB?.warbleIdentity); expect(keyA?.hubDir).toBe(keyB?.hubDir); }, 30_000); + + it("misses when only the verified package version changes", async () => { + const { profileSource, userProject } = await makeProfileAndProject(); + const warbleBin = await makeWarbleStub(); + const loader = await makeLoaderStub("same-shipped-bytes"); + const a = readVerifiedContextLoaderPackage(await makeVerifiedPackage("same-shipped-bytes", "0.1.0")); + const b = readVerifiedContextLoaderPackage(await makeVerifiedPackage("same-shipped-bytes", "0.1.1")); + expect(a).toBeDefined(); + expect(b).toBeDefined(); + const cache = createInMemoryCompileCache(); + const base = { profileSource, userProject, mode: "native" as const, cache, warbleBin, hubDir: "/fixture/hub", contextLoaderBin: loader }; + const first = await compileProfile({ ...base, contextLoaderIdentity: getContextLoaderIdentity(a!) }); + const changedVersion = await compileProfile({ ...base, contextLoaderIdentity: getContextLoaderIdentity(b!) }); + const original = await compileProfile({ ...base, contextLoaderIdentity: getContextLoaderIdentity(a!) }); + expect(first.cacheHit).toBe(false); + expect(changedVersion.cacheHit).toBe(false); + expect(original.cacheHit).toBe(true); + }, 30_000); }); diff --git a/apps/genbi/test/packaging-contract.test.ts b/apps/genbi/test/packaging-contract.test.ts index d69a7b662..3fd3f76f4 100644 --- a/apps/genbi/test/packaging-contract.test.ts +++ b/apps/genbi/test/packaging-contract.test.ts @@ -8,6 +8,7 @@ const repositoryRoot = path.resolve(packageRoot, "..", ".."); const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as { files: string[]; scripts: Record; + dependencies: Record; }; describe("installed-package and vendor-contract CI wiring", () => { @@ -16,6 +17,14 @@ describe("installed-package and vendor-contract CI wiring", () => { expect(packageJson.scripts["check:installed-package"]).toBe("node scripts/installed-package-acceptance.mjs"); }); + it("pins the context loader exactly and runs its no-checkout acceptance on the certified macOS target", () => { + expect(packageJson.dependencies["@wrenai/context-loader"]).toBe("0.1.0"); + const workflow = readFileSync(path.join(repositoryRoot, ".github", "workflows", "genbi-ci.yml"), "utf8"); + expect(workflow).toContain("macOS packed-install context-loader acceptance"); + expect(workflow).toContain("runs-on: macos-14"); + expect(workflow).toContain("pnpm run check:installed-package"); + }); + it("keeps the deterministic vendor-contract entry point explicit", () => { expect(packageJson.scripts["check:vendor-contract"]).toBe("node scripts/run-vendor-contract-probes.mjs"); }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6a23dbeb6..cf814ddeb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4,10 +4,15 @@ settings: autoInstallPeers: true excludeLinksFromLockfile: false +overrides: + '@wrenai/context-loader': link:apps/context-loader + importers: .: {} + apps/context-loader: {} + apps/genbi: dependencies: '@ai-sdk/anthropic': @@ -40,6 +45,9 @@ importers: '@warble/ir-spec': specifier: 0.6.0 version: 0.6.0 + '@wrenai/context-loader': + specifier: link:../context-loader + version: link:../context-loader '@xterm/addon-fit': specifier: ^0.11.0 version: 0.11.0 diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index 0545ceeee..2f55940d1 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -1,6 +1,9 @@ packages: - "apps/*" +overrides: + '@wrenai/context-loader': link:apps/context-loader allowBuilds: + '@wrenai/context-loader': false '@warble/cli': true node-pty: true minimumReleaseAgeExclude: