mirror of
https://github.com/Canner/WrenAI.git
synced 2026-10-16 15:07:41 +08:00
feat(genbi): distribute verified context loader
This commit is contained in:
@@ -12,6 +12,7 @@ on:
|
||||
pull_request:
|
||||
paths:
|
||||
- 'apps/genbi/**'
|
||||
- 'apps/context-loader/**'
|
||||
- 'package.json'
|
||||
- 'pnpm-workspace.yaml'
|
||||
- 'pnpm-lock.yaml'
|
||||
@@ -22,6 +23,7 @@ on:
|
||||
- feat/genbi-app
|
||||
paths:
|
||||
- 'apps/genbi/**'
|
||||
- 'apps/context-loader/**'
|
||||
- 'package.json'
|
||||
- 'pnpm-workspace.yaml'
|
||||
- 'pnpm-lock.yaml'
|
||||
@@ -87,10 +89,33 @@ jobs:
|
||||
- name: Test
|
||||
run: pnpm run test
|
||||
|
||||
installed-package-macos:
|
||||
name: macOS packed-install context-loader acceptance
|
||||
runs-on: macos-14
|
||||
defaults:
|
||||
run:
|
||||
working-directory: apps/genbi
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- name: Setup Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
|
||||
- name: Enable corepack
|
||||
run: corepack enable
|
||||
|
||||
- name: Install workspace dependencies without source-package lifecycle scripts
|
||||
working-directory: .
|
||||
run: pnpm install --frozen-lockfile
|
||||
|
||||
- name: Build packaged application
|
||||
run: pnpm run build
|
||||
|
||||
- name: Packed-install acceptance
|
||||
# This packs the actual artifact, installs it below a fresh temporary
|
||||
# project, and invokes `npx --no-install genbi` with no checkout PATH.
|
||||
# It is deliberately separate from source-tree Vitest coverage.
|
||||
# This packs both real npm artifacts, installs them below a fresh temporary
|
||||
# project, and invokes npx with no checkout PATH or Rust toolchain.
|
||||
run: pnpm run check:installed-package
|
||||
|
||||
vendor-contracts-macos:
|
||||
|
||||
@@ -0,0 +1,98 @@
|
||||
name: Publish context loader
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
- 'context-loader-v*'
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
id-token: write
|
||||
|
||||
jobs:
|
||||
publish-darwin-arm64:
|
||||
name: Build and publish darwin-arm64 context loader
|
||||
runs-on: macos-14
|
||||
environment: npm
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
ref: ${{ github.ref_name }}
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Verify explicit tag version
|
||||
id: version
|
||||
shell: bash
|
||||
run: |
|
||||
version="${GITHUB_REF_NAME#context-loader-v}"
|
||||
test -n "$version"
|
||||
test "$(node -p "require('./apps/context-loader/package.json').version")" = "$version"
|
||||
printf 'version=%s\n' "$version" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Set up locked Rust toolchain
|
||||
uses: dtolnay/rust-toolchain@1.89.0
|
||||
with:
|
||||
targets: aarch64-apple-darwin
|
||||
|
||||
- name: Set up Node
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 24
|
||||
registry-url: https://registry.npmjs.org
|
||||
|
||||
- name: Ensure npm supports trusted publishing
|
||||
run: npm install -g npm@latest
|
||||
|
||||
- name: Test generator source
|
||||
working-directory: core/wren-context-loader
|
||||
run: |
|
||||
cargo fmt --all -- --check
|
||||
cargo clippy --locked --all-targets -- -D warnings
|
||||
cargo test --locked
|
||||
|
||||
- name: Build target archive
|
||||
id: archive
|
||||
shell: bash
|
||||
run: |
|
||||
cargo build --locked --release --target aarch64-apple-darwin --manifest-path core/wren-context-loader/Cargo.toml
|
||||
name="wren-context-loader-${{ steps.version.outputs.version }}-darwin-arm64.tar.gz"
|
||||
mkdir -p release
|
||||
tar -C core/wren-context-loader/target/aarch64-apple-darwin/release -czf "release/$name" wren-context-loader
|
||||
printf 'name=%s\n' "$name" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Create immutable release asset
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
generate_release_notes: true
|
||||
files: release/${{ steps.archive.outputs.name }}
|
||||
|
||||
- name: Generate embedded manifest
|
||||
shell: bash
|
||||
run: |
|
||||
url="https://github.com/${GITHUB_REPOSITORY}/releases/download/${GITHUB_REF_NAME}/${{ steps.archive.outputs.name }}"
|
||||
node apps/context-loader/scripts/generate-release-manifest.mjs apps/context-loader "${{ steps.version.outputs.version }}" "$GITHUB_SHA" "release/${{ steps.archive.outputs.name }}" "$url" > release/context-loader-manifest.json
|
||||
cp release/context-loader-manifest.json "release/${{ steps.archive.outputs.name }}.manifest.json"
|
||||
|
||||
- name: Validate installer contract
|
||||
working-directory: apps/context-loader
|
||||
run: node --test test/*.test.mjs
|
||||
|
||||
- name: Run fresh GenBI package-binding contract
|
||||
run: |
|
||||
corepack enable
|
||||
pnpm install --frozen-lockfile
|
||||
pnpm --filter @wrenai/genbi run build
|
||||
pnpm --filter @wrenai/genbi run check:installed-package
|
||||
|
||||
- name: Pack and publish npm package with provenance
|
||||
working-directory: apps/context-loader
|
||||
run: |
|
||||
npm pack
|
||||
npm publish --provenance --access public
|
||||
|
||||
- name: Upload machine-readable manifest
|
||||
uses: softprops/action-gh-release@v2
|
||||
with:
|
||||
tag_name: ${{ github.ref_name }}
|
||||
files: release/${{ steps.archive.outputs.name }}.manifest.json
|
||||
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"schema": 1,
|
||||
"package": "@wrenai/context-loader",
|
||||
"version": "0.1.0",
|
||||
"artifacts": {}
|
||||
}
|
||||
+13
@@ -0,0 +1,13 @@
|
||||
#!/usr/bin/env node
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { resolveVerifiedBinary } from "../lib/verified.mjs";
|
||||
|
||||
try {
|
||||
const binary = resolveVerifiedBinary();
|
||||
const child = spawnSync(binary, process.argv.slice(2), { stdio: "inherit" });
|
||||
if (child.error) throw child.error;
|
||||
process.exit(child.status ?? 1);
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
export const PACKAGE_NAME = "@wrenai/context-loader";
|
||||
export const STATE_FILE = "install-state.json";
|
||||
export const CANONICAL_BINARY = path.join("bin", "wren-context-loader");
|
||||
|
||||
export class ContextLoaderPackageError extends Error {
|
||||
constructor(code, detail) {
|
||||
super(`@wrenai/context-loader ${code}: ${detail}`);
|
||||
this.name = "ContextLoaderPackageError";
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
export function sha256File(file) {
|
||||
return createHash("sha256").update(readFileSync(file)).digest("hex");
|
||||
}
|
||||
|
||||
export function targetFor(platform = process.platform, arch = process.arch) {
|
||||
return `${platform}-${arch}`;
|
||||
}
|
||||
|
||||
export function packageRootFrom(moduleUrl = import.meta.url) {
|
||||
return path.resolve(path.dirname(fileURLToPath(moduleUrl)), "..");
|
||||
}
|
||||
|
||||
export function readVerifiedState(root = packageRootFrom()) {
|
||||
const packageJson = readJson(path.join(root, "package.json"), "package.json");
|
||||
if (packageJson.name !== PACKAGE_NAME || typeof packageJson.version !== "string") {
|
||||
throw new ContextLoaderPackageError("invalid-package", "package.json does not identify this package");
|
||||
}
|
||||
const state = readJson(path.join(root, STATE_FILE), STATE_FILE);
|
||||
const target = targetFor();
|
||||
if (state.package !== PACKAGE_NAME || state.version !== packageJson.version || state.target !== target) {
|
||||
throw new ContextLoaderPackageError("stale-state", "verification record does not match package version or platform");
|
||||
}
|
||||
if (!isSha256(state.binarySha256)) {
|
||||
throw new ContextLoaderPackageError("invalid-state", "verification record has no binary SHA-256");
|
||||
}
|
||||
if (state.binaryPath !== CANONICAL_BINARY) {
|
||||
throw new ContextLoaderPackageError("invalid-state", "verification record does not name the canonical package-local binary");
|
||||
}
|
||||
const binary = path.resolve(root, state.binaryPath);
|
||||
if (!binary.startsWith(`${root}${path.sep}`) || !existsSync(binary)) {
|
||||
throw new ContextLoaderPackageError("missing-binary", "verified package-local binary is absent");
|
||||
}
|
||||
if (sha256File(binary) !== state.binarySha256) {
|
||||
throw new ContextLoaderPackageError("binary-digest-mismatch", "verified package-local binary was modified");
|
||||
}
|
||||
return { binary, identity: `package:${PACKAGE_NAME}@${packageJson.version}:${target}:${state.binarySha256}`, state };
|
||||
}
|
||||
|
||||
export function resolveVerifiedBinary(root = packageRootFrom()) {
|
||||
return readVerifiedState(root).binary;
|
||||
}
|
||||
|
||||
export function readJson(file, label) {
|
||||
try {
|
||||
return JSON.parse(readFileSync(file, "utf8"));
|
||||
} catch {
|
||||
throw new ContextLoaderPackageError("invalid-state", `${label} is missing or invalid JSON`);
|
||||
}
|
||||
}
|
||||
|
||||
export function isSha256(value) {
|
||||
return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value);
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
{
|
||||
"name": "@wrenai/context-loader",
|
||||
"version": "0.1.0",
|
||||
"description": "Verified native context-loader binary for @wrenai/genbi",
|
||||
"license": "Apache-2.0",
|
||||
"type": "module",
|
||||
"files": [
|
||||
"artifacts.json",
|
||||
"bin",
|
||||
"install-state.json",
|
||||
"lib",
|
||||
"scripts"
|
||||
],
|
||||
"scripts": {
|
||||
"postinstall": "node scripts/postinstall.mjs",
|
||||
"test": "node --test test/*.test.mjs"
|
||||
},
|
||||
"bin": {
|
||||
"wren-context-loader": "bin/wren-context-loader.mjs"
|
||||
},
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const [packageDir, version, sourceCommit, archivePath, assetUrl] = process.argv.slice(2);
|
||||
if (![packageDir, version, sourceCommit, archivePath, assetUrl].every(Boolean)) {
|
||||
throw new Error("usage: generate-release-manifest.mjs <package-dir> <version> <source-commit> <archive> <asset-url>");
|
||||
}
|
||||
const archive = await readFile(archivePath);
|
||||
const binaryPath = "wren-context-loader";
|
||||
const binary = await readBinaryFromTarGz(archive, binaryPath);
|
||||
const sha256 = (data) => createHash("sha256").update(data).digest("hex");
|
||||
const manifest = {
|
||||
schema: 1,
|
||||
package: "@wrenai/context-loader",
|
||||
version,
|
||||
sourceCommit,
|
||||
artifacts: {
|
||||
"darwin-arm64": { url: assetUrl, archiveSha256: sha256(archive), binarySha256: sha256(binary), binaryPath },
|
||||
},
|
||||
};
|
||||
await writeFile(path.join(packageDir, "artifacts.json"), `${JSON.stringify(manifest, null, 2)}\n`);
|
||||
process.stdout.write(`${JSON.stringify(manifest)}\n`);
|
||||
|
||||
async function readBinaryFromTarGz(archive, expectedPath) {
|
||||
const { gunzipSync } = await import("node:zlib");
|
||||
const tar = gunzipSync(archive);
|
||||
for (let offset = 0; offset + 512 <= tar.length;) {
|
||||
const header = tar.subarray(offset, offset + 512);
|
||||
if (header.every((byte) => byte === 0)) break;
|
||||
const name = header.subarray(0, 100).toString("utf8").replace(/\0.*$/u, "");
|
||||
const size = Number.parseInt(header.subarray(124, 136).toString("utf8").replace(/\0.*$/u, "").trim() || "0", 8);
|
||||
if (!Number.isSafeInteger(size) || size < 0 || offset + 512 + size > tar.length) throw new Error("invalid release archive layout");
|
||||
if (name === expectedPath) return tar.subarray(offset + 512, offset + 512 + size);
|
||||
offset += 512 + Math.ceil(size / 512) * 512;
|
||||
}
|
||||
throw new Error(`release archive lacks ${expectedPath}`);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { gunzipSync } from "node:zlib";
|
||||
import { CANONICAL_BINARY, ContextLoaderPackageError, isSha256, readJson, sha256File, targetFor } from "../lib/verified.mjs";
|
||||
|
||||
/** Downloads, verifies, and atomically installs the exact manifest row for this package. */
|
||||
export async function installContextLoader({ packageRoot, fetchImpl = fetch, platform = process.platform, arch = process.arch } = {}) {
|
||||
if (typeof packageRoot !== "string") throw new TypeError("packageRoot is required");
|
||||
const packageJson = readJson(path.join(packageRoot, "package.json"), "package.json");
|
||||
const manifest = readJson(path.join(packageRoot, "artifacts.json"), "artifacts.json");
|
||||
const target = targetFor(platform, arch);
|
||||
const artifact = manifest.artifacts?.[target];
|
||||
validateManifest(packageJson, manifest, artifact, target);
|
||||
|
||||
const binary = path.join(packageRoot, CANONICAL_BINARY);
|
||||
const statePath = path.join(packageRoot, "install-state.json");
|
||||
const reusable = await existingVerified(binary, statePath, packageJson.version, target, artifact.binarySha256);
|
||||
if (reusable) return { binary, reused: true };
|
||||
|
||||
const staging = `${binary}.tmp-${process.pid}-${Date.now()}`;
|
||||
try {
|
||||
const response = await fetchImpl(artifact.url);
|
||||
if (!response?.ok) throw new ContextLoaderPackageError("download-failed", `could not download the ${target} artifact`);
|
||||
const archive = Buffer.from(await response.arrayBuffer());
|
||||
if (sha256(archive) !== artifact.archiveSha256) {
|
||||
throw new ContextLoaderPackageError("archive-digest-mismatch", `downloaded ${target} artifact did not match its SHA-256`);
|
||||
}
|
||||
const content = extractSingleTarGz(archive, artifact.binaryPath);
|
||||
if (sha256(content) !== artifact.binarySha256) {
|
||||
throw new ContextLoaderPackageError("binary-digest-mismatch", `extracted ${target} binary did not match its SHA-256`);
|
||||
}
|
||||
await mkdir(path.dirname(binary), { recursive: true });
|
||||
await writeFile(staging, content, { mode: 0o755 });
|
||||
await chmod(staging, 0o755);
|
||||
await rename(staging, binary);
|
||||
const state = { package: packageJson.name, version: packageJson.version, target, archiveSha256: artifact.archiveSha256, binarySha256: artifact.binarySha256, binaryPath: CANONICAL_BINARY };
|
||||
const stateTmp = `${statePath}.tmp-${process.pid}-${Date.now()}`;
|
||||
await writeFile(stateTmp, `${JSON.stringify(state)}\n`, { mode: 0o600 });
|
||||
await rename(stateTmp, statePath);
|
||||
return { binary, reused: false };
|
||||
} catch (error) {
|
||||
await rm(staging, { force: true });
|
||||
throw error instanceof ContextLoaderPackageError ? error : new ContextLoaderPackageError("download-failed", "could not download the required artifact (offline cache miss)");
|
||||
}
|
||||
}
|
||||
|
||||
function validateManifest(packageJson, manifest, artifact, target) {
|
||||
if (manifest.schema !== 1 || manifest.package !== packageJson.name || manifest.version !== packageJson.version) {
|
||||
throw new ContextLoaderPackageError("invalid-manifest", "manifest does not match package identity");
|
||||
}
|
||||
if (!artifact) throw new ContextLoaderPackageError("unsupported-platform", `${target} has no certified artifact row`);
|
||||
if (typeof artifact.url !== "string" || !artifact.url.startsWith("https://") || !isSha256(artifact.archiveSha256) || !isSha256(artifact.binarySha256) || typeof artifact.binaryPath !== "string" || artifact.binaryPath.includes("..") || path.isAbsolute(artifact.binaryPath)) {
|
||||
throw new ContextLoaderPackageError("invalid-manifest", `${target} row is incomplete or unsafe`);
|
||||
}
|
||||
}
|
||||
|
||||
async function existingVerified(binary, statePath, version, target, digest) {
|
||||
try {
|
||||
const state = JSON.parse(await readFile(statePath, "utf8"));
|
||||
return state.version === version && state.target === target && state.binaryPath === CANONICAL_BINARY && state.binarySha256 === digest && sha256File(binary) === digest;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function extractSingleTarGz(archive, expectedPath) {
|
||||
let tar;
|
||||
try { tar = gunzipSync(archive); } catch { throw new ContextLoaderPackageError("invalid-archive", "artifact is not a gzip tar archive"); }
|
||||
let offset = 0;
|
||||
let found;
|
||||
while (offset + 512 <= tar.length) {
|
||||
const header = tar.subarray(offset, offset + 512);
|
||||
if (header.every((byte) => byte === 0)) break;
|
||||
const name = header.subarray(0, 100).toString("utf8").replace(/\0.*$/u, "");
|
||||
const sizeText = header.subarray(124, 136).toString("utf8").replace(/\0.*$/u, "").trim();
|
||||
const size = Number.parseInt(sizeText || "0", 8);
|
||||
const type = String.fromCharCode(header[156] || 48);
|
||||
if (!Number.isSafeInteger(size) || size < 0 || offset + 512 + size > tar.length) throw new ContextLoaderPackageError("invalid-archive", "artifact has an invalid tar layout");
|
||||
if (name === expectedPath) {
|
||||
if (type !== "0" || found !== undefined) throw new ContextLoaderPackageError("invalid-archive", "artifact has an unexpected binary layout");
|
||||
found = Buffer.from(tar.subarray(offset + 512, offset + 512 + size));
|
||||
}
|
||||
offset += 512 + Math.ceil(size / 512) * 512;
|
||||
}
|
||||
if (found === undefined) throw new ContextLoaderPackageError("invalid-archive", "artifact does not contain the declared binary path");
|
||||
return found;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { installContextLoader } from "./installer.mjs";
|
||||
|
||||
const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
try {
|
||||
await installContextLoader({ packageRoot });
|
||||
} catch (error) {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,78 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { gzipSync } from "node:zlib";
|
||||
import { installContextLoader } from "../scripts/installer.mjs";
|
||||
import { readVerifiedState } from "../lib/verified.mjs";
|
||||
|
||||
const sha256 = (content) => createHash("sha256").update(content).digest("hex");
|
||||
|
||||
function tarEntry(name, content) {
|
||||
const header = Buffer.alloc(512);
|
||||
header.write(name);
|
||||
header.write(content.length.toString(8).padStart(11, "0"), 124);
|
||||
header[135] = 0;
|
||||
header[156] = "0".charCodeAt(0);
|
||||
header.write("ustar", 257);
|
||||
return Buffer.concat([header, content, Buffer.alloc((512 - (content.length % 512)) % 512), Buffer.alloc(1024)]);
|
||||
}
|
||||
|
||||
async function fixture({ artifact = Buffer.from("#!/bin/sh\necho loader\n"), target = "darwin-arm64", row = {} } = {}) {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "context-loader-package-"));
|
||||
const archive = gzipSync(tarEntry("wren-context-loader", artifact));
|
||||
const manifest = {
|
||||
schema: 1,
|
||||
package: "@wrenai/context-loader",
|
||||
version: "0.1.0",
|
||||
artifacts: {
|
||||
[target]: {
|
||||
url: "https://example.invalid/context-loader.tar.gz",
|
||||
archiveSha256: sha256(archive),
|
||||
binarySha256: sha256(artifact),
|
||||
binaryPath: "wren-context-loader",
|
||||
...row,
|
||||
},
|
||||
},
|
||||
};
|
||||
await writeFile(path.join(root, "package.json"), JSON.stringify({ name: manifest.package, version: manifest.version }));
|
||||
await writeFile(path.join(root, "artifacts.json"), JSON.stringify(manifest));
|
||||
return { root, archive, artifact };
|
||||
}
|
||||
|
||||
const fetchArchive = (archive) => async () => ({ ok: true, arrayBuffer: async () => archive });
|
||||
|
||||
test("installs a digest-verified darwin-arm64 binary atomically and records canonical state", async () => {
|
||||
const { root, archive, artifact } = await fixture();
|
||||
const first = await installContextLoader({ packageRoot: root, fetchImpl: fetchArchive(archive), platform: "darwin", arch: "arm64" });
|
||||
assert.equal(first.reused, false);
|
||||
assert.deepEqual(await readFile(first.binary), artifact);
|
||||
const state = readVerifiedState(root);
|
||||
assert.equal(state.binary, first.binary);
|
||||
assert.match(state.identity, /package:@wrenai\/context-loader@0\.1\.0:darwin-arm64:[a-f0-9]{64}/);
|
||||
const second = await installContextLoader({ packageRoot: root, fetchImpl: async () => { throw new Error("offline"); }, platform: "darwin", arch: "arm64" });
|
||||
assert.equal(second.reused, true);
|
||||
});
|
||||
|
||||
test("fails closed for unsupported targets, archive tampering, and binary digest tampering", async () => {
|
||||
const supported = await fixture();
|
||||
await assert.rejects(
|
||||
installContextLoader({ packageRoot: supported.root, fetchImpl: fetchArchive(supported.archive), platform: "linux", arch: "x64" }),
|
||||
/unsupported-platform/,
|
||||
);
|
||||
const archiveTampered = await fixture({ row: { archiveSha256: "0".repeat(64) } });
|
||||
await assert.rejects(installContextLoader({ packageRoot: archiveTampered.root, fetchImpl: fetchArchive(archiveTampered.archive), platform: "darwin", arch: "arm64" }), /archive-digest-mismatch/);
|
||||
const binaryTampered = await fixture({ row: { binarySha256: "0".repeat(64) } });
|
||||
await assert.rejects(installContextLoader({ packageRoot: binaryTampered.root, fetchImpl: fetchArchive(binaryTampered.archive), platform: "darwin", arch: "arm64" }), /binary-digest-mismatch/);
|
||||
});
|
||||
|
||||
test("rejects stale or altered installed state at runtime", async () => {
|
||||
const { root, archive } = await fixture();
|
||||
await installContextLoader({ packageRoot: root, fetchImpl: fetchArchive(archive), platform: "darwin", arch: "arm64" });
|
||||
const statePath = path.join(root, "install-state.json");
|
||||
const state = JSON.parse(await readFile(statePath, "utf8"));
|
||||
await writeFile(statePath, JSON.stringify({ ...state, version: "0.1.1" }));
|
||||
assert.throws(() => readVerifiedState(root), /stale-state/);
|
||||
});
|
||||
+23
-22
@@ -17,20 +17,18 @@ answers (tables, charts, KPIs, narrative text).
|
||||
|
||||
## Supported platforms
|
||||
|
||||
`@wrenai/genbi` depends on `@warble/cli`, a Rust binary fetched from GitHub
|
||||
Releases during install. That binary is published for exactly these four
|
||||
platforms:
|
||||
`@wrenai/genbi` has one additional native dependency for user-project binding:
|
||||
`@wrenai/context-loader`. Its initial certified target is macOS Apple Silicon,
|
||||
so the GenBI package is currently supported only on that platform. This limit is
|
||||
intentional; the Warble package supports more platforms, but GenBI must not
|
||||
claim a target until its context-loader artifact has been built and verified.
|
||||
|
||||
| OS | Architecture |
|
||||
| --- | --- |
|
||||
| macOS | Apple Silicon (`aarch64-apple-darwin`) |
|
||||
| macOS | Intel (`x86_64-apple-darwin`) |
|
||||
| Linux (glibc ≥ 2.31) | x86_64 (`x86_64-unknown-linux-gnu`) |
|
||||
| Linux (glibc ≥ 2.31) | aarch64 (`aarch64-unknown-linux-gnu`) |
|
||||
| macOS | Apple Silicon (`darwin-arm64`) |
|
||||
|
||||
**Windows is not supported.** **musl-based Linux (e.g. Alpine) is not
|
||||
supported**, nor is glibc older than 2.31 — install fails outright on these;
|
||||
see [Troubleshooting](#troubleshooting).
|
||||
**Intel macOS, Linux, and Windows are not supported.** Installation fails
|
||||
before GenBI starts; see [Troubleshooting](#troubleshooting).
|
||||
|
||||
## Prerequisites
|
||||
|
||||
@@ -42,8 +40,9 @@ see [Troubleshooting](#troubleshooting).
|
||||
`@wrenai/genbi` shells out to it for live questions and context
|
||||
inspection; it does not install or configure `wren` for you. See the
|
||||
[Wren project](https://github.com/Canner/WrenAI).
|
||||
- **Network access to `github.com`** during install (to fetch the Warble
|
||||
binary) and on first use (see [Network](#network) below). Fully offline /
|
||||
- **Network access to `github.com`** during install (to fetch the Warble and
|
||||
exact context-loader binaries) and on first use (see [Network](#network)
|
||||
below). Fully offline /
|
||||
air-gapped installs are not supported and are out of scope for this guide.
|
||||
|
||||
## Install
|
||||
@@ -60,10 +59,11 @@ Or install it once, if you would rather not re-fetch:
|
||||
npm install -g @wrenai/genbi
|
||||
```
|
||||
|
||||
Either way, `npm` runs `@warble/cli`'s postinstall script, which downloads the
|
||||
Warble binary for your platform (about 12 MB) from GitHub Releases and verifies
|
||||
its checksum against the release manifest. That is where an unsupported platform
|
||||
fails, before any of the app's own code runs.
|
||||
Either way, `npm` runs the binary-package postinstall scripts. The context
|
||||
loader downloads its exact manifest row, verifies archive and binary SHA-256,
|
||||
then writes the package-local executable atomically. Unsupported targets,
|
||||
offline first installs, altered downloads, and incomplete extraction fail
|
||||
before any app code runs.
|
||||
|
||||
## Run
|
||||
|
||||
@@ -118,7 +118,10 @@ here because they aren't needed for a first install and run.
|
||||
1. **At install time** — `@warble/cli`'s postinstall script downloads the
|
||||
Warble executable for your platform (~12 MB) from a GitHub Release and
|
||||
checksum-verifies it.
|
||||
2. **At runtime, on first use** — the first time the app compiles a
|
||||
2. **At install time** — `@wrenai/context-loader` downloads the exact
|
||||
darwin-arm64 generator artifact and verifies its archive and binary
|
||||
digests. Runtime never downloads it.
|
||||
3. **At runtime, on first use** — the first time the app compiles a
|
||||
profile, it downloads a Hub component archive from GitHub. After that
|
||||
first download it's cached locally and no further network access to
|
||||
GitHub is needed for subsequent compiles.
|
||||
@@ -127,11 +130,9 @@ Offline or air-gapped environments are not supported by this guide.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
**"Platform ... is not supported by @warble/cli"** (during `npm install`) —
|
||||
your OS/architecture isn't one of the four supported platforms listed
|
||||
above. This includes Windows and musl-based Linux distributions (e.g.
|
||||
Alpine), where the installer additionally reports a libc mismatch before
|
||||
failing. There is no workaround; a matching platform is required.
|
||||
**"@wrenai/context-loader unsupported-platform"** (during `npm install`) —
|
||||
your OS/architecture is not the certified macOS Apple Silicon target. There is
|
||||
no fallback to a nearby architecture or an ambient binary.
|
||||
|
||||
**Node version errors, or unexpected syntax/runtime errors on startup** —
|
||||
confirm your Node version is 20 or later (`node --version`). Older
|
||||
|
||||
+17
-11
@@ -128,18 +128,24 @@ Verify a regeneration rather than trusting it: recompiling an unchanged profile
|
||||
did not intend it to means something else moved — the pin, the Hub, or the generator — and the diff
|
||||
should be understood before it is committed.
|
||||
|
||||
### Open question: distributing the generator
|
||||
### Distributing the context-loader generator
|
||||
|
||||
`wren-context-loader` is resolved from an **in-repo Rust build** (or an explicit
|
||||
`WREN_HARNESS_CONTEXT_LOADER_BIN` override), and resolution loud-fails when neither is present.
|
||||
That is sufficient for development in this monorepo and for CI, but it is **not** a distribution
|
||||
story: an installed `@wrenai/genbi` has no Rust toolchain and no `core/` tree, so a
|
||||
user-project-bound compile cannot resolve the generator at all. Warble faced the same problem with
|
||||
its own compiler and solved it by publishing a pinned `@warble/cli` npm package whose postinstall
|
||||
downloads a prebuilt platform binary. The analogue for this generator — publish a
|
||||
platform-binary npm package, vendor prebuilt binaries, statically link it into an existing
|
||||
distributed artifact, or drop the separate binary and expose the projection some other way — is a
|
||||
real packaging decision with release-process consequences, and is deliberately **not** settled here.
|
||||
`@wrenai/genbi` pins `@wrenai/context-loader` to one exact version. The package postinstall
|
||||
downloads only the immutable manifest row for the current target, verifies both archive and binary
|
||||
SHA-256 values, writes the canonical package-local executable atomically, and records the verified
|
||||
package/version/target/digest tuple. Runtime re-verifies that record before execution; it never
|
||||
downloads a binary on first bind or uses PATH.
|
||||
|
||||
The first certified row is `darwin-arm64`. Treat every other target as unsupported until it has a
|
||||
reviewed artifact row and release evidence. The context-loader version is independent of both the
|
||||
GenBI application and managed Wren runtime: release it only from an explicit `context-loader-v*`
|
||||
tag using `.github/workflows/publish-context-loader.yml`, then separately bump GenBI's exact pin.
|
||||
|
||||
Development order remains explicit `WREN_HARNESS_CONTEXT_LOADER_BIN`, then an in-repo Rust build,
|
||||
then the verified package. Neither a missing/altered package record nor a failed development
|
||||
override may fall through to a legacy binding. The compile cache identity includes that resolver
|
||||
kind and the package tuple or development binary digest, so changing the shipped generator moves
|
||||
the key.
|
||||
|
||||
### Why `@warble/ir-spec` stays behind
|
||||
|
||||
|
||||
+11
-8
@@ -22,20 +22,23 @@ Beyond the prerequisites in [README.md](./README.md#prerequisites):
|
||||
local and gateway runtimes are not accepted. Claude is by far the better
|
||||
exercised of the two — see [Codex status](#codex-status).
|
||||
- The Wren CLI on `PATH`:
|
||||
- **The `wren-context-loader` generator, built from this repo.** Binding a user's project
|
||||
compiles a prepared-context document first, and the generator is a Rust binary rather than an
|
||||
npm dependency, so `pnpm install` does not supply it. Build it once with
|
||||
`cargo build --release --manifest-path ../../core/wren-context-loader/Cargo.toml`, or point
|
||||
`WREN_HARNESS_CONTEXT_LOADER_BIN` at an existing build. Without either, the first user-project
|
||||
bind fails loudly — by design, since a silent fallback would leave no way to tell which path
|
||||
produced an IR. Restart the BFF after rebuilding the binary: its identity is memoized for the
|
||||
process lifetime.
|
||||
|
||||
```bash
|
||||
uv tool install wrenai
|
||||
wren --version
|
||||
```
|
||||
|
||||
- For an in-repo user-project bind, build the `wren-context-loader` generator once:
|
||||
|
||||
```bash
|
||||
cargo build --release --manifest-path ../../core/wren-context-loader/Cargo.toml
|
||||
```
|
||||
|
||||
An explicit `WREN_HARNESS_CONTEXT_LOADER_BIN` overrides that development build. Installed
|
||||
packages instead use their exact pinned, digest-verified `@wrenai/context-loader` dependency.
|
||||
There is no PATH or legacy-binding fallback. Restart the BFF after rebuilding the development
|
||||
binary because its identity is memoized for the process lifetime.
|
||||
|
||||
|
||||
## Where Warble comes from
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { existsSync } from "node:fs";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { createHash } from "node:crypto";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import path from "node:path";
|
||||
import { ContextLoaderFailedError, ContextLoaderNotFoundError } from "./errors.js";
|
||||
import { resolveInstalledPackageRoot } from "../npm-package-resolve.js";
|
||||
|
||||
/** Environment override for {@link resolveContextLoaderBinary}, named like the other `WREN_HARNESS_*` binary overrides. */
|
||||
const CONTEXT_LOADER_BIN_ENV = "WREN_HARNESS_CONTEXT_LOADER_BIN";
|
||||
@@ -18,18 +20,30 @@ const IN_REPO_BUILD_PATHS = [
|
||||
* document — guards against a hang, not meant to ever fire in normal operation.
|
||||
*/
|
||||
const CONTEXT_LOADER_TIMEOUT_MS = 60 * 1000;
|
||||
const CONTEXT_LOADER_PACKAGE = "@wrenai/context-loader";
|
||||
const CANONICAL_PACKAGE_BINARY = path.join("bin", "wren-context-loader");
|
||||
|
||||
export interface ResolvedContextLoader {
|
||||
readonly bin: string;
|
||||
readonly kind: "explicit" | "in-repo" | "package";
|
||||
/** Package resolution is already digest-verified. Development modes use the binary hash below. */
|
||||
readonly verifiedIdentity?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolves the `wren-context-loader` binary — the generator that renders a wren project as the
|
||||
* prepared-context document a `kind: prepared` binding reads.
|
||||
*
|
||||
* Two tiers, then a loud failure:
|
||||
* Three deterministic tiers, then a loud failure:
|
||||
*
|
||||
* 1. `explicit`, else the `WREN_HARNESS_CONTEXT_LOADER_BIN` environment variable. Either must
|
||||
* exist on disk; neither falls through to a search.
|
||||
* 2. This repo's own build of the generator crate, found by walking up from this package looking
|
||||
* for `core/wren-context-loader/target/{release,debug}/wren-context-loader`. `release` wins
|
||||
* when both exist, so a stale `debug` build never shadows a deliberate release build.
|
||||
* 3. The verified package-local binary from the exact `@wrenai/context-loader` dependency. Its
|
||||
* install record must match package version and target, name the canonical binary, and pass a
|
||||
* fresh SHA-256 check. Resolution never downloads at runtime.
|
||||
*
|
||||
* There is deliberately **no** `PATH` tier and — more importantly — **no fallback to the old
|
||||
* `wren_project` binding**. A fallback would make the extraction unverifiable: a compiled IR is
|
||||
@@ -39,27 +53,85 @@ const CONTEXT_LOADER_TIMEOUT_MS = 60 * 1000;
|
||||
*
|
||||
* Throws {@link ContextLoaderNotFoundError} with the accumulated attempts when neither tier works.
|
||||
*/
|
||||
export function resolveContextLoaderBinary(explicit?: string): string {
|
||||
export function resolveContextLoader(explicit?: string): ResolvedContextLoader {
|
||||
const override = explicit ?? process.env[CONTEXT_LOADER_BIN_ENV];
|
||||
if (override !== undefined && override.length > 0) {
|
||||
if (!existsSync(override)) {
|
||||
const source = explicit !== undefined ? `explicit contextLoaderBin` : CONTEXT_LOADER_BIN_ENV;
|
||||
throw new ContextLoaderNotFoundError([`${source} "${override}" does not exist`]);
|
||||
}
|
||||
return override;
|
||||
return { bin: override, kind: "explicit" };
|
||||
}
|
||||
|
||||
const inRepo = findInRepoBuild();
|
||||
if (inRepo !== undefined) return inRepo;
|
||||
if (inRepo !== undefined) return { bin: inRepo, kind: "in-repo" };
|
||||
|
||||
const packaged = resolveVerifiedPackageBinary();
|
||||
if (packaged !== undefined) return packaged;
|
||||
|
||||
throw new ContextLoaderNotFoundError([
|
||||
`${CONTEXT_LOADER_BIN_ENV} is not set`,
|
||||
`no in-repo build found (searched ancestors of this package for ` +
|
||||
IN_REPO_BUILD_PATHS.map((segments) => `"${path.join(...segments)}"`).join(" and ") +
|
||||
`) — build it with "cargo build --release --manifest-path core/wren-context-loader/Cargo.toml"`,
|
||||
`) — ${CONTEXT_LOADER_PACKAGE} is not installed with a verified ${targetForCurrentPlatform()} binary`,
|
||||
]);
|
||||
}
|
||||
|
||||
/** Backward-compatible path-only API for callers that do not need cache provenance. */
|
||||
export function resolveContextLoaderBinary(explicit?: string): string {
|
||||
return resolveContextLoader(explicit).bin;
|
||||
}
|
||||
|
||||
/** Stable cache identity: resolver kind plus either a verified package tuple or the dev binary's bytes. */
|
||||
export function getContextLoaderIdentity(resolved: ResolvedContextLoader): string {
|
||||
if (resolved.verifiedIdentity !== undefined) return resolved.verifiedIdentity;
|
||||
return `${resolved.kind}:${createHash("sha256").update(readFileSync(resolved.bin)).digest("hex")}`;
|
||||
}
|
||||
|
||||
function resolveVerifiedPackageBinary(): ResolvedContextLoader | undefined {
|
||||
const packageRoot = resolveInstalledPackageRoot(CONTEXT_LOADER_PACKAGE);
|
||||
if (packageRoot === undefined) return undefined;
|
||||
return readVerifiedContextLoaderPackage(packageRoot);
|
||||
}
|
||||
|
||||
/** Validates a package-local installation record without resolving it from the host environment. */
|
||||
export function readVerifiedContextLoaderPackage(packageRoot: string, target = targetForCurrentPlatform()): ResolvedContextLoader | undefined {
|
||||
try {
|
||||
const packageJson = readJson(path.join(packageRoot, "package.json"));
|
||||
const state = readJson(path.join(packageRoot, "install-state.json"));
|
||||
const bin = path.resolve(packageRoot, CANONICAL_PACKAGE_BINARY);
|
||||
if (
|
||||
packageJson.name !== CONTEXT_LOADER_PACKAGE ||
|
||||
typeof packageJson.version !== "string" ||
|
||||
state.package !== CONTEXT_LOADER_PACKAGE ||
|
||||
state.version !== packageJson.version ||
|
||||
state.target !== target ||
|
||||
state.binaryPath !== CANONICAL_PACKAGE_BINARY ||
|
||||
!isSha256(state.binarySha256) ||
|
||||
!bin.startsWith(`${packageRoot}${path.sep}`) ||
|
||||
!existsSync(bin) ||
|
||||
createHash("sha256").update(readFileSync(bin)).digest("hex") !== state.binarySha256
|
||||
) {
|
||||
return undefined;
|
||||
}
|
||||
return { bin, kind: "package", verifiedIdentity: `package:${CONTEXT_LOADER_PACKAGE}@${packageJson.version}:${target}:${state.binarySha256}` };
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function readJson(file: string): Record<string, unknown> {
|
||||
return JSON.parse(readFileSync(file, "utf8")) as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function isSha256(value: unknown): value is string {
|
||||
return typeof value === "string" && /^[a-f0-9]{64}$/u.test(value);
|
||||
}
|
||||
|
||||
function targetForCurrentPlatform(): string {
|
||||
return `${process.platform}-${process.arch}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Walks up from this package's own directory (`harness/compile/` in source, `dist/compile/` once
|
||||
* built — either way two levels below the package root), returning the first existing generator
|
||||
|
||||
@@ -44,7 +44,7 @@ export class ContextLoaderNotFoundError extends Error {
|
||||
super(
|
||||
`could not resolve the "wren-context-loader" binary:\n` +
|
||||
attempts.map((attempt) => ` - ${attempt}`).join("\n") +
|
||||
`\nfix: build it in-repo, from the repository root ` +
|
||||
`\nfix: install the exact @wrenai/context-loader package, build it in-repo from the repository root ` +
|
||||
`("cargo build --release --manifest-path core/wren-context-loader/Cargo.toml") ` +
|
||||
`or point WREN_HARNESS_CONTEXT_LOADER_BIN at an existing build.`,
|
||||
);
|
||||
|
||||
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { createFileSystemCompileCache } from "./cache.js";
|
||||
import { composeUserProfile } from "./compose-profile.js";
|
||||
import { resolveContextLoaderBinary } from "./context-loader.js";
|
||||
import { getContextLoaderIdentity, resolveContextLoader } from "./context-loader.js";
|
||||
import { WarbleCommandFailedError } from "./errors.js";
|
||||
import { hashDirectory, hashFiles } from "./fingerprint.js";
|
||||
import { resolveHubDir, resolveWarbleBinary } from "./resolve-binary.js";
|
||||
@@ -91,21 +91,21 @@ export async function compileProfile(options: CompileProfileOptions): Promise<Co
|
||||
// Resolving the generator is memoized here the same way `ensureWarbleBin` memoizes warble below:
|
||||
// both the identity computation (a cache-key input) and the compose step (on a miss) need it, and
|
||||
// resolution loud-fails rather than degrading to Warble's built-in adapter.
|
||||
let resolvedLoaderBin: string | undefined;
|
||||
const ensureContextLoaderBin = (): string => {
|
||||
resolvedLoaderBin ??= resolveContextLoaderBinary(options.contextLoaderBin);
|
||||
return resolvedLoaderBin;
|
||||
let resolvedLoader: ReturnType<typeof resolveContextLoader> | undefined;
|
||||
const ensureContextLoader = (): ReturnType<typeof resolveContextLoader> => {
|
||||
resolvedLoader ??= resolveContextLoader(options.contextLoaderBin);
|
||||
return resolvedLoader;
|
||||
};
|
||||
|
||||
return compileProfileSource(
|
||||
options,
|
||||
await hashDirectory(path.resolve(options.userProject)),
|
||||
async () => options.contextLoaderIdentity ?? (await getBinaryIdentity(ensureContextLoaderBin())),
|
||||
async () => options.contextLoaderIdentity ?? getContextLoaderIdentity(ensureContextLoader()),
|
||||
(workDir) => composeUserProfile({
|
||||
profileSource: options.profileSource,
|
||||
userProject: options.userProject,
|
||||
destDir: workDir,
|
||||
contextLoaderBin: ensureContextLoaderBin(),
|
||||
contextLoaderBin: ensureContextLoader().bin,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -96,7 +96,8 @@ export interface CompileProfileOptions {
|
||||
/**
|
||||
* Explicit path to the `wren-context-loader` binary. See `resolveContextLoaderBinary` for the
|
||||
* resolution order when omitted (explicit → `WREN_HARNESS_CONTEXT_LOADER_BIN` → in-repo build →
|
||||
* loud failure; there is no fallback to Warble's built-in `wren_project` adapter).
|
||||
* verified exact `@wrenai/context-loader` package → loud failure; there is no fallback to
|
||||
* Warble's built-in `wren_project` adapter).
|
||||
*/
|
||||
readonly contextLoaderBin?: string;
|
||||
/**
|
||||
|
||||
@@ -22,18 +22,29 @@ import path from "node:path";
|
||||
*/
|
||||
export function resolveInstalledPackageBin(packageName: string, binName: string): string | undefined {
|
||||
try {
|
||||
const require = createRequire(import.meta.url);
|
||||
const pkgJsonPath = require.resolve(`${packageName}/package.json`);
|
||||
const pkg = JSON.parse(readFileSync(pkgJsonPath, "utf8")) as { bin?: string | Record<string, string> };
|
||||
const pkgJsonPath = resolveInstalledPackageRoot(packageName);
|
||||
if (pkgJsonPath === undefined) return undefined;
|
||||
const manifestPath = path.join(pkgJsonPath, "package.json");
|
||||
const pkg = JSON.parse(readFileSync(manifestPath, "utf8")) as { bin?: string | Record<string, string> };
|
||||
const binField = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.[binName];
|
||||
if (binField === undefined) return undefined;
|
||||
const resolved = path.resolve(path.dirname(pkgJsonPath), binField);
|
||||
const resolved = path.resolve(pkgJsonPath, binField);
|
||||
return existsSync(resolved) ? resolved : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** Returns an installed package's canonical root through Node resolution, never by walking PATH. */
|
||||
export function resolveInstalledPackageRoot(packageName: string): string | undefined {
|
||||
try {
|
||||
const require = createRequire(import.meta.url);
|
||||
return path.dirname(require.resolve(`${packageName}/package.json`));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Named, explicit opt-in that re-enables the legacy "sibling `warble` git checkout"
|
||||
* development-mode resolution tier in `resolveWarbleBinary`, `resolveAgentSdkCli`, and
|
||||
|
||||
@@ -48,6 +48,7 @@
|
||||
"@ai-sdk/provider": "~4.0.3",
|
||||
"@ant-design/icons": "^6.3.2",
|
||||
"@hono/node-server": "^2.0.10",
|
||||
"@wrenai/context-loader": "0.1.0",
|
||||
"@warble/claude-agent-sdk": "0.10.0",
|
||||
"@warble/cli": "0.10.0",
|
||||
"@warble/codex-local": "0.10.0",
|
||||
|
||||
@@ -9,18 +9,23 @@
|
||||
* minimal PATH and a require hook that turns any source-checkout read into a
|
||||
* hard failure, so a checkout-only fallback cannot make this pass.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
import { existsSync } from "node:fs";
|
||||
import { mkdtemp, mkdir, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { chmod, cp, mkdtemp, mkdir, readFile, readdir, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer as createHttpServer } from "node:http";
|
||||
import { createServer } from "node:net";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { closeServerBounded, runBounded, spawnProcessGroup, stopProcessTree } from "./process-cleanup.mjs";
|
||||
import { readSseFrames } from "./sse-frames.mjs";
|
||||
|
||||
const packageRoot = path.resolve(process.cwd());
|
||||
const contextLoaderSource = path.resolve(packageRoot, "..", "context-loader");
|
||||
const tempRoot = await mkdtemp(path.join(os.tmpdir(), "genbi-installed-package-"));
|
||||
const packDirectory = path.join(tempRoot, "pack");
|
||||
const contextLoaderPackDirectory = path.join(tempRoot, "context-loader-pack");
|
||||
const contextLoaderStage = path.join(tempRoot, "context-loader-stage");
|
||||
const installRoot = path.join(tempRoot, "fresh-install");
|
||||
const workspaceRoot = path.join(tempRoot, "workspace");
|
||||
const sourceAuditHook = path.join(tempRoot, "block-checkout-access.cjs");
|
||||
@@ -31,21 +36,31 @@ let phase = "initialize";
|
||||
|
||||
try {
|
||||
markPhase("pack");
|
||||
await Promise.all([mkdir(packDirectory), mkdir(installRoot), mkdir(workspaceRoot)]);
|
||||
await Promise.all([mkdir(packDirectory), mkdir(contextLoaderPackDirectory), mkdir(installRoot), mkdir(workspaceRoot)]);
|
||||
await run("pnpm", ["pack", "--pack-destination", packDirectory], { cwd: packageRoot });
|
||||
const contextLoaderTarball = await packVerifiedContextLoaderFixture(contextLoaderSource, contextLoaderStage, contextLoaderPackDirectory);
|
||||
|
||||
const packageTarball = await onlyTarball(packDirectory);
|
||||
const packedFiles = await tarFiles(packageTarball);
|
||||
assertPublishedFiles(packedFiles);
|
||||
const packedManifest = await tarJson(packageTarball, "package/package.json");
|
||||
if (packedManifest.dependencies?.["@wrenai/context-loader"] !== "0.1.0") {
|
||||
throw new Error("packed @wrenai/genbi does not retain an exact @wrenai/context-loader version");
|
||||
}
|
||||
|
||||
markPhase("install");
|
||||
await run("npm", ["init", "--yes"], { cwd: installRoot });
|
||||
await run("npm", ["install", "--no-audit", "--no-fund", packageTarball], { cwd: installRoot });
|
||||
await run("npm", ["install", "--no-audit", "--no-fund", packageTarball, contextLoaderTarball], { cwd: installRoot });
|
||||
|
||||
const installedPackageRoot = path.join(installRoot, "node_modules", "@wrenai", "genbi");
|
||||
if (!existsSync(path.join(installedPackageRoot, "package.json"))) {
|
||||
throw new Error("fresh install did not contain @wrenai/genbi");
|
||||
}
|
||||
const installedContextLoaderRoot = path.join(installRoot, "node_modules", "@wrenai", "context-loader");
|
||||
const installedContextLoaderBin = path.join(installedContextLoaderRoot, "bin", "wren-context-loader");
|
||||
if (!existsSync(path.join(installedContextLoaderRoot, "install-state.json")) || !existsSync(installedContextLoaderBin)) {
|
||||
throw new Error("fresh install did not preserve the verified context-loader package record");
|
||||
}
|
||||
for (const forbidden of ["scripts", "test", "examples", path.join("node_modules", "examples")]) {
|
||||
if (existsSync(path.join(installedPackageRoot, forbidden))) {
|
||||
throw new Error(`published package unexpectedly contains ${forbidden}`);
|
||||
@@ -65,6 +80,7 @@ try {
|
||||
markPhase("setup-connect");
|
||||
await waitForServer(port, output);
|
||||
await verifyFirstRunSetup(port, workspaceRoot, fixtureProvider);
|
||||
await verifyInstalledProjectBind({ installRoot, workspaceRoot, sourceAuditHook, installedPackageRoot, installedContextLoaderBin, port, fixtureEndpoint: fixtureProvider.endpoint });
|
||||
await stopProcessTree(serverProcess);
|
||||
serverProcess = undefined;
|
||||
|
||||
@@ -75,6 +91,7 @@ try {
|
||||
{ name: "tarball excludes checkout-only scripts, tests, fixtures, and examples", ok: true },
|
||||
{ name: "fresh install launches through npx with package-manager PATH", ok: true },
|
||||
{ name: "first-run Setup connect terminal flow works without checkout access or development escapes", ok: true },
|
||||
{ name: "fresh install binds through a verified package-local context loader with no Rust toolchain or checkout access", ok: true },
|
||||
],
|
||||
}, null, 2)}\n`);
|
||||
} finally {
|
||||
@@ -153,6 +170,47 @@ function controlledEnvironment({ installRoot, workspaceRoot, port: selectedPort,
|
||||
return environment;
|
||||
}
|
||||
|
||||
async function packVerifiedContextLoaderFixture(source, stage, destination) {
|
||||
await cp(source, stage, { recursive: true });
|
||||
const binary = path.join(stage, "bin", "wren-context-loader");
|
||||
const content = Buffer.from("#!/bin/sh\nprintf '{\"context_version\":1,\"parseable\":true}' > \"$3\"\n");
|
||||
const binarySha256 = createHash("sha256").update(content).digest("hex");
|
||||
await writeFile(binary, content, { mode: 0o755 });
|
||||
await chmod(binary, 0o755);
|
||||
const packageJson = JSON.parse(await readFile(path.join(stage, "package.json"), "utf8"));
|
||||
await writeFile(path.join(stage, "artifacts.json"), JSON.stringify({ schema: 1, package: packageJson.name, version: packageJson.version, artifacts: { "darwin-arm64": { url: "https://example.invalid/context-loader.tar.gz", archiveSha256: "0".repeat(64), binarySha256, binaryPath: "wren-context-loader" } } }));
|
||||
await writeFile(path.join(stage, "install-state.json"), JSON.stringify({ package: packageJson.name, version: packageJson.version, target: "darwin-arm64", archiveSha256: "0".repeat(64), binarySha256, binaryPath: path.join("bin", "wren-context-loader") }));
|
||||
await run("npm", ["pack", "--ignore-scripts", "--pack-destination", destination], { cwd: stage });
|
||||
return onlyTarball(destination);
|
||||
}
|
||||
|
||||
async function verifyInstalledProjectBind({ installRoot, workspaceRoot, sourceAuditHook, installedPackageRoot, installedContextLoaderBin, port: selectedPort, fixtureEndpoint }) {
|
||||
const project = path.join(workspaceRoot, "bound-project");
|
||||
const warble = path.join(installRoot, "warble-fixture");
|
||||
await mkdir(project);
|
||||
await writeFile(path.join(project, "wren_project.yml"), "name: bound-project\n");
|
||||
await writeFile(warble, "#!/bin/sh\nwhile [ $# -gt 0 ]; do if [ \"$1\" = \"-o\" ]; then shift; printf '{}' > \"$1\"; fi; shift; done\n", { mode: 0o755 });
|
||||
await chmod(warble, 0o755);
|
||||
const pipeline = pathToFileURL(path.join(installedPackageRoot, "dist-server", "harness", "compile", "pipeline.js")).href;
|
||||
const profile = path.join(installedPackageRoot, "profiles", "genbi-default");
|
||||
const bindEnv = controlledEnvironment({ installRoot, workspaceRoot, port: selectedPort, sourceAuditHook, fixtureEndpoint });
|
||||
bindEnv.XDG_CACHE_HOME = path.join(installRoot, "package-cache");
|
||||
const runBind = async () => run(process.execPath, ["--input-type=module", "--eval", `import { compileProfile } from ${JSON.stringify(pipeline)}; const result = await compileProfile({ profileSource: ${JSON.stringify(profile)}, userProject: ${JSON.stringify(project)}, mode: "native", warbleBin: ${JSON.stringify(warble)}, hubDir: "/fixture/hub" }); console.log(JSON.stringify({ cacheHit: result.cacheHit }));`], {
|
||||
cwd: installRoot,
|
||||
env: bindEnv,
|
||||
});
|
||||
const bound = await runBind();
|
||||
if (!/"cacheHit":false/.test(bound.stdout)) throw new Error(`installed package did not bind a user project through its context loader: ${bound.stdout}${bound.stderr}`);
|
||||
await writeFile(installedContextLoaderBin, "tampered", { mode: 0o755 });
|
||||
let failed = false;
|
||||
try {
|
||||
await runBind();
|
||||
} catch (error) {
|
||||
failed = /could not resolve the "wren-context-loader" binary/.test(String(error));
|
||||
}
|
||||
if (!failed) throw new Error("tampered package-local context loader did not fail closed");
|
||||
}
|
||||
|
||||
function createSourceAuditHook() {
|
||||
return [
|
||||
'const fs = require("node:fs");',
|
||||
@@ -314,6 +372,11 @@ async function tarFiles(tarball) {
|
||||
return result.stdout.split("\n").filter(Boolean);
|
||||
}
|
||||
|
||||
async function tarJson(tarball, entry) {
|
||||
const result = await run("tar", ["-xOzf", tarball, entry]);
|
||||
return JSON.parse(result.stdout);
|
||||
}
|
||||
|
||||
function assertPublishedFiles(files) {
|
||||
if (files.length === 0) throw new Error("package tarball is empty");
|
||||
const forbidden = /(^|\/)(scripts|test|tests|fixtures|examples|\.git)(\/|$)|(^|\/)node_modules(\/|$)/;
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import { createHash } from "node:crypto";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createInMemoryCompileCache } from "../harness/compile/cache.js";
|
||||
import { resolveContextLoaderBinary } from "../harness/compile/context-loader.js";
|
||||
import { getContextLoaderIdentity, readVerifiedContextLoaderPackage, resolveContextLoaderBinary } from "../harness/compile/context-loader.js";
|
||||
import { ContextLoaderNotFoundError } from "../harness/compile/errors.js";
|
||||
import { compileProfile } from "../harness/compile/pipeline.js";
|
||||
import type { CompileCache, CompileCacheKey } from "../harness/compile/types.js";
|
||||
@@ -41,6 +42,18 @@ async function makeWarbleStub(): Promise<string> {
|
||||
return bin;
|
||||
}
|
||||
|
||||
async function makeVerifiedPackage(marker: string, version = "0.1.0"): Promise<string> {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "wren-harness-loader-package-"));
|
||||
const binary = path.join(root, "bin", "wren-context-loader");
|
||||
const content = Buffer.from(`#!/bin/sh\n# ${marker}\n`);
|
||||
const digest = createHash("sha256").update(content).digest("hex");
|
||||
await mkdir(path.dirname(binary), { recursive: true });
|
||||
await writeFile(binary, content, { mode: 0o755 });
|
||||
await writeFile(path.join(root, "package.json"), JSON.stringify({ name: "@wrenai/context-loader", version }));
|
||||
await writeFile(path.join(root, "install-state.json"), JSON.stringify({ package: "@wrenai/context-loader", version, target: `${process.platform}-${process.arch}`, binaryPath: path.join("bin", "wren-context-loader"), binarySha256: digest }));
|
||||
return root;
|
||||
}
|
||||
|
||||
describe("resolveContextLoaderBinary", () => {
|
||||
it("returns an explicit override that exists, without searching further", async () => {
|
||||
const bin = await makeLoaderStub("explicit");
|
||||
@@ -93,6 +106,29 @@ describe("getBinaryIdentity over the generator", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("verified package context loader", () => {
|
||||
it("accepts only a canonical package-local binary and folds package/version/digest into identity", async () => {
|
||||
const aRoot = await makeVerifiedPackage("a", "0.1.0");
|
||||
const bRoot = await makeVerifiedPackage("b", "0.1.1");
|
||||
const a = readVerifiedContextLoaderPackage(aRoot);
|
||||
const b = readVerifiedContextLoaderPackage(bRoot);
|
||||
expect(a?.kind).toBe("package");
|
||||
expect(a?.bin).toBe(path.join(aRoot, "bin", "wren-context-loader"));
|
||||
expect(a).toBeDefined();
|
||||
expect(b).toBeDefined();
|
||||
expect(getContextLoaderIdentity(a!)).not.toBe(getContextLoaderIdentity(b!));
|
||||
});
|
||||
|
||||
it("rejects a tampered binary or stale package record instead of selecting an ambient fallback", async () => {
|
||||
const root = await makeVerifiedPackage("before");
|
||||
await writeFile(path.join(root, "bin", "wren-context-loader"), "altered", { mode: 0o755 });
|
||||
expect(readVerifiedContextLoaderPackage(root)).toBeUndefined();
|
||||
const stale = await makeVerifiedPackage("stale");
|
||||
await writeFile(path.join(stale, "install-state.json"), JSON.stringify({ package: "@wrenai/context-loader", version: "0.0.0", target: `${process.platform}-${process.arch}`, binaryPath: path.join("bin", "wren-context-loader"), binarySha256: "0".repeat(64) }));
|
||||
expect(readVerifiedContextLoaderPackage(stale)).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* Criterion: a generator change must invalidate the compile cache **with the user project held
|
||||
* constant**. `contextFingerprint` already content-hashes the project, so it cannot carry this —
|
||||
@@ -168,4 +204,22 @@ describe("compile cache key vs a generator change (project held constant)", () =
|
||||
expect(keyA?.warbleIdentity).toBe(keyB?.warbleIdentity);
|
||||
expect(keyA?.hubDir).toBe(keyB?.hubDir);
|
||||
}, 30_000);
|
||||
|
||||
it("misses when only the verified package version changes", async () => {
|
||||
const { profileSource, userProject } = await makeProfileAndProject();
|
||||
const warbleBin = await makeWarbleStub();
|
||||
const loader = await makeLoaderStub("same-shipped-bytes");
|
||||
const a = readVerifiedContextLoaderPackage(await makeVerifiedPackage("same-shipped-bytes", "0.1.0"));
|
||||
const b = readVerifiedContextLoaderPackage(await makeVerifiedPackage("same-shipped-bytes", "0.1.1"));
|
||||
expect(a).toBeDefined();
|
||||
expect(b).toBeDefined();
|
||||
const cache = createInMemoryCompileCache();
|
||||
const base = { profileSource, userProject, mode: "native" as const, cache, warbleBin, hubDir: "/fixture/hub", contextLoaderBin: loader };
|
||||
const first = await compileProfile({ ...base, contextLoaderIdentity: getContextLoaderIdentity(a!) });
|
||||
const changedVersion = await compileProfile({ ...base, contextLoaderIdentity: getContextLoaderIdentity(b!) });
|
||||
const original = await compileProfile({ ...base, contextLoaderIdentity: getContextLoaderIdentity(a!) });
|
||||
expect(first.cacheHit).toBe(false);
|
||||
expect(changedVersion.cacheHit).toBe(false);
|
||||
expect(original.cacheHit).toBe(true);
|
||||
}, 30_000);
|
||||
});
|
||||
|
||||
@@ -8,6 +8,7 @@ const repositoryRoot = path.resolve(packageRoot, "..", "..");
|
||||
const packageJson = JSON.parse(readFileSync(path.join(packageRoot, "package.json"), "utf8")) as {
|
||||
files: string[];
|
||||
scripts: Record<string, string>;
|
||||
dependencies: Record<string, string>;
|
||||
};
|
||||
|
||||
describe("installed-package and vendor-contract CI wiring", () => {
|
||||
@@ -16,6 +17,14 @@ describe("installed-package and vendor-contract CI wiring", () => {
|
||||
expect(packageJson.scripts["check:installed-package"]).toBe("node scripts/installed-package-acceptance.mjs");
|
||||
});
|
||||
|
||||
it("pins the context loader exactly and runs its no-checkout acceptance on the certified macOS target", () => {
|
||||
expect(packageJson.dependencies["@wrenai/context-loader"]).toBe("0.1.0");
|
||||
const workflow = readFileSync(path.join(repositoryRoot, ".github", "workflows", "genbi-ci.yml"), "utf8");
|
||||
expect(workflow).toContain("macOS packed-install context-loader acceptance");
|
||||
expect(workflow).toContain("runs-on: macos-14");
|
||||
expect(workflow).toContain("pnpm run check:installed-package");
|
||||
});
|
||||
|
||||
it("keeps the deterministic vendor-contract entry point explicit", () => {
|
||||
expect(packageJson.scripts["check:vendor-contract"]).toBe("node scripts/run-vendor-contract-probes.mjs");
|
||||
});
|
||||
|
||||
Generated
+8
@@ -4,10 +4,15 @@ settings:
|
||||
autoInstallPeers: true
|
||||
excludeLinksFromLockfile: false
|
||||
|
||||
overrides:
|
||||
'@wrenai/context-loader': link:apps/context-loader
|
||||
|
||||
importers:
|
||||
|
||||
.: {}
|
||||
|
||||
apps/context-loader: {}
|
||||
|
||||
apps/genbi:
|
||||
dependencies:
|
||||
'@ai-sdk/anthropic':
|
||||
@@ -40,6 +45,9 @@ importers:
|
||||
'@warble/ir-spec':
|
||||
specifier: 0.6.0
|
||||
version: 0.6.0
|
||||
'@wrenai/context-loader':
|
||||
specifier: link:../context-loader
|
||||
version: link:../context-loader
|
||||
'@xterm/addon-fit':
|
||||
specifier: ^0.11.0
|
||||
version: 0.11.0
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
packages:
|
||||
- "apps/*"
|
||||
overrides:
|
||||
'@wrenai/context-loader': link:apps/context-loader
|
||||
allowBuilds:
|
||||
'@wrenai/context-loader': false
|
||||
'@warble/cli': true
|
||||
node-pty: true
|
||||
minimumReleaseAgeExclude:
|
||||
|
||||
Reference in New Issue
Block a user