bundle tts

.gitignore

@@ -63,3 +63,6 @@ result-*
 #others
 
 **/*.import
+
+# Auto-caption model + ORT wasm — regenerated at build by scripts/fetch-caption-model.mjs
+/caption-assets/

electron-builder.json5

@@ -8,6 +8,8 @@
     "**/*.node"
   ],
   "productName": "Openscreen",
+  // Fetch the auto-caption model + ORT wasm into caption-assets/ before packaging (idempotent).
+  "beforePack": "scripts/before-pack.cjs",
   "npmRebuild": true,
   // sharp ships ABI-stable (napi) prebuilt binaries with bundled libvips. Building it from source
   // needs a system libvips we don't provide and breaks on CI/local ("vips-cpp.42 not found"), so we
@@ -37,6 +39,10 @@
     {
       "from": "public/cursors",
       "to": "cursors"
+    },
+    {
+      "from": "caption-assets",
+      "to": "caption-assets"
     }
   ],
 

scripts/before-pack.cjs

@@ -0,0 +1,14 @@
+// electron-builder beforePack hook: ensure the auto-caption assets (Whisper model + ORT wasm) exist
+// before packaging, so the `caption-assets` extraResources entry has something to copy. Runs for
+// every package invocation — local `npm run build:*` and CI's bare `electron-builder` alike. The
+// fetch script is idempotent, so this is a no-op once the assets are present.
+
+const { execFileSync } = require("node:child_process");
+const path = require("node:path");
+
+exports.default = async function beforePack() {
+	execFileSync("node", [path.join(__dirname, "fetch-caption-model.mjs")], {
+		stdio: "inherit",
+		cwd: path.join(__dirname, ".."),
+	});
+};

scripts/fetch-caption-model.mjs

@@ -0,0 +1,107 @@
+// Populates `caption-assets/` with everything the auto-caption worker needs at runtime so the
+// packaged app can transcribe fully offline (under file://), instead of fetching the Whisper model
+// from HuggingFace and the onnxruntime wasm from a CDN.
+//
+//   caption-assets/
+//     models/Xenova/whisper-tiny/...   ← downloaded from HuggingFace (config + quantized ONNX)
+//     ort/ort-wasm*.wasm               ← copied from @xenova/transformers/dist
+//
+// Idempotent: existing, non-empty files are left alone, so re-runs (and CI cache hits) are no-ops.
+// `caption-assets/` is gitignored and shipped via electron-builder `extraResources`.
+
+import { createWriteStream } from "node:fs";
+import { copyFile, mkdir, stat } from "node:fs/promises";
+import path from "node:path";
+import { Readable } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import { fileURLToPath } from "node:url";
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const OUT = path.join(ROOT, "caption-assets");
+const MODEL_ID = "Xenova/whisper-tiny";
+const HF_BASE = `https://huggingface.co/${MODEL_ID}/resolve/main`;
+
+// Config/tokenizer/preprocessor files (all small) plus the quantized ONNX the ASR pipeline loads by
+// default (encoder + merged decoder). We grab all the small metadata files so transformers never
+// requests one we forgot to bundle.
+const MODEL_FILES = [
+	"config.json",
+	"generation_config.json",
+	"preprocessor_config.json",
+	"tokenizer.json",
+	"tokenizer_config.json",
+	"added_tokens.json",
+	"special_tokens_map.json",
+	"normalizer.json",
+	"merges.txt",
+	"vocab.json",
+	"quantize_config.json",
+	"onnx/encoder_model_quantized.onnx",
+	"onnx/decoder_model_merged_quantized.onnx",
+];
+
+async function exists(filePath) {
+	try {
+		const s = await stat(filePath);
+		return s.isFile() && s.size > 0;
+	} catch {
+		return false;
+	}
+}
+
+async function download(url, dest) {
+	if (await exists(dest)) {
+		console.log(`  ✓ cached  ${path.relative(OUT, dest)}`);
+		return;
+	}
+	await mkdir(path.dirname(dest), { recursive: true });
+	const res = await fetch(url);
+	if (!res.ok || !res.body) {
+		throw new Error(`Failed to download ${url}: HTTP ${res.status} ${res.statusText}`);
+	}
+	const tmp = `${dest}.partial`;
+	await pipeline(Readable.fromWeb(res.body), createWriteStream(tmp));
+	const { rename } = await import("node:fs/promises");
+	await rename(tmp, dest);
+	const mb = ((await stat(dest)).size / 1_000_000).toFixed(1);
+	console.log(`  ↓ ${path.relative(OUT, dest)} (${mb} MB)`);
+}
+
+async function copyOrtWasm() {
+	const distDir = path.join(ROOT, "node_modules", "@xenova", "transformers", "dist");
+	// Non-threaded variants only — the worker runs ORT with numThreads=1 (SharedArrayBuffer isn't
+	// available under file://), so the threaded wasm is never loaded. Saves ~20MB.
+	const wasm = ["ort-wasm.wasm", "ort-wasm-simd.wasm"];
+	const ortOut = path.join(OUT, "ort");
+	await mkdir(ortOut, { recursive: true });
+	for (const name of wasm) {
+		const src = path.join(distDir, name);
+		const dest = path.join(ortOut, name);
+		if (!(await exists(src))) {
+			throw new Error(`Missing ${src} — is @xenova/transformers installed? Run npm ci first.`);
+		}
+		if (await exists(dest)) {
+			console.log(`  ✓ cached  ort/${name}`);
+			continue;
+		}
+		await copyFile(src, dest);
+		console.log(`  + copied ort/${name}`);
+	}
+}
+
+async function main() {
+	console.log(`Fetching caption assets → ${path.relative(ROOT, OUT)}/`);
+	console.log("ONNX Runtime wasm:");
+	await copyOrtWasm();
+	console.log(`Whisper model (${MODEL_ID}):`);
+	const modelDir = path.join(OUT, "models", ...MODEL_ID.split("/"));
+	for (const rel of MODEL_FILES) {
+		await download(`${HF_BASE}/${rel}`, path.join(modelDir, rel));
+	}
+	console.log("Caption assets ready.");
+}
+
+main().catch((err) => {
+	console.error(`\nfetch-caption-model failed: ${err.message}`);
+	process.exit(1);
+});

src/lib/captioning/transcribe.ts

@@ -18,6 +18,14 @@ export interface TranscribeMono16kResult {
 export interface TranscribeWorkerRequest {
 	samples: Float32Array;
 	trimRegions: TrimRegion[];
+	/**
+	 * When true, the worker loads the Whisper model + ORT wasm from the app's bundled `caption-assets`
+	 * instead of remote CDNs — required in the packaged app, which runs from `file://` where remote
+	 * fetches fail. The worker can't read `window.electronAPI`, so the renderer resolves these here.
+	 */
+	useLocalModels: boolean;
+	/** Base URL of bundled resources (packaged: resourcesPath file:// URL); used when `useLocalModels`. */
+	assetBaseUrl?: string;
 }
 
 /** Messages the transcription worker posts back to the renderer. */
@@ -80,11 +88,19 @@ export function transcribeMono16kToSegments(
 			finish(() => reject(new Error(e.message || "Caption transcription worker failed")));
 		};
 
+		// Packaged app runs from file:// (remote model/wasm fetches fail there) → load bundled assets.
+		// Dev runs from http://localhost where the remote path works, so keep using it.
+		const useLocalModels = typeof window !== "undefined" && window.location?.protocol === "file:";
+		const assetBaseUrl =
+			typeof window !== "undefined" ? window.electronAPI?.assetBaseUrl : undefined;
+
 		// Structured-clone copy (not a transfer): the caller may reuse `samples`
 		// for the full-buffer retry pass, so the buffer must stay valid here.
 		const request: TranscribeWorkerRequest = {
 			samples,
 			trimRegions: options?.trimRegions ?? [],
+			useLocalModels,
+			assetBaseUrl,
 		};
 		worker.postMessage(request);
 	});

src/lib/captioning/transcribe.worker.ts

@@ -47,10 +47,26 @@ function withoutNodeVersion<T>(fn: () => Promise<T>): Promise<T> {
 	});
 }
 
-async function loadTranscriber(): Promise<TranscriberFn> {
+async function loadTranscriber(opts: {
+	useLocalModels: boolean;
+	assetBaseUrl?: string;
+}): Promise<TranscriberFn> {
 	return withoutNodeVersion(async () => {
 		const { pipeline, env } = await import("@xenova/transformers");
-		env.allowLocalModels = false;
+		if (opts.useLocalModels && opts.assetBaseUrl) {
+			// Packaged app: load the bundled model + ORT wasm from disk so transcription needs no
+			// network and resolves under file:// (remote HuggingFace/CDN fetches fail there).
+			const base = new URL("caption-assets/", opts.assetBaseUrl).href;
+			env.allowLocalModels = true;
+			env.allowRemoteModels = false;
+			env.localModelPath = new URL("models/", base).href;
+			env.backends.onnx.wasm.wasmPaths = new URL("ort/", base).href;
+			// Non-threaded wasm: SharedArrayBuffer isn't available under file:// (no cross-origin isolation).
+			env.backends.onnx.wasm.numThreads = 1;
+		} else {
+			// Dev (http://localhost): fetch from the remote CDN, which works there.
+			env.allowLocalModels = false;
+		}
 		// Default tiny weights only: the `output_attentions` revision has regressed inference for
 		// some environments (empty chunks / thrown errors) while phrase mode works on this model.
 		const transcriber = (await pipeline(
@@ -62,10 +78,10 @@ async function loadTranscriber(): Promise<TranscriberFn> {
 }
 
 self.onmessage = async (event: MessageEvent<TranscribeWorkerRequest>) => {
-	const { samples, trimRegions } = event.data;
+	const { samples, trimRegions, useLocalModels, assetBaseUrl } = event.data;
 	try {
 		post({ type: "status", phase: "model" });
-		const transcriber = await loadTranscriber();
+		const transcriber = await loadTranscriber({ useLocalModels, assetBaseUrl });
 
 		post({ type: "status", phase: "transcribe" });
 		const { segments, granularity } = await runTranscription(