feat(site): add smooth streaming text engine for LLM responses (#22503)

## Problem LLM responses currently stream in bulk chunks — multiple `message_part` events arrive per WebSocket frame, get batched into a single `startTransition` state update, and render as a visual jump. This looks janky compared to smooth character-by-character reveal. ## Solution Port the jitter-buffer approach from [coder/mux](https://github.com/coder/mux) into a single self-contained file: `SmoothText.ts`. ### What's in the file | Component | Purpose | |---|---| | `STREAM_SMOOTHING` constants | Tuning knobs (72–420 cps adaptive rate, 120 char max visual lag, 48 char frame cap) | | `SmoothTextEngine` class | Pure state machine — two-clock model (ingestion vs presentation) with budget-gated adaptive reveal | | `useSmoothStreamingText` hook | React bridge via `requestAnimationFrame` loop, single `useState<number>`, grapheme-safe slicing | ### How the engine works - **Adaptive rate:** Linear interpolation from 72 → 420 chars/sec based on backlog pressure (how far behind the display is from ingested text) - **Budget accumulation:** Fractional character budget accrues per RAF tick. Only reveals when ≥1 whole character is ready. This makes it frame-rate invariant — 60Hz and 240Hz displays reveal the same amount over wall-clock time (tested to ≤2 char deviation) - **Max visual lag:** Hard cap of 120 chars. If the gap exceeds this, the visible pointer jumps forward immediately - **Clean flush:** When streaming ends, remaining buffer appears instantly — no trailing animation - **Grapheme safety:** Uses `Intl.Segmenter` (with codepoint fallback) to never split emoji mid-animation ### Integration To wire this up, wrap the `<Response>` component in `ConversationTimeline.tsx` with the hook: ```tsx const SmoothedResponse: FC<{text: string; isStreaming: boolean; streamKey: string}> = ({ text, isStreaming, streamKey }) => { const { visibleText } = useSmoothStreamingText({ fullText: text, isStreaming, bypassSmoothing: false, streamKey, }); return <Response>{visibleText}</Response>; }; ``` ### Tests 8 engine tests covering: steady reveal, adaptive acceleration, max lag cap, immediate flush on stream end, bypass mode, content shrink, sub-char budget gating, and frame-rate invariance. --------- Co-authored-by: Danielle Maywood <danielle@themaywoods.com>
2026-09-24 15:04:27 +08:00 · 2026-03-02 15:18:30 -05:00
parent 2f684002b8
commit c4a4ad6008
3 changed files with 589 additions and 1 deletions
@@ -18,6 +18,7 @@ import {
 	useState,
 } from "react";
 import { cn } from "utils/cn";
+import { useSmoothStreamingText } from "./SmoothText";
 import type {
 	MergedTool,
 	ParsedMessageContent,
@@ -109,6 +110,22 @@ type RenderBlockListParams = {
 	subagentStatusOverrides?: Map<string, TypesGen.ChatStatus>;
 };

+// Wrapper that runs the smooth-streaming jitter buffer on a single
+// response block. Only used during live streaming — historical
+// messages render through <Response> directly.
+const SmoothedResponse: FC<{
+	text: string;
+	streamKey: string;
+}> = ({ text, streamKey }) => {
+	const { visibleText } = useSmoothStreamingText({
+		fullText: text,
+		isStreaming: true,
+		bypassSmoothing: false,
+		streamKey,
+	});
+	return <Response>{visibleText}</Response>;
+};
+
 type RenderBlockListResult = {
 	elements: ReactNode[];
 	renderedToolIDs: ReadonlySet<string>;
@@ -127,7 +144,13 @@ function renderBlockList({
 		.map((block, index) => {
 			switch (block.type) {
 				case "response":
-					return (
+					return isStreaming ? (
+						<SmoothedResponse
+							key={`${keyPrefix}-response-${index}`}
+							text={block.text}
+							streamKey={keyPrefix}
+						/>
+					) : (
 						<Response key={`${keyPrefix}-response-${index}`}>
 							{block.text}
 						</Response>
@@ -0,0 +1,152 @@
+import { describe, expect, it } from "vitest";
+import { SmoothTextEngine, STREAM_SMOOTHING } from "./SmoothText";
+
+function makeText(length: number): string {
+	return "x".repeat(length);
+}
+
+describe("SmoothTextEngine", () => {
+	it("reveals text steadily and reaches full length", () => {
+		const engine = new SmoothTextEngine();
+		const fullText = makeText(200);
+
+		engine.update(fullText, true, false);
+
+		let previousLength = engine.visibleLength;
+		let reachedFullLength = false;
+
+		for (let i = 0; i < 600; i++) {
+			const nextLength = engine.tick(16);
+			expect(nextLength).toBeGreaterThanOrEqual(previousLength);
+			previousLength = nextLength;
+
+			if (nextLength === fullText.length) {
+				reachedFullLength = true;
+				break;
+			}
+		}
+
+		expect(reachedFullLength).toBe(true);
+		expect(engine.visibleLength).toBe(fullText.length);
+		expect(engine.isCaughtUp).toBe(true);
+	});
+
+	it("accelerates reveal speed when backlog is large", () => {
+		const engine = new SmoothTextEngine();
+		const fullText = makeText(500);
+
+		engine.update(fullText, true, false);
+
+		let previousLength = engine.visibleLength;
+		let revealedCharsInFirst20Ticks = 0;
+
+		for (let i = 0; i < 20; i++) {
+			const nextLength = engine.tick(16);
+			revealedCharsInFirst20Ticks += nextLength - previousLength;
+			previousLength = nextLength;
+		}
+
+		// Baseline low-backlog behavior reveals ~1 char/frame with MIN_FRAME_CHARS.
+		// A large backlog should reveal multiple chars/frame on average.
+		expect(revealedCharsInFirst20Ticks).toBeGreaterThan(20);
+	});
+
+	it("caps visual lag when incoming text jumps ahead", () => {
+		const engine = new SmoothTextEngine();
+
+		engine.update(makeText(40), true, false);
+
+		while (!engine.isCaughtUp) {
+			engine.tick(16);
+		}
+
+		engine.update(makeText(420), true, false);
+
+		expect(420 - engine.visibleLength).toBeLessThanOrEqual(
+			STREAM_SMOOTHING.MAX_VISUAL_LAG_CHARS,
+		);
+	});
+
+	it("flushes immediately when streaming ends", () => {
+		const engine = new SmoothTextEngine();
+		const fullText = makeText(120);
+
+		engine.update(fullText, true, false);
+
+		for (let i = 0; i < 15; i++) {
+			engine.tick(16);
+		}
+
+		expect(engine.visibleLength).toBeLessThan(fullText.length);
+
+		engine.update(fullText, false, false);
+
+		expect(engine.visibleLength).toBe(fullText.length);
+		expect(engine.isCaughtUp).toBe(true);
+	});
+
+	it("bypasses smoothing and returns full length immediately", () => {
+		const engine = new SmoothTextEngine();
+		const fullText = makeText(80);
+
+		engine.update(fullText, true, true);
+
+		expect(engine.visibleLength).toBe(fullText.length);
+		expect(engine.isCaughtUp).toBe(true);
+	});
+
+	it("clamps visible length when content shrinks", () => {
+		const engine = new SmoothTextEngine();
+
+		engine.update(makeText(100), true, false);
+
+		while (engine.visibleLength < 50) {
+			engine.tick(16);
+		}
+
+		engine.update(makeText(30), true, false);
+
+		expect(engine.visibleLength).toBe(30);
+	});
+
+	it("does not force reveal when budget is below one char", () => {
+		const engine = new SmoothTextEngine();
+		// With a 1-char backlog, adaptive rate is at floor (~24 cps).
+		// At 4ms per tick: 24 * 0.004 = 0.096 budget per tick.
+		// Budget reaches 1.0 after ceil(1 / 0.096) ≈ 11 ticks.
+		engine.update("x", true, false);
+
+		// First tick at 4ms should not reveal (budget ~0.10).
+		const afterFirstTick = engine.tick(4);
+		expect(afterFirstTick).toBe(0);
+
+		// Several more small ticks should still not reveal.
+		engine.tick(4);
+		engine.tick(4);
+		expect(engine.visibleLength).toBe(0);
+
+		// After enough ticks to accumulate >= 1 char, it should reveal.
+		for (let i = 0; i < 20; i++) {
+			engine.tick(4);
+		}
+		expect(engine.visibleLength).toBeGreaterThan(0);
+	});
+
+	it("keeps reveal near frame-rate invariant over equal wall time", () => {
+		const run = (frameMs: number) => {
+			const engine = new SmoothTextEngine();
+			engine.update(makeText(400), true, false);
+			for (let t = 0; t < 1000; t += frameMs) {
+				engine.tick(frameMs);
+			}
+			return engine.visibleLength;
+		};
+
+		const at60Hz = run(16);
+		const at240Hz = run(4);
+
+		// Over 1 second of wall time, both refresh rates should reveal
+		// approximately the same number of characters.
+		expect(Math.abs(at60Hz - at240Hz)).toBeLessThanOrEqual(2);
+	});
+});
@@ -0,0 +1,413 @@
+import { useEffect, useState, useSyncExternalStore } from "react";
+
+// Smooth streaming presentation constants. These control the jitter
+// buffer that makes streamed text appear at a steady cadence instead
+// of bursty token clumps. Internal-only; no user-facing setting.
+export const STREAM_SMOOTHING = {
+	/** Baseline reveal speed in characters per second. */
+	BASE_CHARS_PER_SEC: 72,
+	/** Floor — never slower than this even when buffer is nearly empty. */
+	MIN_CHARS_PER_SEC: 24,
+	/** Ceiling — hard cap to prevent overwhelming the markdown renderer. */
+	MAX_CHARS_PER_SEC: 420,
+	/** Backlog level where adaptive reveal runs at MAX_CHARS_PER_SEC. */
+	CATCHUP_BACKLOG_CHARS: 180,
+	/**
+	 * Keep the rendered transcript close to live output even during
+	 * bursty streams.
+	 */
+	MAX_VISUAL_LAG_CHARS: 120,
+	/** Max characters revealed in a single animation frame. */
+	MAX_FRAME_CHARS: 48,
+	/**
+	 * Min characters revealed per tick once budget permits. Avoids
+	 * sub-character stalls.
+	 */
+	MIN_FRAME_CHARS: 1,
+} as const;
+
+function clamp(value: number, min: number, max: number): number {
+	return Math.max(min, Math.min(max, value));
+}
+
+function getAdaptiveRate(backlog: number): number {
+	const backlogPressure = clamp(
+		backlog / STREAM_SMOOTHING.CATCHUP_BACKLOG_CHARS,
+		0,
+		1,
+	);
+
+	const targetRate =
+		STREAM_SMOOTHING.BASE_CHARS_PER_SEC +
+		backlogPressure *
+			(STREAM_SMOOTHING.MAX_CHARS_PER_SEC -
+				STREAM_SMOOTHING.BASE_CHARS_PER_SEC);
+
+	return clamp(
+		targetRate,
+		STREAM_SMOOTHING.MIN_CHARS_PER_SEC,
+		STREAM_SMOOTHING.MAX_CHARS_PER_SEC,
+	);
+}
+
+/**
+ * Deterministic text reveal engine for smoothing streamed output.
+ *
+ * The ingestion clock (incoming full text) is external; this class
+ * manages the presentation clock (visible prefix length) using a
+ * character budget model, and owns the RAF loop that drives it.
+ *
+ * Implements a subscribe/getSnapshot contract for use with
+ * useSyncExternalStore.
+ */
+export class SmoothTextEngine {
+	private fullLength = 0;
+	private visibleLengthValue = 0;
+	private charBudget = 0;
+	private isStreaming = false;
+	private bypassSmoothing = false;
+
+	private rafId: number | null = null;
+	private previousTimestamp: number | null = null;
+	private listeners = new Set<() => void>();
+
+	private enforceMaxVisualLag(): void {
+		if (!this.isStreaming || this.bypassSmoothing) {
+			return;
+		}
+
+		// Keep visible output near the ingested stream so interruption
+		// doesn't reveal a large hidden tail all at once.
+		const minVisibleLength = Math.max(
+			0,
+			this.fullLength - STREAM_SMOOTHING.MAX_VISUAL_LAG_CHARS,
+		);
+		if (this.visibleLengthValue < minVisibleLength) {
+			this.visibleLengthValue = minVisibleLength;
+			this.charBudget = 0;
+		}
+	}
+
+	private notify(): void {
+		for (const listener of this.listeners) {
+			listener();
+		}
+	}
+
+	private stopLoop(): void {
+		if (this.rafId !== null) {
+			cancelAnimationFrame(this.rafId);
+			this.rafId = null;
+		}
+		this.previousTimestamp = null;
+	}
+
+	private startLoop(): void {
+		if (this.rafId !== null) {
+			return;
+		}
+		this.rafId = requestAnimationFrame(this.frame);
+	}
+
+	private frame = (timestampMs: number): void => {
+		if (this.previousTimestamp !== null) {
+			const dtMs = timestampMs - this.previousTimestamp;
+			const prevLength = this.visibleLengthValue;
+			this.tick(dtMs);
+			if (this.visibleLengthValue !== prevLength) {
+				this.notify();
+			}
+		}
+		this.previousTimestamp = timestampMs;
+
+		if (!this.isCaughtUp) {
+			this.rafId = requestAnimationFrame(this.frame);
+		} else {
+			this.rafId = null;
+			this.previousTimestamp = null;
+		}
+	};
+
+	/**
+	 * Update the ingested text and stream state. Starts or stops the
+	 * internal RAF loop as needed, and notifies subscribers when the
+	 * visible length changes synchronously (e.g. bypass, stream end,
+	 * content shrink).
+	 */
+	update(
+		fullText: string,
+		isStreaming: boolean,
+		bypassSmoothing: boolean,
+	): void {
+		const prevVisible = this.visibleLengthValue;
+
+		this.fullLength = fullText.length;
+		this.isStreaming = isStreaming;
+		this.bypassSmoothing = bypassSmoothing;
+
+		if (this.fullLength < this.visibleLengthValue) {
+			this.visibleLengthValue = this.fullLength;
+			this.charBudget = 0;
+		}
+
+		if (!isStreaming || bypassSmoothing) {
+			this.visibleLengthValue = this.fullLength;
+			this.charBudget = 0;
+			this.stopLoop();
+		} else {
+			this.enforceMaxVisualLag();
+			if (!this.isCaughtUp) {
+				this.startLoop();
+			}
+		}
+
+		if (this.visibleLengthValue !== prevVisible) {
+			this.notify();
+		}
+	}
+
+	/**
+	 * Advance the presentation clock by a timestep.
+	 */
+	tick(dtMs: number): number {
+		if (dtMs <= 0) {
+			return this.visibleLengthValue;
+		}
+
+		if (!this.isStreaming || this.bypassSmoothing) {
+			return this.visibleLengthValue;
+		}
+
+		if (this.visibleLengthValue > this.fullLength) {
+			this.visibleLengthValue = this.fullLength;
+			this.charBudget = 0;
+		}
+
+		if (this.visibleLengthValue === this.fullLength) {
+			return this.visibleLengthValue;
+		}
+
+		const backlog = this.fullLength - this.visibleLengthValue;
+		const adaptiveRate = getAdaptiveRate(backlog);
+
+		this.charBudget += adaptiveRate * (dtMs / 1000);
+
+		// Budget-gated reveal: only reveal when at least one whole
+		// character has accrued. This makes cadence frame-rate
+		// invariant — a 240Hz display accumulates budget across
+		// several frames before revealing, rather than forcing
+		// 1 char/frame at any refresh rate.
+		const wholeCharsReady = Math.floor(this.charBudget);
+		if (wholeCharsReady < STREAM_SMOOTHING.MIN_FRAME_CHARS) {
+			return this.visibleLengthValue;
+		}
+
+		const reveal = Math.min(wholeCharsReady, STREAM_SMOOTHING.MAX_FRAME_CHARS);
+		this.visibleLengthValue = Math.min(
+			this.fullLength,
+			this.visibleLengthValue + reveal,
+		);
+		this.charBudget -= reveal;
+
+		return this.visibleLengthValue;
+	}
+
+	get visibleLength(): number {
+		return this.visibleLengthValue;
+	}
+
+	get isCaughtUp(): boolean {
+		return this.visibleLengthValue === this.fullLength;
+	}
+
+	/**
+	 * Subscribe to visible length changes. Returns an unsubscribe
+	 * function, matching the contract useSyncExternalStore expects.
+	 */
+	subscribe = (listener: () => void): (() => void) => {
+		this.listeners.add(listener);
+		return () => {
+			this.listeners.delete(listener);
+		};
+	};
+
+	/**
+	 * Reset all engine state, typically when a new stream starts.
+	 */
+	reset(): void {
+		this.stopLoop();
+		this.fullLength = 0;
+		this.visibleLengthValue = 0;
+		this.charBudget = 0;
+		this.isStreaming = false;
+		this.bypassSmoothing = false;
+	}
+
+	/**
+	 * Stop the animation loop and release resources. Call when the
+	 * engine instance is being discarded.
+	 */
+	dispose(): void {
+		this.stopLoop();
+		this.listeners.clear();
+	}
+}
+
+// ── Hook ────────────────────────────────────────────────────────────
+
+interface UseSmoothStreamingTextOptions {
+	fullText: string;
+	isStreaming: boolean;
+	bypassSmoothing: boolean;
+	/** Changing this resets the engine (new stream). */
+	streamKey: string;
+}
+
+interface UseSmoothStreamingTextResult {
+	visibleText: string;
+	isCaughtUp: boolean;
+}
+
+// Module-scoped grapheme segmenter, created once and shared across
+// all hook instances. Falls back to codepoint iteration when the
+// Intl.Segmenter API is unavailable.
+
+// Minimal type for the Intl.Segmenter API which is widely supported
+// at runtime but not included in all TypeScript lib bundles.
+interface GraphemeSegment {
+	index: number;
+	segment: string;
+}
+
+interface GraphemeSegmenterInstance {
+	segment(input: string): Iterable<GraphemeSegment>;
+}
+
+const graphemeSegmenter: GraphemeSegmenterInstance | null = (() => {
+	try {
+		const Seg = (Intl as Record<string, unknown>).Segmenter;
+		if (typeof Seg === "function") {
+			return new (
+				Seg as new (
+					locales?: string,
+					options?: { granularity?: string },
+				) => GraphemeSegmenterInstance
+			)(undefined, {
+				granularity: "grapheme",
+			});
+		}
+	} catch {
+		// Fallback to null when Intl.Segmenter is unavailable.
+	}
+	return null;
+})();
+
+/**
+ * Slice a string at the largest grapheme-cluster boundary that does
+ * not exceed {@link maxCodeUnitLength} UTF-16 code units. When the
+ * `Intl.Segmenter` API is available it is used for correct grapheme
+ * handling; otherwise the function falls back to iterating by
+ * codepoint which still avoids splitting surrogate pairs.
+ */
+function sliceAtGraphemeBoundary(
+	text: string,
+	maxCodeUnitLength: number,
+): string {
+	if (maxCodeUnitLength <= 0) {
+		return "";
+	}
+
+	if (maxCodeUnitLength >= text.length) {
+		return text;
+	}
+
+	if (graphemeSegmenter) {
+		let safeEnd = 0;
+		const segments = Array.from(graphemeSegmenter.segment(text));
+
+		for (const segment of segments) {
+			const segmentEnd = segment.index + segment.segment.length;
+			if (segmentEnd > maxCodeUnitLength) {
+				break;
+			}
+			safeEnd = segmentEnd;
+		}
+
+		return text.slice(0, safeEnd);
+	}
+
+	// Fallback: iterate by codepoint to avoid splitting surrogate
+	// pairs. This is less precise than grapheme segmentation but
+	// still safe for rendering.
+	let safeEnd = 0;
+	for (const codePoint of Array.from(text)) {
+		const codePointEnd = safeEnd + codePoint.length;
+		if (codePointEnd > maxCodeUnitLength) {
+			break;
+		}
+		safeEnd = codePointEnd;
+	}
+
+	return text.slice(0, safeEnd);
+}
+
+export function useSmoothStreamingText(
+	options: UseSmoothStreamingTextOptions,
+): UseSmoothStreamingTextResult {
+	// Store the engine and the streamKey it was created for together
+	// in a single useState. When the streamKey changes during render,
+	// we dispose the old engine and create a fresh one inline — this
+	// is the "derive state from props" pattern React documents for
+	// useState, avoiding useEffect for reset logic.
+	const [{ engine, streamKey }, setEngineState] = useState(() => ({
+		engine: new SmoothTextEngine(),
+		streamKey: options.streamKey,
+	}));
+
+	if (streamKey !== options.streamKey) {
+		engine.dispose();
+		const next = new SmoothTextEngine();
+		setEngineState({ engine: next, streamKey: options.streamKey });
+		// Use the new engine for the rest of this render.
+		next.update(options.fullText, options.isStreaming, options.bypassSmoothing);
+	} else {
+		engine.update(
+			options.fullText,
+			options.isStreaming,
+			options.bypassSmoothing,
+		);
+	}
+
+	// Dispose on unmount.
+	useEffect(() => {
+		return () => engine.dispose();
+	}, [engine]);
+
+	const visibleLength = useSyncExternalStore(
+		engine.subscribe,
+		() => engine.visibleLength,
+	);
+
+	if (!options.isStreaming || options.bypassSmoothing) {
+		return {
+			visibleText: options.fullText,
+			isCaughtUp: true,
+		};
+	}
+
+	const visiblePrefixLength = Math.min(
+		visibleLength,
+		engine.visibleLength,
+		options.fullText.length,
+	);
+
+	const visibleText = sliceAtGraphemeBoundary(
+		options.fullText,
+		visiblePrefixLength,
+	);
+
+	return {
+		visibleText,
+		isCaughtUp: visibleText.length === options.fullText.length,
+	};
+}