From c4a4ad6008382af4afcb2c9d47720f9f167a3304 Mon Sep 17 00:00:00 2001 From: Kyle Carberry Date: Mon, 2 Mar 2026 15:18:30 -0500 Subject: [PATCH] feat(site): add smooth streaming text engine for LLM responses (#22503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Problem LLM responses currently stream in bulk chunks — multiple `message_part` events arrive per WebSocket frame, get batched into a single `startTransition` state update, and render as a visual jump. This looks janky compared to smooth character-by-character reveal. ## Solution Port the jitter-buffer approach from [coder/mux](https://github.com/coder/mux) into a single self-contained file: `SmoothText.ts`. ### What's in the file | Component | Purpose | |---|---| | `STREAM_SMOOTHING` constants | Tuning knobs (72–420 cps adaptive rate, 120 char max visual lag, 48 char frame cap) | | `SmoothTextEngine` class | Pure state machine — two-clock model (ingestion vs presentation) with budget-gated adaptive reveal | | `useSmoothStreamingText` hook | React bridge via `requestAnimationFrame` loop, single `useState`, grapheme-safe slicing | ### How the engine works - **Adaptive rate:** Linear interpolation from 72 → 420 chars/sec based on backlog pressure (how far behind the display is from ingested text) - **Budget accumulation:** Fractional character budget accrues per RAF tick. Only reveals when ≥1 whole character is ready. This makes it frame-rate invariant — 60Hz and 240Hz displays reveal the same amount over wall-clock time (tested to ≤2 char deviation) - **Max visual lag:** Hard cap of 120 chars. If the gap exceeds this, the visible pointer jumps forward immediately - **Clean flush:** When streaming ends, remaining buffer appears instantly — no trailing animation - **Grapheme safety:** Uses `Intl.Segmenter` (with codepoint fallback) to never split emoji mid-animation ### Integration To wire this up, wrap the `` component in `ConversationTimeline.tsx` with the hook: ```tsx const SmoothedResponse: FC<{text: string; isStreaming: boolean; streamKey: string}> = ({ text, isStreaming, streamKey }) => { const { visibleText } = useSmoothStreamingText({ fullText: text, isStreaming, bypassSmoothing: false, streamKey, }); return {visibleText}; }; ``` ### Tests 8 engine tests covering: steady reveal, adaptive acceleration, max lag cap, immediate flush on stream end, bypass mode, content shrink, sub-char budget gating, and frame-rate invariance. --------- Co-authored-by: Danielle Maywood --- .../AgentDetail/ConversationTimeline.tsx | 25 +- .../AgentsPage/AgentDetail/SmoothText.test.ts | 152 +++++++ .../AgentsPage/AgentDetail/SmoothText.ts | 413 ++++++++++++++++++ 3 files changed, 589 insertions(+), 1 deletion(-) create mode 100644 site/src/pages/AgentsPage/AgentDetail/SmoothText.test.ts create mode 100644 site/src/pages/AgentsPage/AgentDetail/SmoothText.ts diff --git a/site/src/pages/AgentsPage/AgentDetail/ConversationTimeline.tsx b/site/src/pages/AgentsPage/AgentDetail/ConversationTimeline.tsx index 62a1010a02..805d690c76 100644 --- a/site/src/pages/AgentsPage/AgentDetail/ConversationTimeline.tsx +++ b/site/src/pages/AgentsPage/AgentDetail/ConversationTimeline.tsx @@ -18,6 +18,7 @@ import { useState, } from "react"; import { cn } from "utils/cn"; +import { useSmoothStreamingText } from "./SmoothText"; import type { MergedTool, ParsedMessageContent, @@ -109,6 +110,22 @@ type RenderBlockListParams = { subagentStatusOverrides?: Map; }; +// Wrapper that runs the smooth-streaming jitter buffer on a single +// response block. Only used during live streaming — historical +// messages render through directly. +const SmoothedResponse: FC<{ + text: string; + streamKey: string; +}> = ({ text, streamKey }) => { + const { visibleText } = useSmoothStreamingText({ + fullText: text, + isStreaming: true, + bypassSmoothing: false, + streamKey, + }); + return {visibleText}; +}; + type RenderBlockListResult = { elements: ReactNode[]; renderedToolIDs: ReadonlySet; @@ -127,7 +144,13 @@ function renderBlockList({ .map((block, index) => { switch (block.type) { case "response": - return ( + return isStreaming ? ( + + ) : ( {block.text} diff --git a/site/src/pages/AgentsPage/AgentDetail/SmoothText.test.ts b/site/src/pages/AgentsPage/AgentDetail/SmoothText.test.ts new file mode 100644 index 0000000000..8fd93b2c01 --- /dev/null +++ b/site/src/pages/AgentsPage/AgentDetail/SmoothText.test.ts @@ -0,0 +1,152 @@ +import { describe, expect, it } from "vitest"; +import { SmoothTextEngine, STREAM_SMOOTHING } from "./SmoothText"; + +function makeText(length: number): string { + return "x".repeat(length); +} + +describe("SmoothTextEngine", () => { + it("reveals text steadily and reaches full length", () => { + const engine = new SmoothTextEngine(); + const fullText = makeText(200); + + engine.update(fullText, true, false); + + let previousLength = engine.visibleLength; + let reachedFullLength = false; + + for (let i = 0; i < 600; i++) { + const nextLength = engine.tick(16); + expect(nextLength).toBeGreaterThanOrEqual(previousLength); + previousLength = nextLength; + + if (nextLength === fullText.length) { + reachedFullLength = true; + break; + } + } + + expect(reachedFullLength).toBe(true); + expect(engine.visibleLength).toBe(fullText.length); + expect(engine.isCaughtUp).toBe(true); + }); + + it("accelerates reveal speed when backlog is large", () => { + const engine = new SmoothTextEngine(); + const fullText = makeText(500); + + engine.update(fullText, true, false); + + let previousLength = engine.visibleLength; + let revealedCharsInFirst20Ticks = 0; + + for (let i = 0; i < 20; i++) { + const nextLength = engine.tick(16); + revealedCharsInFirst20Ticks += nextLength - previousLength; + previousLength = nextLength; + } + + // Baseline low-backlog behavior reveals ~1 char/frame with MIN_FRAME_CHARS. + // A large backlog should reveal multiple chars/frame on average. + expect(revealedCharsInFirst20Ticks).toBeGreaterThan(20); + }); + + it("caps visual lag when incoming text jumps ahead", () => { + const engine = new SmoothTextEngine(); + + engine.update(makeText(40), true, false); + + while (!engine.isCaughtUp) { + engine.tick(16); + } + + engine.update(makeText(420), true, false); + + expect(420 - engine.visibleLength).toBeLessThanOrEqual( + STREAM_SMOOTHING.MAX_VISUAL_LAG_CHARS, + ); + }); + + it("flushes immediately when streaming ends", () => { + const engine = new SmoothTextEngine(); + const fullText = makeText(120); + + engine.update(fullText, true, false); + + for (let i = 0; i < 15; i++) { + engine.tick(16); + } + + expect(engine.visibleLength).toBeLessThan(fullText.length); + + engine.update(fullText, false, false); + + expect(engine.visibleLength).toBe(fullText.length); + expect(engine.isCaughtUp).toBe(true); + }); + + it("bypasses smoothing and returns full length immediately", () => { + const engine = new SmoothTextEngine(); + const fullText = makeText(80); + + engine.update(fullText, true, true); + + expect(engine.visibleLength).toBe(fullText.length); + expect(engine.isCaughtUp).toBe(true); + }); + + it("clamps visible length when content shrinks", () => { + const engine = new SmoothTextEngine(); + + engine.update(makeText(100), true, false); + + while (engine.visibleLength < 50) { + engine.tick(16); + } + + engine.update(makeText(30), true, false); + + expect(engine.visibleLength).toBe(30); + }); + + it("does not force reveal when budget is below one char", () => { + const engine = new SmoothTextEngine(); + // With a 1-char backlog, adaptive rate is at floor (~24 cps). + // At 4ms per tick: 24 * 0.004 = 0.096 budget per tick. + // Budget reaches 1.0 after ceil(1 / 0.096) ≈ 11 ticks. + engine.update("x", true, false); + + // First tick at 4ms should not reveal (budget ~0.10). + const afterFirstTick = engine.tick(4); + expect(afterFirstTick).toBe(0); + + // Several more small ticks should still not reveal. + engine.tick(4); + engine.tick(4); + expect(engine.visibleLength).toBe(0); + + // After enough ticks to accumulate >= 1 char, it should reveal. + for (let i = 0; i < 20; i++) { + engine.tick(4); + } + expect(engine.visibleLength).toBeGreaterThan(0); + }); + + it("keeps reveal near frame-rate invariant over equal wall time", () => { + const run = (frameMs: number) => { + const engine = new SmoothTextEngine(); + engine.update(makeText(400), true, false); + for (let t = 0; t < 1000; t += frameMs) { + engine.tick(frameMs); + } + return engine.visibleLength; + }; + + const at60Hz = run(16); + const at240Hz = run(4); + + // Over 1 second of wall time, both refresh rates should reveal + // approximately the same number of characters. + expect(Math.abs(at60Hz - at240Hz)).toBeLessThanOrEqual(2); + }); +}); diff --git a/site/src/pages/AgentsPage/AgentDetail/SmoothText.ts b/site/src/pages/AgentsPage/AgentDetail/SmoothText.ts new file mode 100644 index 0000000000..cc4c3f9764 --- /dev/null +++ b/site/src/pages/AgentsPage/AgentDetail/SmoothText.ts @@ -0,0 +1,413 @@ +import { useEffect, useState, useSyncExternalStore } from "react"; + +// Smooth streaming presentation constants. These control the jitter +// buffer that makes streamed text appear at a steady cadence instead +// of bursty token clumps. Internal-only; no user-facing setting. +export const STREAM_SMOOTHING = { + /** Baseline reveal speed in characters per second. */ + BASE_CHARS_PER_SEC: 72, + /** Floor — never slower than this even when buffer is nearly empty. */ + MIN_CHARS_PER_SEC: 24, + /** Ceiling — hard cap to prevent overwhelming the markdown renderer. */ + MAX_CHARS_PER_SEC: 420, + /** Backlog level where adaptive reveal runs at MAX_CHARS_PER_SEC. */ + CATCHUP_BACKLOG_CHARS: 180, + /** + * Keep the rendered transcript close to live output even during + * bursty streams. + */ + MAX_VISUAL_LAG_CHARS: 120, + /** Max characters revealed in a single animation frame. */ + MAX_FRAME_CHARS: 48, + /** + * Min characters revealed per tick once budget permits. Avoids + * sub-character stalls. + */ + MIN_FRAME_CHARS: 1, +} as const; + +function clamp(value: number, min: number, max: number): number { + return Math.max(min, Math.min(max, value)); +} + +function getAdaptiveRate(backlog: number): number { + const backlogPressure = clamp( + backlog / STREAM_SMOOTHING.CATCHUP_BACKLOG_CHARS, + 0, + 1, + ); + + const targetRate = + STREAM_SMOOTHING.BASE_CHARS_PER_SEC + + backlogPressure * + (STREAM_SMOOTHING.MAX_CHARS_PER_SEC - + STREAM_SMOOTHING.BASE_CHARS_PER_SEC); + + return clamp( + targetRate, + STREAM_SMOOTHING.MIN_CHARS_PER_SEC, + STREAM_SMOOTHING.MAX_CHARS_PER_SEC, + ); +} + +/** + * Deterministic text reveal engine for smoothing streamed output. + * + * The ingestion clock (incoming full text) is external; this class + * manages the presentation clock (visible prefix length) using a + * character budget model, and owns the RAF loop that drives it. + * + * Implements a subscribe/getSnapshot contract for use with + * useSyncExternalStore. + */ +export class SmoothTextEngine { + private fullLength = 0; + private visibleLengthValue = 0; + private charBudget = 0; + private isStreaming = false; + private bypassSmoothing = false; + + private rafId: number | null = null; + private previousTimestamp: number | null = null; + private listeners = new Set<() => void>(); + + private enforceMaxVisualLag(): void { + if (!this.isStreaming || this.bypassSmoothing) { + return; + } + + // Keep visible output near the ingested stream so interruption + // doesn't reveal a large hidden tail all at once. + const minVisibleLength = Math.max( + 0, + this.fullLength - STREAM_SMOOTHING.MAX_VISUAL_LAG_CHARS, + ); + if (this.visibleLengthValue < minVisibleLength) { + this.visibleLengthValue = minVisibleLength; + this.charBudget = 0; + } + } + + private notify(): void { + for (const listener of this.listeners) { + listener(); + } + } + + private stopLoop(): void { + if (this.rafId !== null) { + cancelAnimationFrame(this.rafId); + this.rafId = null; + } + this.previousTimestamp = null; + } + + private startLoop(): void { + if (this.rafId !== null) { + return; + } + this.rafId = requestAnimationFrame(this.frame); + } + + private frame = (timestampMs: number): void => { + if (this.previousTimestamp !== null) { + const dtMs = timestampMs - this.previousTimestamp; + const prevLength = this.visibleLengthValue; + this.tick(dtMs); + if (this.visibleLengthValue !== prevLength) { + this.notify(); + } + } + this.previousTimestamp = timestampMs; + + if (!this.isCaughtUp) { + this.rafId = requestAnimationFrame(this.frame); + } else { + this.rafId = null; + this.previousTimestamp = null; + } + }; + + /** + * Update the ingested text and stream state. Starts or stops the + * internal RAF loop as needed, and notifies subscribers when the + * visible length changes synchronously (e.g. bypass, stream end, + * content shrink). + */ + update( + fullText: string, + isStreaming: boolean, + bypassSmoothing: boolean, + ): void { + const prevVisible = this.visibleLengthValue; + + this.fullLength = fullText.length; + this.isStreaming = isStreaming; + this.bypassSmoothing = bypassSmoothing; + + if (this.fullLength < this.visibleLengthValue) { + this.visibleLengthValue = this.fullLength; + this.charBudget = 0; + } + + if (!isStreaming || bypassSmoothing) { + this.visibleLengthValue = this.fullLength; + this.charBudget = 0; + this.stopLoop(); + } else { + this.enforceMaxVisualLag(); + if (!this.isCaughtUp) { + this.startLoop(); + } + } + + if (this.visibleLengthValue !== prevVisible) { + this.notify(); + } + } + + /** + * Advance the presentation clock by a timestep. + */ + tick(dtMs: number): number { + if (dtMs <= 0) { + return this.visibleLengthValue; + } + + if (!this.isStreaming || this.bypassSmoothing) { + return this.visibleLengthValue; + } + + if (this.visibleLengthValue > this.fullLength) { + this.visibleLengthValue = this.fullLength; + this.charBudget = 0; + } + + if (this.visibleLengthValue === this.fullLength) { + return this.visibleLengthValue; + } + + const backlog = this.fullLength - this.visibleLengthValue; + const adaptiveRate = getAdaptiveRate(backlog); + + this.charBudget += adaptiveRate * (dtMs / 1000); + + // Budget-gated reveal: only reveal when at least one whole + // character has accrued. This makes cadence frame-rate + // invariant — a 240Hz display accumulates budget across + // several frames before revealing, rather than forcing + // 1 char/frame at any refresh rate. + const wholeCharsReady = Math.floor(this.charBudget); + if (wholeCharsReady < STREAM_SMOOTHING.MIN_FRAME_CHARS) { + return this.visibleLengthValue; + } + + const reveal = Math.min(wholeCharsReady, STREAM_SMOOTHING.MAX_FRAME_CHARS); + this.visibleLengthValue = Math.min( + this.fullLength, + this.visibleLengthValue + reveal, + ); + this.charBudget -= reveal; + + return this.visibleLengthValue; + } + + get visibleLength(): number { + return this.visibleLengthValue; + } + + get isCaughtUp(): boolean { + return this.visibleLengthValue === this.fullLength; + } + + /** + * Subscribe to visible length changes. Returns an unsubscribe + * function, matching the contract useSyncExternalStore expects. + */ + subscribe = (listener: () => void): (() => void) => { + this.listeners.add(listener); + return () => { + this.listeners.delete(listener); + }; + }; + + /** + * Reset all engine state, typically when a new stream starts. + */ + reset(): void { + this.stopLoop(); + this.fullLength = 0; + this.visibleLengthValue = 0; + this.charBudget = 0; + this.isStreaming = false; + this.bypassSmoothing = false; + } + + /** + * Stop the animation loop and release resources. Call when the + * engine instance is being discarded. + */ + dispose(): void { + this.stopLoop(); + this.listeners.clear(); + } +} + +// ── Hook ──────────────────────────────────────────────────────────── + +interface UseSmoothStreamingTextOptions { + fullText: string; + isStreaming: boolean; + bypassSmoothing: boolean; + /** Changing this resets the engine (new stream). */ + streamKey: string; +} + +interface UseSmoothStreamingTextResult { + visibleText: string; + isCaughtUp: boolean; +} + +// Module-scoped grapheme segmenter, created once and shared across +// all hook instances. Falls back to codepoint iteration when the +// Intl.Segmenter API is unavailable. + +// Minimal type for the Intl.Segmenter API which is widely supported +// at runtime but not included in all TypeScript lib bundles. +interface GraphemeSegment { + index: number; + segment: string; +} + +interface GraphemeSegmenterInstance { + segment(input: string): Iterable; +} + +const graphemeSegmenter: GraphemeSegmenterInstance | null = (() => { + try { + const Seg = (Intl as Record).Segmenter; + if (typeof Seg === "function") { + return new ( + Seg as new ( + locales?: string, + options?: { granularity?: string }, + ) => GraphemeSegmenterInstance + )(undefined, { + granularity: "grapheme", + }); + } + } catch { + // Fallback to null when Intl.Segmenter is unavailable. + } + return null; +})(); + +/** + * Slice a string at the largest grapheme-cluster boundary that does + * not exceed {@link maxCodeUnitLength} UTF-16 code units. When the + * `Intl.Segmenter` API is available it is used for correct grapheme + * handling; otherwise the function falls back to iterating by + * codepoint which still avoids splitting surrogate pairs. + */ +function sliceAtGraphemeBoundary( + text: string, + maxCodeUnitLength: number, +): string { + if (maxCodeUnitLength <= 0) { + return ""; + } + + if (maxCodeUnitLength >= text.length) { + return text; + } + + if (graphemeSegmenter) { + let safeEnd = 0; + const segments = Array.from(graphemeSegmenter.segment(text)); + + for (const segment of segments) { + const segmentEnd = segment.index + segment.segment.length; + if (segmentEnd > maxCodeUnitLength) { + break; + } + safeEnd = segmentEnd; + } + + return text.slice(0, safeEnd); + } + + // Fallback: iterate by codepoint to avoid splitting surrogate + // pairs. This is less precise than grapheme segmentation but + // still safe for rendering. + let safeEnd = 0; + for (const codePoint of Array.from(text)) { + const codePointEnd = safeEnd + codePoint.length; + if (codePointEnd > maxCodeUnitLength) { + break; + } + safeEnd = codePointEnd; + } + + return text.slice(0, safeEnd); +} + +export function useSmoothStreamingText( + options: UseSmoothStreamingTextOptions, +): UseSmoothStreamingTextResult { + // Store the engine and the streamKey it was created for together + // in a single useState. When the streamKey changes during render, + // we dispose the old engine and create a fresh one inline — this + // is the "derive state from props" pattern React documents for + // useState, avoiding useEffect for reset logic. + const [{ engine, streamKey }, setEngineState] = useState(() => ({ + engine: new SmoothTextEngine(), + streamKey: options.streamKey, + })); + + if (streamKey !== options.streamKey) { + engine.dispose(); + const next = new SmoothTextEngine(); + setEngineState({ engine: next, streamKey: options.streamKey }); + // Use the new engine for the rest of this render. + next.update(options.fullText, options.isStreaming, options.bypassSmoothing); + } else { + engine.update( + options.fullText, + options.isStreaming, + options.bypassSmoothing, + ); + } + + // Dispose on unmount. + useEffect(() => { + return () => engine.dispose(); + }, [engine]); + + const visibleLength = useSyncExternalStore( + engine.subscribe, + () => engine.visibleLength, + ); + + if (!options.isStreaming || options.bypassSmoothing) { + return { + visibleText: options.fullText, + isCaughtUp: true, + }; + } + + const visiblePrefixLength = Math.min( + visibleLength, + engine.visibleLength, + options.fullText.length, + ); + + const visibleText = sliceAtGraphemeBoundary( + options.fullText, + visiblePrefixLength, + ); + + return { + visibleText, + isCaughtUp: visibleText.length === options.fullText.length, + }; +}