From 4a6f4e7a32ce3d3360d4d209d14105acc144d8c7 Mon Sep 17 00:00:00 2001 From: David Luzar <5153846+dwelle@users.noreply.github.com> Date: Tue, 25 Aug 2026 16:08:37 +0200 Subject: [PATCH] feat(app): d2c streaming (#11960) * support d2c streaming * render status spinner while streaming * enable analytics for ai --- excalidraw-app/components/AI.tsx | 65 +++++-- packages/excalidraw/analytics.ts | 6 +- packages/excalidraw/components/App.tsx | 158 ++++++++++-------- .../TTDDialog/utils/TTDStreamFetch.ts | 2 +- packages/excalidraw/css/styles.scss | 26 +++ packages/excalidraw/index.tsx | 6 +- packages/excalidraw/types.ts | 6 + 7 files changed, 184 insertions(+), 85 deletions(-) diff --git a/excalidraw-app/components/AI.tsx b/excalidraw-app/components/AI.tsx index f4f28271b5..9a4472f107 100644 --- a/excalidraw-app/components/AI.tsx +++ b/excalidraw-app/components/AI.tsx @@ -4,12 +4,14 @@ import { getNonDeletedElements, getTextFromElements, MIME_TYPES, + parseSSEStream, TTDDialog, TTDStreamFetch, } from "@excalidraw/excalidraw"; import { getDataURL } from "@excalidraw/excalidraw/data/blob"; import { safelyParseJSON } from "@excalidraw/common"; +import type { StreamChunk } from "@excalidraw/excalidraw"; import type { ExcalidrawImperativeAPI } from "@excalidraw/excalidraw/types"; import { TTDIndexedDBAdapter } from "../data/TTDStorage"; @@ -22,7 +24,7 @@ export const AIComponents = ({ return ( <> { + generate={async ({ frame, children, onPartial }) => { const appState = excalidrawAPI.getAppState(); // SAFETY: This should never happen, but log it just in case @@ -51,11 +53,11 @@ export const AIComponents = ({ const response = await fetch( `${ import.meta.env.VITE_APP_AI_BACKEND - }/v1/ai/diagram-to-code/generate`, + }/v1/ai/diagram-to-code/generate-streaming`, { method: "POST", headers: { - Accept: "application/json", + Accept: "text/event-stream", "Content-Type": "application/json", }, body: JSON.stringify({ @@ -94,18 +96,57 @@ export const AIComponents = ({ throw new Error(errorJSON.message || text); } - try { - const { html } = await response.json(); + const reader = response.body?.getReader(); - if (!html) { - throw new Error("Generation failed (invalid response)"); - } - return { - html, - }; - } catch (error: any) { + if (!reader) { throw new Error("Generation failed (invalid response)"); } + + let html = ""; + let streamError: Error | null = null; + + for await (const data of parseSSEStream(reader)) { + if (data === "[DONE]") { + break; + } + + const chunk = safelyParseJSON(data) as StreamChunk | null; + + if (!chunk) { + continue; + } + + switch (chunk.type) { + case "content": { + if (chunk.delta) { + html += chunk.delta; + onPartial?.(html); + } + break; + } + case "error": { + streamError = new Error( + chunk.error.message || "Generation failed", + ); + break; + } + case "done": { + break; + } + } + } + + if (streamError) { + throw streamError; + } + + if (!html.trim()) { + throw new Error("Generation failed (invalid response)"); + } + + return { + html, + }; }} /> diff --git a/packages/excalidraw/analytics.ts b/packages/excalidraw/analytics.ts index a1d31e4db3..0c5878cc5b 100644 --- a/packages/excalidraw/analytics.ts +++ b/packages/excalidraw/analytics.ts @@ -3,7 +3,11 @@ import { isDevEnv } from "@excalidraw/common"; // small subset of categories at a given time. -const ALLOWED_CATEGORIES_TO_TRACK = new Set(["command_palette", "export"]); +const ALLOWED_CATEGORIES_TO_TRACK = new Set([ + "command_palette", + "export", + "ai", +]); export const trackEvent = ( category: string, diff --git a/packages/excalidraw/components/App.tsx b/packages/excalidraw/components/App.tsx index d0c75ad30f..291e4878ce 100644 --- a/packages/excalidraw/components/App.tsx +++ b/packages/excalidraw/components/App.tsx @@ -444,6 +444,7 @@ import { ViewportStatusBorder } from "./ViewportStatusFrame/ViewportStatusFrame" import LayerUI from "./LayerUI"; import { ElementCanvasButton } from "./MagicButton"; import { SVGLayer } from "./SVGLayer"; +import Spinner from "./Spinner"; import { searchItemInFocusAtom } from "./SearchMenu"; import { isSidebarDockedAtom } from "./Sidebar/Sidebar"; import { StaticCanvas, InteractiveCanvas } from "./canvases"; @@ -1482,6 +1483,20 @@ class App extends React.Component { return this.iFrameRefs.get(element.id); } + /** + * AI-generated iframe elements aren't interactive while their generation + * is still in progress (the partial content is render-only). + */ + private isIframeLikeInteractive(element: ExcalidrawElement): boolean { + if (isIframeElement(element)) { + const data = + element.customData?.generationData ?? + this.magicGenerations.get(element.id); + return data?.status !== "pending"; + } + return true; + } + private handleIframeLikeElementHover = ({ hitElement, scenePointer, @@ -1494,6 +1509,7 @@ class App extends React.Component { if ( hitElement && isIframeLikeElement(hitElement) && + this.isIframeLikeInteractive(hitElement) && (this.state.viewModeEnabled || this.state.activeTool.type === "laser" || this.isIframeLikeElementCenter( @@ -1568,6 +1584,7 @@ class App extends React.Component { 300 && gesture.pointers.size < 2 && isIframeLikeElement(hitElement) && + this.isIframeLikeInteractive(hitElement) && (this.state.viewModeEnabled || this.state.activeTool.type === "laser" || this.isIframeLikeElementCenter( @@ -1787,6 +1804,7 @@ class App extends React.Component { } let src: IframeData | null; + let isPendingGeneration = false; if (isIframeElement(el)) { src = null; @@ -1817,78 +1835,46 @@ class App extends React.Component { html, body { width: 100%; height: 100%; - color: ${ - this.state.theme === THEME.DARK ? "white" : "black" - }; - } - body { - display: flex; - align-items: center; - justify-content: center; - flex-direction: column; - gap: 1rem; - } - - .Spinner { - display: flex; - align-items: center; - justify-content: center; - margin-left: auto; - margin-right: auto; - } - - .Spinner svg { - animation: rotate 1.6s linear infinite; - transform-origin: center center; - width: 40px; - height: 40px; - } - - .Spinner circle { - stroke: currentColor; - animation: dash 1.6s linear 0s infinite; - stroke-linecap: round; - } - - @keyframes rotate { - 100% { - transform: rotate(360deg); - } - } - - @keyframes dash { - 0% { - stroke-dasharray: 1, 300; - stroke-dashoffset: 0; - } - 50% { - stroke-dasharray: 150, 300; - stroke-dashoffset: -200; - } - 100% { - stroke-dasharray: 1, 300; - stroke-dashoffset: -280; - } + margin: 0; } -
- - - -
-
Generating...
+ `); }, } as const; + isPendingGeneration = true; } else { let message: string; if (data.code === "ERR_GENERATION_INTERRUPTED") { @@ -2049,6 +2035,12 @@ class App extends React.Component { )} + {isPendingGeneration && ( +
+ + Generating… +
+ )} ); @@ -2844,6 +2836,30 @@ class App extends React.Component { const { html } = await generateDiagramToCode({ frame: magicFrame, children: magicFrameChildren, + onPartial: (partialResponse) => { + // only stream into the frame while this generation is still pending + if ( + this.magicGenerations.get(frameElement.id)?.status !== "pending" + ) { + return; + } + const htmlStartIndex = partialResponse.search( + /]/i, + ); + if (htmlStartIndex === -1) { + return; + } + // the pending iframe document renders the partial html itself + // (see the streaming shell in `renderEmbeddables`) — we only feed + // it snapshots of the html received so far + this.getHTMLIFrameElement(frameElement)?.contentWindow?.postMessage( + { + type: "excalidraw:diagramToCode:partial", + html: partialResponse.slice(htmlStartIndex), + }, + "*", + ); + }, }); trackEvent("ai", "generate (success)", "d2c"); @@ -7125,9 +7141,11 @@ class App extends React.Component { const hitElement = this.getElementAtPosition(sceneX, sceneY); if (isIframeLikeElement(hitElement)) { - this.setState({ - activeEmbeddable: { element: hitElement, state: "active" }, - }); + if (this.isIframeLikeInteractive(hitElement)) { + this.setState({ + activeEmbeddable: { element: hitElement, state: "active" }, + }); + } return; } diff --git a/packages/excalidraw/components/TTDDialog/utils/TTDStreamFetch.ts b/packages/excalidraw/components/TTDDialog/utils/TTDStreamFetch.ts index a2c421a867..d37aca3684 100644 --- a/packages/excalidraw/components/TTDDialog/utils/TTDStreamFetch.ts +++ b/packages/excalidraw/components/TTDDialog/utils/TTDStreamFetch.ts @@ -45,7 +45,7 @@ function extractRateLimitHeaders(headers: Headers): RateLimitInfo { }; } -async function* parseSSEStream( +export async function* parseSSEStream( reader: ReadableStreamDefaultReader, ): AsyncGenerator { const decoder = new TextDecoder(); diff --git a/packages/excalidraw/css/styles.scss b/packages/excalidraw/css/styles.scss index 147341f6b8..068ba03893 100644 --- a/packages/excalidraw/css/styles.scss +++ b/packages/excalidraw/css/styles.scss @@ -936,6 +936,7 @@ body.excalidraw-cursor-resize * { .excalidraw__embeddable-container { .excalidraw__embeddable-container__inner { + position: relative; overflow: hidden; display: flex; align-items: center; @@ -943,6 +944,31 @@ body.excalidraw-cursor-resize * { border-radius: var(--embeddable-radius); } + .excalidraw__embeddable__generating { + position: absolute; + z-index: 1; + bottom: 1rem; + right: 1rem; + display: flex; + align-items: center; + gap: 0.5rem; + padding: 0.375rem 0.75rem; + border-radius: 12px; + background: rgba(0, 0, 0, 0.5); + color: #fff; + font-size: 0.875rem; + font-family: "Assistant"; + pointer-events: none; + + .Spinner { + --spinner-color: #fff; + + flex: 0 0 auto; + height: auto; + margin: 0; + } + } + .excalidraw__embeddable__outer { width: 100%; height: 100%; diff --git a/packages/excalidraw/index.tsx b/packages/excalidraw/index.tsx index 699e0a4a58..70ac526ef8 100644 --- a/packages/excalidraw/index.tsx +++ b/packages/excalidraw/index.tsx @@ -478,7 +478,11 @@ export { Stats } from "./components/Stats"; export { DefaultSidebar } from "./components/DefaultSidebar"; export { TTDDialog } from "./components/TTDDialog/TTDDialog"; export { TTDDialogTrigger } from "./components/TTDDialog/TTDDialogTrigger"; -export { TTDStreamFetch } from "./components/TTDDialog/utils/TTDStreamFetch"; +export { + TTDStreamFetch, + parseSSEStream, +} from "./components/TTDDialog/utils/TTDStreamFetch"; +export type { StreamChunk } from "./components/TTDDialog/utils/TTDStreamFetch"; export type { TTDPersistenceAdapter, SavedChat, diff --git a/packages/excalidraw/types.ts b/packages/excalidraw/types.ts index 6685ee968d..57429ad150 100644 --- a/packages/excalidraw/types.ts +++ b/packages/excalidraw/types.ts @@ -1375,6 +1375,12 @@ export type NullableGridSize = export type GenerateDiagramToCode = (props: { frame: NonDeleted; children: readonly NonDeletedExcalidrawElement[]; + /** + * Optional streaming hook. Call with the accumulated response text as it + * streams in so the editor can progressively render the partial HTML + * inside the generated frame. + */ + onPartial?: (html: string) => void; }) => MaybePromise<{ html: string }>; export type Offsets = Partial<{