opensource/excalidraw
3
0
Fork 0
mirror of https://github.com/excalidraw/excalidraw.git synced 2026-08-28 17:41:10 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(app): d2c streaming (#11960)

Browse Source 
* support d2c streaming

* render status spinner while streaming

* enable analytics for ai
This commit is contained in:
David Luzar
2026-08-25 16:08:37 +02:00
committed by GitHub
parent e160ff7ba0
commit 4a6f4e7a32
7 changed files with 184 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
excalidraw-app/components/AI.tsx
+53 -12
View File
@@ -4,12 +4,14 @@ import {
getNonDeletedElements,
getTextFromElements,
MIME_TYPES,
parseSSEStream,
TTDDialog,
TTDStreamFetch,
} from "@excalidraw/excalidraw";
import { getDataURL } from "@excalidraw/excalidraw/data/blob";
import { safelyParseJSON } from "@excalidraw/common";
import type { StreamChunk } from "@excalidraw/excalidraw";
import type { ExcalidrawImperativeAPI } from "@excalidraw/excalidraw/types";
import { TTDIndexedDBAdapter } from "../data/TTDStorage";
@@ -22,7 +24,7 @@ export const AIComponents = ({
return (
<>
<DiagramToCodePlugin
generate={async ({ frame, children }) => {
generate={async ({ frame, children, onPartial }) => {
const appState = excalidrawAPI.getAppState();
// SAFETY: This should never happen, but log it just in case
@@ -51,11 +53,11 @@ export const AIComponents = ({
const response = await fetch(
`${
import.meta.env.VITE_APP_AI_BACKEND
}/v1/ai/diagram-to-code/generate`,
}/v1/ai/diagram-to-code/generate-streaming`,
{
method: "POST",
headers: {
Accept: "application/json",
Accept: "text/event-stream",
"Content-Type": "application/json",
},
body: JSON.stringify({
@@ -94,18 +96,57 @@ export const AIComponents = ({
throw new Error(errorJSON.message || text);
}
try {
const { html } = await response.json();
const reader = response.body?.getReader();
if (!html) {
throw new Error("Generation failed (invalid response)");
}
return {
html,
};
} catch (error: any) {
if (!reader) {
throw new Error("Generation failed (invalid response)");
}
let html = "";
let streamError: Error | null = null;
for await (const data of parseSSEStream(reader)) {
if (data === "[DONE]") {
break;
}
const chunk = safelyParseJSON(data) as StreamChunk | null;
if (!chunk) {
continue;
}
switch (chunk.type) {
case "content": {
if (chunk.delta) {
html += chunk.delta;
onPartial?.(html);
}
break;
}
case "error": {
streamError = new Error(
chunk.error.message || "Generation failed",
);
break;
}
case "done": {
break;
}
}
}
if (streamError) {
throw streamError;
}
if (!html.trim()) {
throw new Error("Generation failed (invalid response)");
}
return {
html,
};
}}
/>
packages/excalidraw/analytics.ts
+5 -1
View File
@@ -3,7 +3,11 @@
import { isDevEnv } from "@excalidraw/common";
// small subset of categories at a given time.
const ALLOWED_CATEGORIES_TO_TRACK = new Set(["command_palette", "export"]);
const ALLOWED_CATEGORIES_TO_TRACK = new Set([
"command_palette",
"export",
"ai",
]);
export const trackEvent = (
category: string,
packages/excalidraw/components/App.tsx
+88 -70
View File
@@ -444,6 +444,7 @@ import { ViewportStatusBorder } from "./ViewportStatusFrame/ViewportStatusFrame"
import LayerUI from "./LayerUI";
import { ElementCanvasButton } from "./MagicButton";
import { SVGLayer } from "./SVGLayer";
import Spinner from "./Spinner";
import { searchItemInFocusAtom } from "./SearchMenu";
import { isSidebarDockedAtom } from "./Sidebar/Sidebar";
import { StaticCanvas, InteractiveCanvas } from "./canvases";
@@ -1482,6 +1483,20 @@ class App extends React.Component<AppProps, AppState> {
return this.iFrameRefs.get(element.id);
}
/**
* AI-generated iframe elements aren't interactive while their generation
* is still in progress (the partial content is render-only).
*/
private isIframeLikeInteractive(element: ExcalidrawElement): boolean {
if (isIframeElement(element)) {
const data =
element.customData?.generationData ??
this.magicGenerations.get(element.id);
return data?.status !== "pending";
}
return true;
}
private handleIframeLikeElementHover = ({
hitElement,
scenePointer,
@@ -1494,6 +1509,7 @@ class App extends React.Component<AppProps, AppState> {
if (
hitElement &&
isIframeLikeElement(hitElement) &&
this.isIframeLikeInteractive(hitElement) &&
(this.state.viewModeEnabled ||
this.state.activeTool.type === "laser" ||
this.isIframeLikeElementCenter(
@@ -1568,6 +1584,7 @@ class App extends React.Component<AppProps, AppState> {
300 &&
gesture.pointers.size < 2 &&
isIframeLikeElement(hitElement) &&
this.isIframeLikeInteractive(hitElement) &&
(this.state.viewModeEnabled ||
this.state.activeTool.type === "laser" ||
this.isIframeLikeElementCenter(
@@ -1787,6 +1804,7 @@ class App extends React.Component<AppProps, AppState> {
}
let src: IframeData | null;
let isPendingGeneration = false;
if (isIframeElement(el)) {
src = null;
@@ -1817,78 +1835,46 @@ class App extends React.Component<AppProps, AppState> {
html, body {
width: 100%;
height: 100%;
color: ${
this.state.theme === THEME.DARK ? "white" : "black"
};
}
body {
display: flex;
align-items: center;
justify-content: center;
flex-direction: column;
gap: 1rem;
}
.Spinner {
display: flex;
align-items: center;
justify-content: center;
margin-left: auto;
margin-right: auto;
}
.Spinner svg {
animation: rotate 1.6s linear infinite;
transform-origin: center center;
width: 40px;
height: 40px;
}
.Spinner circle {
stroke: currentColor;
animation: dash 1.6s linear 0s infinite;
stroke-linecap: round;
}
@keyframes rotate {
100% {
transform: rotate(360deg);
}
}
@keyframes dash {
0% {
stroke-dasharray: 1, 300;
stroke-dashoffset: 0;
}
50% {
stroke-dasharray: 150, 300;
stroke-dashoffset: -200;
}
100% {
stroke-dasharray: 1, 300;
stroke-dashoffset: -280;
}
margin: 0;
}
</style>
<div class="Spinner">
<svg
viewBox="0 0 100 100"
>
<circle
cx="50"
cy="50"
r="46"
stroke-width="8"
fill="none"
stroke-miter-limit="10"
/>
</svg>
</div>
<div>Generating...</div>
<script>
// progressively renders the partial HTML snapshots the
// editor streams in during generation by document.write-
// ing them into this very document — feeding the
// browser's incremental HTML parser, so incomplete
// markup renders progressively
// (see App.onMagicFrameGenerate)
let writtenLength = 0;
let opened = false;
const onPartialMessage = (event) => {
const data = event.data;
if (
!data ||
data.type !== "excalidraw:diagramToCode:partial" ||
typeof data.html !== "string" ||
data.html.length <= writtenLength
) {
return;
}
if (!opened) {
opened = true;
document.open();
// document.open() wipes all listeners from both the
// document and the window, so re-register
window.addEventListener("message", onPartialMessage);
}
document.write(data.html.slice(writtenLength));
writtenLength = data.html.length;
};
window.addEventListener("message", onPartialMessage);
</script>
`);
},
} as const;
isPendingGeneration = true;
} else {
let message: string;
if (data.code === "ERR_GENERATION_INTERRUPTED") {
@@ -2049,6 +2035,12 @@ class App extends React.Component<AppProps, AppState> {
)}
</div>
</div>
{isPendingGeneration && (
<div className="excalidraw__embeddable__generating">
<Spinner size="1em" />
Generating…
</div>
)}
</div>
</div>
);
@@ -2844,6 +2836,30 @@ class App extends React.Component<AppProps, AppState> {
const { html } = await generateDiagramToCode({
frame: magicFrame,
children: magicFrameChildren,
onPartial: (partialResponse) => {
// only stream into the frame while this generation is still pending
if (
this.magicGenerations.get(frameElement.id)?.status !== "pending"
) {
return;
}
const htmlStartIndex = partialResponse.search(
/<!doctype html|<html[\s>]/i,
);
if (htmlStartIndex === -1) {
return;
}
// the pending iframe document renders the partial html itself
// (see the streaming shell in `renderEmbeddables`) — we only feed
// it snapshots of the html received so far
this.getHTMLIFrameElement(frameElement)?.contentWindow?.postMessage(
{
type: "excalidraw:diagramToCode:partial",
html: partialResponse.slice(htmlStartIndex),
},
"*",
);
},
});
trackEvent("ai", "generate (success)", "d2c");
@@ -7125,9 +7141,11 @@ class App extends React.Component<AppProps, AppState> {
const hitElement = this.getElementAtPosition(sceneX, sceneY);
if (isIframeLikeElement(hitElement)) {
this.setState({
activeEmbeddable: { element: hitElement, state: "active" },
});
if (this.isIframeLikeInteractive(hitElement)) {
this.setState({
activeEmbeddable: { element: hitElement, state: "active" },
});
}
return;
}
packages/excalidraw/components/TTDDialog/utils/TTDStreamFetch.ts
+1 -1
View File
@@ -45,7 +45,7 @@ function extractRateLimitHeaders(headers: Headers): RateLimitInfo {
};
}
async function* parseSSEStream(
export async function* parseSSEStream(
reader: ReadableStreamDefaultReader<Uint8Array>,
): AsyncGenerator<string, void, unknown> {
const decoder = new TextDecoder();
packages/excalidraw/css/styles.scss
+26
View File
@@ -936,6 +936,7 @@ body.excalidraw-cursor-resize * {
.excalidraw__embeddable-container {
.excalidraw__embeddable-container__inner {
position: relative;
overflow: hidden;
display: flex;
align-items: center;
@@ -943,6 +944,31 @@ body.excalidraw-cursor-resize * {
border-radius: var(--embeddable-radius);
}
.excalidraw__embeddable__generating {
position: absolute;
z-index: 1;
bottom: 1rem;
right: 1rem;
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.375rem 0.75rem;
border-radius: 12px;
background: rgba(0, 0, 0, 0.5);
color: #fff;
font-size: 0.875rem;
font-family: "Assistant";
pointer-events: none;
.Spinner {
--spinner-color: #fff;
flex: 0 0 auto;
height: auto;
margin: 0;
}
}
.excalidraw__embeddable__outer {
width: 100%;
height: 100%;
packages/excalidraw/index.tsx
+5 -1
View File
@@ -478,7 +478,11 @@ export { Stats } from "./components/Stats";
export { DefaultSidebar } from "./components/DefaultSidebar";
export { TTDDialog } from "./components/TTDDialog/TTDDialog";
export { TTDDialogTrigger } from "./components/TTDDialog/TTDDialogTrigger";
export { TTDStreamFetch } from "./components/TTDDialog/utils/TTDStreamFetch";
export {
TTDStreamFetch,
parseSSEStream,
} from "./components/TTDDialog/utils/TTDStreamFetch";
export type { StreamChunk } from "./components/TTDDialog/utils/TTDStreamFetch";
export type {
TTDPersistenceAdapter,
SavedChat,
packages/excalidraw/types.ts
+6
View File
@@ -1375,6 +1375,12 @@ export type NullableGridSize =
export type GenerateDiagramToCode = (props: {
frame: NonDeleted<ExcalidrawMagicFrameElement>;
children: readonly NonDeletedExcalidrawElement[];
/**
* Optional streaming hook. Call with the accumulated response text as it
* streams in so the editor can progressively render the partial HTML
* inside the generated frame.
*/
onPartial?: (html: string) => void;
}) => MaybePromise<{ html: string }>;
export type Offsets = Partial<{