feat(desktop): add media player

This commit is contained in:
abeatrix
2026-08-05 11:14:57 -07:00
parent e2b97792b1
commit 17e8c0cbc9
14 changed files with 16853 additions and 2537 deletions
+1
View File
@@ -79,6 +79,7 @@
"embla-carousel-react": "8.6.0",
"input-otp": "1.4.2",
"lucide-react": "^0.564.0",
"media-chrome": "^4.18.1",
"next": "16.2.11",
"next-themes": "^0.4.6",
"pino": "^10.3.1",
@@ -13,6 +13,7 @@ import { materializeUserFiles } from "./attachments";
import {
buildSessionConnectionUpdate,
consumeWorkspaceMetadata,
copySessionGeneratedArtifacts,
handleChatSessionCommand,
hasProviderChanged,
mergeSessionConfig,
@@ -215,6 +216,43 @@ describe("pathless session starts", () => {
});
describe("session forks", () => {
it("copies generated artifacts into the forked session", () => {
const previousSessionDataDir = process.env.CLINE_SESSION_DATA_DIR;
const sessionsDir = mkdtempSync(join(tmpdir(), "desktop-fork-artifacts-"));
const sourceSessionId = "source-session";
const targetSessionId = "forked-session";
const sourceArtifactsDir = join(sessionsDir, sourceSessionId, "artifacts");
try {
process.env.CLINE_SESSION_DATA_DIR = sessionsDir;
mkdirSync(sourceArtifactsDir, { recursive: true });
writeFileSync(join(sourceArtifactsDir, "generated.mp4"), "video");
writeFileSync(join(sourceArtifactsDir, "generated.mp3"), "audio");
copySessionGeneratedArtifacts(sourceSessionId, targetSessionId);
expect(
readFileSync(
join(sessionsDir, targetSessionId, "artifacts", "generated.mp4"),
"utf8",
),
).toBe("video");
expect(
readFileSync(
join(sessionsDir, targetSessionId, "artifacts", "generated.mp3"),
"utf8",
),
).toBe("audio");
} finally {
if (previousSessionDataDir === undefined) {
delete process.env.CLINE_SESSION_DATA_DIR;
} else {
process.env.CLINE_SESSION_DATA_DIR = previousSessionDataDir;
}
rmSync(sessionsDir, { recursive: true, force: true });
}
});
it("restores the selected workspace checkpoint before forking for message editing", async () => {
const sourceSessionId = `source-fork-${Date.now()}`;
const sourceMessages = [
@@ -1,4 +1,10 @@
import { existsSync, readFileSync } from "node:fs";
import {
copyFileSync,
existsSync,
mkdirSync,
readdirSync,
readFileSync,
} from "node:fs";
import { basename, join, resolve } from "node:path";
import { isDeepStrictEqual } from "node:util";
import {
@@ -241,6 +247,22 @@ function readPersistedChatMessages(sessionId: string): unknown[] | null {
}
}
export function copySessionGeneratedArtifacts(
sourceSessionId: string,
targetSessionId: string,
): void {
if (sourceSessionId === targetSessionId) return;
const sourceDir = join(sharedSessionDataDir(), sourceSessionId, "artifacts");
if (!existsSync(sourceDir)) return;
const targetDir = join(sharedSessionDataDir(), targetSessionId, "artifacts");
mkdirSync(targetDir, { recursive: true });
for (const entry of readdirSync(sourceDir, { withFileTypes: true })) {
if (!entry.isFile()) continue;
copyFileSync(join(sourceDir, entry.name), join(targetDir, entry.name));
}
}
function readSessionMetadataTitle(sessionId: string): string | undefined {
const manifest = readSessionManifest(sessionId);
const metadata =
@@ -1255,6 +1277,7 @@ async function handleForkUnlocked(
});
newSessionId = started.sessionId;
}
copySessionGeneratedArtifacts(sourceSessionId, newSessionId);
try {
const read = await manager.readMessages(newSessionId);
if (forkBeforeRunCount !== undefined || read.length > 0) {
@@ -1350,6 +1373,7 @@ async function handleRestoreCheckpoint(
if (!sessionId || !restoredMessages) {
throw new Error("Checkpoint restore did not return a new session");
}
copySessionGeneratedArtifacts(sourceSessionId, sessionId);
discardAllTrackedAttachments(
sourceSessionId,
ctx.liveSessions.get(sourceSessionId),
@@ -21,6 +21,29 @@
}
}
.cline-audio-player {
--media-background-color: transparent;
--media-button-icon-height: 1.25rem;
--media-button-icon-width: 1.25rem;
--media-control-background: transparent;
--media-control-hover-background: var(--color-accent);
--media-control-padding: 0;
--media-font: var(--font-sans);
--media-font-size: 10px;
--media-icon-color: currentColor;
--media-preview-time-background: var(--color-background);
--media-preview-time-border-radius: var(--radius-md);
--media-preview-time-text-shadow: none;
--media-primary-color: var(--color-primary);
--media-range-bar-color: var(--color-primary);
--media-range-track-background: var(--color-secondary);
--media-secondary-color: var(--color-secondary);
--media-text-color: var(--color-foreground);
--media-tooltip-arrow-display: none;
--media-tooltip-background: var(--color-background);
--media-tooltip-border-radius: var(--radius-md);
}
/*
* Chat Markdown (components/ui/markdown.tsx). Streamdown ships its defaults
* as Tailwind utilities, which land in the layered @source output; these
@@ -0,0 +1,134 @@
"use client";
import type { SpeechResult } from "ai";
import {
MediaControlBar,
MediaController,
MediaPlayButton,
MediaTimeDisplay,
MediaTimeRange,
} from "media-chrome/react";
import type { ComponentProps } from "react";
import { Button } from "@/components/ui/button";
import { ButtonGroup, ButtonGroupText } from "@/components/ui/button-group";
import { cn } from "@/lib/utils";
export type AudioPlayerProps = Omit<
ComponentProps<typeof MediaController>,
"audio"
>;
export function AudioPlayer({
children,
className,
...props
}: AudioPlayerProps) {
return (
<MediaController
audio
className={cn("cline-audio-player", className)}
data-slot="audio-player"
{...props}
>
{children}
</MediaController>
);
}
export type AudioPlayerElementProps = Omit<ComponentProps<"audio">, "src"> &
({ data: SpeechResult["audio"] } | { src: string });
export function AudioPlayerElement(props: AudioPlayerElementProps) {
if ("src" in props) {
return <audio data-slot="audio-player-element" slot="media" {...props} />;
}
const { data, ...audioProps } = props;
return (
<audio
data-slot="audio-player-element"
slot="media"
src={`data:${data.mediaType};base64,${data.base64}`}
{...audioProps}
/>
);
}
export type AudioPlayerControlBarProps = ComponentProps<typeof MediaControlBar>;
export function AudioPlayerControlBar({
children,
...props
}: AudioPlayerControlBarProps) {
return (
<MediaControlBar data-slot="audio-player-control-bar" {...props}>
<ButtonGroup className="w-full" orientation="horizontal">
{children}
</ButtonGroup>
</MediaControlBar>
);
}
export type AudioPlayerPlayButtonProps = ComponentProps<typeof MediaPlayButton>;
export function AudioPlayerPlayButton({
className,
...props
}: AudioPlayerPlayButtonProps) {
return (
<Button
asChild
className="size-8 border border-input bg-transparent hover:bg-transparent dark:bg-transparent dark:hover:bg-transparent"
size="icon-lg"
variant="ghost"
>
<MediaPlayButton
className={cn(
"size-2 bg-transparent [--media-button-icon-height:0.8rem] [--media-button-icon-width:1rem]",
className,
)}
data-slot="audio-player-play-button"
{...props}
/>
</Button>
);
}
export type AudioPlayerTimeDisplayProps = ComponentProps<
typeof MediaTimeDisplay
>;
export function AudioPlayerTimeDisplay({
className,
...props
}: AudioPlayerTimeDisplayProps) {
return (
<ButtonGroupText asChild className="h-8 bg-transparent px-2">
<MediaTimeDisplay
className={cn("tabular-nums", className)}
data-slot="audio-player-time-display"
{...props}
/>
</ButtonGroupText>
);
}
export type AudioPlayerTimeRangeProps = ComponentProps<typeof MediaTimeRange>;
export function AudioPlayerTimeRange({
className,
...props
}: AudioPlayerTimeRangeProps) {
return (
<ButtonGroupText
asChild
className="h-8 min-w-20 flex-1 bg-transparent px-2"
>
<MediaTimeRange
className={cn("w-full", className)}
data-slot="audio-player-time-range"
{...props}
/>
</ButtonGroupText>
);
}
@@ -1166,7 +1166,16 @@ describe("ChatMessages generated audio", () => {
expect(audio?.src).toBe(
"http://127.0.0.1:3126/api/session-artifacts/session-1/audio%20result.mp3",
);
expect(audio?.controls).toBe(true);
expect(audio?.controls).toBe(false);
expect(
container.querySelector('[data-slot="audio-player"]'),
).not.toBeNull();
expect(
container.querySelector('[data-slot="audio-player-play-button"]'),
).not.toBeNull();
expect(
container.querySelector('[data-slot="audio-player-time-range"]'),
).not.toBeNull();
});
});
});
@@ -33,8 +33,8 @@ import {
FilesIcon,
LibraryIcon,
Loader2,
Maximize2,
type LucideIcon,
Maximize2,
MessageCircleQuestionMarkIcon,
PanelsTopLeftIcon,
PencilIcon,
@@ -62,6 +62,14 @@ import {
AlertDialogHeader,
AlertDialogTitle,
} from "@/components/ui/alert-dialog";
import {
AudioPlayer,
AudioPlayerControlBar,
AudioPlayerElement,
AudioPlayerPlayButton,
AudioPlayerTimeDisplay,
AudioPlayerTimeRange,
} from "@/components/ui/audio-player";
import { Button } from "@/components/ui/button";
import { toast } from "@/hooks/use-toast";
import type {
@@ -368,14 +376,29 @@ function GeneratedAudio({
}, [audio.artifactName, sessionId]);
return source ? (
// biome-ignore lint/a11y/useMediaCaption: Generated audio does not include a separate caption track.
<audio
aria-label="Generated audio"
className="h-10 w-full max-w-md"
controls
preload="metadata"
src={source}
/>
<AudioPlayer className="w-full max-w-md text-foreground">
<AudioPlayerElement
aria-label="Generated audio"
muted={false}
onPlay={(event) => {
event.currentTarget.muted = false;
if (event.currentTarget.volume === 0) {
event.currentTarget.volume = 1;
}
}}
preload="metadata"
src={source}
/>
<AudioPlayerControlBar className="w-full">
<AudioPlayerPlayButton aria-label="Play or pause generated audio" />
<AudioPlayerTimeRange />
<AudioPlayerTimeDisplay
aria-label="Generated audio time remaining"
noToggle
remaining
/>
</AudioPlayerControlBar>
</AudioPlayer>
) : (
<div className="flex h-10 w-72 items-center justify-center rounded-lg border border-border text-sm text-muted-foreground">
<Loader2 className="mr-2 size-4 animate-spin" />
@@ -1017,10 +1017,14 @@ export function useChatSession() {
mediaType: audio.mediaType as string,
artifactName: audio.artifactName as string,
};
const updated = updateMessageById(previous, assistantId, (message) => ({
...message,
audios: [...(message.audios ?? []), generatedAudio],
}));
const updated = updateMessageById(
previous,
assistantId,
(message) => ({
...message,
audios: [...(message.audios ?? []), generatedAudio],
}),
);
if (updated !== previous) return updated;
return sliceMessages([
...previous,
+7
View File
@@ -218,6 +218,7 @@
"embla-carousel-react": "8.6.0",
"input-otp": "1.4.2",
"lucide-react": "^0.564.0",
"media-chrome": "^4.18.1",
"next": "16.2.11",
"next-themes": "^0.4.6",
"pino": "^10.3.1",
@@ -5138,6 +5139,12 @@
"@cline/code/zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
"@cline/llms/@ai-sdk/gateway": ["@ai-sdk/gateway@4.0.40", "", { "dependencies": { "@ai-sdk/provider": "4.0.5", "@ai-sdk/provider-utils": "5.0.20", "@vercel/oidc": "3.2.0" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-poNySlk+zSe04M4v0wgJKt+mzY6Vk6abcQid58YZnFUBJYo6VwfsFUaW7oINoR1vYYB7Ne6eqY1doSxYKt0KDg=="],
"@cline/llms/@ai-sdk/google": ["@ai-sdk/google@4.0.33", "", { "dependencies": { "@ai-sdk/provider": "4.0.5", "@ai-sdk/provider-utils": "5.0.20" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-xj3rhHYT3Dj8Xixjo4ZqGPqXz2GkubrikoqSN5IOyM9pEmNasSw303stcV7CeW8VS9BJX4oqbY5ohSFclDckYg=="],
"@cline/llms/@ai-sdk/openai": ["@ai-sdk/openai@4.0.29", "", { "dependencies": { "@ai-sdk/provider": "4.0.5", "@ai-sdk/provider-utils": "5.0.20" }, "peerDependencies": { "zod": "^3.25.76 || ^4.1.8" } }, "sha512-CtwkH7S0IzCKryZ+yLSuqOR0UhzzaNT1vyVlGvfiba43qQvyHUiYFS5/ayJRXiVOWJjUkSfqfKLBZ8LI+SqqgQ=="],
"@cline/ui/@types/react-dom": ["@types/react-dom@19.2.3", "", { "peerDependencies": { "@types/react": "^19.2.0" } }, "sha512-jp2L/eY6fn+KgVVQAOqYItbF0VY/YApe5Mz2F0aykSO8gx31bYCZyvSeYxCHKvzHG5eZjc+zyaS5BrBWya2+kQ=="],
"@cline/ui/vite": ["vite@7.3.6", "", { "dependencies": { "esbuild": "^0.27.0 || ^0.28.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-4XP60spRGjSZFf1qYH+dJIkK2znL3zQfl9KkOV9MkkRR/3Dls0dxaBsQPTloEc5BLXWPL9vsOxopxyKoMmDueg=="],
File diff suppressed because it is too large Load Diff
+3 -3
View File
@@ -1262,11 +1262,11 @@ async function* emitAiSdkEvents(
const data = file?.base64;
if (typeof data === "string" && data.length > 0) {
if (file?.mediaType?.startsWith("image/")) emittedImages += 1;
yield {
type: "file",
yield {
type: "file",
data,
mediaType: file?.mediaType ?? "application/octet-stream",
};
};
}
continue;
}
@@ -1495,9 +1495,7 @@ describe("sdk-gateway", () => {
mockSuccessfulStream();
const gateway = createGateway({
providerConfigs: [
{ providerId: "deepseek", apiKey: "deepseek-key" },
],
providerConfigs: [{ providerId: "deepseek", apiKey: "deepseek-key" }],
});
await collect(
@@ -1558,9 +1556,7 @@ describe("sdk-gateway", () => {
mockSuccessfulStream();
const gateway = createGateway({
providerConfigs: [
{ providerId: "deepseek", apiKey: "deepseek-key" },
],
providerConfigs: [{ providerId: "deepseek", apiKey: "deepseek-key" }],
});
await collect(
@@ -119,7 +119,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "alibaba",
defaultModelId: "qwen3.7-plus",
defaultModelId: "qwen3.8-max",
apiKeyEnv: ["DASHSCOPE_API_KEY"],
docsUrl: "https://www.alibabacloud.com/help/en/model-studio/models",
defaults: {
@@ -133,7 +133,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "alibaba-cn",
defaultModelId: "qwen3.7-flash",
defaultModelId: "qwen3.8-max",
apiKeyEnv: ["DASHSCOPE_API_KEY"],
docsUrl: "https://www.alibabacloud.com/help/en/model-studio/models",
defaults: {
@@ -443,7 +443,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "cortecs",
defaultModelId: "kimi-k3",
defaultModelId: "deepseek-v4-flash-0731",
apiKeyEnv: ["CORTECS_API_KEY"],
docsUrl: "https://api.cortecs.ai/v1/models",
defaults: {
@@ -457,7 +457,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "crof",
defaultModelId: "kimi-k3",
defaultModelId: "deepseek-v4-flash-0731",
apiKeyEnv: ["CROF_API_KEY"],
docsUrl: "https://crof.ai/docs",
defaults: {
@@ -471,7 +471,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "crossmodel",
defaultModelId: "anthropic/claude-opus-5",
defaultModelId: "qwen/qwen3.8-max",
apiKeyEnv: ["CROSSMODEL_API_KEY"],
docsUrl: "https://www.crossmodel.ai/docs",
defaults: {
@@ -726,9 +726,9 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
name: "GreenPT",
description: "GreenPT model provider from models.dev",
family: "openai-compatible",
capabilities: ["tools", "reasoning"],
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "greenpt",
defaultModelId: "glm-5.2",
defaultModelId: "deepseek-v4-flash-0731",
apiKeyEnv: ["GREENPT_API_KEY"],
docsUrl: "https://docs.greenpt.ai",
defaults: {
@@ -742,7 +742,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "groq",
defaultModelId: "openai/gpt-oss-safeguard-20b",
defaultModelId: "qwen/qwen3.6-27b",
apiKeyEnv: ["GROQ_API_KEY"],
docsUrl: "https://console.groq.com/docs/models",
},
@@ -900,6 +900,22 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
baseUrl: "https://model.inferx.net/endpoints/v1",
},
},
{
id: "infomaniak",
name: "Infomaniak",
description: "Infomaniak model provider from models.dev",
family: "openai-compatible",
capabilities: ["tools", "reasoning"],
modelsProviderId: "infomaniak",
defaultModelId: "swiss-ai/Apertus-v1.5-70B",
apiKeyEnv: ["INFOMANIAK_API_KEY", "INFOMANIAK_PRODUCT_ID"],
docsUrl:
"https://www.infomaniak.com/en/hosting/ai-services/open-source-models",
defaults: {
baseUrl:
"https://api.infomaniak.com/2/ai/${INFOMANIAK_PRODUCT_ID}/openai/v1",
},
},
{
id: "io-net",
name: "IO.NET",
@@ -1328,7 +1344,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "nano-gpt",
defaultModelId: "qwen3.8-max",
defaultModelId: "pokee-isaac",
apiKeyEnv: ["NANO_GPT_API_KEY"],
docsUrl: "https://docs.nano-gpt.com",
defaults: {
@@ -1384,7 +1400,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "neuralwatt",
defaultModelId: "glm-5.2",
defaultModelId: "kimi-k3",
apiKeyEnv: ["NEURALWATT_API_KEY"],
docsUrl: "https://portal.neuralwatt.com/docs",
defaults: {
@@ -1662,7 +1678,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "requesty",
defaultModelId: "openai/gpt-5.4",
defaultModelId: "deepseek-v4-flash-0731",
apiKeyEnv: ["REQUESTY_API_KEY"],
docsUrl: "https://requesty.ai/solution/llm-routing/models",
defaults: {
@@ -1736,6 +1752,20 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
baseUrl: "https://api.scaleway.ai/v1",
},
},
{
id: "scx",
name: "SCX.ai",
description: "SCX.ai model provider from models.dev",
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "scx",
defaultModelId: "MiniMax-M2.7",
apiKeyEnv: ["SCX_API_KEY"],
docsUrl: "https://platform.scx.ai/docs",
defaults: {
baseUrl: "https://api.scx.ai/v1",
},
},
{
id: "siliconflow",
name: "SiliconFlow",
@@ -2003,7 +2033,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "together",
defaultModelId: "moonshotai/Kimi-K3",
defaultModelId: "deepseek-ai/DeepSeek-V4-Flash-0731",
apiKeyEnv: ["TOGETHER_API_KEY"],
docsUrl: "https://docs.together.ai/docs/serverless-models",
},
@@ -2164,7 +2194,7 @@ export const GENERATED_PROVIDER_SPECS: readonly BuiltinSpec[] = [
family: "openai-compatible",
capabilities: ["tools", "reasoning", "prompt-cache"],
modelsProviderId: "wandb",
defaultModelId: "moonshotai/Kimi-K3",
defaultModelId: "deepseek-ai/DeepSeek-V4-Flash-0731",
apiKeyEnv: ["WANDB_API_KEY"],
docsUrl: "https://docs.wandb.ai/guides/integrations/inference/",
defaults: {
@@ -410,9 +410,7 @@ describe("anthropic-compatible routing helpers", () => {
"us.anthropic.claude-future-9-20990101-v1:0",
];
for (const modelId of unknownClaudeIds) {
expect(resolveClaudeThinkingEra(modelId), modelId).toBe(
"unknown-claude",
);
expect(resolveClaudeThinkingEra(modelId), modelId).toBe("unknown-claude");
}
const nonClaudeIds = ["custom/anthropic-alias", "gpt-5.4", undefined];