From 0e0f5003ff52a7057c67c0535736b329018b72b6 Mon Sep 17 00:00:00 2001 From: Siddharth Date: Sat, 22 Nov 2025 23:40:39 -0700 Subject: [PATCH] fix screen recording, optimize exporting pipeline --- src/components/video-editor/VideoEditor.tsx | 11 ++++- src/hooks/useScreenRecorder.ts | 15 ++++--- src/lib/exporter/frameRenderer.ts | 18 ++++---- src/lib/exporter/videoDecoder.ts | 29 ------------ src/lib/exporter/videoExporter.ts | 50 +++++++++------------ 5 files changed, 48 insertions(+), 75 deletions(-) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 485fa6c4..25b75711 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -184,8 +184,15 @@ export default function VideoEditor() { videoPlaybackRef.current?.pause(); } - const width = 1920; - const height = 1080; + // Get actual video dimensions to match recording resolution + const video = videoPlaybackRef.current?.video; + if (!video) { + toast.error('Video not ready'); + return; + } + + const width = video.videoWidth || 1920; + const height = video.videoHeight || 1080; // Calculate visually lossless bitrate matching screen recording optimization const totalPixels = width * height; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index 23c141d7..f38a8043 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -55,17 +55,13 @@ export function useScreenRecorder(): UseScreenRecorderReturn { return; } await window.electronAPI.startMouseTracking(); - // Enable hardware acceleration and set optimal resolution/framerate constraints + // Capture screen at source resolution without constraints const mediaStream = await (navigator.mediaDevices as any).getUserMedia({ audio: false, video: { mandatory: { chromeMediaSource: "desktop", chromeMediaSourceId: selectedSource.id, - minWidth: 1920, - minHeight: 1080, - maxWidth: 3840, - maxHeight: 2160, frameRate: { ideal: 60, max: 60 } }, }, @@ -75,7 +71,14 @@ export function useScreenRecorder(): UseScreenRecorderReturn { throw new Error("Media stream is not available."); } const videoTrack = stream.current.getVideoTracks()[0]; - const { width = 1920, height = 1080 } = videoTrack.getSettings(); + let { width = 1920, height = 1080 } = videoTrack.getSettings(); + + // Ensure dimensions are divisible by 2 for VP9/AV1 codec compatibility + width = Math.floor(width / 2) * 2; + height = Math.floor(height / 2) * 2; + + console.log(`Recording at ${width}x${height}`); + const totalPixels = width * height; // Use visually lossless bitrates optimized for quality and file size balance let bitrate = 30_000_000; diff --git a/src/lib/exporter/frameRenderer.ts b/src/lib/exporter/frameRenderer.ts index 08b5ea0e..20035e45 100644 --- a/src/lib/exporter/frameRenderer.ts +++ b/src/lib/exporter/frameRenderer.ts @@ -69,15 +69,15 @@ export class FrameRenderer { console.warn('[FrameRenderer] colorSpace not supported on this platform:', error); } - // Initialize PixiJS app with transparent background (background rendered separately) + // Initialize PixiJS with optimized settings for export performance this.app = new PIXI.Application(); await this.app.init({ canvas, width: this.config.width, height: this.config.height, backgroundAlpha: 0, - antialias: true, - resolution: 2, + antialias: false, + resolution: 1, autoDensity: true, }); @@ -249,15 +249,17 @@ export class FrameRenderer { this.currentVideoTime = timestamp / 1000000; - // Create or update video sprite from VideoFrame + // Create or update video sprite from VideoFrame (optimized to reuse sprite) if (!this.videoSprite) { const texture = PIXI.Texture.from(videoFrame as any); this.videoSprite = new PIXI.Sprite(texture); this.videoContainer.addChild(this.videoSprite); } else { - // Update texture with new frame - const texture = PIXI.Texture.from(videoFrame as any); - this.videoSprite.texture = texture; + // Destroy old texture to avoid memory leaks, then create new one + const oldTexture = this.videoSprite.texture; + const newTexture = PIXI.Texture.from(videoFrame as any); + this.videoSprite.texture = newTexture; + oldTexture.destroy(true); } // Apply layout @@ -442,7 +444,7 @@ export class FrameRenderer { console.warn('[FrameRenderer] No background sprite found during compositing!'); } - // Step 2: Draw video layer with shadows on top of background + // Draw video layer with shadows on top of background (using CSS filter for accuracy) if (this.config.showShadow && this.shadowCanvas && this.shadowCtx) { const shadowCtx = this.shadowCtx; shadowCtx.clearRect(0, 0, w, h); diff --git a/src/lib/exporter/videoDecoder.ts b/src/lib/exporter/videoDecoder.ts index 4f3c590a..34ece776 100644 --- a/src/lib/exporter/videoDecoder.ts +++ b/src/lib/exporter/videoDecoder.ts @@ -7,7 +7,6 @@ export interface DecodedVideoInfo { } export class VideoFileDecoder { - private decoder: VideoDecoder | null = null; private info: DecodedVideoInfo | null = null; private videoElement: HTMLVideoElement | null = null; @@ -44,27 +43,6 @@ export class VideoFileDecoder { return this.videoElement; } - /** - * Seek to a specific time and wait for the frame to be ready - */ - async seekToTime(timeInSeconds: number): Promise { - if (!this.videoElement) { - throw new Error('Video not loaded'); - } - - return new Promise((resolve) => { - const video = this.videoElement!; - - const onSeeked = () => { - video.removeEventListener('seeked', onSeeked); - resolve(); - }; - - video.addEventListener('seeked', onSeeked); - video.currentTime = timeInSeconds; - }); - } - getInfo(): DecodedVideoInfo | null { return this.info; } @@ -75,12 +53,5 @@ export class VideoFileDecoder { this.videoElement.src = ''; this.videoElement = null; } - - if (this.decoder) { - if (this.decoder.state !== 'closed') { - this.decoder.close(); - } - this.decoder = null; - } } } diff --git a/src/lib/exporter/videoExporter.ts b/src/lib/exporter/videoExporter.ts index c7f06aee..769e918f 100644 --- a/src/lib/exporter/videoExporter.ts +++ b/src/lib/exporter/videoExporter.ts @@ -23,7 +23,8 @@ export class VideoExporter { private cancelled = false; private encodedChunks: EncodedVideoChunk[] = []; private encodeQueue = 0; - private readonly MAX_ENCODE_QUEUE = 60; + // Increased queue size for better throughput with hardware encoding + private readonly MAX_ENCODE_QUEUE = 120; private videoDescription: Uint8Array | undefined; private videoColorSpace: VideoColorSpaceInit | undefined; @@ -68,38 +69,25 @@ export class VideoExporter { throw new Error('Video element not available'); } - // Process frames with optimized seeking + // Process frames with optimized seeking (no unnecessary timeouts) const frameDuration = 1_000_000 / this.config.frameRate; // in microseconds let frameIndex = 0; const timeStep = 1 / this.config.frameRate; - // Pre-load first frame - videoElement.currentTime = 0; - await new Promise(resolve => { - const onSeeked = () => { - videoElement.removeEventListener('seeked', onSeeked); - resolve(null); - }; - videoElement.addEventListener('seeked', onSeeked); - }); - while (frameIndex < totalFrames && !this.cancelled) { const timestamp = frameIndex * frameDuration; const videoTime = frameIndex * timeStep; - // Seek to frame (only seek if not already there) - if (Math.abs(videoElement.currentTime - videoTime) > 0.001) { - videoElement.currentTime = videoTime; - await Promise.race([ - new Promise(resolve => { - const onSeeked = () => { - videoElement.removeEventListener('seeked', onSeeked); - // Wait for video to render the frame - videoElement.requestVideoFrameCallback(() => resolve(null)); - }; - videoElement.addEventListener('seeked', onSeeked, { once: true }); - }), - new Promise(resolve => setTimeout(resolve, 200)) // higher this number, slower the export, but better capture/ no frame drops - ]); + + // Seek if needed or wait for first frame to be ready + const needsSeek = Math.abs(videoElement.currentTime - videoTime) > 0.001; + if (needsSeek || frameIndex === 0) { + if (needsSeek) { + videoElement.currentTime = videoTime; + } + // Wait for video frame to be ready + await new Promise(resolve => { + videoElement.requestVideoFrameCallback(() => resolve()); + }); } // Create a VideoFrame from the video element (on GPU!) @@ -112,16 +100,17 @@ export class VideoExporter { videoFrame.close(); + // Wait for encoder queue to have space (yield immediately instead of 1ms timeout) while (this.encodeQueue >= this.MAX_ENCODE_QUEUE && !this.cancelled) { - await new Promise(resolve => setTimeout(resolve, 1)); + await new Promise(resolve => setTimeout(resolve, 0)); } if (this.cancelled) break; const canvas = this.renderer!.getCanvas(); - - // @ts-ignore - TypeScript definitions may not include all VideoFrameInit properties + // Create VideoFrame from canvas on GPU without reading pixels + // @ts-ignore - colorSpace not in TypeScript definitions but works at runtime const exportFrame = new VideoFrame(canvas, { timestamp, duration: frameDuration, @@ -141,7 +130,8 @@ export class VideoExporter { frameIndex++; - if (this.config.onProgress) { + // Batch progress updates to reduce callback overhead (every 5 frames) + if (frameIndex % 5 === 0 && this.config.onProgress) { this.config.onProgress({ currentFrame: frameIndex, totalFrames,