diff --git a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts index 95ed10ce..7c799b9f 100644 --- a/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts +++ b/electron/native-bridge/cursor/recording/macNativeCursorRecordingSession.ts @@ -6,10 +6,21 @@ import { type Rectangle, screen, systemPreferences } from "electron"; import type { CursorRecordingData, CursorRecordingSample, + NativeCursorAsset, NativeCursorType, } from "../../../../src/native/contracts"; import type { CursorRecordingSession } from "./session"; +interface MacCursorAssetPayload { + id: string; + imageDataUrl: string; + width: number; + height: number; + hotspotX: number; + hotspotY: number; + scaleFactor?: number; +} + interface MacNativeCursorRecordingSessionOptions { getDisplayBounds: () => Rectangle | null; maxSamples: number; @@ -28,6 +39,8 @@ type MacCursorEvent = type: "sample"; timestampMs: number; cursorType?: NativeCursorType | null; + assetId?: string | null; + asset?: MacCursorAssetPayload | null; leftButtonDown?: boolean; leftButtonPressed?: boolean; leftButtonReleased?: boolean; @@ -170,6 +183,7 @@ function normalizeCursorType(value: unknown): NativeCursorType | null { export class MacNativeCursorRecordingSession implements CursorRecordingSession { private samples: CursorRecordingSample[] = []; + private assets = new Map(); private process: ChildProcessByStdio | null = null; private lineBuffer = ""; private startTimeMs = 0; @@ -187,6 +201,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { async start(): Promise { this.samples = []; + this.assets.clear(); this.lineBuffer = ""; this.startTimeMs = this.options.startTimeMs ?? Date.now(); this.previousLeftButtonDown = false; @@ -263,19 +278,38 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { return { version: 2, - provider: "none", + provider: this.assets.size > 0 ? "native" : "none", samples: this.samples, - assets: [], + assets: [...this.assets.values()], }; } private startPositionOnlyFallback() { - this.captureSample(Date.now(), null, false, false, false); + this.captureSample(Date.now(), null, null, false, false, false); this.fallbackInterval = setInterval(() => { - this.captureSample(Date.now(), null, false, false, false); + this.captureSample(Date.now(), null, null, false, false, false); }, this.options.sampleIntervalMs); } + private rememberAsset(asset: MacCursorAssetPayload | null | undefined) { + if (!asset?.id || this.assets.has(asset.id)) { + return; + } + + const cursor = screen.getCursorScreenPoint(); + const displayScaleFactor = screen.getDisplayNearestPoint(cursor).scaleFactor; + this.assets.set(asset.id, { + id: asset.id, + platform: "darwin", + imageDataUrl: asset.imageDataUrl, + width: asset.width, + height: asset.height, + hotspotX: asset.hotspotX, + hotspotY: asset.hotspotY, + scaleFactor: asset.scaleFactor ?? displayScaleFactor, + }); + } + private handleStdoutChunk(chunk: string) { this.lineBuffer += chunk; const lines = this.lineBuffer.split(/\r?\n/); @@ -307,9 +341,11 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { } if (payload.type === "sample") { + this.rememberAsset(payload.asset); this.captureSample( payload.timestampMs, normalizeCursorType(payload.cursorType), + payload.assetId ?? null, payload.leftButtonDown === true, payload.leftButtonPressed === true, payload.leftButtonReleased === true, @@ -320,6 +356,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { private captureSample( timestampMs: number, cursorType: NativeCursorType | null, + assetId: string | null, leftButtonDown: boolean, leftButtonPressed: boolean, leftButtonReleased: boolean, @@ -357,6 +394,7 @@ export class MacNativeCursorRecordingSession implements CursorRecordingSession { cy: clamp(normalizedY, 0, 1), visible, interactionType, + ...(assetId ? { assetId } : {}), ...(cursorType ? { cursorType } : {}), }); diff --git a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift index 672e86f7..0daaf2b5 100644 --- a/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift +++ b/electron/native/screencapturekit/Sources/OpenScreenMacOSCursorHelper/main.swift @@ -1,11 +1,22 @@ import AppKit import ApplicationServices +import CryptoKit import Foundation struct CursorHelperRequest: Decodable { let sampleIntervalMs: Int? } +struct CapturedCursorAsset { + let id: String + let imageDataUrl: String + let width: Int + let height: Int + let hotspotX: Double + let hotspotY: Double + let scaleFactor: Double +} + final class MouseButtonTracker { private let lock = NSLock() private var leftDownCount = 0 @@ -211,10 +222,60 @@ func currentCursorType() -> String? { ) guard result == .success, let element else { - return "arrow" + return nil } - return cursorTypeForElement(element) ?? "arrow" + // Returns nil for anything that is not a text/pointer affordance so the + // renderer falls through to the natively captured cursor bitmap (this is + // what makes default and custom cursors render as their real images). + return cursorTypeForElement(element) +} + +func currentCursorAsset() -> CapturedCursorAsset? { + guard let cursor = NSCursor.currentSystem ?? NSCursor.current as NSCursor? else { + return nil + } + + let image = cursor.image + let pointSize = image.size + guard pointSize.width > 0, pointSize.height > 0 else { + return nil + } + + var proposedRect = NSRect(origin: .zero, size: pointSize) + guard let cgImage = image.cgImage(forProposedRect: &proposedRect, context: nil, hints: nil) else { + return nil + } + + let bitmap = NSBitmapImageRep(cgImage: cgImage) + guard let png = bitmap.representation(using: .png, properties: [:]) else { + return nil + } + + let pixelsWide = bitmap.pixelsWide + let pixelsHigh = bitmap.pixelsHigh + guard pixelsWide > 0, pixelsHigh > 0 else { + return nil + } + + // Intrinsic backing scale of the cursor image (e.g. 2.0 on Retina). The + // renderer divides pixel dimensions/hotspot by this to recover point sizes. + let scaleFactor = Double(pixelsWide) / Double(pointSize.width) + let hotSpot = cursor.hotSpot + + let digest = SHA256.hash(data: png) + let id = digest.map { String(format: "%02x", $0) }.joined() + let imageDataUrl = "data:image/png;base64,\(png.base64EncodedString())" + + return CapturedCursorAsset( + id: id, + imageDataUrl: imageDataUrl, + width: pixelsWide, + height: pixelsHigh, + hotspotX: hotSpot.x * scaleFactor, + hotspotY: hotSpot.y * scaleFactor, + scaleFactor: scaleFactor + ) } func timestampMs() -> Int { @@ -253,13 +314,33 @@ emit([ "mouseTapReady": mouseTapReady, ]) +var lastEmittedAssetId: String? + while true { mouseTracker.pump() let mouseEvents = mouseTracker.consume() + let asset = currentCursorAsset() + // Only ship the (large) base64 payload the first time a cursor shape is seen; + // subsequent samples reference it by assetId so stdout stays small. + var assetPayload: [String: Any]? + if let asset, asset.id != lastEmittedAssetId { + lastEmittedAssetId = asset.id + assetPayload = [ + "id": asset.id, + "imageDataUrl": asset.imageDataUrl, + "width": asset.width, + "height": asset.height, + "hotspotX": asset.hotspotX, + "hotspotY": asset.hotspotY, + "scaleFactor": asset.scaleFactor, + ] + } emit([ "type": "sample", "timestampMs": timestampMs(), "cursorType": currentCursorType(), + "assetId": asset?.id, + "asset": assetPayload, "leftButtonDown": leftButtonDown(), "leftButtonPressed": mouseEvents.leftDownCount > 0, "leftButtonReleased": mouseEvents.leftUpCount > 0, diff --git a/src/lib/cursor/nativeCursor.test.ts b/src/lib/cursor/nativeCursor.test.ts index 75cdc77b..3e7a6760 100644 --- a/src/lib/cursor/nativeCursor.test.ts +++ b/src/lib/cursor/nativeCursor.test.ts @@ -1,9 +1,11 @@ import { describe, expect, it } from "vitest"; +import type { NativeCursorAsset } from "@/native/contracts"; import { getNativeCursorClickBounceProgress, getNativeCursorClickBounceScale, hasNativeCursorRecordingData, resolveInterpolatedNativeCursorFrame, + resolveNativeCursorRenderAsset, } from "./nativeCursor"; describe("native cursor click bounce", () => { @@ -52,6 +54,38 @@ describe("native cursor click bounce", () => { expect(frame?.sample.cy).toBeCloseTo(0.5); }); + it("renders the natively captured cursor bitmap for untyped macOS samples", () => { + const capturedAsset: NativeCursorAsset = { + id: "sha-custom", + platform: "darwin", + imageDataUrl: "data:image/png;base64,CUSTOMCURSOR", + width: 48, + height: 48, + hotspotX: 8, + hotspotY: 8, + scaleFactor: 2, + }; + const recordingData = { + version: 2, + provider: "native" as const, + assets: [capturedAsset], + samples: [ + { timeMs: 0, cx: 0.4, cy: 0.4, visible: true, assetId: "sha-custom" }, + { timeMs: 100, cx: 0.6, cy: 0.6, visible: true, assetId: "sha-custom" }, + ], + }; + + const frame = resolveInterpolatedNativeCursorFrame(recordingData, 50); + expect(frame?.asset.id).toBe("sha-custom"); + + // No cursorType => no bundled pretty SVG substitution => real captured bitmap, + // downscaled from pixels to points via the asset scale factor. + const rendered = resolveNativeCursorRenderAsset(capturedAsset, 1, frame?.sample); + expect(rendered.imageDataUrl).toBe("data:image/png;base64,CUSTOMCURSOR"); + expect(rendered.width).toBe(24); + expect(rendered.hotspotX).toBe(4); + }); + it("applies click bounce to telemetry-only macOS recordings", () => { const recordingData = { version: 2,