From 6f36331b7a979efb7c3db09a65a92829e290c63a Mon Sep 17 00:00:00 2001 From: Siddharth Ganesan Date: Thu, 13 Aug 2026 15:25:32 -0700 Subject: [PATCH] feat(desktop): coordinate clicks, caret insertion, and drag for the browser agent --- apps/desktop/src/main/browser-agent/cdp.ts | 192 +++++++- .../src/main/browser-agent/driver.test.ts | 209 ++++++++ apps/desktop/src/main/browser-agent/driver.ts | 298 +++++++++++- .../main/browser-agent/page-functions.test.ts | 81 ++++ .../src/main/browser-agent/page-functions.ts | 152 ++++++ .../lib/copilot/generated/tool-catalog-v1.ts | 372 ++++++++++++++ .../lib/copilot/generated/tool-schemas-v1.ts | 455 ++++++++++++++++++ apps/sim/lib/copilot/tools/tool-display.ts | 5 + packages/browser-protocol/src/index.ts | 3 + 9 files changed, 1743 insertions(+), 24 deletions(-) diff --git a/apps/desktop/src/main/browser-agent/cdp.ts b/apps/desktop/src/main/browser-agent/cdp.ts index 4a7088e1b2..6387461e80 100644 --- a/apps/desktop/src/main/browser-agent/cdp.ts +++ b/apps/desktop/src/main/browser-agent/cdp.ts @@ -10,6 +10,7 @@ */ import type { BrowserTheme } from '@sim/browser-protocol' import { createLogger } from '@sim/logger' +import { sleep } from '@sim/utils/helpers' import type { WebContents, WebFrameMain } from 'electron' const logger = createLogger('BrowserAgentCdp') @@ -127,12 +128,29 @@ export async function setColorScheme(contents: WebContents, theme: BrowserTheme) }) } +/** Live drag-interception state while a dragPointer call is in flight. */ +interface DragInterception { + intercepted: boolean + data: Record | null +} +const dragInterceptionsByContents = new WeakMap() + function handleDebuggerEvent( contents: WebContents, method: string, params: Record, parentSessionId?: string ): void { + if (method === 'Input.dragIntercepted') { + const interception = dragInterceptionsByContents.get(contents) + if (interception) { + interception.intercepted = true + const data = params.data + interception.data = + data && typeof data === 'object' ? (data as Record) : null + } + return + } if (method === 'Target.attachedToTarget') { const sessionId = typeof params.sessionId === 'string' ? params.sessionId : '' const targetInfo = params.targetInfo @@ -367,7 +385,9 @@ interface CdpViewport { * edge within {@link MAX_SCREENSHOT_EDGE}. Falls back to an unclipped capture * when layout metrics are unavailable. */ -export async function captureScreenshot(contents: WebContents): Promise { +export async function captureScreenshot( + contents: WebContents +): Promise<{ dataUrl: string; scale: number }> { const metrics = await send<{ cssLayoutViewport?: CdpViewport layoutViewport?: CdpViewport @@ -376,6 +396,8 @@ export async function captureScreenshot(contents: WebContents): Promise const viewport = metrics?.cssLayoutViewport ?? metrics?.layoutViewport const width = viewport?.clientWidth ?? 0 const height = viewport?.clientHeight ?? 0 + const scale = + width > 0 && height > 0 ? Math.min(1, MAX_SCREENSHOT_EDGE / Math.max(width, height)) : 1 const clip = width > 0 && height > 0 ? { @@ -383,7 +405,7 @@ export async function captureScreenshot(contents: WebContents): Promise y: 0, width, height, - scale: Math.min(1, MAX_SCREENSHOT_EDGE / Math.max(width, height)), + scale, } : undefined @@ -392,7 +414,7 @@ export async function captureScreenshot(contents: WebContents): Promise quality: SCREENSHOT_QUALITY, ...(clip ? { clip } : {}), }) - return `data:image/jpeg;base64,${result.data}` + return { dataUrl: `data:image/jpeg;base64,${result.data}`, scale } } /** One half of a trusted key press (`Input.dispatchKeyEvent` params). */ @@ -430,7 +452,8 @@ export async function clickAt( contents: WebContents, x: number, y: number, - moveBeforePress = true + moveBeforePress = true, + clickCount = 1 ): Promise { if (moveBeforePress) await moveMouse(contents, x, y) let pressed = false @@ -439,22 +462,26 @@ export async function clickAt( // response (navigation/process swap). In that ambiguous case a release is // safer than leaving Blink's pointer state stuck down. pressed = true - await sendInput(contents, 'Input.dispatchMouseEvent', { - type: 'mousePressed', - x, - y, - button: 'left', - buttons: 1, - clickCount: 1, - }) - await sendInput(contents, 'Input.dispatchMouseEvent', { - type: 'mouseReleased', - x, - y, - button: 'left', - buttons: 0, - clickCount: 1, - }) + // A multi-click is a sequence of press/release pairs with an increasing + // clickCount — Blink synthesizes dblclick from the pair whose count is 2. + for (let count = 1; count <= clickCount; count++) { + await sendInput(contents, 'Input.dispatchMouseEvent', { + type: 'mousePressed', + x, + y, + button: 'left', + buttons: 1, + clickCount: count, + }) + await sendInput(contents, 'Input.dispatchMouseEvent', { + type: 'mouseReleased', + x, + y, + button: 'left', + buttons: 0, + clickCount: count, + }) + } pressed = false } finally { if (pressed && !contents.isDestroyed()) { @@ -473,6 +500,131 @@ export async function clickAt( } } +/** + * Drags the pointer from one viewport point to another through the trusted + * pipeline: press, a threshold-crossing nudge, interpolated moves with the + * button held, a settle hold over the target, then release or drop. + * + * Two drag models are covered by the one call. Pointer-sensor libraries + * (dnd-kit, react-beautiful-dnd, canvas apps) treat the held-button move + * sequence exactly like a human drag. Native HTML5 `draggable="true"` + * sources instead START a Blink drag session on the press+move — with + * `Input.setInterceptDrags` enabled, Chromium reports it as + * `Input.dragIntercepted` and the remaining movement is delivered as trusted + * `Input.dispatchDragEvent` dragEnter/dragOver events ending in a `drop` + * (the technique Playwright uses). Both paths are trusted input. + */ +export async function dragPointer( + contents: WebContents, + from: { x: number; y: number }, + to: { x: number; y: number }, + steps = 12, + stepDelayMs = 20 +): Promise<{ nativeDragIntercepted: boolean }> { + const interception: DragInterception = { intercepted: false, data: null } + dragInterceptionsByContents.set(contents, interception) + let interceptEnabled = false + try { + await send(contents, 'Input.setInterceptDrags', { enabled: true }) + interceptEnabled = true + } catch { + // Chromium without drag interception: the pointer-only path still works + // for pointer-sensor drags; native HTML5 sources will report no effect. + } + await moveMouse(contents, from.x, from.y) + let pressed = false + let dragEnterSent = false + const dragMove = async (x: number, y: number) => { + if (interception.intercepted && interception.data) { + await sendInput(contents, 'Input.dispatchDragEvent', { + type: dragEnterSent ? 'dragOver' : 'dragEnter', + x, + y, + data: interception.data, + }) + dragEnterSent = true + } else { + await sendInput(contents, 'Input.dispatchMouseEvent', { + type: 'mouseMoved', + x, + y, + button: 'left', + buttons: 1, + }) + } + } + try { + pressed = true + await sendInput(contents, 'Input.dispatchMouseEvent', { + type: 'mousePressed', + x: from.x, + y: from.y, + button: 'left', + buttons: 1, + clickCount: 1, + }) + // Small first nudge so libraries with a start threshold (commonly 3-8px) + // register the drag before the pointer sweeps across the page. + await dragMove(from.x + Math.sign(to.x - from.x || 1) * 4, from.y + 2) + await sleep(stepDelayMs) + const stepCount = Math.max(2, steps) + for (let step = 1; step <= stepCount; step++) { + const progress = step / stepCount + await dragMove(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress) + await sleep(stepDelayMs) + } + // Hold over the target so drop zones running enter/over animations settle + // before the release lands. + await sleep(120) + if (interception.intercepted && interception.data) { + await sendInput(contents, 'Input.dispatchDragEvent', { + type: 'drop', + x: to.x, + y: to.y, + data: interception.data, + }) + // Blink ended the intercepted drag session itself; a trailing + // mouseReleased would be a stray click on the drop target. + pressed = false + } else { + await sendInput(contents, 'Input.dispatchMouseEvent', { + type: 'mouseReleased', + x: to.x, + y: to.y, + button: 'left', + buttons: 0, + clickCount: 1, + }) + pressed = false + } + return { nativeDragIntercepted: interception.intercepted } + } finally { + if (pressed && !contents.isDestroyed()) { + if (interception.intercepted && interception.data) { + await sendInput(contents, 'Input.dispatchDragEvent', { + type: 'dragCancel', + x: to.x, + y: to.y, + data: interception.data, + }).catch(() => {}) + } else { + await sendInput(contents, 'Input.dispatchMouseEvent', { + type: 'mouseReleased', + x: to.x, + y: to.y, + button: 'left', + buttons: 0, + clickCount: 1, + }).catch(() => {}) + } + } + dragInterceptionsByContents.delete(contents) + if (interceptEnabled && !contents.isDestroyed()) { + await send(contents, 'Input.setInterceptDrags', { enabled: false }).catch(() => {}) + } + } +} + /** * Inserts text at the focused element's selection (replacing it) through the * native IME path — works in plain fields and code editors alike. diff --git a/apps/desktop/src/main/browser-agent/driver.test.ts b/apps/desktop/src/main/browser-agent/driver.test.ts index 5970afec61..38eea541b6 100644 --- a/apps/desktop/src/main/browser-agent/driver.test.ts +++ b/apps/desktop/src/main/browser-agent/driver.test.ts @@ -1874,4 +1874,213 @@ describe('credential protection', () => { expect(result).toMatchObject({ ok: true, result: { dispatched: true } }) expect(clickReads).toBeGreaterThan(1) }) + + it('clicks a coordinate point with native input and reports the target', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { + found: true, + element: 'button "Send"', + editable: false, + secret: false, + fileInput: false, + cursor: 'pointer', + }, + readActiveElementState: {}, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 120, y: 240 }) + + expect(result).toMatchObject({ + ok: true, + result: { + dispatched: true, + trusted: true, + clickedAt: { x: 120, y: 240 }, + target: 'button "Send"', + }, + }) + const presses = cdpCalls(contents, 'Input.dispatchMouseEvent').filter( + ([, event]) => (event as { type?: string }).type === 'mousePressed' + ) + expect(presses).toHaveLength(1) + }) + + it('double-clicks a coordinate point as a rising clickCount sequence', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { found: true, element: 'canvas', editable: false }, + readActiveElementState: {}, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_click_at', { + x: 10, + y: 20, + clickCount: 2, + }) + + expect(result).toMatchObject({ ok: true, result: { clickCount: 2 } }) + const counts = cdpCalls(contents, 'Input.dispatchMouseEvent') + .filter(([, event]) => (event as { type?: string }).type === 'mousePressed') + .map(([, event]) => (event as { clickCount?: number }).clickCount) + expect(counts).toEqual([1, 2]) + }) + + it('refuses a coordinate click on a file input', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { found: true, element: 'input', fileInput: true }, + }) + + const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 5, y: 5 }) + + expect(result.ok).toBe(false) + expect(result.error).toMatch(/file input/) + expect(cdpCalls(contents, 'Input.dispatchMouseEvent')).toHaveLength(0) + }) + + it('rejects a coordinate click outside the viewport with mapping guidance', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { error: 'outside-viewport' }, + }) + + const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 9999, y: 5 }) + + expect(result.ok).toBe(false) + expect(result.error).toMatch(/divide image pixels by its scale/) + }) + + it('inserts text into the focused editable at the caret', async () => { + const contents = await openPage() + respondWith(contents, { + activeElementSecrecy: 'safe', + describeFocusedEditable: { editable: true, kind: 'contenteditable' }, + readActiveElementState: { activeElement: 'div', valueLength: 12 }, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_insert_text', { + text: 'hello world', + }) + + expect(result).toMatchObject({ + ok: true, + result: { dispatched: true, trusted: true, kind: 'contenteditable', insertedChars: 11 }, + }) + expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(1) + }) + + it('refuses insertion when nothing editable holds focus', async () => { + const contents = await openPage() + respondWith(contents, { + activeElementSecrecy: 'safe', + describeFocusedEditable: { editable: false, reason: 'none' }, + }) + + const result = await driver.executeTool('chat-test', 'browser_insert_text', { text: 'x' }) + + expect(result.ok).toBe(false) + expect(result.error).toMatch(/No element is focused/) + expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0) + }) + + it('refuses insertion while a password field holds focus', async () => { + const contents = await openPage() + respondWith(contents, { activeElementSecrecy: 'secret' }) + + const result = await driver.executeTool('chat-test', 'browser_insert_text', { text: 'x' }) + + expect(result.ok).toBe(false) + expect(result.error).toMatch(/Refusing to act on a password field/) + expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0) + }) + + it('drags between coordinate points through the trusted pointer pipeline', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { found: true, element: 'div "Card"' }, + readActiveElementState: {}, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_drag', { + fromX: 40, + fromY: 50, + toX: 200, + toY: 260, + }) + + expect(result).toMatchObject({ + ok: true, + result: { dispatched: true, trusted: true, from: { x: 40, y: 50 }, to: { x: 200, y: 260 } }, + }) + const events = cdpCalls(contents, 'Input.dispatchMouseEvent').map( + ([, event]) => (event as { type?: string }).type + ) + expect(events[0]).toBe('mouseMoved') + expect(events).toContain('mousePressed') + expect(events[events.length - 1]).toBe('mouseReleased') + expect(cdpCalls(contents, 'Input.setInterceptDrags').length).toBeGreaterThan(0) + }) + + it('drags from a snapshot element to a coordinate target', async () => { + const contents = await openPage() + respondWith(contents, { + clickElement: { dispatched: false, x: 24, y: 48, element: 'Card "Ship it"' }, + describePointTarget: { found: true, element: 'section "Done"' }, + readActiveElementState: {}, + readPageActionState: {}, + }) + + const result = await driver.executeTool('chat-test', 'browser_drag', { + fromElementId: 0, + toX: 300, + toY: 60, + }) + + expect(result).toMatchObject({ + ok: true, + result: { dispatched: true, from: { x: 24, y: 48, element: 'Card "Ship it"' } }, + }) + }) + + it('rejects a drag whose endpoints are the same point', async () => { + const contents = await openPage() + respondWith(contents, { + describePointTarget: { found: true, element: 'div' }, + }) + + const result = await driver.executeTool('chat-test', 'browser_drag', { + fromX: 10, + fromY: 10, + toX: 10, + toY: 10, + }) + + expect(result.ok).toBe(false) + expect(result.error).toMatch(/same point/) + }) + + it('returns the screenshot scale for coordinate mapping', async () => { + const contents = await openPage() + vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => { + if (method === 'Page.getLayoutMetrics') { + return Promise.resolve({ + cssLayoutViewport: { clientWidth: 2048, clientHeight: 1024 }, + }) + } + if (method === 'Page.captureScreenshot') { + return Promise.resolve({ data: 'c2lt' }) + } + return Promise.resolve(undefined) + }) + respondWith(contents, { getViewportInfo: { width: 2048, height: 1024 } }) + + const result = await driver.executeTool('chat-test', 'browser_screenshot', {}) + + expect(result).toMatchObject({ ok: true, result: { scale: 0.5 } }) + }) }) diff --git a/apps/desktop/src/main/browser-agent/driver.ts b/apps/desktop/src/main/browser-agent/driver.ts index f6c43f21c6..1c1003f36e 100644 --- a/apps/desktop/src/main/browser-agent/driver.ts +++ b/apps/desktop/src/main/browser-agent/driver.ts @@ -46,6 +46,8 @@ import { activeElementSecrecy, clickElement, collectSnapshot, + describeFocusedEditable, + describePointTarget, focusElementForTyping, getViewportInfo, hoverElement, @@ -892,6 +894,11 @@ function unwrapPageResult(result: unknown): unknown { if (code === 'not-editable') { throw new ToolError('That element is not a text input — pick an editable element.') } + if (code === 'outside-viewport') { + throw new ToolError( + 'That point is outside the visible viewport. Coordinates are CSS pixels within the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale, and scroll the target into view first.' + ) + } if (code === 'ambiguous-editable') { throw new ToolError( 'That composite control contains multiple editable fields. Take a fresh browser_snapshot and target the exact field.' @@ -2002,19 +2009,21 @@ async function executeToolInner( case 'browser_screenshot': { const contents = session.requireAutomationTab().view.webContents - const dataUrl = await cdp.captureScreenshot(contents).catch(() => null) - if (dataUrl === null) { + const shot = await cdp.captureScreenshot(contents).catch(() => null) + if (shot === null) { throw new ToolError( 'Could not capture the page. Use browser_snapshot or browser_read_text instead.' ) } - if (dataUrl.length > 8_000_000) { + if (shot.dataUrl.length > 8_000_000) { throw new ToolError( 'The screenshot result was too large to return safely. Use browser_snapshot or browser_read_text instead.' ) } const viewport = await execInPage(contents, getViewportInfo, []).catch(() => null) - return { dataUrl, viewport } + // scale maps image pixels back to CSS viewport pixels for the + // coordinate tools: cssX = imageX / scale. + return { dataUrl: shot.dataUrl, viewport, scale: shot.scale } } case 'browser_extract': { @@ -3106,6 +3115,287 @@ async function executeToolInner( } } + case 'browser_click_at': { + const clickedTab = session.requireAutomationTab() + const contents = clickedTab.view.webContents + const x = requireNum(params, 'x') + const y = requireNum(params, 'y') + const clickCount = num(params, 'clickCount') ?? 1 + if (![1, 2, 3].includes(clickCount)) { + throw new ToolError('clickCount must be 1 (click), 2 (double-click), or 3 (triple-click).') + } + const clickNavigationEpoch = navigationEpoch(contents) + assertCurrentExecution() + assertActiveContents(contents) + const pointTarget = unwrapPageResult( + await execInPage(contents, describePointTarget, [x, y], false, executionDeadline) + ) + if (!isRecordLike(pointTarget) || pointTarget.found !== true) { + throw new ToolError( + 'Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.' + ) + } + if (pointTarget.fileInput === true) { + throw new ToolError( + 'Refusing to click a file input because it opens a native chooser the browser agent cannot inspect or complete. Ask the user to upload the file themselves.' + ) + } + const beforePage = await pageActionState(contents, true) + const beforeElement = await activeElementState(contents) + assertCurrentExecution() + assertActiveContents(contents, clickNavigationEpoch) + try { + await cdp.clickAt(contents, x, y, true, clickCount) + } catch (error) { + throw new ToolError( + `Native click dispatch failed (${getErrorMessage(error)}). The action was not retried because a partial pointer press may already have reached the page. Take a fresh snapshot before continuing.` + ) + } + await sleep(150) + const afterElement = await activeElementState(contents) + const afterPage = await pageActionState(contents) + const observation = pageEffect(beforePage, afterPage, beforeElement, afterElement) + const activeTab = session.automationTab() + const tabChanged = activeTab?.id !== clickedTab.id + const effectObserved = + observation.effect.urlChanged || + observation.effect.dialogChanged || + observation.effect.popupChanged || + observation.effect.targetChanged || + (pointTarget.editable === true && observation.effect.focusChanged) || + tabChanged + const dialogs = Array.isArray(afterPage.dialogs) ? afterPage.dialogs.map(String) : [] + const notes: string[] = [] + if (pointTarget.secret === true) { + notes.push( + 'The point resolves to a password field. Focusing it is fine, but typing there is refused — call browser_request_takeover for credentials.' + ) + } + if (pointTarget.crossOriginFrame === true) { + notes.push( + 'The point lands inside an embedded frame that could not be inspected; the click was dispatched but its target is unverified.' + ) + } + if (!effectObserved) { + notes.push( + observation.possibleEffectObserved || tabChanged + ? 'Only background DOM/title churn followed the click; inspect the page before treating it as successful.' + : 'No strong observable page change followed the click; inspect the page before treating it as successful.' + ) + } + return { + dispatched: true, + trusted: true, + activation: 'native-pointer', + clickedAt: { x, y }, + clickCount, + target: pointTarget.element, + targetCursor: pointTarget.cursor, + effectObserved, + possibleEffectObserved: observation.possibleEffectObserved || tabChanged, + effect: { ...observation.effect, tabChanged }, + dialogs, + ...(tabChanged && activeTab + ? { activeTab: { tabId: activeTab.id, url: activeTab.view.webContents.getURL() } } + : {}), + ...(notes.length > 0 ? { note: notes.join(' ') } : {}), + } + } + + case 'browser_insert_text': { + const text = requireStr(params, 'text') + const submit = params.submit === true + const contents = session.requireAutomationTab().view.webContents + const insertNavigationEpoch = navigationEpoch(contents) + const target: PageExecutionTarget = focusedPageTarget(contents) + const secrecy = await execInPage(target, activeElementSecrecy, []).catch(() => 'opaque') + if (secrecy === 'secret') throw new ToolError(PASSWORD_REFUSAL) + if (secrecy === 'opaque') { + throw new ToolError( + 'Focus is inside a cross-origin frame whose contents cannot be inspected, so this ' + + 'insertion could reach a password field. Call browser_request_takeover if the user needs to type here.' + ) + } + const focusState = unwrapPageResult( + await execInPage(target, describeFocusedEditable, [], false, executionDeadline) + ) + if (!isRecordLike(focusState) || focusState.editable !== true) { + const reason = isRecordLike(focusState) ? String(focusState.reason || '') : '' + throw new ToolError( + reason === 'none' + ? 'No element is focused. Click the field first (browser_click or browser_click_at), then insert text.' + : `The focused element does not accept text${reason ? ` (${reason})` : ''}. Focus an editable field first.` + ) + } + const beforePage = await pageActionState(target, true) + const beforeElement = await activeElementState(target) + assertCurrentExecution() + assertActiveContents(contents, insertNavigationEpoch) + assertFocusedTargetUnchanged(contents, target) + try { + await cdp.insertText(contents, text) + } catch (error) { + throw new ToolError( + `Native text insertion failed (${getErrorMessage(error)}). Take a fresh snapshot before retrying.` + ) + } + let submitDispatched = false + if (submit) { + await sleep(25) + assertCurrentExecution() + assertActiveContents(contents, insertNavigationEpoch) + try { + await dispatchKeyCombo(contents, parseKeyCombo('Enter')) + submitDispatched = true + } catch { + // Reported below through submitDispatched: false. + } + } + await sleep(150) + const state = await activeElementState(target) + const afterPage = await pageActionState(target) + const observation = pageEffect(beforePage, afterPage, beforeElement, state) + const effectObserved = + observation.effect.fieldChanged || + observation.effect.urlChanged || + observation.effect.dialogChanged || + observation.effect.targetChanged + return { + dispatched: true, + trusted: true, + kind: focusState.kind, + insertedChars: text.length, + ...state, + effectObserved, + possibleEffectObserved: observation.possibleEffectObserved, + effect: observation.effect, + submitRequested: submit, + submitDispatched, + ...(focusState.kind === 'canvas' || focusState.kind === 'textbox-role' + ? { + note: 'The focused editor is canvas/model-backed, so field readback cannot confirm the text — verify visually with browser_screenshot.', + } + : !effectObserved + ? { + note: 'Insertion produced no observable field or page change; inspect the page before continuing.', + } + : {}), + } + } + + case 'browser_drag': { + const draggedTab = session.requireAutomationTab() + const contents = draggedTab.view.webContents + const dragNavigationEpoch = navigationEpoch(contents) + + const resolveEndpoint = async ( + which: 'from' | 'to' + ): Promise<{ x: number; y: number; element?: string }> => { + const elementId = num(params, `${which}ElementId`) + if (elementId !== undefined) { + const target = pageTargetForElement(contents, elementId) + if (frameExecutionTarget(target, contents)) { + throw new ToolError( + `Dragging elements inside embedded frames is not supported. Use ${which}X/${which}Y viewport coordinates instead.` + ) + } + const prepared = unwrapPageResult( + await execInPageWithSettleGrace( + target, + clickElement, + [elementId, false, false], + executionDeadline + ) + ) + if ( + !isRecordLike(prepared) || + typeof prepared.x !== 'number' || + typeof prepared.y !== 'number' + ) { + throw new ToolError(`Could not resolve the ${which} element for the drag.`) + } + return { + x: prepared.x, + y: prepared.y, + ...(typeof prepared.element === 'string' ? { element: prepared.element } : {}), + } + } + const pointX = num(params, `${which}X`) + const pointY = num(params, `${which}Y`) + if (pointX === undefined || pointY === undefined) { + throw new ToolError( + `Provide either ${which}ElementId or both ${which}X and ${which}Y for the drag ${which === 'from' ? 'source' : 'target'}.` + ) + } + const probe = unwrapPageResult( + await execInPage( + contents, + describePointTarget, + [pointX, pointY], + false, + executionDeadline + ) + ) + if (!isRecordLike(probe) || probe.found !== true) { + throw new ToolError( + `Nothing is rendered at the ${which} point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.` + ) + } + return { + x: pointX, + y: pointY, + ...(typeof probe.element === 'string' ? { element: probe.element } : {}), + } + } + + assertCurrentExecution() + assertActiveContents(contents) + const from = await resolveEndpoint('from') + const to = await resolveEndpoint('to') + if (Math.abs(from.x - to.x) < 1 && Math.abs(from.y - to.y) < 1) { + throw new ToolError('The drag source and target are the same point; nothing to drag.') + } + const beforePage = await pageActionState(contents, true) + const beforeElement = await activeElementState(contents) + assertCurrentExecution() + assertActiveContents(contents, dragNavigationEpoch) + let interception: { nativeDragIntercepted: boolean } + try { + interception = await cdp.dragPointer(contents, from, to) + } catch (error) { + throw new ToolError( + `Native drag dispatch failed (${getErrorMessage(error)}). The pointer may have been mid-drag; take a fresh snapshot to see the page's current state before retrying.` + ) + } + await sleep(200) + const afterElement = await activeElementState(contents) + const afterPage = await pageActionState(contents) + const observation = pageEffect(beforePage, afterPage, beforeElement, afterElement) + const effectObserved = + observation.effect.domChanged || + observation.effect.urlChanged || + observation.effect.dialogChanged || + observation.effect.targetChanged || + observation.effect.scrollChanged + const dialogs = Array.isArray(afterPage.dialogs) ? afterPage.dialogs.map(String) : [] + return { + dispatched: true, + trusted: true, + nativeHtml5Drag: interception.nativeDragIntercepted, + from, + to, + effectObserved, + possibleEffectObserved: observation.possibleEffectObserved, + effect: observation.effect, + dialogs, + ...(!effectObserved + ? { + note: 'No observable page change followed the drag. Verify with browser_snapshot or browser_screenshot; some drop targets only commit on their own animation frame.', + } + : {}), + } + } + case 'browser_request_takeover': { // The reason renders in the chat's tool row, not here — but require it // so the model always tells the user why control was handed over. diff --git a/apps/desktop/src/main/browser-agent/page-functions.test.ts b/apps/desktop/src/main/browser-agent/page-functions.test.ts index a74f95bf9c..a8f6b8816e 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.test.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.test.ts @@ -5,6 +5,8 @@ import { activeElementSecrecy, clickElement, collectSnapshot, + describeFocusedEditable, + describePointTarget, focusElementForTyping, getViewportInfo, hoverElement, @@ -155,6 +157,8 @@ describe('serialization contract', () => { ['readPageText', readPageText, []], ['pageContainsText', pageContainsText, ['needle']], ['getViewportInfo', getViewportInfo, []], + ['describePointTarget', describePointTarget, [10, 10]], + ['describeFocusedEditable', describeFocusedEditable, []], ] it.each(cases)('%s is self-contained', (_name, fn, args) => { @@ -1373,3 +1377,80 @@ describe('pressKeyOnPage', () => { expect(seen).toEqual(['a']) }) }) + +describe('describePointTarget', () => { + function pointAt(el: Element | null): void { + Object.defineProperty(document, 'elementFromPoint', { + configurable: true, + value: () => el, + }) + } + + it('describes the element at a viewport point', () => { + document.body.innerHTML = '' + pointAt(document.querySelector('button')) + + expect(describePointTarget(10, 10)).toMatchObject({ + found: true, + tag: 'button', + element: 'button "Send message"', + editable: false, + fileInput: false, + secret: false, + }) + }) + + it('flags file inputs and password fields at the point', () => { + document.body.innerHTML = '' + pointAt(document.querySelector('input')) + expect(describePointTarget(10, 10)).toMatchObject({ found: true, fileInput: true }) + + document.body.innerHTML = '' + pointAt(document.querySelector('input')) + expect(describePointTarget(10, 10)).toMatchObject({ + found: true, + secret: true, + editable: true, + }) + }) + + it('rejects points outside the viewport', () => { + expect(describePointTarget(-5, 10)).toEqual({ error: 'outside-viewport' }) + expect(describePointTarget(10, window.innerHeight + 5)).toEqual({ + error: 'outside-viewport', + }) + }) +}) + +describe('describeFocusedEditable', () => { + it('reports no focus when the body holds focus', () => { + setActiveElement(document, document.body) + expect(describeFocusedEditable()).toEqual({ editable: false, reason: 'none' }) + }) + + it('reports a writable input as insertable', () => { + document.body.innerHTML = '' + setActiveElement(document, document.querySelector('input')) + expect(describeFocusedEditable()).toEqual({ editable: true, kind: 'input:text' }) + }) + + it('reports a read-only input as not insertable', () => { + document.body.innerHTML = '' + setActiveElement(document, document.querySelector('input')) + expect(describeFocusedEditable()).toEqual({ editable: false, reason: 'readonly' }) + }) + + it('reports a focused contenteditable editor as insertable', () => { + document.body.innerHTML = '
' + const editor = document.querySelector('div') as HTMLElement + Object.defineProperty(editor, 'isContentEditable', { get: () => true }) + setActiveElement(document, editor) + expect(describeFocusedEditable()).toEqual({ editable: true, kind: 'contenteditable' }) + }) + + it('treats a focused canvas editor surface as insertable', () => { + document.body.innerHTML = '' + setActiveElement(document, document.querySelector('canvas')) + expect(describeFocusedEditable()).toEqual({ editable: true, kind: 'canvas' }) + }) +}) diff --git a/apps/desktop/src/main/browser-agent/page-functions.ts b/apps/desktop/src/main/browser-agent/page-functions.ts index 3bdb88a64c..33af0d6f96 100644 --- a/apps/desktop/src/main/browser-agent/page-functions.ts +++ b/apps/desktop/src/main/browser-agent/page-functions.ts @@ -2676,3 +2676,155 @@ export function getViewportInfo(): unknown { height: window.innerHeight, } } + +/** + * Describes whatever sits at a viewport point, descending through open shadow + * roots and same-origin iframes. Coordinate-addressed actions have no + * snapshot ref to revalidate, so this probe is their safety check: the driver + * refuses file inputs outright and reports what the point resolves to so the + * model can confirm it hit what the screenshot showed. + */ +export function describePointTarget(x: number, y: number): unknown { + if ( + !Number.isFinite(x) || + !Number.isFinite(y) || + x < 0 || + y < 0 || + x >= window.innerWidth || + y >= window.innerHeight + ) { + return { error: 'outside-viewport' } + } + + let doc: Document = document + let localX = x + let localY = y + let element: Element | null = null + for (let depth = 0; depth < 10; depth++) { + if (typeof doc.elementFromPoint !== 'function') break + let found: Element | null = doc.elementFromPoint(localX, localY) + // Open shadow roots re-hit-test at the same point until a leaf host. + for (let shadowDepth = 0; shadowDepth < 10; shadowDepth++) { + const shadow = (found as HTMLElement | null)?.shadowRoot + const inner = shadow?.elementFromPoint(localX, localY) + if (!inner || inner === found) break + found = inner + } + element = found + const tag = String(found?.tagName || '').toUpperCase() + if ((tag !== 'IFRAME' && tag !== 'FRAME') || !found) break + try { + const innerDoc = (found as HTMLIFrameElement).contentDocument + if (!innerDoc) break + const rect = found.getBoundingClientRect() + localX -= rect.left + (found as HTMLIFrameElement).clientLeft + localY -= rect.top + (found as HTMLIFrameElement).clientTop + doc = innerDoc + } catch { + // Cross-origin frame — cannot inspect further; report the frame itself. + break + } + } + if (!element) return { found: false } + + const tag = String(element.tagName || '').toUpperCase() + const inputType = + tag === 'INPUT' ? String((element as HTMLInputElement).type || 'text').toLowerCase() : '' + const secret = + tag === 'INPUT' && + (inputType === 'password' || + String(element.getAttribute('autocomplete') || '') + .toLowerCase() + .split(/\s+/) + .some((token) => token === 'current-password' || token === 'new-password')) + const editable = Boolean( + tag === 'TEXTAREA' || + (tag === 'INPUT' && + ['text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(inputType)) || + (element as HTMLElement).isContentEditable + ) + + const name = ( + element.getAttribute('aria-label') || + element.getAttribute('title') || + element.getAttribute('alt') || + ((element as HTMLElement).innerText ?? element.textContent ?? '') + ) + .replace(/\s+/g, ' ') + .trim() + .slice(0, 80) + const role = element.getAttribute('role') || '' + const style = element.ownerDocument.defaultView?.getComputedStyle(element) + + return { + found: true, + element: name + ? `${tag.toLowerCase()}${role ? `[${role}]` : ''} "${name}"` + : `${tag.toLowerCase()}${role ? `[${role}]` : ''}`, + tag: tag.toLowerCase(), + role, + editable, + secret, + fileInput: tag === 'INPUT' && inputType === 'file', + disabled: + (element as HTMLInputElement).disabled === true || + element.getAttribute('aria-disabled') === 'true', + canvas: tag === 'CANVAS', + crossOriginFrame: tag === 'IFRAME' || tag === 'FRAME', + cursor: style?.cursor || '', + } +} + +/** + * Reports whether the currently-focused element accepts text insertion, for + * browser_insert_text — which types at the caret instead of addressing a + * snapshot ref. Secrecy is separately (and authoritatively) probed by + * activeElementSecrecy before any insertion. + */ +export function describeFocusedEditable(): unknown { + let active = document.activeElement as HTMLElement | null + for (let depth = 0; active && depth < 10; depth++) { + const shadow = active.shadowRoot + if (shadow?.activeElement) { + active = shadow.activeElement as HTMLElement + continue + } + break + } + if (!active || active === document.body) return { editable: false, reason: 'none' } + const tag = String(active.tagName || '').toUpperCase() + const inputType = + tag === 'INPUT' ? String((active as HTMLInputElement).type || 'text').toLowerCase() : '' + if (tag === 'INPUT' || tag === 'TEXTAREA') { + const field = active as HTMLInputElement | HTMLTextAreaElement + if (field.disabled || active.getAttribute('aria-disabled') === 'true') { + return { editable: false, reason: 'disabled' } + } + if (field.readOnly || active.getAttribute('aria-readonly') === 'true') { + return { editable: false, reason: 'readonly' } + } + if ( + tag === 'INPUT' && + !['text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(inputType) + ) { + return { editable: false, reason: 'not-text' } + } + return { editable: true, kind: tag === 'TEXTAREA' ? 'textarea' : `input:${inputType}` } + } + if (active.isContentEditable) { + if (active.getAttribute('aria-disabled') === 'true') { + return { editable: false, reason: 'disabled' } + } + if (active.getAttribute('aria-readonly') === 'true') { + return { editable: false, reason: 'readonly' } + } + return { editable: true, kind: 'contenteditable' } + } + // A canvas-rendered editor (Google Docs) focuses a hidden proxy or the + // canvas region itself; trusted IME insertion still reaches it, so report + // it as insertable rather than refusing. + if (tag === 'CANVAS' || active.getAttribute('role') === 'textbox') { + return { editable: true, kind: tag === 'CANVAS' ? 'canvas' : 'textbox-role' } + } + return { editable: false, reason: 'not-editable' } +} diff --git a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts index 9ba13a5aab..7648644e3f 100644 --- a/apps/sim/lib/copilot/generated/tool-catalog-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-catalog-v1.ts @@ -11,11 +11,14 @@ export interface ToolCatalogEntry { | 'auth' | 'browser' | 'browser_click' + | 'browser_click_at' | 'browser_close_tab' + | 'browser_drag' | 'browser_extract' | 'browser_go_back' | 'browser_go_forward' | 'browser_hover' + | 'browser_insert_text' | 'browser_list_sessions' | 'browser_list_tabs' | 'browser_navigate' @@ -132,11 +135,14 @@ export interface ToolCatalogEntry { | 'auth' | 'browser' | 'browser_click' + | 'browser_click_at' | 'browser_close_tab' + | 'browser_drag' | 'browser_extract' | 'browser_go_back' | 'browser_go_forward' | 'browser_hover' + | 'browser_insert_text' | 'browser_list_sessions' | 'browser_list_tabs' | 'browser_navigate' @@ -450,6 +456,135 @@ export const BrowserClick: ToolCatalogEntry = { clientExecutable: true, } +export const BrowserClickAt: ToolCatalogEntry = { + id: 'browser_click_at', + name: 'browser_click_at', + route: 'client', + mode: 'async', + parameters: { + type: 'object', + properties: { + clickCount: { + type: 'number', + description: '1 = single click (default), 2 = double-click, 3 = triple-click.', + }, + x: { + type: 'number', + description: + "X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by the screenshot's scale.", + }, + y: { + type: 'number', + description: + 'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.', + }, + }, + required: ['x', 'y'], + }, + resultSchema: { + type: 'object', + properties: { + activeTab: { + type: 'object', + description: 'New active tab after a tab-changing click.', + properties: { + tabId: { type: 'string', description: 'Stable browser tab id.' }, + url: { type: 'string', description: 'New active tab URL.' }, + }, + }, + clickCount: { + type: 'number', + description: 'The click count that was dispatched (1, 2, or 3).', + }, + clickedAt: { + type: 'object', + description: 'The CSS-pixel viewport point that was clicked.', + properties: { + x: { type: 'number', description: 'Clicked X in CSS viewport pixels.' }, + y: { type: 'number', description: 'Clicked Y in CSS viewport pixels.' }, + }, + }, + dialogs: { + type: 'array', + description: 'Visible DOM dialogs remaining after the click.', + items: { type: 'string' }, + }, + dispatched: { + type: 'boolean', + description: 'Whether the native pointer click was dispatched at the point.', + }, + effect: { + type: 'object', + description: + 'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.', + properties: { + dialogChanged: { type: 'boolean', description: 'The visible DOM dialog set changed.' }, + domChanged: { + type: 'boolean', + description: 'The DOM mutation revision changed; weak evidence on its own.', + }, + fieldChanged: { + type: 'boolean', + description: 'The safely inspectable focused-field state changed.', + }, + focusChanged: { type: 'boolean', description: 'The focused element changed.' }, + popupChanged: { type: 'boolean', description: 'The visible popup/menu set changed.' }, + scrollChanged: { + type: 'boolean', + description: 'A tracked scroll offset changed; weak evidence except for scroll keys.', + }, + tabChanged: { type: 'boolean', description: 'The active browser tab changed.' }, + targetChanged: { + type: 'boolean', + description: "The requested target's checked/selected/expanded/open state changed.", + }, + titleChanged: { + type: 'boolean', + description: 'The document title changed; weak evidence on its own.', + }, + urlChanged: { type: 'boolean', description: 'The observed URL changed.' }, + }, + }, + effectObserved: { + type: 'boolean', + description: + 'Whether a strong page change (URL/dialog/popup/target, or focus into an editable) followed the click.', + }, + note: { + type: 'string', + description: 'Caution or follow-up guidance, present when the click needs verification.', + }, + notices: { + type: 'array', + description: + 'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.', + items: { type: 'string' }, + }, + possibleEffectObserved: { + type: 'boolean', + description: 'Whether only weaker background churn (DOM/title) was observed.', + }, + target: { + type: 'string', + description: + 'What the point resolved to before the click (tag/role and accessible name) — confirm it matches the intended target.', + }, + targetCursor: { + type: 'string', + description: + "The CSS cursor at the point (e.g. 'pointer', 'text', 'crosshair') — a hint about what kind of surface was hit.", + }, + trusted: { + type: 'boolean', + description: + 'True — coordinate and drag input always use the trusted Chromium pointer pipeline.', + }, + }, + required: ['dispatched'], + }, + clientExecutable: true, +} + export const BrowserCloseTab: ToolCatalogEntry = { id: 'browser_close_tab', name: 'browser_close_tab', @@ -468,6 +603,130 @@ export const BrowserCloseTab: ToolCatalogEntry = { clientExecutable: true, } +export const BrowserDrag: ToolCatalogEntry = { + id: 'browser_drag', + name: 'browser_drag', + route: 'client', + mode: 'async', + parameters: { + type: 'object', + properties: { + fromElementId: { + type: 'number', + description: 'Drag source element id from the latest snapshot. Alternative to fromX/fromY.', + }, + fromX: { + type: 'number', + description: + 'Drag source X in CSS viewport pixels (paired with fromY) when no source element id is available.', + }, + fromY: { type: 'number', description: 'Drag source Y in CSS viewport pixels.' }, + toElementId: { + type: 'number', + description: 'Drop target element id from the latest snapshot. Alternative to toX/toY.', + }, + toX: { + type: 'number', + description: 'Drop target X in CSS viewport pixels (paired with toY).', + }, + toY: { type: 'number', description: 'Drop target Y in CSS viewport pixels.' }, + }, + }, + resultSchema: { + type: 'object', + properties: { + dialogs: { + type: 'array', + description: 'Visible DOM dialogs remaining after the click.', + items: { type: 'string' }, + }, + dispatched: { type: 'boolean', description: 'Whether the full drag gesture was dispatched.' }, + effect: { + type: 'object', + description: + 'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.', + properties: { + dialogChanged: { type: 'boolean', description: 'The visible DOM dialog set changed.' }, + domChanged: { + type: 'boolean', + description: 'The DOM mutation revision changed; weak evidence on its own.', + }, + fieldChanged: { + type: 'boolean', + description: 'The safely inspectable focused-field state changed.', + }, + focusChanged: { type: 'boolean', description: 'The focused element changed.' }, + popupChanged: { type: 'boolean', description: 'The visible popup/menu set changed.' }, + scrollChanged: { + type: 'boolean', + description: 'A tracked scroll offset changed; weak evidence except for scroll keys.', + }, + tabChanged: { type: 'boolean', description: 'The active browser tab changed.' }, + targetChanged: { + type: 'boolean', + description: "The requested target's checked/selected/expanded/open state changed.", + }, + titleChanged: { + type: 'boolean', + description: 'The document title changed; weak evidence on its own.', + }, + urlChanged: { type: 'boolean', description: 'The observed URL changed.' }, + }, + }, + effectObserved: { + type: 'boolean', + description: + 'Whether an observable page change (DOM/URL/dialog/target/scroll) followed the drag.', + }, + from: { + type: 'object', + description: 'The resolved drag source.', + properties: { + element: { type: 'string', description: 'What that endpoint resolved to, when known.' }, + x: { type: 'number', description: 'Endpoint X in CSS viewport pixels.' }, + y: { type: 'number', description: 'Endpoint Y in CSS viewport pixels.' }, + }, + }, + nativeHtml5Drag: { + type: 'boolean', + description: + 'True when the page started a native HTML5 drag and it was completed as a real drag-and-drop; false for a pointer-sensor drag.', + }, + note: { + type: 'string', + description: + 'Caution or follow-up guidance — e.g. verify the drop with a fresh snapshot when no effect was observed.', + }, + notices: { + type: 'array', + description: + 'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.', + items: { type: 'string' }, + }, + possibleEffectObserved: { + type: 'boolean', + description: 'Whether only weaker background churn (DOM/title) was observed.', + }, + to: { + type: 'object', + description: 'The resolved drop target.', + properties: { + element: { type: 'string', description: 'What that endpoint resolved to, when known.' }, + x: { type: 'number', description: 'Endpoint X in CSS viewport pixels.' }, + y: { type: 'number', description: 'Endpoint Y in CSS viewport pixels.' }, + }, + }, + trusted: { + type: 'boolean', + description: + 'True — coordinate and drag input always use the trusted Chromium pointer pipeline.', + }, + }, + required: ['dispatched'], + }, + clientExecutable: true, +} + export const BrowserExtract: ToolCatalogEntry = { id: 'browser_extract', name: 'browser_extract', @@ -630,6 +889,116 @@ export const BrowserHover: ToolCatalogEntry = { clientExecutable: true, } +export const BrowserInsertText: ToolCatalogEntry = { + id: 'browser_insert_text', + name: 'browser_insert_text', + route: 'client', + mode: 'async', + parameters: { + type: 'object', + properties: { + submit: { type: 'boolean', description: 'Press Enter after inserting. Default false.' }, + text: { type: 'string', description: 'The text to insert at the caret. Must be non-empty.' }, + }, + required: ['text'], + }, + resultSchema: { + type: 'object', + properties: { + activeElement: { type: 'string', description: 'Focused element kind after the action.' }, + dispatched: { + type: 'boolean', + description: 'Whether the text was inserted through the native IME pipeline.', + }, + effect: { + type: 'object', + description: + 'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.', + properties: { + dialogChanged: { type: 'boolean', description: 'The visible DOM dialog set changed.' }, + domChanged: { + type: 'boolean', + description: 'The DOM mutation revision changed; weak evidence on its own.', + }, + fieldChanged: { + type: 'boolean', + description: 'The safely inspectable focused-field state changed.', + }, + focusChanged: { type: 'boolean', description: 'The focused element changed.' }, + popupChanged: { type: 'boolean', description: 'The visible popup/menu set changed.' }, + scrollChanged: { + type: 'boolean', + description: 'A tracked scroll offset changed; weak evidence except for scroll keys.', + }, + tabChanged: { type: 'boolean', description: 'The active browser tab changed.' }, + targetChanged: { + type: 'boolean', + description: "The requested target's checked/selected/expanded/open state changed.", + }, + titleChanged: { + type: 'boolean', + description: 'The document title changed; weak evidence on its own.', + }, + urlChanged: { type: 'boolean', description: 'The observed URL changed.' }, + }, + }, + effectObserved: { + type: 'boolean', + description: + 'Whether a field or page change confirmed the insertion. Canvas editors cannot echo one — verify visually.', + }, + insertedChars: { type: 'number', description: 'How many characters were inserted.' }, + kind: { + type: 'string', + description: + 'The kind of focused editable that received the text (input:*, textarea, contenteditable, canvas, textbox-role).', + }, + note: { + type: 'string', + description: + 'Caution or follow-up guidance, e.g. that a canvas editor needs visual verification.', + }, + notices: { + type: 'array', + description: + 'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.', + items: { type: 'string' }, + }, + possibleEffectObserved: { + type: 'boolean', + description: 'Whether only weaker background churn was observed.', + }, + redacted: { + type: 'boolean', + description: 'Whether sensitive focused-field details were withheld.', + }, + selectedChars: { + type: 'number', + description: 'Number of selected characters when safely inspectable.', + }, + submitDispatched: { type: 'boolean', description: 'Whether Enter was actually dispatched.' }, + submitRequested: { + type: 'boolean', + description: 'Whether Enter was requested after insertion.', + }, + trusted: { + type: 'boolean', + description: 'True — insertion uses the trusted input pipeline.', + }, + valueLength: { + type: 'number', + description: 'Focused non-secret field length when safely inspectable.', + }, + valuePreview: { + type: 'string', + description: 'Bounded focused non-secret field preview when safely inspectable.', + }, + }, + required: ['dispatched'], + }, + clientExecutable: true, +} + export const BrowserListSessions: ToolCatalogEntry = { id: 'browser_list_sessions', name: 'browser_list_sessions', @@ -6617,11 +6986,14 @@ export const TOOL_CATALOG: Record = { [Auth.id]: Auth, [Browser.id]: Browser, [BrowserClick.id]: BrowserClick, + [BrowserClickAt.id]: BrowserClickAt, [BrowserCloseTab.id]: BrowserCloseTab, + [BrowserDrag.id]: BrowserDrag, [BrowserExtract.id]: BrowserExtract, [BrowserGoBack.id]: BrowserGoBack, [BrowserGoForward.id]: BrowserGoForward, [BrowserHover.id]: BrowserHover, + [BrowserInsertText.id]: BrowserInsertText, [BrowserListSessions.id]: BrowserListSessions, [BrowserListTabs.id]: BrowserListTabs, [BrowserNavigate.id]: BrowserNavigate, diff --git a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts index 3bebb7cd52..a75d3ddd63 100644 --- a/apps/sim/lib/copilot/generated/tool-schemas-v1.ts +++ b/apps/sim/lib/copilot/generated/tool-schemas-v1.ts @@ -214,6 +214,160 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['dispatched'], }, }, + browser_click_at: { + parameters: { + type: 'object', + properties: { + clickCount: { + type: 'number', + description: '1 = single click (default), 2 = double-click, 3 = triple-click.', + }, + x: { + type: 'number', + description: + "X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by the screenshot's scale.", + }, + y: { + type: 'number', + description: + 'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.', + }, + }, + required: ['x', 'y'], + }, + resultSchema: { + type: 'object', + properties: { + activeTab: { + type: 'object', + description: 'New active tab after a tab-changing click.', + properties: { + tabId: { + type: 'string', + description: 'Stable browser tab id.', + }, + url: { + type: 'string', + description: 'New active tab URL.', + }, + }, + }, + clickCount: { + type: 'number', + description: 'The click count that was dispatched (1, 2, or 3).', + }, + clickedAt: { + type: 'object', + description: 'The CSS-pixel viewport point that was clicked.', + properties: { + x: { + type: 'number', + description: 'Clicked X in CSS viewport pixels.', + }, + y: { + type: 'number', + description: 'Clicked Y in CSS viewport pixels.', + }, + }, + }, + dialogs: { + type: 'array', + description: 'Visible DOM dialogs remaining after the click.', + items: { + type: 'string', + }, + }, + dispatched: { + type: 'boolean', + description: 'Whether the native pointer click was dispatched at the point.', + }, + effect: { + type: 'object', + description: + 'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.', + properties: { + dialogChanged: { + type: 'boolean', + description: 'The visible DOM dialog set changed.', + }, + domChanged: { + type: 'boolean', + description: 'The DOM mutation revision changed; weak evidence on its own.', + }, + fieldChanged: { + type: 'boolean', + description: 'The safely inspectable focused-field state changed.', + }, + focusChanged: { + type: 'boolean', + description: 'The focused element changed.', + }, + popupChanged: { + type: 'boolean', + description: 'The visible popup/menu set changed.', + }, + scrollChanged: { + type: 'boolean', + description: 'A tracked scroll offset changed; weak evidence except for scroll keys.', + }, + tabChanged: { + type: 'boolean', + description: 'The active browser tab changed.', + }, + targetChanged: { + type: 'boolean', + description: "The requested target's checked/selected/expanded/open state changed.", + }, + titleChanged: { + type: 'boolean', + description: 'The document title changed; weak evidence on its own.', + }, + urlChanged: { + type: 'boolean', + description: 'The observed URL changed.', + }, + }, + }, + effectObserved: { + type: 'boolean', + description: + 'Whether a strong page change (URL/dialog/popup/target, or focus into an editable) followed the click.', + }, + note: { + type: 'string', + description: 'Caution or follow-up guidance, present when the click needs verification.', + }, + notices: { + type: 'array', + description: + 'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.', + items: { + type: 'string', + }, + }, + possibleEffectObserved: { + type: 'boolean', + description: 'Whether only weaker background churn (DOM/title) was observed.', + }, + target: { + type: 'string', + description: + 'What the point resolved to before the click (tag/role and accessible name) — confirm it matches the intended target.', + }, + targetCursor: { + type: 'string', + description: + "The CSS cursor at the point (e.g. 'pointer', 'text', 'crosshair') — a hint about what kind of surface was hit.", + }, + trusted: { + type: 'boolean', + description: + 'True — coordinate and drag input always use the trusted Chromium pointer pipeline.', + }, + }, + required: ['dispatched'], + }, + }, browser_close_tab: { parameters: { type: 'object', @@ -227,6 +381,171 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { }, resultSchema: undefined, }, + browser_drag: { + parameters: { + type: 'object', + properties: { + fromElementId: { + type: 'number', + description: + 'Drag source element id from the latest snapshot. Alternative to fromX/fromY.', + }, + fromX: { + type: 'number', + description: + 'Drag source X in CSS viewport pixels (paired with fromY) when no source element id is available.', + }, + fromY: { + type: 'number', + description: 'Drag source Y in CSS viewport pixels.', + }, + toElementId: { + type: 'number', + description: 'Drop target element id from the latest snapshot. Alternative to toX/toY.', + }, + toX: { + type: 'number', + description: 'Drop target X in CSS viewport pixels (paired with toY).', + }, + toY: { + type: 'number', + description: 'Drop target Y in CSS viewport pixels.', + }, + }, + }, + resultSchema: { + type: 'object', + properties: { + dialogs: { + type: 'array', + description: 'Visible DOM dialogs remaining after the click.', + items: { + type: 'string', + }, + }, + dispatched: { + type: 'boolean', + description: 'Whether the full drag gesture was dispatched.', + }, + effect: { + type: 'object', + description: + 'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.', + properties: { + dialogChanged: { + type: 'boolean', + description: 'The visible DOM dialog set changed.', + }, + domChanged: { + type: 'boolean', + description: 'The DOM mutation revision changed; weak evidence on its own.', + }, + fieldChanged: { + type: 'boolean', + description: 'The safely inspectable focused-field state changed.', + }, + focusChanged: { + type: 'boolean', + description: 'The focused element changed.', + }, + popupChanged: { + type: 'boolean', + description: 'The visible popup/menu set changed.', + }, + scrollChanged: { + type: 'boolean', + description: 'A tracked scroll offset changed; weak evidence except for scroll keys.', + }, + tabChanged: { + type: 'boolean', + description: 'The active browser tab changed.', + }, + targetChanged: { + type: 'boolean', + description: "The requested target's checked/selected/expanded/open state changed.", + }, + titleChanged: { + type: 'boolean', + description: 'The document title changed; weak evidence on its own.', + }, + urlChanged: { + type: 'boolean', + description: 'The observed URL changed.', + }, + }, + }, + effectObserved: { + type: 'boolean', + description: + 'Whether an observable page change (DOM/URL/dialog/target/scroll) followed the drag.', + }, + from: { + type: 'object', + description: 'The resolved drag source.', + properties: { + element: { + type: 'string', + description: 'What that endpoint resolved to, when known.', + }, + x: { + type: 'number', + description: 'Endpoint X in CSS viewport pixels.', + }, + y: { + type: 'number', + description: 'Endpoint Y in CSS viewport pixels.', + }, + }, + }, + nativeHtml5Drag: { + type: 'boolean', + description: + 'True when the page started a native HTML5 drag and it was completed as a real drag-and-drop; false for a pointer-sensor drag.', + }, + note: { + type: 'string', + description: + 'Caution or follow-up guidance — e.g. verify the drop with a fresh snapshot when no effect was observed.', + }, + notices: { + type: 'array', + description: + 'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.', + items: { + type: 'string', + }, + }, + possibleEffectObserved: { + type: 'boolean', + description: 'Whether only weaker background churn (DOM/title) was observed.', + }, + to: { + type: 'object', + description: 'The resolved drop target.', + properties: { + element: { + type: 'string', + description: 'What that endpoint resolved to, when known.', + }, + x: { + type: 'number', + description: 'Endpoint X in CSS viewport pixels.', + }, + y: { + type: 'number', + description: 'Endpoint Y in CSS viewport pixels.', + }, + }, + }, + trusted: { + type: 'boolean', + description: + 'True — coordinate and drag input always use the trusted Chromium pointer pipeline.', + }, + }, + required: ['dispatched'], + }, + }, browser_extract: { parameters: { type: 'object', @@ -410,6 +729,142 @@ export const TOOL_RUNTIME_SCHEMAS: Record = { required: ['hovered'], }, }, + browser_insert_text: { + parameters: { + type: 'object', + properties: { + submit: { + type: 'boolean', + description: 'Press Enter after inserting. Default false.', + }, + text: { + type: 'string', + description: 'The text to insert at the caret. Must be non-empty.', + }, + }, + required: ['text'], + }, + resultSchema: { + type: 'object', + properties: { + activeElement: { + type: 'string', + description: 'Focused element kind after the action.', + }, + dispatched: { + type: 'boolean', + description: 'Whether the text was inserted through the native IME pipeline.', + }, + effect: { + type: 'object', + description: + 'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.', + properties: { + dialogChanged: { + type: 'boolean', + description: 'The visible DOM dialog set changed.', + }, + domChanged: { + type: 'boolean', + description: 'The DOM mutation revision changed; weak evidence on its own.', + }, + fieldChanged: { + type: 'boolean', + description: 'The safely inspectable focused-field state changed.', + }, + focusChanged: { + type: 'boolean', + description: 'The focused element changed.', + }, + popupChanged: { + type: 'boolean', + description: 'The visible popup/menu set changed.', + }, + scrollChanged: { + type: 'boolean', + description: 'A tracked scroll offset changed; weak evidence except for scroll keys.', + }, + tabChanged: { + type: 'boolean', + description: 'The active browser tab changed.', + }, + targetChanged: { + type: 'boolean', + description: "The requested target's checked/selected/expanded/open state changed.", + }, + titleChanged: { + type: 'boolean', + description: 'The document title changed; weak evidence on its own.', + }, + urlChanged: { + type: 'boolean', + description: 'The observed URL changed.', + }, + }, + }, + effectObserved: { + type: 'boolean', + description: + 'Whether a field or page change confirmed the insertion. Canvas editors cannot echo one — verify visually.', + }, + insertedChars: { + type: 'number', + description: 'How many characters were inserted.', + }, + kind: { + type: 'string', + description: + 'The kind of focused editable that received the text (input:*, textarea, contenteditable, canvas, textbox-role).', + }, + note: { + type: 'string', + description: + 'Caution or follow-up guidance, e.g. that a canvas editor needs visual verification.', + }, + notices: { + type: 'array', + description: + 'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.', + items: { + type: 'string', + }, + }, + possibleEffectObserved: { + type: 'boolean', + description: 'Whether only weaker background churn was observed.', + }, + redacted: { + type: 'boolean', + description: 'Whether sensitive focused-field details were withheld.', + }, + selectedChars: { + type: 'number', + description: 'Number of selected characters when safely inspectable.', + }, + submitDispatched: { + type: 'boolean', + description: 'Whether Enter was actually dispatched.', + }, + submitRequested: { + type: 'boolean', + description: 'Whether Enter was requested after insertion.', + }, + trusted: { + type: 'boolean', + description: 'True — insertion uses the trusted input pipeline.', + }, + valueLength: { + type: 'number', + description: 'Focused non-secret field length when safely inspectable.', + }, + valuePreview: { + type: 'string', + description: 'Bounded focused non-secret field preview when safely inspectable.', + }, + }, + required: ['dispatched'], + }, + }, browser_list_sessions: { parameters: { type: 'object', diff --git a/apps/sim/lib/copilot/tools/tool-display.ts b/apps/sim/lib/copilot/tools/tool-display.ts index b274fdd2bd..5981910cd0 100644 --- a/apps/sim/lib/copilot/tools/tool-display.ts +++ b/apps/sim/lib/copilot/tools/tool-display.ts @@ -525,7 +525,10 @@ const TOOL_TITLES: Record = { browser_read_text: 'Reading page', browser_screenshot: 'Taking screenshot', browser_click: 'Clicking element', + browser_click_at: 'Clicking point', browser_type: 'Typing text', + browser_insert_text: 'Inserting text', + browser_drag: 'Dragging element', browser_select_option: 'Selecting option', browser_hover: 'Hovering element', // Subagent trigger tools, when surfaced as a tool call. @@ -1015,6 +1018,8 @@ const COMPLETED_VERB_REWRITES: Record = { Creating: 'Created', Deleting: 'Deleted', Deploying: 'Deployed', + Dragging: 'Dragged', + Inserting: 'Inserted', Publishing: 'Published', Unpublishing: 'Unpublished', Analyzing: 'Analyzed', diff --git a/packages/browser-protocol/src/index.ts b/packages/browser-protocol/src/index.ts index 5158764dd0..ca7dd65cbd 100644 --- a/packages/browser-protocol/src/index.ts +++ b/packages/browser-protocol/src/index.ts @@ -33,11 +33,14 @@ export const BROWSER_TOOL_NAMES = [ 'browser_screenshot', 'browser_extract', 'browser_click', + 'browser_click_at', 'browser_type', + 'browser_insert_text', 'browser_press_key', 'browser_scroll', 'browser_select_option', 'browser_hover', + 'browser_drag', 'browser_request_takeover', ] as const