feat(desktop): coordinate clicks, caret insertion, and drag for the browser agent

apps/desktop/src/main/browser-agent/cdp.ts

@@ -10,6 +10,7 @@
  */
 import type { BrowserTheme } from '@sim/browser-protocol'
 import { createLogger } from '@sim/logger'
+import { sleep } from '@sim/utils/helpers'
 import type { WebContents, WebFrameMain } from 'electron'
 
 const logger = createLogger('BrowserAgentCdp')
@@ -127,12 +128,29 @@ export async function setColorScheme(contents: WebContents, theme: BrowserTheme)
   })
 }
 
+/** Live drag-interception state while a dragPointer call is in flight. */
+interface DragInterception {
+  intercepted: boolean
+  data: Record<string, unknown> | null
+}
+const dragInterceptionsByContents = new WeakMap<WebContents, DragInterception>()
+
 function handleDebuggerEvent(
   contents: WebContents,
   method: string,
   params: Record<string, unknown>,
   parentSessionId?: string
 ): void {
+  if (method === 'Input.dragIntercepted') {
+    const interception = dragInterceptionsByContents.get(contents)
+    if (interception) {
+      interception.intercepted = true
+      const data = params.data
+      interception.data =
+        data && typeof data === 'object' ? (data as Record<string, unknown>) : null
+    }
+    return
+  }
   if (method === 'Target.attachedToTarget') {
     const sessionId = typeof params.sessionId === 'string' ? params.sessionId : ''
     const targetInfo = params.targetInfo
@@ -367,7 +385,9 @@ interface CdpViewport {
  * edge within {@link MAX_SCREENSHOT_EDGE}. Falls back to an unclipped capture
  * when layout metrics are unavailable.
  */
-export async function captureScreenshot(contents: WebContents): Promise<string> {
+export async function captureScreenshot(
+  contents: WebContents
+): Promise<{ dataUrl: string; scale: number }> {
   const metrics = await send<{
     cssLayoutViewport?: CdpViewport
     layoutViewport?: CdpViewport
@@ -376,6 +396,8 @@ export async function captureScreenshot(contents: WebContents): Promise<string>
   const viewport = metrics?.cssLayoutViewport ?? metrics?.layoutViewport
   const width = viewport?.clientWidth ?? 0
   const height = viewport?.clientHeight ?? 0
+  const scale =
+    width > 0 && height > 0 ? Math.min(1, MAX_SCREENSHOT_EDGE / Math.max(width, height)) : 1
   const clip =
     width > 0 && height > 0
       ? {
@@ -383,7 +405,7 @@ export async function captureScreenshot(contents: WebContents): Promise<string>
           y: 0,
           width,
           height,
-          scale: Math.min(1, MAX_SCREENSHOT_EDGE / Math.max(width, height)),
+          scale,
         }
       : undefined
 
@@ -392,7 +414,7 @@ export async function captureScreenshot(contents: WebContents): Promise<string>
     quality: SCREENSHOT_QUALITY,
     ...(clip ? { clip } : {}),
   })
-  return `data:image/jpeg;base64,${result.data}`
+  return { dataUrl: `data:image/jpeg;base64,${result.data}`, scale }
 }
 
 /** One half of a trusted key press (`Input.dispatchKeyEvent` params). */
@@ -430,7 +452,8 @@ export async function clickAt(
   contents: WebContents,
   x: number,
   y: number,
-  moveBeforePress = true
+  moveBeforePress = true,
+  clickCount = 1
 ): Promise<void> {
   if (moveBeforePress) await moveMouse(contents, x, y)
   let pressed = false
@@ -439,22 +462,26 @@ export async function clickAt(
     // response (navigation/process swap). In that ambiguous case a release is
     // safer than leaving Blink's pointer state stuck down.
     pressed = true
-    await sendInput(contents, 'Input.dispatchMouseEvent', {
-      type: 'mousePressed',
-      x,
-      y,
-      button: 'left',
-      buttons: 1,
-      clickCount: 1,
-    })
-    await sendInput(contents, 'Input.dispatchMouseEvent', {
-      type: 'mouseReleased',
-      x,
-      y,
-      button: 'left',
-      buttons: 0,
-      clickCount: 1,
-    })
+    // A multi-click is a sequence of press/release pairs with an increasing
+    // clickCount — Blink synthesizes dblclick from the pair whose count is 2.
+    for (let count = 1; count <= clickCount; count++) {
+      await sendInput(contents, 'Input.dispatchMouseEvent', {
+        type: 'mousePressed',
+        x,
+        y,
+        button: 'left',
+        buttons: 1,
+        clickCount: count,
+      })
+      await sendInput(contents, 'Input.dispatchMouseEvent', {
+        type: 'mouseReleased',
+        x,
+        y,
+        button: 'left',
+        buttons: 0,
+        clickCount: count,
+      })
+    }
     pressed = false
   } finally {
     if (pressed && !contents.isDestroyed()) {
@@ -473,6 +500,131 @@ export async function clickAt(
   }
 }
 
+/**
+ * Drags the pointer from one viewport point to another through the trusted
+ * pipeline: press, a threshold-crossing nudge, interpolated moves with the
+ * button held, a settle hold over the target, then release or drop.
+ *
+ * Two drag models are covered by the one call. Pointer-sensor libraries
+ * (dnd-kit, react-beautiful-dnd, canvas apps) treat the held-button move
+ * sequence exactly like a human drag. Native HTML5 `draggable="true"`
+ * sources instead START a Blink drag session on the press+move — with
+ * `Input.setInterceptDrags` enabled, Chromium reports it as
+ * `Input.dragIntercepted` and the remaining movement is delivered as trusted
+ * `Input.dispatchDragEvent` dragEnter/dragOver events ending in a `drop`
+ * (the technique Playwright uses). Both paths are trusted input.
+ */
+export async function dragPointer(
+  contents: WebContents,
+  from: { x: number; y: number },
+  to: { x: number; y: number },
+  steps = 12,
+  stepDelayMs = 20
+): Promise<{ nativeDragIntercepted: boolean }> {
+  const interception: DragInterception = { intercepted: false, data: null }
+  dragInterceptionsByContents.set(contents, interception)
+  let interceptEnabled = false
+  try {
+    await send(contents, 'Input.setInterceptDrags', { enabled: true })
+    interceptEnabled = true
+  } catch {
+    // Chromium without drag interception: the pointer-only path still works
+    // for pointer-sensor drags; native HTML5 sources will report no effect.
+  }
+  await moveMouse(contents, from.x, from.y)
+  let pressed = false
+  let dragEnterSent = false
+  const dragMove = async (x: number, y: number) => {
+    if (interception.intercepted && interception.data) {
+      await sendInput(contents, 'Input.dispatchDragEvent', {
+        type: dragEnterSent ? 'dragOver' : 'dragEnter',
+        x,
+        y,
+        data: interception.data,
+      })
+      dragEnterSent = true
+    } else {
+      await sendInput(contents, 'Input.dispatchMouseEvent', {
+        type: 'mouseMoved',
+        x,
+        y,
+        button: 'left',
+        buttons: 1,
+      })
+    }
+  }
+  try {
+    pressed = true
+    await sendInput(contents, 'Input.dispatchMouseEvent', {
+      type: 'mousePressed',
+      x: from.x,
+      y: from.y,
+      button: 'left',
+      buttons: 1,
+      clickCount: 1,
+    })
+    // Small first nudge so libraries with a start threshold (commonly 3-8px)
+    // register the drag before the pointer sweeps across the page.
+    await dragMove(from.x + Math.sign(to.x - from.x || 1) * 4, from.y + 2)
+    await sleep(stepDelayMs)
+    const stepCount = Math.max(2, steps)
+    for (let step = 1; step <= stepCount; step++) {
+      const progress = step / stepCount
+      await dragMove(from.x + (to.x - from.x) * progress, from.y + (to.y - from.y) * progress)
+      await sleep(stepDelayMs)
+    }
+    // Hold over the target so drop zones running enter/over animations settle
+    // before the release lands.
+    await sleep(120)
+    if (interception.intercepted && interception.data) {
+      await sendInput(contents, 'Input.dispatchDragEvent', {
+        type: 'drop',
+        x: to.x,
+        y: to.y,
+        data: interception.data,
+      })
+      // Blink ended the intercepted drag session itself; a trailing
+      // mouseReleased would be a stray click on the drop target.
+      pressed = false
+    } else {
+      await sendInput(contents, 'Input.dispatchMouseEvent', {
+        type: 'mouseReleased',
+        x: to.x,
+        y: to.y,
+        button: 'left',
+        buttons: 0,
+        clickCount: 1,
+      })
+      pressed = false
+    }
+    return { nativeDragIntercepted: interception.intercepted }
+  } finally {
+    if (pressed && !contents.isDestroyed()) {
+      if (interception.intercepted && interception.data) {
+        await sendInput(contents, 'Input.dispatchDragEvent', {
+          type: 'dragCancel',
+          x: to.x,
+          y: to.y,
+          data: interception.data,
+        }).catch(() => {})
+      } else {
+        await sendInput(contents, 'Input.dispatchMouseEvent', {
+          type: 'mouseReleased',
+          x: to.x,
+          y: to.y,
+          button: 'left',
+          buttons: 0,
+          clickCount: 1,
+        }).catch(() => {})
+      }
+    }
+    dragInterceptionsByContents.delete(contents)
+    if (interceptEnabled && !contents.isDestroyed()) {
+      await send(contents, 'Input.setInterceptDrags', { enabled: false }).catch(() => {})
+    }
+  }
+}
+
 /**
  * Inserts text at the focused element's selection (replacing it) through the
  * native IME path — works in plain fields and code editors alike.

apps/desktop/src/main/browser-agent/driver.test.ts

@@ -1874,4 +1874,213 @@ describe('credential protection', () => {
     expect(result).toMatchObject({ ok: true, result: { dispatched: true } })
     expect(clickReads).toBeGreaterThan(1)
   })
+
+  it('clicks a coordinate point with native input and reports the target', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: {
+        found: true,
+        element: 'button "Send"',
+        editable: false,
+        secret: false,
+        fileInput: false,
+        cursor: 'pointer',
+      },
+      readActiveElementState: {},
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 120, y: 240 })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: {
+        dispatched: true,
+        trusted: true,
+        clickedAt: { x: 120, y: 240 },
+        target: 'button "Send"',
+      },
+    })
+    const presses = cdpCalls(contents, 'Input.dispatchMouseEvent').filter(
+      ([, event]) => (event as { type?: string }).type === 'mousePressed'
+    )
+    expect(presses).toHaveLength(1)
+  })
+
+  it('double-clicks a coordinate point as a rising clickCount sequence', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { found: true, element: 'canvas', editable: false },
+      readActiveElementState: {},
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_click_at', {
+      x: 10,
+      y: 20,
+      clickCount: 2,
+    })
+
+    expect(result).toMatchObject({ ok: true, result: { clickCount: 2 } })
+    const counts = cdpCalls(contents, 'Input.dispatchMouseEvent')
+      .filter(([, event]) => (event as { type?: string }).type === 'mousePressed')
+      .map(([, event]) => (event as { clickCount?: number }).clickCount)
+    expect(counts).toEqual([1, 2])
+  })
+
+  it('refuses a coordinate click on a file input', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { found: true, element: 'input', fileInput: true },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 5, y: 5 })
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toMatch(/file input/)
+    expect(cdpCalls(contents, 'Input.dispatchMouseEvent')).toHaveLength(0)
+  })
+
+  it('rejects a coordinate click outside the viewport with mapping guidance', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { error: 'outside-viewport' },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_click_at', { x: 9999, y: 5 })
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toMatch(/divide image pixels by its scale/)
+  })
+
+  it('inserts text into the focused editable at the caret', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      activeElementSecrecy: 'safe',
+      describeFocusedEditable: { editable: true, kind: 'contenteditable' },
+      readActiveElementState: { activeElement: 'div', valueLength: 12 },
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_insert_text', {
+      text: 'hello world',
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: { dispatched: true, trusted: true, kind: 'contenteditable', insertedChars: 11 },
+    })
+    expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(1)
+  })
+
+  it('refuses insertion when nothing editable holds focus', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      activeElementSecrecy: 'safe',
+      describeFocusedEditable: { editable: false, reason: 'none' },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_insert_text', { text: 'x' })
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toMatch(/No element is focused/)
+    expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0)
+  })
+
+  it('refuses insertion while a password field holds focus', async () => {
+    const contents = await openPage()
+    respondWith(contents, { activeElementSecrecy: 'secret' })
+
+    const result = await driver.executeTool('chat-test', 'browser_insert_text', { text: 'x' })
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toMatch(/Refusing to act on a password field/)
+    expect(cdpCalls(contents, 'Input.insertText')).toHaveLength(0)
+  })
+
+  it('drags between coordinate points through the trusted pointer pipeline', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { found: true, element: 'div "Card"' },
+      readActiveElementState: {},
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_drag', {
+      fromX: 40,
+      fromY: 50,
+      toX: 200,
+      toY: 260,
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: { dispatched: true, trusted: true, from: { x: 40, y: 50 }, to: { x: 200, y: 260 } },
+    })
+    const events = cdpCalls(contents, 'Input.dispatchMouseEvent').map(
+      ([, event]) => (event as { type?: string }).type
+    )
+    expect(events[0]).toBe('mouseMoved')
+    expect(events).toContain('mousePressed')
+    expect(events[events.length - 1]).toBe('mouseReleased')
+    expect(cdpCalls(contents, 'Input.setInterceptDrags').length).toBeGreaterThan(0)
+  })
+
+  it('drags from a snapshot element to a coordinate target', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      clickElement: { dispatched: false, x: 24, y: 48, element: 'Card "Ship it"' },
+      describePointTarget: { found: true, element: 'section "Done"' },
+      readActiveElementState: {},
+      readPageActionState: {},
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_drag', {
+      fromElementId: 0,
+      toX: 300,
+      toY: 60,
+    })
+
+    expect(result).toMatchObject({
+      ok: true,
+      result: { dispatched: true, from: { x: 24, y: 48, element: 'Card "Ship it"' } },
+    })
+  })
+
+  it('rejects a drag whose endpoints are the same point', async () => {
+    const contents = await openPage()
+    respondWith(contents, {
+      describePointTarget: { found: true, element: 'div' },
+    })
+
+    const result = await driver.executeTool('chat-test', 'browser_drag', {
+      fromX: 10,
+      fromY: 10,
+      toX: 10,
+      toY: 10,
+    })
+
+    expect(result.ok).toBe(false)
+    expect(result.error).toMatch(/same point/)
+  })
+
+  it('returns the screenshot scale for coordinate mapping', async () => {
+    const contents = await openPage()
+    vi.mocked(contents.debugger.sendCommand).mockImplementation((method: string) => {
+      if (method === 'Page.getLayoutMetrics') {
+        return Promise.resolve({
+          cssLayoutViewport: { clientWidth: 2048, clientHeight: 1024 },
+        })
+      }
+      if (method === 'Page.captureScreenshot') {
+        return Promise.resolve({ data: 'c2lt' })
+      }
+      return Promise.resolve(undefined)
+    })
+    respondWith(contents, { getViewportInfo: { width: 2048, height: 1024 } })
+
+    const result = await driver.executeTool('chat-test', 'browser_screenshot', {})
+
+    expect(result).toMatchObject({ ok: true, result: { scale: 0.5 } })
+  })
 })

apps/desktop/src/main/browser-agent/driver.ts

@@ -46,6 +46,8 @@ import {
   activeElementSecrecy,
   clickElement,
   collectSnapshot,
+  describeFocusedEditable,
+  describePointTarget,
   focusElementForTyping,
   getViewportInfo,
   hoverElement,
@@ -892,6 +894,11 @@ function unwrapPageResult(result: unknown): unknown {
     if (code === 'not-editable') {
       throw new ToolError('That element is not a text input — pick an editable element.')
     }
+    if (code === 'outside-viewport') {
+      throw new ToolError(
+        'That point is outside the visible viewport. Coordinates are CSS pixels within the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale, and scroll the target into view first.'
+      )
+    }
     if (code === 'ambiguous-editable') {
       throw new ToolError(
         'That composite control contains multiple editable fields. Take a fresh browser_snapshot and target the exact field.'
@@ -2002,19 +2009,21 @@ async function executeToolInner(
 
     case 'browser_screenshot': {
       const contents = session.requireAutomationTab().view.webContents
-      const dataUrl = await cdp.captureScreenshot(contents).catch(() => null)
-      if (dataUrl === null) {
+      const shot = await cdp.captureScreenshot(contents).catch(() => null)
+      if (shot === null) {
         throw new ToolError(
           'Could not capture the page. Use browser_snapshot or browser_read_text instead.'
         )
       }
-      if (dataUrl.length > 8_000_000) {
+      if (shot.dataUrl.length > 8_000_000) {
         throw new ToolError(
           'The screenshot result was too large to return safely. Use browser_snapshot or browser_read_text instead.'
         )
       }
       const viewport = await execInPage(contents, getViewportInfo, []).catch(() => null)
-      return { dataUrl, viewport }
+      // scale maps image pixels back to CSS viewport pixels for the
+      // coordinate tools: cssX = imageX / scale.
+      return { dataUrl: shot.dataUrl, viewport, scale: shot.scale }
     }
 
     case 'browser_extract': {
@@ -3106,6 +3115,287 @@ async function executeToolInner(
       }
     }
 
+    case 'browser_click_at': {
+      const clickedTab = session.requireAutomationTab()
+      const contents = clickedTab.view.webContents
+      const x = requireNum(params, 'x')
+      const y = requireNum(params, 'y')
+      const clickCount = num(params, 'clickCount') ?? 1
+      if (![1, 2, 3].includes(clickCount)) {
+        throw new ToolError('clickCount must be 1 (click), 2 (double-click), or 3 (triple-click).')
+      }
+      const clickNavigationEpoch = navigationEpoch(contents)
+      assertCurrentExecution()
+      assertActiveContents(contents)
+      const pointTarget = unwrapPageResult(
+        await execInPage(contents, describePointTarget, [x, y], false, executionDeadline)
+      )
+      if (!isRecordLike(pointTarget) || pointTarget.found !== true) {
+        throw new ToolError(
+          'Nothing is rendered at that point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.'
+        )
+      }
+      if (pointTarget.fileInput === true) {
+        throw new ToolError(
+          'Refusing to click a file input because it opens a native chooser the browser agent cannot inspect or complete. Ask the user to upload the file themselves.'
+        )
+      }
+      const beforePage = await pageActionState(contents, true)
+      const beforeElement = await activeElementState(contents)
+      assertCurrentExecution()
+      assertActiveContents(contents, clickNavigationEpoch)
+      try {
+        await cdp.clickAt(contents, x, y, true, clickCount)
+      } catch (error) {
+        throw new ToolError(
+          `Native click dispatch failed (${getErrorMessage(error)}). The action was not retried because a partial pointer press may already have reached the page. Take a fresh snapshot before continuing.`
+        )
+      }
+      await sleep(150)
+      const afterElement = await activeElementState(contents)
+      const afterPage = await pageActionState(contents)
+      const observation = pageEffect(beforePage, afterPage, beforeElement, afterElement)
+      const activeTab = session.automationTab()
+      const tabChanged = activeTab?.id !== clickedTab.id
+      const effectObserved =
+        observation.effect.urlChanged ||
+        observation.effect.dialogChanged ||
+        observation.effect.popupChanged ||
+        observation.effect.targetChanged ||
+        (pointTarget.editable === true && observation.effect.focusChanged) ||
+        tabChanged
+      const dialogs = Array.isArray(afterPage.dialogs) ? afterPage.dialogs.map(String) : []
+      const notes: string[] = []
+      if (pointTarget.secret === true) {
+        notes.push(
+          'The point resolves to a password field. Focusing it is fine, but typing there is refused — call browser_request_takeover for credentials.'
+        )
+      }
+      if (pointTarget.crossOriginFrame === true) {
+        notes.push(
+          'The point lands inside an embedded frame that could not be inspected; the click was dispatched but its target is unverified.'
+        )
+      }
+      if (!effectObserved) {
+        notes.push(
+          observation.possibleEffectObserved || tabChanged
+            ? 'Only background DOM/title churn followed the click; inspect the page before treating it as successful.'
+            : 'No strong observable page change followed the click; inspect the page before treating it as successful.'
+        )
+      }
+      return {
+        dispatched: true,
+        trusted: true,
+        activation: 'native-pointer',
+        clickedAt: { x, y },
+        clickCount,
+        target: pointTarget.element,
+        targetCursor: pointTarget.cursor,
+        effectObserved,
+        possibleEffectObserved: observation.possibleEffectObserved || tabChanged,
+        effect: { ...observation.effect, tabChanged },
+        dialogs,
+        ...(tabChanged && activeTab
+          ? { activeTab: { tabId: activeTab.id, url: activeTab.view.webContents.getURL() } }
+          : {}),
+        ...(notes.length > 0 ? { note: notes.join(' ') } : {}),
+      }
+    }
+
+    case 'browser_insert_text': {
+      const text = requireStr(params, 'text')
+      const submit = params.submit === true
+      const contents = session.requireAutomationTab().view.webContents
+      const insertNavigationEpoch = navigationEpoch(contents)
+      const target: PageExecutionTarget = focusedPageTarget(contents)
+      const secrecy = await execInPage(target, activeElementSecrecy, []).catch(() => 'opaque')
+      if (secrecy === 'secret') throw new ToolError(PASSWORD_REFUSAL)
+      if (secrecy === 'opaque') {
+        throw new ToolError(
+          'Focus is inside a cross-origin frame whose contents cannot be inspected, so this ' +
+            'insertion could reach a password field. Call browser_request_takeover if the user needs to type here.'
+        )
+      }
+      const focusState = unwrapPageResult(
+        await execInPage(target, describeFocusedEditable, [], false, executionDeadline)
+      )
+      if (!isRecordLike(focusState) || focusState.editable !== true) {
+        const reason = isRecordLike(focusState) ? String(focusState.reason || '') : ''
+        throw new ToolError(
+          reason === 'none'
+            ? 'No element is focused. Click the field first (browser_click or browser_click_at), then insert text.'
+            : `The focused element does not accept text${reason ? ` (${reason})` : ''}. Focus an editable field first.`
+        )
+      }
+      const beforePage = await pageActionState(target, true)
+      const beforeElement = await activeElementState(target)
+      assertCurrentExecution()
+      assertActiveContents(contents, insertNavigationEpoch)
+      assertFocusedTargetUnchanged(contents, target)
+      try {
+        await cdp.insertText(contents, text)
+      } catch (error) {
+        throw new ToolError(
+          `Native text insertion failed (${getErrorMessage(error)}). Take a fresh snapshot before retrying.`
+        )
+      }
+      let submitDispatched = false
+      if (submit) {
+        await sleep(25)
+        assertCurrentExecution()
+        assertActiveContents(contents, insertNavigationEpoch)
+        try {
+          await dispatchKeyCombo(contents, parseKeyCombo('Enter'))
+          submitDispatched = true
+        } catch {
+          // Reported below through submitDispatched: false.
+        }
+      }
+      await sleep(150)
+      const state = await activeElementState(target)
+      const afterPage = await pageActionState(target)
+      const observation = pageEffect(beforePage, afterPage, beforeElement, state)
+      const effectObserved =
+        observation.effect.fieldChanged ||
+        observation.effect.urlChanged ||
+        observation.effect.dialogChanged ||
+        observation.effect.targetChanged
+      return {
+        dispatched: true,
+        trusted: true,
+        kind: focusState.kind,
+        insertedChars: text.length,
+        ...state,
+        effectObserved,
+        possibleEffectObserved: observation.possibleEffectObserved,
+        effect: observation.effect,
+        submitRequested: submit,
+        submitDispatched,
+        ...(focusState.kind === 'canvas' || focusState.kind === 'textbox-role'
+          ? {
+              note: 'The focused editor is canvas/model-backed, so field readback cannot confirm the text — verify visually with browser_screenshot.',
+            }
+          : !effectObserved
+            ? {
+                note: 'Insertion produced no observable field or page change; inspect the page before continuing.',
+              }
+            : {}),
+      }
+    }
+
+    case 'browser_drag': {
+      const draggedTab = session.requireAutomationTab()
+      const contents = draggedTab.view.webContents
+      const dragNavigationEpoch = navigationEpoch(contents)
+
+      const resolveEndpoint = async (
+        which: 'from' | 'to'
+      ): Promise<{ x: number; y: number; element?: string }> => {
+        const elementId = num(params, `${which}ElementId`)
+        if (elementId !== undefined) {
+          const target = pageTargetForElement(contents, elementId)
+          if (frameExecutionTarget(target, contents)) {
+            throw new ToolError(
+              `Dragging elements inside embedded frames is not supported. Use ${which}X/${which}Y viewport coordinates instead.`
+            )
+          }
+          const prepared = unwrapPageResult(
+            await execInPageWithSettleGrace(
+              target,
+              clickElement,
+              [elementId, false, false],
+              executionDeadline
+            )
+          )
+          if (
+            !isRecordLike(prepared) ||
+            typeof prepared.x !== 'number' ||
+            typeof prepared.y !== 'number'
+          ) {
+            throw new ToolError(`Could not resolve the ${which} element for the drag.`)
+          }
+          return {
+            x: prepared.x,
+            y: prepared.y,
+            ...(typeof prepared.element === 'string' ? { element: prepared.element } : {}),
+          }
+        }
+        const pointX = num(params, `${which}X`)
+        const pointY = num(params, `${which}Y`)
+        if (pointX === undefined || pointY === undefined) {
+          throw new ToolError(
+            `Provide either ${which}ElementId or both ${which}X and ${which}Y for the drag ${which === 'from' ? 'source' : 'target'}.`
+          )
+        }
+        const probe = unwrapPageResult(
+          await execInPage(
+            contents,
+            describePointTarget,
+            [pointX, pointY],
+            false,
+            executionDeadline
+          )
+        )
+        if (!isRecordLike(probe) || probe.found !== true) {
+          throw new ToolError(
+            `Nothing is rendered at the ${which} point. Coordinates are CSS pixels in the current viewport — when reading them off a browser_screenshot, divide image pixels by its scale.`
+          )
+        }
+        return {
+          x: pointX,
+          y: pointY,
+          ...(typeof probe.element === 'string' ? { element: probe.element } : {}),
+        }
+      }
+
+      assertCurrentExecution()
+      assertActiveContents(contents)
+      const from = await resolveEndpoint('from')
+      const to = await resolveEndpoint('to')
+      if (Math.abs(from.x - to.x) < 1 && Math.abs(from.y - to.y) < 1) {
+        throw new ToolError('The drag source and target are the same point; nothing to drag.')
+      }
+      const beforePage = await pageActionState(contents, true)
+      const beforeElement = await activeElementState(contents)
+      assertCurrentExecution()
+      assertActiveContents(contents, dragNavigationEpoch)
+      let interception: { nativeDragIntercepted: boolean }
+      try {
+        interception = await cdp.dragPointer(contents, from, to)
+      } catch (error) {
+        throw new ToolError(
+          `Native drag dispatch failed (${getErrorMessage(error)}). The pointer may have been mid-drag; take a fresh snapshot to see the page's current state before retrying.`
+        )
+      }
+      await sleep(200)
+      const afterElement = await activeElementState(contents)
+      const afterPage = await pageActionState(contents)
+      const observation = pageEffect(beforePage, afterPage, beforeElement, afterElement)
+      const effectObserved =
+        observation.effect.domChanged ||
+        observation.effect.urlChanged ||
+        observation.effect.dialogChanged ||
+        observation.effect.targetChanged ||
+        observation.effect.scrollChanged
+      const dialogs = Array.isArray(afterPage.dialogs) ? afterPage.dialogs.map(String) : []
+      return {
+        dispatched: true,
+        trusted: true,
+        nativeHtml5Drag: interception.nativeDragIntercepted,
+        from,
+        to,
+        effectObserved,
+        possibleEffectObserved: observation.possibleEffectObserved,
+        effect: observation.effect,
+        dialogs,
+        ...(!effectObserved
+          ? {
+              note: 'No observable page change followed the drag. Verify with browser_snapshot or browser_screenshot; some drop targets only commit on their own animation frame.',
+            }
+          : {}),
+      }
+    }
+
     case 'browser_request_takeover': {
       // The reason renders in the chat's tool row, not here — but require it
       // so the model always tells the user why control was handed over.

apps/desktop/src/main/browser-agent/page-functions.test.ts

@@ -5,6 +5,8 @@ import {
   activeElementSecrecy,
   clickElement,
   collectSnapshot,
+  describeFocusedEditable,
+  describePointTarget,
   focusElementForTyping,
   getViewportInfo,
   hoverElement,
@@ -155,6 +157,8 @@ describe('serialization contract', () => {
     ['readPageText', readPageText, []],
     ['pageContainsText', pageContainsText, ['needle']],
     ['getViewportInfo', getViewportInfo, []],
+    ['describePointTarget', describePointTarget, [10, 10]],
+    ['describeFocusedEditable', describeFocusedEditable, []],
   ]
 
   it.each(cases)('%s is self-contained', (_name, fn, args) => {
@@ -1373,3 +1377,80 @@ describe('pressKeyOnPage', () => {
     expect(seen).toEqual(['a'])
   })
 })
+
+describe('describePointTarget', () => {
+  function pointAt(el: Element | null): void {
+    Object.defineProperty(document, 'elementFromPoint', {
+      configurable: true,
+      value: () => el,
+    })
+  }
+
+  it('describes the element at a viewport point', () => {
+    document.body.innerHTML = '<button aria-label="Send message">Send</button>'
+    pointAt(document.querySelector('button'))
+
+    expect(describePointTarget(10, 10)).toMatchObject({
+      found: true,
+      tag: 'button',
+      element: 'button "Send message"',
+      editable: false,
+      fileInput: false,
+      secret: false,
+    })
+  })
+
+  it('flags file inputs and password fields at the point', () => {
+    document.body.innerHTML = '<input type="file" />'
+    pointAt(document.querySelector('input'))
+    expect(describePointTarget(10, 10)).toMatchObject({ found: true, fileInput: true })
+
+    document.body.innerHTML = '<input type="password" />'
+    pointAt(document.querySelector('input'))
+    expect(describePointTarget(10, 10)).toMatchObject({
+      found: true,
+      secret: true,
+      editable: true,
+    })
+  })
+
+  it('rejects points outside the viewport', () => {
+    expect(describePointTarget(-5, 10)).toEqual({ error: 'outside-viewport' })
+    expect(describePointTarget(10, window.innerHeight + 5)).toEqual({
+      error: 'outside-viewport',
+    })
+  })
+})
+
+describe('describeFocusedEditable', () => {
+  it('reports no focus when the body holds focus', () => {
+    setActiveElement(document, document.body)
+    expect(describeFocusedEditable()).toEqual({ editable: false, reason: 'none' })
+  })
+
+  it('reports a writable input as insertable', () => {
+    document.body.innerHTML = '<input type="text" />'
+    setActiveElement(document, document.querySelector('input'))
+    expect(describeFocusedEditable()).toEqual({ editable: true, kind: 'input:text' })
+  })
+
+  it('reports a read-only input as not insertable', () => {
+    document.body.innerHTML = '<input type="text" readonly />'
+    setActiveElement(document, document.querySelector('input'))
+    expect(describeFocusedEditable()).toEqual({ editable: false, reason: 'readonly' })
+  })
+
+  it('reports a focused contenteditable editor as insertable', () => {
+    document.body.innerHTML = '<div></div>'
+    const editor = document.querySelector('div') as HTMLElement
+    Object.defineProperty(editor, 'isContentEditable', { get: () => true })
+    setActiveElement(document, editor)
+    expect(describeFocusedEditable()).toEqual({ editable: true, kind: 'contenteditable' })
+  })
+
+  it('treats a focused canvas editor surface as insertable', () => {
+    document.body.innerHTML = '<canvas></canvas>'
+    setActiveElement(document, document.querySelector('canvas'))
+    expect(describeFocusedEditable()).toEqual({ editable: true, kind: 'canvas' })
+  })
+})

apps/desktop/src/main/browser-agent/page-functions.ts

@@ -2676,3 +2676,155 @@ export function getViewportInfo(): unknown {
     height: window.innerHeight,
   }
 }
+
+/**
+ * Describes whatever sits at a viewport point, descending through open shadow
+ * roots and same-origin iframes. Coordinate-addressed actions have no
+ * snapshot ref to revalidate, so this probe is their safety check: the driver
+ * refuses file inputs outright and reports what the point resolves to so the
+ * model can confirm it hit what the screenshot showed.
+ */
+export function describePointTarget(x: number, y: number): unknown {
+  if (
+    !Number.isFinite(x) ||
+    !Number.isFinite(y) ||
+    x < 0 ||
+    y < 0 ||
+    x >= window.innerWidth ||
+    y >= window.innerHeight
+  ) {
+    return { error: 'outside-viewport' }
+  }
+
+  let doc: Document = document
+  let localX = x
+  let localY = y
+  let element: Element | null = null
+  for (let depth = 0; depth < 10; depth++) {
+    if (typeof doc.elementFromPoint !== 'function') break
+    let found: Element | null = doc.elementFromPoint(localX, localY)
+    // Open shadow roots re-hit-test at the same point until a leaf host.
+    for (let shadowDepth = 0; shadowDepth < 10; shadowDepth++) {
+      const shadow = (found as HTMLElement | null)?.shadowRoot
+      const inner = shadow?.elementFromPoint(localX, localY)
+      if (!inner || inner === found) break
+      found = inner
+    }
+    element = found
+    const tag = String(found?.tagName || '').toUpperCase()
+    if ((tag !== 'IFRAME' && tag !== 'FRAME') || !found) break
+    try {
+      const innerDoc = (found as HTMLIFrameElement).contentDocument
+      if (!innerDoc) break
+      const rect = found.getBoundingClientRect()
+      localX -= rect.left + (found as HTMLIFrameElement).clientLeft
+      localY -= rect.top + (found as HTMLIFrameElement).clientTop
+      doc = innerDoc
+    } catch {
+      // Cross-origin frame — cannot inspect further; report the frame itself.
+      break
+    }
+  }
+  if (!element) return { found: false }
+
+  const tag = String(element.tagName || '').toUpperCase()
+  const inputType =
+    tag === 'INPUT' ? String((element as HTMLInputElement).type || 'text').toLowerCase() : ''
+  const secret =
+    tag === 'INPUT' &&
+    (inputType === 'password' ||
+      String(element.getAttribute('autocomplete') || '')
+        .toLowerCase()
+        .split(/\s+/)
+        .some((token) => token === 'current-password' || token === 'new-password'))
+  const editable = Boolean(
+    tag === 'TEXTAREA' ||
+      (tag === 'INPUT' &&
+        ['text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(inputType)) ||
+      (element as HTMLElement).isContentEditable
+  )
+
+  const name = (
+    element.getAttribute('aria-label') ||
+    element.getAttribute('title') ||
+    element.getAttribute('alt') ||
+    ((element as HTMLElement).innerText ?? element.textContent ?? '')
+  )
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, 80)
+  const role = element.getAttribute('role') || ''
+  const style = element.ownerDocument.defaultView?.getComputedStyle(element)
+
+  return {
+    found: true,
+    element: name
+      ? `${tag.toLowerCase()}${role ? `[${role}]` : ''} "${name}"`
+      : `${tag.toLowerCase()}${role ? `[${role}]` : ''}`,
+    tag: tag.toLowerCase(),
+    role,
+    editable,
+    secret,
+    fileInput: tag === 'INPUT' && inputType === 'file',
+    disabled:
+      (element as HTMLInputElement).disabled === true ||
+      element.getAttribute('aria-disabled') === 'true',
+    canvas: tag === 'CANVAS',
+    crossOriginFrame: tag === 'IFRAME' || tag === 'FRAME',
+    cursor: style?.cursor || '',
+  }
+}
+
+/**
+ * Reports whether the currently-focused element accepts text insertion, for
+ * browser_insert_text — which types at the caret instead of addressing a
+ * snapshot ref. Secrecy is separately (and authoritatively) probed by
+ * activeElementSecrecy before any insertion.
+ */
+export function describeFocusedEditable(): unknown {
+  let active = document.activeElement as HTMLElement | null
+  for (let depth = 0; active && depth < 10; depth++) {
+    const shadow = active.shadowRoot
+    if (shadow?.activeElement) {
+      active = shadow.activeElement as HTMLElement
+      continue
+    }
+    break
+  }
+  if (!active || active === document.body) return { editable: false, reason: 'none' }
+  const tag = String(active.tagName || '').toUpperCase()
+  const inputType =
+    tag === 'INPUT' ? String((active as HTMLInputElement).type || 'text').toLowerCase() : ''
+  if (tag === 'INPUT' || tag === 'TEXTAREA') {
+    const field = active as HTMLInputElement | HTMLTextAreaElement
+    if (field.disabled || active.getAttribute('aria-disabled') === 'true') {
+      return { editable: false, reason: 'disabled' }
+    }
+    if (field.readOnly || active.getAttribute('aria-readonly') === 'true') {
+      return { editable: false, reason: 'readonly' }
+    }
+    if (
+      tag === 'INPUT' &&
+      !['text', 'search', 'email', 'url', 'tel', 'number', 'password'].includes(inputType)
+    ) {
+      return { editable: false, reason: 'not-text' }
+    }
+    return { editable: true, kind: tag === 'TEXTAREA' ? 'textarea' : `input:${inputType}` }
+  }
+  if (active.isContentEditable) {
+    if (active.getAttribute('aria-disabled') === 'true') {
+      return { editable: false, reason: 'disabled' }
+    }
+    if (active.getAttribute('aria-readonly') === 'true') {
+      return { editable: false, reason: 'readonly' }
+    }
+    return { editable: true, kind: 'contenteditable' }
+  }
+  // A canvas-rendered editor (Google Docs) focuses a hidden proxy or the
+  // canvas region itself; trusted IME insertion still reaches it, so report
+  // it as insertable rather than refusing.
+  if (tag === 'CANVAS' || active.getAttribute('role') === 'textbox') {
+    return { editable: true, kind: tag === 'CANVAS' ? 'canvas' : 'textbox-role' }
+  }
+  return { editable: false, reason: 'not-editable' }
+}

apps/sim/lib/copilot/generated/tool-catalog-v1.ts

@@ -11,11 +11,14 @@ export interface ToolCatalogEntry {
     | 'auth'
     | 'browser'
     | 'browser_click'
+    | 'browser_click_at'
     | 'browser_close_tab'
+    | 'browser_drag'
     | 'browser_extract'
     | 'browser_go_back'
     | 'browser_go_forward'
     | 'browser_hover'
+    | 'browser_insert_text'
     | 'browser_list_sessions'
     | 'browser_list_tabs'
     | 'browser_navigate'
@@ -132,11 +135,14 @@ export interface ToolCatalogEntry {
     | 'auth'
     | 'browser'
     | 'browser_click'
+    | 'browser_click_at'
     | 'browser_close_tab'
+    | 'browser_drag'
     | 'browser_extract'
     | 'browser_go_back'
     | 'browser_go_forward'
     | 'browser_hover'
+    | 'browser_insert_text'
     | 'browser_list_sessions'
     | 'browser_list_tabs'
     | 'browser_navigate'
@@ -450,6 +456,135 @@ export const BrowserClick: ToolCatalogEntry = {
   clientExecutable: true,
 }
 
+export const BrowserClickAt: ToolCatalogEntry = {
+  id: 'browser_click_at',
+  name: 'browser_click_at',
+  route: 'client',
+  mode: 'async',
+  parameters: {
+    type: 'object',
+    properties: {
+      clickCount: {
+        type: 'number',
+        description: '1 = single click (default), 2 = double-click, 3 = triple-click.',
+      },
+      x: {
+        type: 'number',
+        description:
+          "X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by the screenshot's scale.",
+      },
+      y: {
+        type: 'number',
+        description:
+          'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.',
+      },
+    },
+    required: ['x', 'y'],
+  },
+  resultSchema: {
+    type: 'object',
+    properties: {
+      activeTab: {
+        type: 'object',
+        description: 'New active tab after a tab-changing click.',
+        properties: {
+          tabId: { type: 'string', description: 'Stable browser tab id.' },
+          url: { type: 'string', description: 'New active tab URL.' },
+        },
+      },
+      clickCount: {
+        type: 'number',
+        description: 'The click count that was dispatched (1, 2, or 3).',
+      },
+      clickedAt: {
+        type: 'object',
+        description: 'The CSS-pixel viewport point that was clicked.',
+        properties: {
+          x: { type: 'number', description: 'Clicked X in CSS viewport pixels.' },
+          y: { type: 'number', description: 'Clicked Y in CSS viewport pixels.' },
+        },
+      },
+      dialogs: {
+        type: 'array',
+        description: 'Visible DOM dialogs remaining after the click.',
+        items: { type: 'string' },
+      },
+      dispatched: {
+        type: 'boolean',
+        description: 'Whether the native pointer click was dispatched at the point.',
+      },
+      effect: {
+        type: 'object',
+        description:
+          'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.',
+        properties: {
+          dialogChanged: { type: 'boolean', description: 'The visible DOM dialog set changed.' },
+          domChanged: {
+            type: 'boolean',
+            description: 'The DOM mutation revision changed; weak evidence on its own.',
+          },
+          fieldChanged: {
+            type: 'boolean',
+            description: 'The safely inspectable focused-field state changed.',
+          },
+          focusChanged: { type: 'boolean', description: 'The focused element changed.' },
+          popupChanged: { type: 'boolean', description: 'The visible popup/menu set changed.' },
+          scrollChanged: {
+            type: 'boolean',
+            description: 'A tracked scroll offset changed; weak evidence except for scroll keys.',
+          },
+          tabChanged: { type: 'boolean', description: 'The active browser tab changed.' },
+          targetChanged: {
+            type: 'boolean',
+            description: "The requested target's checked/selected/expanded/open state changed.",
+          },
+          titleChanged: {
+            type: 'boolean',
+            description: 'The document title changed; weak evidence on its own.',
+          },
+          urlChanged: { type: 'boolean', description: 'The observed URL changed.' },
+        },
+      },
+      effectObserved: {
+        type: 'boolean',
+        description:
+          'Whether a strong page change (URL/dialog/popup/target, or focus into an editable) followed the click.',
+      },
+      note: {
+        type: 'string',
+        description: 'Caution or follow-up guidance, present when the click needs verification.',
+      },
+      notices: {
+        type: 'array',
+        description:
+          'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.',
+        items: { type: 'string' },
+      },
+      possibleEffectObserved: {
+        type: 'boolean',
+        description: 'Whether only weaker background churn (DOM/title) was observed.',
+      },
+      target: {
+        type: 'string',
+        description:
+          'What the point resolved to before the click (tag/role and accessible name) — confirm it matches the intended target.',
+      },
+      targetCursor: {
+        type: 'string',
+        description:
+          "The CSS cursor at the point (e.g. 'pointer', 'text', 'crosshair') — a hint about what kind of surface was hit.",
+      },
+      trusted: {
+        type: 'boolean',
+        description:
+          'True — coordinate and drag input always use the trusted Chromium pointer pipeline.',
+      },
+    },
+    required: ['dispatched'],
+  },
+  clientExecutable: true,
+}
+
 export const BrowserCloseTab: ToolCatalogEntry = {
   id: 'browser_close_tab',
   name: 'browser_close_tab',
@@ -468,6 +603,130 @@ export const BrowserCloseTab: ToolCatalogEntry = {
   clientExecutable: true,
 }
 
+export const BrowserDrag: ToolCatalogEntry = {
+  id: 'browser_drag',
+  name: 'browser_drag',
+  route: 'client',
+  mode: 'async',
+  parameters: {
+    type: 'object',
+    properties: {
+      fromElementId: {
+        type: 'number',
+        description: 'Drag source element id from the latest snapshot. Alternative to fromX/fromY.',
+      },
+      fromX: {
+        type: 'number',
+        description:
+          'Drag source X in CSS viewport pixels (paired with fromY) when no source element id is available.',
+      },
+      fromY: { type: 'number', description: 'Drag source Y in CSS viewport pixels.' },
+      toElementId: {
+        type: 'number',
+        description: 'Drop target element id from the latest snapshot. Alternative to toX/toY.',
+      },
+      toX: {
+        type: 'number',
+        description: 'Drop target X in CSS viewport pixels (paired with toY).',
+      },
+      toY: { type: 'number', description: 'Drop target Y in CSS viewport pixels.' },
+    },
+  },
+  resultSchema: {
+    type: 'object',
+    properties: {
+      dialogs: {
+        type: 'array',
+        description: 'Visible DOM dialogs remaining after the click.',
+        items: { type: 'string' },
+      },
+      dispatched: { type: 'boolean', description: 'Whether the full drag gesture was dispatched.' },
+      effect: {
+        type: 'object',
+        description:
+          'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.',
+        properties: {
+          dialogChanged: { type: 'boolean', description: 'The visible DOM dialog set changed.' },
+          domChanged: {
+            type: 'boolean',
+            description: 'The DOM mutation revision changed; weak evidence on its own.',
+          },
+          fieldChanged: {
+            type: 'boolean',
+            description: 'The safely inspectable focused-field state changed.',
+          },
+          focusChanged: { type: 'boolean', description: 'The focused element changed.' },
+          popupChanged: { type: 'boolean', description: 'The visible popup/menu set changed.' },
+          scrollChanged: {
+            type: 'boolean',
+            description: 'A tracked scroll offset changed; weak evidence except for scroll keys.',
+          },
+          tabChanged: { type: 'boolean', description: 'The active browser tab changed.' },
+          targetChanged: {
+            type: 'boolean',
+            description: "The requested target's checked/selected/expanded/open state changed.",
+          },
+          titleChanged: {
+            type: 'boolean',
+            description: 'The document title changed; weak evidence on its own.',
+          },
+          urlChanged: { type: 'boolean', description: 'The observed URL changed.' },
+        },
+      },
+      effectObserved: {
+        type: 'boolean',
+        description:
+          'Whether an observable page change (DOM/URL/dialog/target/scroll) followed the drag.',
+      },
+      from: {
+        type: 'object',
+        description: 'The resolved drag source.',
+        properties: {
+          element: { type: 'string', description: 'What that endpoint resolved to, when known.' },
+          x: { type: 'number', description: 'Endpoint X in CSS viewport pixels.' },
+          y: { type: 'number', description: 'Endpoint Y in CSS viewport pixels.' },
+        },
+      },
+      nativeHtml5Drag: {
+        type: 'boolean',
+        description:
+          'True when the page started a native HTML5 drag and it was completed as a real drag-and-drop; false for a pointer-sensor drag.',
+      },
+      note: {
+        type: 'string',
+        description:
+          'Caution or follow-up guidance — e.g. verify the drop with a fresh snapshot when no effect was observed.',
+      },
+      notices: {
+        type: 'array',
+        description:
+          'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.',
+        items: { type: 'string' },
+      },
+      possibleEffectObserved: {
+        type: 'boolean',
+        description: 'Whether only weaker background churn (DOM/title) was observed.',
+      },
+      to: {
+        type: 'object',
+        description: 'The resolved drop target.',
+        properties: {
+          element: { type: 'string', description: 'What that endpoint resolved to, when known.' },
+          x: { type: 'number', description: 'Endpoint X in CSS viewport pixels.' },
+          y: { type: 'number', description: 'Endpoint Y in CSS viewport pixels.' },
+        },
+      },
+      trusted: {
+        type: 'boolean',
+        description:
+          'True — coordinate and drag input always use the trusted Chromium pointer pipeline.',
+      },
+    },
+    required: ['dispatched'],
+  },
+  clientExecutable: true,
+}
+
 export const BrowserExtract: ToolCatalogEntry = {
   id: 'browser_extract',
   name: 'browser_extract',
@@ -630,6 +889,116 @@ export const BrowserHover: ToolCatalogEntry = {
   clientExecutable: true,
 }
 
+export const BrowserInsertText: ToolCatalogEntry = {
+  id: 'browser_insert_text',
+  name: 'browser_insert_text',
+  route: 'client',
+  mode: 'async',
+  parameters: {
+    type: 'object',
+    properties: {
+      submit: { type: 'boolean', description: 'Press Enter after inserting. Default false.' },
+      text: { type: 'string', description: 'The text to insert at the caret. Must be non-empty.' },
+    },
+    required: ['text'],
+  },
+  resultSchema: {
+    type: 'object',
+    properties: {
+      activeElement: { type: 'string', description: 'Focused element kind after the action.' },
+      dispatched: {
+        type: 'boolean',
+        description: 'Whether the text was inserted through the native IME pipeline.',
+      },
+      effect: {
+        type: 'object',
+        description:
+          'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.',
+        properties: {
+          dialogChanged: { type: 'boolean', description: 'The visible DOM dialog set changed.' },
+          domChanged: {
+            type: 'boolean',
+            description: 'The DOM mutation revision changed; weak evidence on its own.',
+          },
+          fieldChanged: {
+            type: 'boolean',
+            description: 'The safely inspectable focused-field state changed.',
+          },
+          focusChanged: { type: 'boolean', description: 'The focused element changed.' },
+          popupChanged: { type: 'boolean', description: 'The visible popup/menu set changed.' },
+          scrollChanged: {
+            type: 'boolean',
+            description: 'A tracked scroll offset changed; weak evidence except for scroll keys.',
+          },
+          tabChanged: { type: 'boolean', description: 'The active browser tab changed.' },
+          targetChanged: {
+            type: 'boolean',
+            description: "The requested target's checked/selected/expanded/open state changed.",
+          },
+          titleChanged: {
+            type: 'boolean',
+            description: 'The document title changed; weak evidence on its own.',
+          },
+          urlChanged: { type: 'boolean', description: 'The observed URL changed.' },
+        },
+      },
+      effectObserved: {
+        type: 'boolean',
+        description:
+          'Whether a field or page change confirmed the insertion. Canvas editors cannot echo one — verify visually.',
+      },
+      insertedChars: { type: 'number', description: 'How many characters were inserted.' },
+      kind: {
+        type: 'string',
+        description:
+          'The kind of focused editable that received the text (input:*, textarea, contenteditable, canvas, textbox-role).',
+      },
+      note: {
+        type: 'string',
+        description:
+          'Caution or follow-up guidance, e.g. that a canvas editor needs visual verification.',
+      },
+      notices: {
+        type: 'array',
+        description:
+          'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.',
+        items: { type: 'string' },
+      },
+      possibleEffectObserved: {
+        type: 'boolean',
+        description: 'Whether only weaker background churn was observed.',
+      },
+      redacted: {
+        type: 'boolean',
+        description: 'Whether sensitive focused-field details were withheld.',
+      },
+      selectedChars: {
+        type: 'number',
+        description: 'Number of selected characters when safely inspectable.',
+      },
+      submitDispatched: { type: 'boolean', description: 'Whether Enter was actually dispatched.' },
+      submitRequested: {
+        type: 'boolean',
+        description: 'Whether Enter was requested after insertion.',
+      },
+      trusted: {
+        type: 'boolean',
+        description: 'True — insertion uses the trusted input pipeline.',
+      },
+      valueLength: {
+        type: 'number',
+        description: 'Focused non-secret field length when safely inspectable.',
+      },
+      valuePreview: {
+        type: 'string',
+        description: 'Bounded focused non-secret field preview when safely inspectable.',
+      },
+    },
+    required: ['dispatched'],
+  },
+  clientExecutable: true,
+}
+
 export const BrowserListSessions: ToolCatalogEntry = {
   id: 'browser_list_sessions',
   name: 'browser_list_sessions',
@@ -6617,11 +6986,14 @@ export const TOOL_CATALOG: Record<string, ToolCatalogEntry> = {
   [Auth.id]: Auth,
   [Browser.id]: Browser,
   [BrowserClick.id]: BrowserClick,
+  [BrowserClickAt.id]: BrowserClickAt,
   [BrowserCloseTab.id]: BrowserCloseTab,
+  [BrowserDrag.id]: BrowserDrag,
   [BrowserExtract.id]: BrowserExtract,
   [BrowserGoBack.id]: BrowserGoBack,
   [BrowserGoForward.id]: BrowserGoForward,
   [BrowserHover.id]: BrowserHover,
+  [BrowserInsertText.id]: BrowserInsertText,
   [BrowserListSessions.id]: BrowserListSessions,
   [BrowserListTabs.id]: BrowserListTabs,
   [BrowserNavigate.id]: BrowserNavigate,

apps/sim/lib/copilot/generated/tool-schemas-v1.ts

@@ -214,6 +214,160 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
       required: ['dispatched'],
     },
   },
+  browser_click_at: {
+    parameters: {
+      type: 'object',
+      properties: {
+        clickCount: {
+          type: 'number',
+          description: '1 = single click (default), 2 = double-click, 3 = triple-click.',
+        },
+        x: {
+          type: 'number',
+          description:
+            "X in CSS pixels within the current viewport. When read off a browser_screenshot, divide the image pixel value by the screenshot's scale.",
+        },
+        y: {
+          type: 'number',
+          description:
+            'Y in CSS pixels within the current viewport, converted from screenshot pixels the same way as x.',
+        },
+      },
+      required: ['x', 'y'],
+    },
+    resultSchema: {
+      type: 'object',
+      properties: {
+        activeTab: {
+          type: 'object',
+          description: 'New active tab after a tab-changing click.',
+          properties: {
+            tabId: {
+              type: 'string',
+              description: 'Stable browser tab id.',
+            },
+            url: {
+              type: 'string',
+              description: 'New active tab URL.',
+            },
+          },
+        },
+        clickCount: {
+          type: 'number',
+          description: 'The click count that was dispatched (1, 2, or 3).',
+        },
+        clickedAt: {
+          type: 'object',
+          description: 'The CSS-pixel viewport point that was clicked.',
+          properties: {
+            x: {
+              type: 'number',
+              description: 'Clicked X in CSS viewport pixels.',
+            },
+            y: {
+              type: 'number',
+              description: 'Clicked Y in CSS viewport pixels.',
+            },
+          },
+        },
+        dialogs: {
+          type: 'array',
+          description: 'Visible DOM dialogs remaining after the click.',
+          items: {
+            type: 'string',
+          },
+        },
+        dispatched: {
+          type: 'boolean',
+          description: 'Whether the native pointer click was dispatched at the point.',
+        },
+        effect: {
+          type: 'object',
+          description:
+            'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.',
+          properties: {
+            dialogChanged: {
+              type: 'boolean',
+              description: 'The visible DOM dialog set changed.',
+            },
+            domChanged: {
+              type: 'boolean',
+              description: 'The DOM mutation revision changed; weak evidence on its own.',
+            },
+            fieldChanged: {
+              type: 'boolean',
+              description: 'The safely inspectable focused-field state changed.',
+            },
+            focusChanged: {
+              type: 'boolean',
+              description: 'The focused element changed.',
+            },
+            popupChanged: {
+              type: 'boolean',
+              description: 'The visible popup/menu set changed.',
+            },
+            scrollChanged: {
+              type: 'boolean',
+              description: 'A tracked scroll offset changed; weak evidence except for scroll keys.',
+            },
+            tabChanged: {
+              type: 'boolean',
+              description: 'The active browser tab changed.',
+            },
+            targetChanged: {
+              type: 'boolean',
+              description: "The requested target's checked/selected/expanded/open state changed.",
+            },
+            titleChanged: {
+              type: 'boolean',
+              description: 'The document title changed; weak evidence on its own.',
+            },
+            urlChanged: {
+              type: 'boolean',
+              description: 'The observed URL changed.',
+            },
+          },
+        },
+        effectObserved: {
+          type: 'boolean',
+          description:
+            'Whether a strong page change (URL/dialog/popup/target, or focus into an editable) followed the click.',
+        },
+        note: {
+          type: 'string',
+          description: 'Caution or follow-up guidance, present when the click needs verification.',
+        },
+        notices: {
+          type: 'array',
+          description:
+            'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.',
+          items: {
+            type: 'string',
+          },
+        },
+        possibleEffectObserved: {
+          type: 'boolean',
+          description: 'Whether only weaker background churn (DOM/title) was observed.',
+        },
+        target: {
+          type: 'string',
+          description:
+            'What the point resolved to before the click (tag/role and accessible name) — confirm it matches the intended target.',
+        },
+        targetCursor: {
+          type: 'string',
+          description:
+            "The CSS cursor at the point (e.g. 'pointer', 'text', 'crosshair') — a hint about what kind of surface was hit.",
+        },
+        trusted: {
+          type: 'boolean',
+          description:
+            'True — coordinate and drag input always use the trusted Chromium pointer pipeline.',
+        },
+      },
+      required: ['dispatched'],
+    },
+  },
   browser_close_tab: {
     parameters: {
       type: 'object',
@@ -227,6 +381,171 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
     },
     resultSchema: undefined,
   },
+  browser_drag: {
+    parameters: {
+      type: 'object',
+      properties: {
+        fromElementId: {
+          type: 'number',
+          description:
+            'Drag source element id from the latest snapshot. Alternative to fromX/fromY.',
+        },
+        fromX: {
+          type: 'number',
+          description:
+            'Drag source X in CSS viewport pixels (paired with fromY) when no source element id is available.',
+        },
+        fromY: {
+          type: 'number',
+          description: 'Drag source Y in CSS viewport pixels.',
+        },
+        toElementId: {
+          type: 'number',
+          description: 'Drop target element id from the latest snapshot. Alternative to toX/toY.',
+        },
+        toX: {
+          type: 'number',
+          description: 'Drop target X in CSS viewport pixels (paired with toY).',
+        },
+        toY: {
+          type: 'number',
+          description: 'Drop target Y in CSS viewport pixels.',
+        },
+      },
+    },
+    resultSchema: {
+      type: 'object',
+      properties: {
+        dialogs: {
+          type: 'array',
+          description: 'Visible DOM dialogs remaining after the click.',
+          items: {
+            type: 'string',
+          },
+        },
+        dispatched: {
+          type: 'boolean',
+          description: 'Whether the full drag gesture was dispatched.',
+        },
+        effect: {
+          type: 'object',
+          description:
+            'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.',
+          properties: {
+            dialogChanged: {
+              type: 'boolean',
+              description: 'The visible DOM dialog set changed.',
+            },
+            domChanged: {
+              type: 'boolean',
+              description: 'The DOM mutation revision changed; weak evidence on its own.',
+            },
+            fieldChanged: {
+              type: 'boolean',
+              description: 'The safely inspectable focused-field state changed.',
+            },
+            focusChanged: {
+              type: 'boolean',
+              description: 'The focused element changed.',
+            },
+            popupChanged: {
+              type: 'boolean',
+              description: 'The visible popup/menu set changed.',
+            },
+            scrollChanged: {
+              type: 'boolean',
+              description: 'A tracked scroll offset changed; weak evidence except for scroll keys.',
+            },
+            tabChanged: {
+              type: 'boolean',
+              description: 'The active browser tab changed.',
+            },
+            targetChanged: {
+              type: 'boolean',
+              description: "The requested target's checked/selected/expanded/open state changed.",
+            },
+            titleChanged: {
+              type: 'boolean',
+              description: 'The document title changed; weak evidence on its own.',
+            },
+            urlChanged: {
+              type: 'boolean',
+              description: 'The observed URL changed.',
+            },
+          },
+        },
+        effectObserved: {
+          type: 'boolean',
+          description:
+            'Whether an observable page change (DOM/URL/dialog/target/scroll) followed the drag.',
+        },
+        from: {
+          type: 'object',
+          description: 'The resolved drag source.',
+          properties: {
+            element: {
+              type: 'string',
+              description: 'What that endpoint resolved to, when known.',
+            },
+            x: {
+              type: 'number',
+              description: 'Endpoint X in CSS viewport pixels.',
+            },
+            y: {
+              type: 'number',
+              description: 'Endpoint Y in CSS viewport pixels.',
+            },
+          },
+        },
+        nativeHtml5Drag: {
+          type: 'boolean',
+          description:
+            'True when the page started a native HTML5 drag and it was completed as a real drag-and-drop; false for a pointer-sensor drag.',
+        },
+        note: {
+          type: 'string',
+          description:
+            'Caution or follow-up guidance — e.g. verify the drop with a fresh snapshot when no effect was observed.',
+        },
+        notices: {
+          type: 'array',
+          description:
+            'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.',
+          items: {
+            type: 'string',
+          },
+        },
+        possibleEffectObserved: {
+          type: 'boolean',
+          description: 'Whether only weaker background churn (DOM/title) was observed.',
+        },
+        to: {
+          type: 'object',
+          description: 'The resolved drop target.',
+          properties: {
+            element: {
+              type: 'string',
+              description: 'What that endpoint resolved to, when known.',
+            },
+            x: {
+              type: 'number',
+              description: 'Endpoint X in CSS viewport pixels.',
+            },
+            y: {
+              type: 'number',
+              description: 'Endpoint Y in CSS viewport pixels.',
+            },
+          },
+        },
+        trusted: {
+          type: 'boolean',
+          description:
+            'True — coordinate and drag input always use the trusted Chromium pointer pipeline.',
+        },
+      },
+      required: ['dispatched'],
+    },
+  },
   browser_extract: {
     parameters: {
       type: 'object',
@@ -410,6 +729,142 @@ export const TOOL_RUNTIME_SCHEMAS: Record<string, ToolRuntimeSchemaEntry> = {
       required: ['hovered'],
     },
   },
+  browser_insert_text: {
+    parameters: {
+      type: 'object',
+      properties: {
+        submit: {
+          type: 'boolean',
+          description: 'Press Enter after inserting. Default false.',
+        },
+        text: {
+          type: 'string',
+          description: 'The text to insert at the caret. Must be non-empty.',
+        },
+      },
+      required: ['text'],
+    },
+    resultSchema: {
+      type: 'object',
+      properties: {
+        activeElement: {
+          type: 'string',
+          description: 'Focused element kind after the action.',
+        },
+        dispatched: {
+          type: 'boolean',
+          description: 'Whether the text was inserted through the native IME pipeline.',
+        },
+        effect: {
+          type: 'object',
+          description:
+            'Detailed postcondition signals; generic title/DOM/scroll churn is weak evidence unless the tool documents otherwise.',
+          properties: {
+            dialogChanged: {
+              type: 'boolean',
+              description: 'The visible DOM dialog set changed.',
+            },
+            domChanged: {
+              type: 'boolean',
+              description: 'The DOM mutation revision changed; weak evidence on its own.',
+            },
+            fieldChanged: {
+              type: 'boolean',
+              description: 'The safely inspectable focused-field state changed.',
+            },
+            focusChanged: {
+              type: 'boolean',
+              description: 'The focused element changed.',
+            },
+            popupChanged: {
+              type: 'boolean',
+              description: 'The visible popup/menu set changed.',
+            },
+            scrollChanged: {
+              type: 'boolean',
+              description: 'A tracked scroll offset changed; weak evidence except for scroll keys.',
+            },
+            tabChanged: {
+              type: 'boolean',
+              description: 'The active browser tab changed.',
+            },
+            targetChanged: {
+              type: 'boolean',
+              description: "The requested target's checked/selected/expanded/open state changed.",
+            },
+            titleChanged: {
+              type: 'boolean',
+              description: 'The document title changed; weak evidence on its own.',
+            },
+            urlChanged: {
+              type: 'boolean',
+              description: 'The observed URL changed.',
+            },
+          },
+        },
+        effectObserved: {
+          type: 'boolean',
+          description:
+            'Whether a field or page change confirmed the insertion. Canvas editors cannot echo one — verify visually.',
+        },
+        insertedChars: {
+          type: 'number',
+          description: 'How many characters were inserted.',
+        },
+        kind: {
+          type: 'string',
+          description:
+            'The kind of focused editable that received the text (input:*, textarea, contenteditable, canvas, textbox-role).',
+        },
+        note: {
+          type: 'string',
+          description:
+            'Caution or follow-up guidance, e.g. that a canvas editor needs visual verification.',
+        },
+        notices: {
+          type: 'array',
+          description:
+            'Pending auto-handled JavaScript alert/confirm/prompt notices since the previous successful browser result.',
+          items: {
+            type: 'string',
+          },
+        },
+        possibleEffectObserved: {
+          type: 'boolean',
+          description: 'Whether only weaker background churn was observed.',
+        },
+        redacted: {
+          type: 'boolean',
+          description: 'Whether sensitive focused-field details were withheld.',
+        },
+        selectedChars: {
+          type: 'number',
+          description: 'Number of selected characters when safely inspectable.',
+        },
+        submitDispatched: {
+          type: 'boolean',
+          description: 'Whether Enter was actually dispatched.',
+        },
+        submitRequested: {
+          type: 'boolean',
+          description: 'Whether Enter was requested after insertion.',
+        },
+        trusted: {
+          type: 'boolean',
+          description: 'True — insertion uses the trusted input pipeline.',
+        },
+        valueLength: {
+          type: 'number',
+          description: 'Focused non-secret field length when safely inspectable.',
+        },
+        valuePreview: {
+          type: 'string',
+          description: 'Bounded focused non-secret field preview when safely inspectable.',
+        },
+      },
+      required: ['dispatched'],
+    },
+  },
   browser_list_sessions: {
     parameters: {
       type: 'object',

apps/sim/lib/copilot/tools/tool-display.ts

@@ -525,7 +525,10 @@ const TOOL_TITLES: Record<string, string> = {
   browser_read_text: 'Reading page',
   browser_screenshot: 'Taking screenshot',
   browser_click: 'Clicking element',
+  browser_click_at: 'Clicking point',
   browser_type: 'Typing text',
+  browser_insert_text: 'Inserting text',
+  browser_drag: 'Dragging element',
   browser_select_option: 'Selecting option',
   browser_hover: 'Hovering element',
   // Subagent trigger tools, when surfaced as a tool call.
@@ -1015,6 +1018,8 @@ const COMPLETED_VERB_REWRITES: Record<string, string> = {
   Creating: 'Created',
   Deleting: 'Deleted',
   Deploying: 'Deployed',
+  Dragging: 'Dragged',
+  Inserting: 'Inserted',
   Publishing: 'Published',
   Unpublishing: 'Unpublished',
   Analyzing: 'Analyzed',

packages/browser-protocol/src/index.ts

@@ -33,11 +33,14 @@ export const BROWSER_TOOL_NAMES = [
   'browser_screenshot',
   'browser_extract',
   'browser_click',
+  'browser_click_at',
   'browser_type',
+  'browser_insert_text',
   'browser_press_key',
   'browser_scroll',
   'browser_select_option',
   'browser_hover',
+  'browser_drag',
   'browser_request_takeover',
 ] as const