feat: tools optimize

.gitignore

@@ -36,4 +36,8 @@ dist
 false/
 metadata-v1.3/
 registry.npmmirror.com/
-registry.npmjs.com/
+registry.npmjs.com/
+
+other/
+tools_optimize.md
+Agents.md

app/chrome-extension/common/message-types.ts

@@ -41,6 +41,7 @@ export const CONTENT_MESSAGE_TYPES = {
   KEYBOARD_HELPER_PING: 'keyboard_helper_ping',
   SCREENSHOT_HELPER_PING: 'screenshot_helper_ping',
   INTERACTIVE_ELEMENTS_HELPER_PING: 'interactive_elements_helper_ping',
+  ACCESSIBILITY_TREE_HELPER_PING: 'chrome_read_page_ping',
 } as const;
 
 // Tool action message types (for chrome.runtime.sendMessage)
@@ -64,6 +65,11 @@ export const TOOL_MESSAGE_TYPES = {
   // Interactive elements
   GET_INTERACTIVE_ELEMENTS: 'getInteractiveElements',
 
+  // Accessibility tree
+  GENERATE_ACCESSIBILITY_TREE: 'generateAccessibilityTree',
+  RESOLVE_REF: 'resolveRef',
+  ENSURE_REF_FOR_SELECTOR: 'ensureRefForSelector',
+
   // Network requests
   NETWORK_SEND_REQUEST: 'sendPureNetworkRequest',
 

app/chrome-extension/entrypoints/background/tools/browser/computer.ts

@@ -0,0 +1,859 @@
+import { createErrorResponse, ToolResult } from '@/common/tool-handler';
+import { BaseBrowserToolExecutor } from '../base-browser';
+import { TOOL_NAMES } from 'chrome-mcp-shared';
+import { ERROR_MESSAGES, TIMEOUTS } from '@/common/constants';
+import { TOOL_MESSAGE_TYPES } from '@/common/message-types';
+import { clickTool, fillTool } from './interaction';
+import { keyboardTool } from './keyboard';
+import { screenshotTool } from './screenshot';
+import { screenshotContextManager, scaleCoordinates } from '@/utils/screenshot-context';
+
+type MouseButton = 'left' | 'right' | 'middle';
+
+interface Coordinates {
+  x: number;
+  y: number;
+}
+
+interface ComputerParams {
+  action:
+    | 'left_click'
+    | 'right_click'
+    | 'double_click'
+    | 'triple_click'
+    | 'left_click_drag'
+    | 'scroll'
+    | 'type'
+    | 'key'
+    | 'hover'
+    | 'wait'
+    | 'fill'
+    | 'screenshot';
+  // click/scroll coordinates in screenshot space (if screenshot context exists) or viewport space
+  coordinates?: Coordinates; // for click/scroll; for drag, this is endCoordinates
+  startCoordinates?: Coordinates; // for drag start
+  // Optional element refs (from chrome_read_page) as alternative to coordinates
+  ref?: string; // click target or drag end
+  startRef?: string; // drag start
+  scrollDirection?: 'up' | 'down' | 'left' | 'right';
+  scrollAmount?: number;
+  text?: string; // for type/key
+  duration?: number; // seconds for wait
+  // For fill
+  selector?: string;
+  value?: string;
+}
+
+// Minimal CDP helper encapsulated here to avoid scattering CDP code
+class CDPHelper {
+  private static active = new Set<number>();
+
+  static async attach(tabId: number): Promise<void> {
+    // If already attached by us, skip
+    const targets = await chrome.debugger.getTargets();
+    const existing = targets.find((t) => t.tabId === tabId && t.attached);
+    if (existing) {
+      if (existing.extensionId === chrome.runtime.id) {
+        this.active.add(tabId);
+        return;
+      }
+      throw new Error(
+        `Debugger is already attached to tab ${tabId} by another tool (e.g., DevTools/extension)`,
+      );
+    }
+    await chrome.debugger.attach({ tabId }, '1.3');
+    this.active.add(tabId);
+  }
+
+  static async detach(tabId: number): Promise<void> {
+    if (!this.active.has(tabId)) return;
+    try {
+      await chrome.debugger.detach({ tabId });
+    } finally {
+      this.active.delete(tabId);
+    }
+  }
+
+  static async send(tabId: number, method: string, params?: object): Promise<any> {
+    try {
+      return await chrome.debugger.sendCommand({ tabId }, method, params);
+    } catch (e: any) {
+      // Try reattach once if lost
+      if (
+        String(e?.message || e)
+          .toLowerCase()
+          .includes('not attached')
+      ) {
+        await this.attach(tabId);
+        return await chrome.debugger.sendCommand({ tabId }, method, params);
+      }
+      throw e;
+    }
+  }
+
+  static async dispatchMouseEvent(tabId: number, opts: any) {
+    const params: any = {
+      type: opts.type,
+      x: Math.round(opts.x),
+      y: Math.round(opts.y),
+      modifiers: opts.modifiers || 0,
+    };
+    if (
+      opts.type === 'mousePressed' ||
+      opts.type === 'mouseReleased' ||
+      opts.type === 'mouseMoved'
+    ) {
+      params.button = opts.button || 'none';
+      if (opts.type === 'mousePressed' || opts.type === 'mouseReleased') {
+        params.clickCount = opts.clickCount || 1;
+      }
+      // Per CDP: buttons is ignored for mouseWheel
+      params.buttons = opts.buttons !== undefined ? opts.buttons : 0;
+    }
+    if (opts.type === 'mouseWheel') {
+      params.deltaX = opts.deltaX || 0;
+      params.deltaY = opts.deltaY || 0;
+    }
+    await this.send(tabId, 'Input.dispatchMouseEvent', params);
+  }
+
+  static async insertText(tabId: number, text: string) {
+    await this.send(tabId, 'Input.insertText', { text });
+  }
+
+  static modifierMask(mods: string[]): number {
+    const map: Record<string, number> = {
+      alt: 1,
+      ctrl: 2,
+      control: 2,
+      meta: 4,
+      cmd: 4,
+      command: 4,
+      win: 4,
+      windows: 4,
+      shift: 8,
+    };
+    let mask = 0;
+    for (const m of mods) mask |= map[m] || 0;
+    return mask;
+  }
+
+  // Enhanced key mapping for common non-character keys
+  private static KEY_ALIASES: Record<string, { key: string; code?: string; text?: string }> = {
+    enter: { key: 'Enter', code: 'Enter' },
+    return: { key: 'Enter', code: 'Enter' },
+    backspace: { key: 'Backspace', code: 'Backspace' },
+    delete: { key: 'Delete', code: 'Delete' },
+    tab: { key: 'Tab', code: 'Tab' },
+    escape: { key: 'Escape', code: 'Escape' },
+    esc: { key: 'Escape', code: 'Escape' },
+    space: { key: ' ', code: 'Space', text: ' ' },
+    pageup: { key: 'PageUp', code: 'PageUp' },
+    pagedown: { key: 'PageDown', code: 'PageDown' },
+    home: { key: 'Home', code: 'Home' },
+    end: { key: 'End', code: 'End' },
+    arrowup: { key: 'ArrowUp', code: 'ArrowUp' },
+    arrowdown: { key: 'ArrowDown', code: 'ArrowDown' },
+    arrowleft: { key: 'ArrowLeft', code: 'ArrowLeft' },
+    arrowright: { key: 'ArrowRight', code: 'ArrowRight' },
+  };
+
+  private static resolveKeyDef(token: string): { key: string; code?: string; text?: string } {
+    const t = (token || '').toLowerCase();
+    if (this.KEY_ALIASES[t]) return this.KEY_ALIASES[t];
+    if (/^f([1-9]|1[0-2])$/.test(t)) {
+      return { key: t.toUpperCase(), code: t.toUpperCase() };
+    }
+    if (t.length === 1) {
+      const upper = t.toUpperCase();
+      return { key: upper, code: `Key${upper}`, text: t };
+    }
+    return { key: token };
+  }
+
+  static async dispatchSimpleKey(tabId: number, token: string) {
+    const def = this.resolveKeyDef(token);
+    if (def.text && def.text.length === 1) {
+      await this.insertText(tabId, def.text);
+      return;
+    }
+    await this.send(tabId, 'Input.dispatchKeyEvent', {
+      type: 'rawKeyDown',
+      key: def.key,
+      code: def.code,
+    });
+    await this.send(tabId, 'Input.dispatchKeyEvent', {
+      type: 'keyUp',
+      key: def.key,
+      code: def.code,
+    });
+  }
+
+  static async dispatchKeyChord(tabId: number, chord: string) {
+    const parts = chord.split('+');
+    const modifiers: string[] = [];
+    let keyToken = '';
+    for (const pRaw of parts) {
+      const p = pRaw.trim().toLowerCase();
+      if (
+        ['ctrl', 'control', 'alt', 'shift', 'cmd', 'meta', 'command', 'win', 'windows'].includes(p)
+      )
+        modifiers.push(p);
+      else keyToken = pRaw.trim();
+    }
+    const mask = this.modifierMask(modifiers);
+    const def = this.resolveKeyDef(keyToken);
+    await this.send(tabId, 'Input.dispatchKeyEvent', {
+      type: 'rawKeyDown',
+      key: def.key,
+      code: def.code,
+      text: def.text,
+      modifiers: mask,
+    });
+    await this.send(tabId, 'Input.dispatchKeyEvent', {
+      type: 'keyUp',
+      key: def.key,
+      code: def.code,
+      modifiers: mask,
+    });
+  }
+}
+
+class ComputerTool extends BaseBrowserToolExecutor {
+  name = TOOL_NAMES.BROWSER.COMPUTER;
+
+  async execute(args: ComputerParams): Promise<ToolResult> {
+    const params = args || ({} as ComputerParams);
+    if (!params.action) return createErrorResponse('Action parameter is required');
+
+    try {
+      const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
+      if (!tabs[0]) return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND);
+      const tab = tabs[0];
+      if (!tab.id)
+        return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID');
+
+      // Helper to project coordinates using screenshot context when available
+      const project = (c?: Coordinates): Coordinates | undefined => {
+        if (!c) return undefined;
+        const ctx = screenshotContextManager.getContext(tab.id!);
+        if (!ctx) return c;
+        const scaled = scaleCoordinates(c.x, c.y, ctx);
+        return { x: scaled.x, y: scaled.y };
+      };
+
+      switch (params.action) {
+        case 'hover': {
+          // Resolve target point from ref | selector | coordinates
+          let coord: Coordinates | undefined = undefined;
+          let resolvedBy: 'ref' | 'selector' | 'coordinates' | undefined;
+
+          try {
+            if (params.ref) {
+              await this.injectContentScript(tab.id, [
+                'inject-scripts/accessibility-tree-helper.js',
+              ]);
+              const resolved = await this.sendMessageToTab(tab.id, {
+                action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
+                ref: params.ref,
+              });
+              if (resolved && resolved.success) {
+                coord = project({ x: resolved.center.x, y: resolved.center.y });
+                resolvedBy = 'ref';
+              }
+            } else if (params.selector) {
+              await this.injectContentScript(tab.id, [
+                'inject-scripts/accessibility-tree-helper.js',
+              ]);
+              const ensured = await this.sendMessageToTab(tab.id, {
+                action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
+                selector: params.selector,
+              });
+              if (ensured && ensured.success) {
+                coord = project({ x: ensured.center.x, y: ensured.center.y });
+                resolvedBy = 'selector';
+              }
+            } else if (params.coordinates) {
+              coord = project(params.coordinates);
+              resolvedBy = 'coordinates';
+            }
+          } catch (e) {
+            // fall through to error handling below
+          }
+
+          if (!coord)
+            return createErrorResponse(
+              'Provide ref or selector or coordinates for hover, or failed to resolve target',
+            );
+          {
+            const stale = ((): any => {
+              if (!params.coordinates) return null;
+              const getHostname = (url: string): string => {
+                try {
+                  return new URL(url).hostname;
+                } catch {
+                  return '';
+                }
+              };
+              const currentHostname = getHostname(tab.url || '');
+              const ctx = screenshotContextManager.getContext(tab.id!);
+              const contextHostname = (ctx as any)?.hostname as string | undefined;
+              if (contextHostname && contextHostname !== currentHostname) {
+                return createErrorResponse(
+                  `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during hover. Capture a new screenshot or use ref/selector.`,
+                );
+              }
+              return null;
+            })();
+            if (stale) return stale;
+          }
+
+          try {
+            await CDPHelper.attach(tab.id);
+            // Move pointer to target. We can dispatch a single mouseMoved; browsers will generate mouseover/mouseenter as needed.
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseMoved',
+              x: coord.x,
+              y: coord.y,
+              button: 'none',
+              buttons: 0,
+            });
+            await CDPHelper.detach(tab.id);
+
+            // Optional hold to allow UI (menus/tooltips) to appear
+            const holdMs = Math.max(
+              0,
+              Math.min(params.duration ? params.duration * 1000 : 400, 5000),
+            );
+            if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
+
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({
+                    success: true,
+                    action: 'hover',
+                    coordinates: coord,
+                    resolvedBy,
+                  }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            return createErrorResponse(
+              `Hover failed: ${e instanceof Error ? e.message : String(e)}`,
+            );
+          }
+        }
+        case 'left_click':
+        case 'right_click': {
+          if (params.ref) {
+            // Prefer DOM click via ref
+            const domResult = await clickTool.execute({
+              ref: params.ref,
+              waitForNavigation: false,
+              timeout: TIMEOUTS.DEFAULT_WAIT * 5,
+            });
+            return domResult;
+          }
+          if (!params.coordinates)
+            return createErrorResponse('Coordinate parameter is required for click action');
+          {
+            const stale = ((): any => {
+              const getHostname = (url: string): string => {
+                try {
+                  return new URL(url).hostname;
+                } catch {
+                  return '';
+                }
+              };
+              const currentHostname = getHostname(tab.url || '');
+              const ctx = screenshotContextManager.getContext(tab.id!);
+              const contextHostname = (ctx as any)?.hostname as string | undefined;
+              if (contextHostname && contextHostname !== currentHostname) {
+                return createErrorResponse(
+                  `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during ${params.action}. Capture a new screenshot or use ref/selector.`,
+                );
+              }
+              return null;
+            })();
+            if (stale) return stale;
+          }
+          const coord = project(params.coordinates)!;
+          // Prefer DOM path via existing click tool
+          const domResult = await clickTool.execute({
+            coordinates: coord,
+            waitForNavigation: false,
+            timeout: TIMEOUTS.DEFAULT_WAIT * 5,
+          });
+          if (!domResult.isError) {
+            return domResult; // Standardized response from click tool
+          }
+          // Fallback to CDP if DOM failed
+          try {
+            await CDPHelper.attach(tab.id);
+            const button: MouseButton = params.action === 'right_click' ? 'right' : 'left';
+            const clickCount = 1;
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseMoved',
+              x: coord.x,
+              y: coord.y,
+              button: 'none',
+              buttons: 0,
+            });
+            for (let i = 1; i <= clickCount; i++) {
+              await CDPHelper.dispatchMouseEvent(tab.id, {
+                type: 'mousePressed',
+                x: coord.x,
+                y: coord.y,
+                button,
+                buttons: button === 'left' ? 1 : 2,
+                clickCount: i,
+              });
+              await CDPHelper.dispatchMouseEvent(tab.id, {
+                type: 'mouseReleased',
+                x: coord.x,
+                y: coord.y,
+                button,
+                buttons: 0,
+                clickCount: i,
+              });
+            }
+            await CDPHelper.detach(tab.id);
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({
+                    success: true,
+                    action: params.action,
+                    coordinates: coord,
+                  }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            return createErrorResponse(
+              `CDP click failed: ${e instanceof Error ? e.message : String(e)}`,
+            );
+          }
+        }
+        case 'double_click':
+        case 'triple_click': {
+          if (!params.coordinates && !params.ref)
+            return createErrorResponse('Provide ref or coordinates for double/triple click');
+          let coord = params.coordinates ? project(params.coordinates)! : (undefined as any);
+          // If ref is provided, resolve center via accessibility helper
+          if (params.ref) {
+            try {
+              await this.injectContentScript(tab.id, [
+                'inject-scripts/accessibility-tree-helper.js',
+              ]);
+              const resolved = await this.sendMessageToTab(tab.id, {
+                action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
+                ref: params.ref,
+              });
+              if (resolved && resolved.success) {
+                coord = project({ x: resolved.center.x, y: resolved.center.y })!;
+              }
+            } catch (e) {
+              // ignore and use provided coordinates
+            }
+          }
+          if (!coord) return createErrorResponse('Failed to resolve coordinates from ref');
+          {
+            const stale = ((): any => {
+              if (!params.coordinates) return null;
+              const getHostname = (url: string): string => {
+                try {
+                  return new URL(url).hostname;
+                } catch {
+                  return '';
+                }
+              };
+              const currentHostname = getHostname(tab.url || '');
+              const ctx = screenshotContextManager.getContext(tab.id!);
+              const contextHostname = (ctx as any)?.hostname as string | undefined;
+              if (contextHostname && contextHostname !== currentHostname) {
+                return createErrorResponse(
+                  `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during ${params.action}. Capture a new screenshot or use ref/selector.`,
+                );
+              }
+              return null;
+            })();
+            if (stale) return stale;
+          }
+          try {
+            await CDPHelper.attach(tab.id);
+            const button: MouseButton = 'left';
+            const clickCount = params.action === 'double_click' ? 2 : 3;
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseMoved',
+              x: coord.x,
+              y: coord.y,
+              button: 'none',
+              buttons: 0,
+            });
+            for (let i = 1; i <= clickCount; i++) {
+              await CDPHelper.dispatchMouseEvent(tab.id, {
+                type: 'mousePressed',
+                x: coord.x,
+                y: coord.y,
+                button,
+                buttons: 1,
+                clickCount: i,
+              });
+              await CDPHelper.dispatchMouseEvent(tab.id, {
+                type: 'mouseReleased',
+                x: coord.x,
+                y: coord.y,
+                button,
+                buttons: 0,
+                clickCount: i,
+              });
+            }
+            await CDPHelper.detach(tab.id);
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({
+                    success: true,
+                    action: params.action,
+                    coordinates: coord,
+                  }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            return createErrorResponse(
+              `CDP ${params.action} failed: ${e instanceof Error ? e.message : String(e)}`,
+            );
+          }
+        }
+        case 'left_click_drag': {
+          if (!params.startCoordinates && !params.startRef)
+            return createErrorResponse('Provide startRef or startCoordinates for drag');
+          if (!params.coordinates && !params.ref)
+            return createErrorResponse('Provide ref or end coordinates for drag');
+          let start = params.startCoordinates
+            ? project(params.startCoordinates)!
+            : (undefined as any);
+          let end = params.coordinates ? project(params.coordinates)! : (undefined as any);
+          {
+            const stale = ((): any => {
+              if (!params.startCoordinates && !params.coordinates) return null;
+              const getHostname = (url: string): string => {
+                try {
+                  return new URL(url).hostname;
+                } catch {
+                  return '';
+                }
+              };
+              const currentHostname = getHostname(tab.url || '');
+              const ctx = screenshotContextManager.getContext(tab.id!);
+              const contextHostname = (ctx as any)?.hostname as string | undefined;
+              if (contextHostname && contextHostname !== currentHostname) {
+                return createErrorResponse(
+                  `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during left_click_drag. Capture a new screenshot or use ref/selector.`,
+                );
+              }
+              return null;
+            })();
+            if (stale) return stale;
+          }
+          if (params.startRef || params.ref) {
+            await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
+          }
+          if (params.startRef) {
+            try {
+              const resolved = await this.sendMessageToTab(tab.id, {
+                action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
+                ref: params.startRef,
+              });
+              if (resolved && resolved.success)
+                start = project({ x: resolved.center.x, y: resolved.center.y })!;
+            } catch {
+              // ignore
+            }
+          }
+          if (params.ref) {
+            try {
+              const resolved = await this.sendMessageToTab(tab.id, {
+                action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
+                ref: params.ref,
+              });
+              if (resolved && resolved.success)
+                end = project({ x: resolved.center.x, y: resolved.center.y })!;
+            } catch {
+              // ignore
+            }
+          }
+          if (!start || !end) return createErrorResponse('Failed to resolve drag coordinates');
+          try {
+            await CDPHelper.attach(tab.id);
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseMoved',
+              x: start.x,
+              y: start.y,
+              button: 'none',
+              buttons: 0,
+            });
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mousePressed',
+              x: start.x,
+              y: start.y,
+              button: 'left',
+              buttons: 1,
+              clickCount: 1,
+            });
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseMoved',
+              x: end.x,
+              y: end.y,
+              button: 'left',
+              buttons: 1,
+            });
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseReleased',
+              x: end.x,
+              y: end.y,
+              button: 'left',
+              buttons: 0,
+              clickCount: 1,
+            });
+            await CDPHelper.detach(tab.id);
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({ success: true, action: 'left_click_drag', start, end }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            return createErrorResponse(
+              `Drag failed: ${e instanceof Error ? e.message : String(e)}`,
+            );
+          }
+        }
+        case 'scroll': {
+          if (!params.coordinates && !params.ref)
+            return createErrorResponse('Provide ref or coordinates for scroll');
+          let coord = params.coordinates ? project(params.coordinates)! : (undefined as any);
+          if (params.ref) {
+            try {
+              await this.injectContentScript(tab.id, [
+                'inject-scripts/accessibility-tree-helper.js',
+              ]);
+              const resolved = await this.sendMessageToTab(tab.id, {
+                action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
+                ref: params.ref,
+              });
+              if (resolved && resolved.success)
+                coord = project({ x: resolved.center.x, y: resolved.center.y })!;
+            } catch {
+              // ignore
+            }
+          }
+          if (!coord) return createErrorResponse('Failed to resolve scroll coordinates');
+          {
+            const stale = ((): any => {
+              if (!params.coordinates) return null;
+              const getHostname = (url: string): string => {
+                try {
+                  return new URL(url).hostname;
+                } catch {
+                  return '';
+                }
+              };
+              const currentHostname = getHostname(tab.url || '');
+              const ctx = screenshotContextManager.getContext(tab.id!);
+              const contextHostname = (ctx as any)?.hostname as string | undefined;
+              if (contextHostname && contextHostname !== currentHostname) {
+                return createErrorResponse(
+                  `Security check failed: Domain changed since last screenshot (from ${contextHostname} to ${currentHostname}) during scroll. Capture a new screenshot or use ref/selector.`,
+                );
+              }
+              return null;
+            })();
+            if (stale) return stale;
+          }
+          const direction = params.scrollDirection || 'down';
+          const amount = Math.max(1, Math.min(params.scrollAmount || 3, 10));
+          // Convert to deltas (~100px per tick)
+          const unit = 100;
+          let deltaX = 0,
+            deltaY = 0;
+          if (direction === 'up') deltaY = -amount * unit;
+          if (direction === 'down') deltaY = amount * unit;
+          if (direction === 'left') deltaX = -amount * unit;
+          if (direction === 'right') deltaX = amount * unit;
+          try {
+            await CDPHelper.attach(tab.id);
+            await CDPHelper.dispatchMouseEvent(tab.id, {
+              type: 'mouseWheel',
+              x: coord.x,
+              y: coord.y,
+              deltaX,
+              deltaY,
+            });
+            await CDPHelper.detach(tab.id);
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({
+                    success: true,
+                    action: 'scroll',
+                    coordinates: coord,
+                    deltaX,
+                    deltaY,
+                  }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            return createErrorResponse(
+              `Scroll failed: ${e instanceof Error ? e.message : String(e)}`,
+            );
+          }
+        }
+        case 'type': {
+          if (!params.text)
+            return createErrorResponse('Text parameter is required for type action');
+          try {
+            // Optional focus via ref before typing
+            if (params.ref) {
+              await clickTool.execute({
+                ref: params.ref,
+                waitForNavigation: false,
+                timeout: TIMEOUTS.DEFAULT_WAIT * 5,
+              });
+            }
+            await CDPHelper.attach(tab.id);
+            // Use CDP insertText to avoid complex KeyboardEvent emulation for long text
+            await CDPHelper.insertText(tab.id, params.text);
+            await CDPHelper.detach(tab.id);
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({
+                    success: true,
+                    action: 'type',
+                    length: params.text.length,
+                  }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            // Fallback to DOM-based keyboard tool
+            const res = await keyboardTool.execute({
+              keys: params.text.split('').join(','),
+              delay: 0,
+              selector: undefined,
+            });
+            return res;
+          }
+        }
+        case 'fill': {
+          if (!params.ref && !params.selector) {
+            return createErrorResponse('Provide ref or selector and a value for fill');
+          }
+          // Reuse existing fill tool to leverage robust DOM event behavior
+          const res = await fillTool.execute({
+            selector: params.selector as any,
+            ref: params.ref as any,
+            value: params.value as any,
+          } as any);
+          return res;
+        }
+        case 'key': {
+          if (!params.text)
+            return createErrorResponse(
+              'text is required for key action (e.g., "Backspace Backspace Enter" or "cmd+a")',
+            );
+          const tokens = params.text.trim().split(/\s+/).filter(Boolean);
+          try {
+            // Optional focus via ref before key events
+            if (params.ref) {
+              await clickTool.execute({
+                ref: params.ref,
+                waitForNavigation: false,
+                timeout: TIMEOUTS.DEFAULT_WAIT * 5,
+              });
+            }
+            await CDPHelper.attach(tab.id);
+            for (const t of tokens) {
+              if (t.includes('+')) await CDPHelper.dispatchKeyChord(tab.id, t);
+              else await CDPHelper.dispatchSimpleKey(tab.id, t);
+            }
+            await CDPHelper.detach(tab.id);
+            return {
+              content: [
+                {
+                  type: 'text',
+                  text: JSON.stringify({ success: true, action: 'key', keys: tokens }),
+                },
+              ],
+              isError: false,
+            };
+          } catch (e) {
+            await CDPHelper.detach(tab.id);
+            // Fallback to DOM keyboard simulation (comma-separated combinations)
+            const keysStr = tokens.join(',');
+            const res = await keyboardTool.execute({ keys: keysStr });
+            return res;
+          }
+        }
+        case 'wait': {
+          const seconds = Math.max(0, Math.min(params.duration || 0, 30));
+          if (!seconds)
+            return createErrorResponse('Duration parameter is required and must be > 0');
+          await new Promise((r) => setTimeout(r, seconds * 1000));
+          return {
+            content: [
+              {
+                type: 'text',
+                text: JSON.stringify({ success: true, action: 'wait', duration: seconds }),
+              },
+            ],
+            isError: false,
+          };
+        }
+        case 'screenshot': {
+          // Reuse existing screenshot tool; it already supports base64 save option
+          const result = await screenshotTool.execute({
+            name: 'computer',
+            storeBase64: true,
+            fullPage: false,
+          });
+          return result;
+        }
+        default:
+          return createErrorResponse(`Unsupported action: ${params.action}`);
+      }
+    } catch (error) {
+      console.error('Error in computer tool:', error);
+      return createErrorResponse(
+        `Failed to execute action: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+export const computerTool = new ComputerTool();

app/chrome-extension/entrypoints/background/tools/browser/index.ts

@@ -13,3 +13,5 @@ export { bookmarkSearchTool, bookmarkAddTool, bookmarkDeleteTool } from './bookm
 export { injectScriptTool, sendCommandToInjectScriptTool } from './inject-script';
 export { consoleTool } from './console';
 export { fileUploadTool } from './file-upload';
+export { readPageTool } from './read-page';
+export { computerTool } from './computer';

app/chrome-extension/entrypoints/background/tools/browser/interaction.ts

@@ -11,6 +11,7 @@ interface Coordinates {
 
 interface ClickToolParams {
   selector?: string; // CSS selector for the element to click
+  ref?: string; // Element ref from accessibility tree (window.__claudeElementMap)
   coordinates?: Coordinates; // Coordinates to click at (x, y relative to viewport)
   waitForNavigation?: boolean; // Whether to wait for navigation to complete after click
   timeout?: number; // Timeout in milliseconds for waiting for the element or navigation
@@ -35,9 +36,9 @@ class ClickTool extends BaseBrowserToolExecutor {
 
     console.log(`Starting click operation with options:`, args);
 
-    if (!selector && !coordinates) {
+    if (!selector && !coordinates && !args.ref) {
       return createErrorResponse(
-        ERROR_MESSAGES.INVALID_PARAMETERS + ': Either selector or coordinates must be provided',
+        ERROR_MESSAGES.INVALID_PARAMETERS + ': Provide ref or selector or coordinates',
       );
     }
 
@@ -60,6 +61,7 @@ class ClickTool extends BaseBrowserToolExecutor {
         action: TOOL_MESSAGE_TYPES.CLICK_ELEMENT,
         selector,
         coordinates,
+        ref: args.ref,
         waitForNavigation,
         timeout,
       });
@@ -91,8 +93,10 @@ class ClickTool extends BaseBrowserToolExecutor {
 export const clickTool = new ClickTool();
 
 interface FillToolParams {
-  selector: string;
-  value: string;
+  selector?: string;
+  ref?: string; // Element ref from accessibility tree
+  // Accept string | number | boolean for broader form input coverage
+  value: string | number | boolean;
 }
 
 /**
@@ -105,12 +109,12 @@ class FillTool extends BaseBrowserToolExecutor {
    * Execute fill operation
    */
   async execute(args: FillToolParams): Promise<ToolResult> {
-    const { selector, value } = args;
+    const { selector, ref, value } = args;
 
     console.log(`Starting fill operation with options:`, args);
 
-    if (!selector) {
-      return createErrorResponse(ERROR_MESSAGES.INVALID_PARAMETERS + ': Selector must be provided');
+    if (!selector && !ref) {
+      return createErrorResponse(ERROR_MESSAGES.INVALID_PARAMETERS + ': Provide ref or selector');
     }
 
     if (value === undefined || value === null) {
@@ -135,10 +139,11 @@ class FillTool extends BaseBrowserToolExecutor {
       const result = await this.sendMessageToTab(tab.id, {
         action: TOOL_MESSAGE_TYPES.FILL_ELEMENT,
         selector,
+        ref,
         value,
       });
 
-      if (result.error) {
+      if (result && result.error) {
         return createErrorResponse(result.error);
       }
 

app/chrome-extension/entrypoints/background/tools/browser/read-page.ts

@@ -0,0 +1,111 @@
+import { createErrorResponse, ToolResult } from '@/common/tool-handler';
+import { BaseBrowserToolExecutor } from '../base-browser';
+import { TOOL_NAMES } from 'chrome-mcp-shared';
+import { TOOL_MESSAGE_TYPES } from '@/common/message-types';
+import { ERROR_MESSAGES } from '@/common/constants';
+
+interface ReadPageParams {
+  filter?: 'interactive'; // when omitted, return all visible elements
+}
+
+class ReadPageTool extends BaseBrowserToolExecutor {
+  name = TOOL_NAMES.BROWSER.READ_PAGE;
+
+  // Execute read page
+  async execute(args: ReadPageParams): Promise<ToolResult> {
+    const { filter } = args || {};
+
+    try {
+      // Tip text returned to callers to guide next action
+      const standardTips =
+        "If the specific element you need is missing from the returned data, use the 'screenshot' tool to capture the current viewport and confirm the element's on-screen coordinates.";
+
+      const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
+      if (!tabs[0]) return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND);
+      const tab = tabs[0];
+      if (!tab.id)
+        return createErrorResponse(ERROR_MESSAGES.TAB_NOT_FOUND + ': Active tab has no ID');
+
+      // Inject helper in ISOLATED world to enable chrome.runtime messaging
+      await this.injectContentScript(tab.id, ['inject-scripts/accessibility-tree-helper.js']);
+
+      // Ask content script to generate accessibility tree
+      const resp = await this.sendMessageToTab(tab.id, {
+        action: TOOL_MESSAGE_TYPES.GENERATE_ACCESSIBILITY_TREE,
+        filter: filter || null,
+      });
+
+      // Evaluate tree result and decide whether to fallback
+      const treeOk = resp && resp.success === true;
+      const pageContent: string =
+        resp && typeof resp.pageContent === 'string' ? resp.pageContent : '';
+      const lines = pageContent
+        ? pageContent.split('\n').filter((l: string) => l.trim().length > 0).length
+        : 0;
+      const refCount = Array.isArray(resp?.refMap) ? resp.refMap.length : 0;
+      const isSparse = lines < 10 && refCount < 3; // heuristic threshold for sparse trees
+
+      if (treeOk && !isSparse) {
+        // Normal path: return tree
+        const resultPayload = {
+          success: true,
+          filter: filter || 'all',
+          pageContent: resp.pageContent,
+          tips: standardTips,
+          viewport: resp.viewport,
+          refMapCount: refCount,
+          sparse: false,
+        };
+
+        return {
+          content: [{ type: 'text', text: JSON.stringify(resultPayload) }],
+          isError: false,
+        };
+      }
+
+      // Fallback path: try get_interactive_elements once
+      try {
+        await this.injectContentScript(tab.id, ['inject-scripts/interactive-elements-helper.js']);
+        const fallback = await this.sendMessageToTab(tab.id, {
+          action: TOOL_MESSAGE_TYPES.GET_INTERACTIVE_ELEMENTS,
+          includeCoordinates: true,
+        });
+
+        if (fallback && fallback.success && Array.isArray(fallback.elements)) {
+          const limited = fallback.elements.slice(0, 150);
+          const fallbackPayload = {
+            success: true,
+            fallbackUsed: true,
+            fallbackSource: 'get_interactive_elements',
+            reason: treeOk ? 'sparse_tree' : resp?.error || 'tree_failed',
+            treeStats: { lines, refCount },
+            elements: limited,
+            count: fallback.elements.length,
+            tips: standardTips,
+          };
+
+          return {
+            content: [{ type: 'text', text: JSON.stringify(fallbackPayload) }],
+            isError: false,
+          };
+        }
+      } catch (fallbackErr) {
+        console.warn('read_page fallback failed:', fallbackErr);
+      }
+
+      // If we reach here, both tree (usable) and fallback failed
+      return createErrorResponse(
+        treeOk
+          ? 'Accessibility tree is too sparse and fallback failed'
+          : resp?.error || 'Failed to generate accessibility tree and fallback failed',
+      );
+    } catch (error) {
+      console.error('Error in read page tool:', error);
+      return createErrorResponse(
+        `Error generating accessibility tree: ${error instanceof Error ? error.message : String(error)}`,
+      );
+    }
+  }
+}
+
+export const readPageTool = new ReadPageTool();

app/chrome-extension/entrypoints/background/tools/browser/screenshot.ts

@@ -10,6 +10,7 @@ import {
   stitchImages,
   compressImage,
 } from '../../../../utils/image-utils';
+import { screenshotContextManager } from '@/utils/screenshot-context';
 
 // Screenshot-specific constants
 const SCREENSHOT_CONSTANTS = {
@@ -28,7 +29,20 @@ const SCREENSHOT_CONSTANTS = {
   readonly SCRIPT_INIT_DELAY: number;
 };
 
-SCREENSHOT_CONSTANTS["CAPTURE_STITCH_DELAY_MS"] = Math.max(1000 / chrome.tabs.MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND - SCREENSHOT_CONSTANTS.SCROLL_DELAY_MS, SCREENSHOT_CONSTANTS.CAPTURE_STITCH_DELAY_MS)
+// Adjust CAPTURE_STITCH_DELAY_MS to respect Chrome's capture rate if available in runtime
+// Some TS typings don't expose MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND; use a safe cast with a sane fallback.
+const __MAX_CAP_RATE: number | undefined = (chrome.tabs as any)
+  ?.MAX_CAPTURE_VISIBLE_TAB_CALLS_PER_SECOND;
+if (typeof __MAX_CAP_RATE === 'number' && __MAX_CAP_RATE > 0) {
+  // Minimum interval between consecutive captureVisibleTab calls (ms)
+  const minIntervalMs = Math.ceil(1000 / __MAX_CAP_RATE);
+  // Our capture loop already waits SCROLL_DELAY_MS between scroll and capture; add any extra delay needed
+  const requiredExtraDelay = Math.max(0, minIntervalMs - SCREENSHOT_CONSTANTS.SCROLL_DELAY_MS);
+  SCREENSHOT_CONSTANTS.CAPTURE_STITCH_DELAY_MS = Math.max(
+    requiredExtraDelay,
+    SCREENSHOT_CONSTANTS.CAPTURE_STITCH_DELAY_MS,
+  );
+}
 
 interface ScreenshotToolParams {
   name: string;
@@ -81,6 +95,8 @@ class ScreenshotTool extends BaseBrowserToolExecutor {
     }
 
     let finalImageDataUrl: string | undefined;
+    let finalImageWidthCss: number | undefined;
+    let finalImageHeightCss: number | undefined;
     const results: any = { base64: null, fileSaved: false };
     let originalScroll = { x: 0, y: 0 };
 
@@ -102,14 +118,45 @@ class ScreenshotTool extends BaseBrowserToolExecutor {
 
       if (fullPage) {
         this.logInfo('Capturing full page...');
-        finalImageDataUrl = await this._captureFullPage(tab.id!, args, pageDetails);
+        const dataUrl = await this._captureFullPage(tab.id!, args, pageDetails);
+        finalImageDataUrl = dataUrl;
+        // For full page, compute final CSS size from provided width/height fallback to pageDetails
+        if (args.width && args.height) {
+          finalImageWidthCss = args.width;
+          finalImageHeightCss = args.height;
+        } else if (args.width && !args.height) {
+          finalImageWidthCss = args.width;
+          // height will be scaled by aspect ratio; approximate with page ratio
+          const ratio = pageDetails.totalHeight / pageDetails.totalWidth;
+          finalImageHeightCss = Math.round(args.width * ratio);
+        } else if (!args.width && args.height) {
+          finalImageHeightCss = args.height;
+          const ratio = pageDetails.totalWidth / pageDetails.totalHeight;
+          finalImageWidthCss = Math.round(args.height * ratio);
+        } else {
+          finalImageWidthCss = pageDetails.totalWidth;
+          finalImageHeightCss = pageDetails.totalHeight;
+        }
       } else if (selector) {
         this.logInfo(`Capturing element: ${selector}`);
-        finalImageDataUrl = await this._captureElement(tab.id!, args, pageDetails.devicePixelRatio);
+        const cropped = await this._captureElement(tab.id!, args, pageDetails.devicePixelRatio);
+        finalImageDataUrl = cropped;
+        // For element capture, if target width/height provided, respect them; otherwise use element rect size
+        if (args.width && args.height) {
+          finalImageWidthCss = args.width;
+          finalImageHeightCss = args.height;
+        } else {
+          // Fallback to visible element rect (already used for crop)
+          // We do not have easy access to rect here; use viewport as conservative default
+          finalImageWidthCss = pageDetails.viewportWidth;
+          finalImageHeightCss = pageDetails.viewportHeight;
+        }
       } else {
         // Visible area only
         this.logInfo('Capturing visible area...');
         finalImageDataUrl = await chrome.tabs.captureVisibleTab(tab.windowId, { format: 'png' });
+        finalImageWidthCss = pageDetails.viewportWidth;
+        finalImageHeightCss = pageDetails.viewportHeight;
       }
 
       if (!finalImageDataUrl) {
@@ -117,6 +164,27 @@ class ScreenshotTool extends BaseBrowserToolExecutor {
       }
 
       // 2. Process output
+      // Update screenshot context for coordinate scaling by tools like chrome_computer
+      try {
+        if (typeof finalImageWidthCss === 'number' && typeof finalImageHeightCss === 'number') {
+          let hostname = '';
+          try {
+            hostname = tab.url ? new URL(tab.url).hostname : '';
+          } catch {
+            // ignore
+          }
+          screenshotContextManager.setContext(tab.id!, {
+            screenshotWidth: finalImageWidthCss,
+            screenshotHeight: finalImageHeightCss,
+            viewportWidth: pageDetails.viewportWidth,
+            viewportHeight: pageDetails.viewportHeight,
+            devicePixelRatio: pageDetails.devicePixelRatio,
+            hostname,
+          });
+        }
+      } catch (e) {
+        console.warn('Failed to set screenshot context:', e);
+      }
       if (storeBase64 === true) {
         // Compress image for base64 output to reduce size
         const compressed = await compressImage(finalImageDataUrl, {

app/chrome-extension/inject-scripts/accessibility-tree-helper.js

@@ -0,0 +1,593 @@
+/* eslint-disable */
+// accessibility-tree-helper.js
+// Injected script to generate an accessibility-like tree of the visible page
+// Elements receive stable refs (ref_*) via WeakRef mapping for later reference.
+
+(function () {
+  if (window.__ACCESSIBILITY_TREE_HELPER_INITIALIZED__) return;
+  window.__ACCESSIBILITY_TREE_HELPER_INITIALIZED__ = true;
+
+  // Traversal and output limits to ensure stability on very large/complex pages
+  const MAX_DEPTH = 30; // maximum DOM depth to traverse
+  const MAX_NODES = 4000; // hard limit to avoid long blocking on huge DOMs
+  const MAX_LINE_LABEL = 100; // max characters for a single label in output
+  const REF_MAP_LIMIT = 1000; // limit size of the ref map to keep payload small
+
+  // Keep a weak map from ref id to elements
+  if (!window.__claudeElementMap) window.__claudeElementMap = {};
+  if (!window.__claudeRefCounter) window.__claudeRefCounter = 0;
+
+  /**
+   * Infer ARIA-like role from element
+   * @param {Element} el
+   * @returns {string}
+   */
+  function inferRole(el) {
+    const role = el.getAttribute('role');
+    if (role) return role;
+    const tag = el.tagName.toLowerCase();
+    const type = el.getAttribute('type') || '';
+    const map = {
+      a: 'link',
+      button: 'button',
+      input:
+        type === 'submit' || type === 'button'
+          ? 'button'
+          : type === 'checkbox'
+            ? 'checkbox'
+            : type === 'radio'
+              ? 'radio'
+              : type === 'file'
+                ? 'button'
+                : 'textbox',
+      select: 'combobox',
+      textarea: 'textbox',
+      h1: 'heading',
+      h2: 'heading',
+      h3: 'heading',
+      h4: 'heading',
+      h5: 'heading',
+      h6: 'heading',
+      img: 'image',
+      nav: 'navigation',
+      main: 'main',
+      header: 'banner',
+      footer: 'contentinfo',
+      section: 'region',
+      article: 'article',
+      aside: 'complementary',
+      form: 'form',
+      table: 'table',
+      ul: 'list',
+      ol: 'list',
+      li: 'listitem',
+      label: 'label',
+    };
+    return map[tag] || 'generic';
+  }
+
+  /**
+   * Derive readable label for element
+   * @param {Element} el
+   * @returns {string}
+   */
+  function inferLabel(el) {
+    const tag = el.tagName.toLowerCase();
+    if (tag === 'select') {
+      const sel = /** @type {HTMLSelectElement} */ (el);
+      const opt = sel.querySelector('option[selected]') || sel.options[sel.selectedIndex];
+      if (opt && opt.textContent) return opt.textContent.trim();
+    }
+    const aria = el.getAttribute('aria-label');
+    if (aria && aria.trim()) return aria.trim();
+    const placeholder = el.getAttribute('placeholder');
+    if (placeholder && placeholder.trim()) return placeholder.trim();
+    const title = el.getAttribute('title');
+    if (title && title.trim()) return title.trim();
+    const alt = el.getAttribute('alt');
+    if (alt && alt.trim()) return alt.trim();
+    if (/** @type {HTMLElement} */ (el).id) {
+      const lab = document.querySelector(`label[for="${/** @type {HTMLElement} */ (el).id}"]`);
+      if (lab && lab.textContent && lab.textContent.trim()) return lab.textContent.trim();
+    }
+    if (tag === 'input') {
+      const input = /** @type {HTMLInputElement} */ (el);
+      const type = input.getAttribute('type') || '';
+      const val = input.getAttribute('value');
+      if (type === 'submit' && val && val.trim()) return val.trim();
+      if (input.value && input.value.length < 50 && input.value.trim()) return input.value.trim();
+    }
+    if (['button', 'a', 'summary'].includes(tag)) {
+      let text = '';
+      for (let i = 0; i < el.childNodes.length; i++) {
+        const n = el.childNodes[i];
+        if (n.nodeType === Node.TEXT_NODE) text += n.textContent || '';
+      }
+      if (text.trim()) return text.trim();
+    }
+    if (/^h[1-6]$/.test(tag)) {
+      const t = el.textContent;
+      if (t && t.trim()) return t.trim().substring(0, MAX_LINE_LABEL);
+    }
+    if (tag === 'img') {
+      const src = el.getAttribute('src');
+      if (src) {
+        const file = src.split('/').pop()?.split('?')[0];
+        return `Image: ${file}`;
+      }
+    }
+    let agg = '';
+    for (let i = 0; i < el.childNodes.length; i++) {
+      const n = el.childNodes[i];
+      if (n.nodeType === Node.TEXT_NODE) agg += n.textContent || '';
+    }
+    if (agg && agg.trim() && agg.trim().length >= 3) {
+      const v = agg.trim();
+      return v.length > 50 ? v.substring(0, 50) + '...' : v;
+    }
+    return '';
+  }
+
+  /**
+   * Check if element is visible in DOM
+   * @param {Element} el
+   */
+  function isVisible(el) {
+    const cs = window.getComputedStyle(/** @type {HTMLElement} */ (el));
+    if (cs.display === 'none' || cs.visibility === 'hidden' || cs.opacity === '0') return false;
+    const he = /** @type {HTMLElement} */ (el);
+    return he.offsetWidth > 0 && he.offsetHeight > 0;
+  }
+
+  /**
+   * Whether the element is interactive
+   * @param {Element} el
+   */
+  function isInteractive(el) {
+    // Native interactive tags
+    const tag = el.tagName.toLowerCase();
+    if (['a', 'button', 'input', 'select', 'textarea', 'details', 'summary'].includes(tag))
+      return true;
+
+    // Generic interactive hints
+    if (el.getAttribute('onclick') != null) return true;
+    if (
+      el.getAttribute('tabindex') != null &&
+      String(el.getAttribute('tabindex')).trim() !== '' &&
+      !String(el.getAttribute('tabindex')).trim().startsWith('-')
+    )
+      return true;
+    if (el.getAttribute('contenteditable') === 'true') return true;
+
+    // ARIA roles commonly used by custom elements
+    const role = (el.getAttribute && el.getAttribute('role')) || '';
+    const interactiveRoles = new Set([
+      'button',
+      'link',
+      'checkbox',
+      'radio',
+      'switch',
+      'slider',
+      'option',
+      'menuitem',
+      'textbox',
+      'searchbox',
+      'combobox',
+      'spinbutton',
+      'tab',
+      'treeitem',
+    ]);
+    if (role && interactiveRoles.has(role.toLowerCase())) return true;
+
+    // Shadow host case: treat host as interactive if its open shadow root contains
+    // an interactive control (textarea/input/select/button/a or contenteditable).
+    try {
+      const anyEl = /** @type {any} */ (el);
+      const sr = anyEl && anyEl.shadowRoot ? anyEl.shadowRoot : null;
+      if (sr) {
+        const inner = sr.querySelector(
+          'input, textarea, select, button, a[href], [contenteditable="true"], [role="button"], [role="link"], [role="textbox"], [role="combobox"], [role="searchbox"], [role="menuitem"], [role="option"], [role="switch"], [role="radio"], [role="checkbox"], [role="tab"], [role="slider"]',
+        );
+        if (inner) return true;
+      }
+    } catch (_) {
+      /* ignore */
+    }
+    return false;
+  }
+
+  /**
+   * Structural containers useful to include
+   * @param {Element} el
+   */
+  function isStructural(el) {
+    const tag = el.tagName.toLowerCase();
+    if (
+      [
+        'h1',
+        'h2',
+        'h3',
+        'h4',
+        'h5',
+        'h6',
+        'nav',
+        'main',
+        'header',
+        'footer',
+        'section',
+        'article',
+        'aside',
+      ].includes(tag)
+    )
+      return true;
+    return el.getAttribute('role') != null;
+  }
+
+  /**
+   * Form-ish containers to keep
+   * @param {Element} el
+   */
+  function isFormishContainer(el) {
+    const tag = el.tagName.toLowerCase();
+    const role = (el.getAttribute && el.getAttribute('role')) || '';
+    const id = /** @type {HTMLElement} */ (el).id || '';
+    // Normalize className for HTML/SVG elements
+    let cls = '';
+    try {
+      const attr = el.getAttribute && el.getAttribute('class');
+      if (typeof attr === 'string') cls = attr;
+      else {
+        const cn = /** @type {any} */ (el).className;
+        if (typeof cn === 'string') cls = cn;
+        else if (cn && typeof cn.baseVal === 'string') cls = cn.baseVal;
+      }
+    } catch (e) {
+      /* ignore */
+    }
+    return (
+      role === 'search' ||
+      role === 'form' ||
+      role === 'group' ||
+      role === 'toolbar' ||
+      role === 'navigation' ||
+      tag === 'form' ||
+      tag === 'fieldset' ||
+      tag === 'nav' ||
+      tag === 'legend' ||
+      id.includes('search') ||
+      cls.includes('search') ||
+      id.includes('form') ||
+      cls.includes('form') ||
+      id.includes('menu') ||
+      cls.includes('menu') ||
+      id.includes('nav') ||
+      cls.includes('nav')
+    );
+  }
+
+  /**
+   * Whether to include element in tree under config
+   * @param {Element} el
+   * @param {{filter?: 'all'|'interactive'}} cfg
+   */
+  function shouldInclude(el, cfg) {
+    const tag = el.tagName.toLowerCase();
+    if (['script', 'style', 'meta', 'link', 'title', 'noscript'].includes(tag)) return false;
+    if (el.getAttribute('aria-hidden') === 'true') return false;
+    if (!isVisible(el)) return false;
+    if (cfg.filter !== 'all') {
+      const r = /** @type {HTMLElement} */ (el).getBoundingClientRect();
+      if (
+        !(r.top < window.innerHeight && r.bottom > 0 && r.left < window.innerWidth && r.right > 0)
+      )
+        return false;
+    }
+    if (cfg.filter === 'interactive') return isInteractive(el);
+    if (isInteractive(el)) return true;
+    if (isStructural(el)) return true;
+    if (inferLabel(el).length > 0) return true;
+    return isFormishContainer(el);
+  }
+
+  /**
+   * Generate a fairly stable CSS selector
+   * @param {Element} el
+   * @returns {string}
+   */
+  function generateSelector(el) {
+    if (!(el instanceof Element)) return '';
+    if (/** @type {HTMLElement} */ (el).id) {
+      const idSel = `#${CSS.escape(/** @type {HTMLElement} */ (el).id)}`;
+      if (document.querySelectorAll(idSel).length === 1) return idSel;
+    }
+    for (const attr of ['data-testid', 'data-cy', 'name']) {
+      const attrValue = el.getAttribute(attr);
+      if (attrValue) {
+        const s = `[${attr}="${CSS.escape(attrValue)}"]`;
+        if (document.querySelectorAll(s).length === 1) return s;
+      }
+    }
+    let path = '';
+    let current = el;
+    while (current && current.nodeType === Node.ELEMENT_NODE && current.tagName !== 'BODY') {
+      let selector = current.tagName.toLowerCase();
+      const parent = current.parentElement;
+      if (parent) {
+        const siblings = Array.from(parent.children).filter(
+          (child) => child.tagName === current.tagName,
+        );
+        if (siblings.length > 1) {
+          const index = siblings.indexOf(current) + 1;
+          selector += `:nth-of-type(${index})`;
+        }
+      }
+      path = path ? `${selector} > ${path}` : selector;
+      current = parent;
+    }
+    return path ? `body > ${path}` : 'body';
+  }
+
+  /**
+   * Traverse DOM and build pageContent lines; collect ref map for interactive nodes.
+   * @param {Element} el
+   * @param {number} depth
+   * @param {{filter?: 'all'|'interactive'}} cfg
+   * @param {string[]} out
+   * @param {Array<{ref:string, selector:string, rect:{x:number,y:number,width:number,height:number}}>} refMap
+   */
+  function traverse(el, depth, cfg, out, refMap, state) {
+    if (depth > MAX_DEPTH || !el || !el.tagName) return;
+    if (state.processed >= MAX_NODES) return;
+    if (state.visited.has(el)) return;
+    state.visited.add(el);
+    const include = shouldInclude(el, cfg) || depth === 0;
+    if (include) {
+      const role = inferRole(el);
+      let label = inferLabel(el);
+      let refId = null;
+      for (const k in window.__claudeElementMap) {
+        if (window.__claudeElementMap[k].deref && window.__claudeElementMap[k].deref() === el) {
+          refId = k;
+          break;
+        }
+      }
+      if (!refId) {
+        refId = `ref_${++window.__claudeRefCounter}`;
+        window.__claudeElementMap[refId] = new WeakRef(el);
+      }
+      const rect = /** @type {HTMLElement} */ (el).getBoundingClientRect();
+      const cx = Math.round(rect.left + rect.width / 2);
+      const cy = Math.round(rect.top + rect.height / 2);
+      let line = `${'  '.repeat(depth)}- ${role}`;
+      if (label) {
+        label = label.replace(/\s+/g, ' ').substring(0, MAX_LINE_LABEL);
+        line += ` "${label.replace(/"/g, '\\"')}"`;
+      }
+      line += ` [ref=${refId}] (x=${cx},y=${cy})`;
+      if (/** @type {HTMLElement} */ (el).id) line += ` id="${/** @type {HTMLElement} */ (el).id}"`;
+      const href = el.getAttribute('href');
+      if (href) line += ` href="${href}"`;
+      const type = el.getAttribute('type');
+      if (type) line += ` type="${type}"`;
+      const placeholder = el.getAttribute('placeholder');
+      if (placeholder) line += ` placeholder="${placeholder}"`;
+      // Surface disabled/pointer-events for better agent judgement
+      try {
+        const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true';
+        if (disabled) line += ` disabled`;
+        const cs = window.getComputedStyle(/** @type {HTMLElement} */ (el));
+        if (cs && cs.pointerEvents === 'none') line += ` pe=none`;
+      } catch (_) {
+        /* ignore style issues */
+      }
+      out.push(line);
+      state.included++;
+      state.processed++;
+
+      // Only collect ref mapping for interactive elements to limit cost
+      if (isInteractive(el) && refMap.length < REF_MAP_LIMIT) {
+        refMap.push({
+          ref: /** @type {string} */ (refId),
+          selector: generateSelector(el),
+          rect: {
+            x: rect.x,
+            y: rect.y,
+            width: rect.width,
+            height: rect.height,
+          },
+        });
+      }
+    }
+    if (state.processed >= MAX_NODES) return;
+    // Traverse light DOM children
+    if (/** @type {HTMLElement} */ (el).children && depth < MAX_DEPTH) {
+      const children = /** @type {HTMLElement} */ (el).children;
+      for (let i = 0; i < children.length; i++) {
+        if (state.processed >= MAX_NODES) break;
+        traverse(children[i], include ? depth + 1 : depth, cfg, out, refMap, state);
+      }
+    }
+    // Traverse shadow DOM roots (limited by MAX_DEPTH and MAX_NODES)
+    try {
+      const anyEl = /** @type {any} */ (el);
+      if (anyEl && anyEl.shadowRoot && depth < MAX_DEPTH) {
+        const srChildren = anyEl.shadowRoot.children || [];
+        for (let i = 0; i < srChildren.length; i++) {
+          if (state.processed >= MAX_NODES) break;
+          traverse(srChildren[i], include ? depth + 1 : depth, cfg, out, refMap, state);
+        }
+      }
+    } catch (_) {
+      /* ignore shadow errors */
+    }
+  }
+
+  /**
+   * Generate tree and return
+   * @param {'all'|'interactive'|null} filter
+   */
+  function __generateAccessibilityTree(filter) {
+    try {
+      const start = performance && performance.now ? performance.now() : Date.now();
+      const out = [];
+      const cfg = { filter: filter || undefined };
+      const refMap = [];
+      const state = { processed: 0, included: 0, visited: new WeakSet() };
+      if (document.body) traverse(document.body, 0, cfg, out, refMap, state);
+      for (const k in window.__claudeElementMap) {
+        if (!window.__claudeElementMap[k].deref || !window.__claudeElementMap[k].deref())
+          delete window.__claudeElementMap[k];
+      }
+      const pageContent = out
+        .filter((line) => !/^\s*- generic \[ref=ref_\d+\]$/.test(line))
+        .join('\n');
+      const end = performance && performance.now ? performance.now() : Date.now();
+      return {
+        pageContent,
+        viewport: {
+          width: window.innerWidth,
+          height: window.innerHeight,
+          dpr: window.devicePixelRatio || 1,
+        },
+        stats: {
+          processed: state.processed,
+          included: state.included,
+          durationMs: Math.round(end - start),
+        },
+        refMap,
+      };
+    } catch (err) {
+      throw new Error(
+        'Error generating accessibility tree: ' +
+          (err && err.message ? err.message : 'Unknown error'),
+      );
+    }
+  }
+
+  // Expose API on window
+  window.__generateAccessibilityTree = __generateAccessibilityTree;
+
+  // Chrome message bridge for ping and tree generation
+  chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
+    try {
+      if (request && request.action === 'chrome_read_page_ping') {
+        sendResponse({ status: 'pong' });
+        return false;
+      }
+      if (request && request.action === 'generateAccessibilityTree') {
+        const result = __generateAccessibilityTree(request.filter || null);
+        sendResponse({ success: true, ...result });
+        return true;
+      }
+      if (request && request.action === 'ensureRefForSelector') {
+        try {
+          const sel = String(request.selector || '').trim();
+          if (!sel) {
+            sendResponse({ success: false, error: 'selector is required' });
+            return true;
+          }
+          const el = document.querySelector(sel);
+          if (!el) {
+            sendResponse({ success: false, error: `selector not found: ${sel}` });
+            return true;
+          }
+          let refId = null;
+          for (const k in window.__claudeElementMap) {
+            if (window.__claudeElementMap[k].deref && window.__claudeElementMap[k].deref() === el) {
+              refId = k;
+              break;
+            }
+          }
+          if (!refId) {
+            refId = `ref_${++window.__claudeRefCounter}`;
+            window.__claudeElementMap[refId] = new WeakRef(el);
+          }
+          const rect = /** @type {HTMLElement} */ (el).getBoundingClientRect();
+          sendResponse({
+            success: true,
+            ref: refId,
+            center: {
+              x: Math.round(rect.left + rect.width / 2),
+              y: Math.round(rect.top + rect.height / 2),
+            },
+          });
+          return true;
+        } catch (e) {
+          sendResponse({ success: false, error: String(e && e.message ? e.message : e) });
+          return true;
+        }
+      }
+      if (request && request.action === 'resolveRef') {
+        const ref = request.ref;
+        try {
+          const map = window.__claudeElementMap;
+          const weak = map && map[ref];
+          const el = weak && typeof weak.deref === 'function' ? weak.deref() : null;
+          if (!el || !(el instanceof Element)) {
+            sendResponse({ success: false, error: `ref "${ref}" not found or expired` });
+            return true;
+          }
+          const rect = /** @type {HTMLElement} */ (el).getBoundingClientRect();
+          sendResponse({
+            success: true,
+            rect: { x: rect.x, y: rect.y, width: rect.width, height: rect.height },
+            center: {
+              x: Math.round(rect.left + rect.width / 2),
+              y: Math.round(rect.top + rect.height / 2),
+            },
+            selector: (function () {
+              // Simple selector generation inline to avoid duplication
+              const generateSelector = function (node) {
+                if (!(node instanceof Element)) return '';
+                if (node.id) {
+                  const idSel = `#${CSS.escape(node.id)}`;
+                  if (document.querySelectorAll(idSel).length === 1) return idSel;
+                }
+                for (const attr of ['data-testid', 'data-cy', 'name']) {
+                  const val = node.getAttribute(attr);
+                  if (val) {
+                    const s = `[${attr}="${CSS.escape(val)}"]`;
+                    if (document.querySelectorAll(s).length === 1) return s;
+                  }
+                }
+                let path = '';
+                let current = node;
+                while (
+                  current &&
+                  current.nodeType === Node.ELEMENT_NODE &&
+                  current.tagName !== 'BODY'
+                ) {
+                  let sel = current.tagName.toLowerCase();
+                  const parent = current.parentElement;
+                  if (parent) {
+                    const siblings = Array.from(parent.children).filter(
+                      (c) => c.tagName === current.tagName,
+                    );
+                    if (siblings.length > 1) {
+                      const idx = siblings.indexOf(current) + 1;
+                      sel += `:nth-of-type(${idx})`;
+                    }
+                  }
+                  path = path ? `${sel} > ${path}` : sel;
+                  current = parent;
+                }
+                return path ? `body > ${path}` : 'body';
+              };
+              return generateSelector(el);
+            })(),
+          });
+          return true;
+        } catch (e) {
+          sendResponse({ success: false, error: String(e && e.message ? e.message : e) });
+          return true;
+        }
+      }
+    } catch (e) {
+      sendResponse({ success: false, error: e && e.message ? e.message : String(e) });
+      return true;
+    }
+    return false;
+  });
+
+  console.log('Accessibility tree helper script loaded');
+})();

app/chrome-extension/inject-scripts/click-helper.js

@@ -21,13 +21,63 @@ if (window.__CLICK_HELPER_INITIALIZED__) {
     waitForNavigation = false,
     timeout = 5000,
     coordinates = null,
+    ref = null,
   ) {
     try {
       let element = null;
       let elementInfo = null;
       let clickX, clickY;
 
-      if (coordinates && typeof coordinates.x === 'number' && typeof coordinates.y === 'number') {
+      if (ref && typeof ref === 'string') {
+        // Resolve element from weak map
+        let target = null;
+        try {
+          const map = window.__claudeElementMap;
+          const weak = map && map[ref];
+          target = weak && typeof weak.deref === 'function' ? weak.deref() : null;
+        } catch (e) {
+          // ignore
+        }
+
+        if (!target || !(target instanceof Element)) {
+          return {
+            error: `Element ref "${ref}" not found. Please call chrome_read_page first and ensure the ref is still valid.`,
+          };
+        }
+
+        element = target;
+        element.scrollIntoView({ behavior: 'auto', block: 'center', inline: 'center' });
+        await new Promise((resolve) => setTimeout(resolve, 80));
+
+        const rect = element.getBoundingClientRect();
+        clickX = rect.left + rect.width / 2;
+        clickY = rect.top + rect.height / 2;
+        elementInfo = {
+          tagName: element.tagName,
+          id: element.id,
+          className: element.className,
+          text: element.textContent?.trim().substring(0, 100) || '',
+          href: element.href || null,
+          type: element.type || null,
+          isVisible: true,
+          rect: {
+            x: rect.x,
+            y: rect.y,
+            width: rect.width,
+            height: rect.height,
+            top: rect.top,
+            right: rect.right,
+            bottom: rect.bottom,
+            left: rect.left,
+          },
+          clickMethod: 'ref',
+          ref,
+        };
+      } else if (
+        coordinates &&
+        typeof coordinates.x === 'number' &&
+        typeof coordinates.y === 'number'
+      ) {
         clickX = coordinates.x;
         clickY = coordinates.y;
 
@@ -125,7 +175,10 @@ if (window.__CLICK_HELPER_INITIALIZED__) {
         });
       }
 
-      if (element && elementInfo.clickMethod === 'selector') {
+      if (
+        element &&
+        (elementInfo.clickMethod === 'selector' || elementInfo.clickMethod === 'ref')
+      ) {
         element.click();
       } else {
         simulateClick(clickX, clickY);
@@ -217,6 +270,7 @@ if (window.__CLICK_HELPER_INITIALIZED__) {
         request.waitForNavigation,
         request.timeout,
         request.coordinates,
+        request.ref,
       )
         .then(sendResponse)
         .catch((error) => {

app/chrome-extension/inject-scripts/fill-helper.js

@@ -12,13 +12,31 @@ if (window.__FILL_HELPER_INITIALIZED__) {
    * @param {string} value - Value to fill into the element
    * @returns {Promise<Object>} - Result of the fill operation
    */
-  async function fillElement(selector, value) {
+  async function fillElement(selector, value, ref = null) {
     try {
       // Find the element
-      const element = document.querySelector(selector);
+      let element = null;
+      if (ref && typeof ref === 'string') {
+        try {
+          const map = window.__claudeElementMap;
+          const weak = map && map[ref];
+          element = weak && typeof weak.deref === 'function' ? weak.deref() : null;
+        } catch (e) {
+          // ignore
+        }
+        if (!element || !(element instanceof Element)) {
+          return {
+            error: `Element ref "${ref}" not found. Please call chrome_read_page first and ensure the ref is still valid.`,
+          };
+        }
+      } else {
+        element = document.querySelector(selector);
+      }
       if (!element) {
         return {
-          error: `Element with selector "${selector}" not found`,
+          error: selector
+            ? `Element with selector "${selector}" not found`
+            : `Element for ref not found`,
         };
       }
 
@@ -52,6 +70,7 @@ if (window.__FILL_HELPER_INITIALIZED__) {
 
       // Check if element is an input, textarea, or select
       const validTags = ['INPUT', 'TEXTAREA', 'SELECT'];
+      // Keep a permissive list to allow type-specific branches below to handle behavior
       const validInputTypes = [
         'text',
         'email',
@@ -66,6 +85,9 @@ if (window.__FILL_HELPER_INITIALIZED__) {
         'time',
         'week',
         'color',
+        'checkbox',
+        'radio',
+        'range',
       ];
 
       if (!validTags.includes(element.tagName)) {
@@ -75,7 +97,7 @@ if (window.__FILL_HELPER_INITIALIZED__) {
         };
       }
 
-      // For input elements, check if the type is valid
+      // For input elements, check if the type is valid (allow type-specific branches below)
       if (
         element.tagName === 'INPUT' &&
         !validInputTypes.includes(element.type) &&
@@ -94,6 +116,92 @@ if (window.__FILL_HELPER_INITIALIZED__) {
       // Focus the element
       element.focus();
 
+      // Type-specific handling for tricky inputs first
+      if (element.tagName === 'INPUT' && element.type === 'checkbox') {
+        // Accept boolean or string-like boolean
+        let checkedVal;
+        if (typeof value === 'boolean') {
+          checkedVal = value;
+        } else if (typeof value === 'string') {
+          const v = value.trim().toLowerCase();
+          if (['true', '1', 'yes', 'on'].includes(v)) checkedVal = true;
+          else if (['false', '0', 'no', 'off'].includes(v)) checkedVal = false;
+        }
+        if (typeof checkedVal !== 'boolean') {
+          return {
+            error:
+              'Checkbox requires a boolean (true/false) or a boolean-like string ("true"/"false"/"on"/"off").',
+            elementInfo,
+          };
+        }
+        const previous = element.checked;
+        element.checked = checkedVal;
+        element.focus();
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+        element.blur();
+        return {
+          success: true,
+          message: `Checkbox set to ${element.checked}`,
+          elementInfo: { ...elementInfo, checked: element.checked, previousChecked: previous },
+        };
+      }
+
+      if (element.tagName === 'INPUT' && element.type === 'radio') {
+        // For radios, the selector/ref should target the specific input to select
+        const previous = element.checked;
+        element.checked = true;
+        element.focus();
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+        element.blur();
+        return {
+          success: true,
+          message: 'Radio selected',
+          elementInfo: {
+            ...elementInfo,
+            checked: element.checked,
+            previousChecked: previous,
+            name: element.name || null,
+          },
+        };
+      }
+
+      if (element.tagName === 'INPUT' && element.type === 'range') {
+        const numericValue = typeof value === 'number' ? value : Number(value);
+        if (Number.isNaN(numericValue)) {
+          return { error: 'Range input requires a numeric value', elementInfo };
+        }
+        const previous = element.value;
+        element.value = String(numericValue);
+        element.focus();
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+        element.blur();
+        return {
+          success: true,
+          message: `Set range to ${element.value} (min: ${element.min}, max: ${element.max})`,
+          elementInfo: { ...elementInfo, value: element.value },
+        };
+      }
+
+      if (element.tagName === 'INPUT' && element.type === 'number') {
+        if (value !== '' && value !== null && value !== undefined && Number.isNaN(Number(value))) {
+          return { error: 'Number input requires a numeric value', elementInfo };
+        }
+        const previous = element.value;
+        element.value = String(value ?? '');
+        element.focus();
+        element.dispatchEvent(new Event('input', { bubbles: true }));
+        element.dispatchEvent(new Event('change', { bubbles: true }));
+        element.blur();
+        return {
+          success: true,
+          message: `Set number input to ${element.value} (previous: ${previous})`,
+          elementInfo: { ...elementInfo, value: element.value },
+        };
+      }
+
       // Fill the element based on its type
       if (element.tagName === 'SELECT') {
         // For select elements, find the option with matching value or text
@@ -117,15 +225,12 @@ if (window.__FILL_HELPER_INITIALIZED__) {
         element.dispatchEvent(new Event('change', { bubbles: true }));
       } else {
         // For input and textarea elements
-
-        // Clear the current value
+        // Clear the current value then set new value
         element.value = '';
         element.dispatchEvent(new Event('input', { bubbles: true }));
 
-        // Set the new value
-        element.value = value;
+        element.value = String(value);
 
-        // Trigger input and change events
         element.dispatchEvent(new Event('input', { bubbles: true }));
         element.dispatchEvent(new Event('change', { bubbles: true }));
       }
@@ -189,7 +294,7 @@ if (window.__FILL_HELPER_INITIALIZED__) {
   // Listen for messages from the extension
   chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
     if (request.action === 'fillElement') {
-      fillElement(request.selector, request.value)
+      fillElement(request.selector, request.value, request.ref)
         .then(sendResponse)
         .catch((error) => {
           sendResponse({

app/chrome-extension/inject-scripts/interactive-elements-helper.js

@@ -34,17 +34,56 @@
       'input:not([type="button"]):not([type="submit"]):not([type="checkbox"]):not([type="radio"])',
     checkbox: 'input[type="checkbox"], [role="checkbox"]',
     radio: 'input[type="radio"], [role="radio"]',
-    textarea: 'textarea',
-    select: 'select',
+    textarea: 'textarea, [role="textbox"], [role="searchbox"]',
+    select: 'select, [role="combobox"]',
     tab: '[role="tab"]',
     // Generic interactive elements: combines tabindex, common roles, and explicit handlers.
     // This is the key to finding custom-built interactive components.
-    interactive: `[onclick], [tabindex]:not([tabindex^="-"]), [role="menuitem"], [role="slider"], [role="option"], [role="treeitem"]`,
+    interactive: `[onclick], [tabindex]:not([tabindex^="-"]), [role="menuitem"], [role="slider"], [role="option"], [role="treeitem"], [role="switch"]`,
   };
 
   // A combined selector for ANY interactive element, used in the fallback logic.
   const ANY_INTERACTIVE_SELECTOR = Object.values(ELEMENT_CONFIG).join(', ');
 
+  // Query helpers that pierce open shadow roots. These are used only in fallback paths or
+  // when a selector is explicitly provided, to keep costs bounded.
+  function* walkAllNodesDeep(root) {
+    const stack = [root];
+    const MAX = 12000; // safety bound
+    let count = 0;
+    while (stack.length) {
+      const node = stack.pop();
+      if (!node) continue;
+      if (++count > MAX) break;
+      yield node;
+      const anyNode = /** @type {any} */ (node);
+      try {
+        const children = node.children ? Array.from(node.children) : [];
+        for (let i = children.length - 1; i >= 0; i--) stack.push(children[i]);
+        const sr = anyNode && anyNode.shadowRoot ? anyNode.shadowRoot : null;
+        if (sr && sr.children) {
+          const srChildren = Array.from(sr.children);
+          for (let i = srChildren.length - 1; i >= 0; i--) stack.push(srChildren[i]);
+        }
+      } catch (_) {
+        /* ignore */
+      }
+    }
+  }
+
+  function querySelectorAllDeep(selector, root = document) {
+    const results = [];
+    for (const node of walkAllNodesDeep(root)) {
+      if (!(node instanceof Element)) continue;
+      try {
+        if (node.matches && node.matches(selector)) results.push(node);
+      } catch (_) {
+        /* ignore invalid selectors for given node */
+      }
+    }
+    return results;
+  }
+
   // --- Core Helper Functions ---
 
   /**
@@ -221,7 +260,7 @@
       .join(', ');
     if (!selectorsToFind) return [];
 
-    const targetElements = Array.from(document.querySelectorAll(selectorsToFind));
+    const targetElements = querySelectorAllDeep(selectorsToFind);
     const uniqueElements = new Set(targetElements);
     const results = [];
 
@@ -325,7 +364,7 @@
         let elements;
         if (request.selector) {
           // If a selector is provided, bypass the text-based logic and use a direct query.
-          const foundEls = Array.from(document.querySelectorAll(request.selector));
+          const foundEls = querySelectorAllDeep(request.selector);
           elements = foundEls.map((el) =>
             createElementInfo(
               el,

app/chrome-extension/utils/screenshot-context.ts

@@ -0,0 +1,53 @@
+// Simple in-memory screenshot context manager per tab
+// Used to scale coordinates from screenshot space to viewport space
+
+export interface ScreenshotContext {
+  // Final screenshot dimensions (in CSS pixels after any scaling)
+  screenshotWidth: number;
+  screenshotHeight: number;
+  // Viewport dimensions (CSS pixels)
+  viewportWidth: number;
+  viewportHeight: number;
+  // Device pixel ratio at capture time (optional, for reference)
+  devicePixelRatio?: number;
+  // Hostname of the page when the screenshot was taken (used for domain safety checks)
+  hostname?: string;
+  // Timestamp
+  timestamp: number;
+}
+
+const TTL_MS = 5 * 60 * 1000; // 5 minutes
+
+const contexts = new Map<number, ScreenshotContext>();
+
+export const screenshotContextManager = {
+  setContext(tabId: number, ctx: Omit<ScreenshotContext, 'timestamp'>) {
+    contexts.set(tabId, { ...ctx, timestamp: Date.now() });
+  },
+  getContext(tabId: number): ScreenshotContext | undefined {
+    const ctx = contexts.get(tabId);
+    if (!ctx) return undefined;
+    if (Date.now() - ctx.timestamp > TTL_MS) {
+      contexts.delete(tabId);
+      return undefined;
+    }
+    return ctx;
+  },
+  clear(tabId: number) {
+    contexts.delete(tabId);
+  },
+};
+
+// Scale screenshot-space coordinates (x,y) to viewport CSS pixels
+export function scaleCoordinates(
+  x: number,
+  y: number,
+  ctx: ScreenshotContext,
+): { x: number; y: number } {
+  if (!ctx.screenshotWidth || !ctx.screenshotHeight || !ctx.viewportWidth || !ctx.viewportHeight) {
+    return { x, y };
+  }
+  const sx = (x / ctx.screenshotWidth) * ctx.viewportWidth;
+  const sy = (y / ctx.screenshotHeight) * ctx.viewportHeight;
+  return { x: Math.round(sx), y: Math.round(sy) };
+}

app/chrome-extension/wxt.config.ts

@@ -66,6 +66,8 @@ export default defineConfig({
   },
   vite: (env) => ({
     plugins: [
+      // Ensure static assets are available as early as possible to avoid race conditions in dev
+      // Copy workers/_locales/inject-scripts into the build output before other steps
       viteStaticCopy({
         targets: [
           {
@@ -81,6 +83,13 @@ export default defineConfig({
             dest: '_locales',
           },
         ],
+        // Copy at buildStart to avoid ENOENT when extension first boots in dev
+        hook: 'buildStart',
+        // Enable watch so changes to these files are reflected during dev
+        watch: {
+          // Use default patterns inferred from targets; explicit true enables watching
+          // Vite plugin will watch src patterns and re-copy on change
+        } as any,
       }) as any,
     ],
     build: {

app/native-server/package.json

@@ -35,11 +35,13 @@
   "dependencies": {
     "@fastify/cors": "^11.0.1",
     "@modelcontextprotocol/sdk": "^1.11.0",
+    "@types/node-fetch": "2",
     "chalk": "^5.4.1",
     "chrome-mcp-shared": "workspace:*",
     "commander": "^13.1.0",
     "fastify": "^5.3.2",
     "is-admin": "^4.0.0",
+    "node-fetch": "2",
     "pino": "^9.6.0",
     "uuid": "^11.1.0"
   },

docs/TOOLS.md

@@ -247,6 +247,24 @@ Send custom HTTP requests.
 
 ## 🔍 Content Analysis
 
+### `chrome_read_page`
+
+Build an accessibility-like tree of the current page (visible viewport by default) with stable `ref_*` identifiers and viewport info. Useful for semantic element discovery or agent planning.
+
+Parameters:
+
+- `filter` (string, optional): `interactive` to only include interactive elements; default includes structural and labeled nodes.
+
+Example:
+
+```json
+{
+  "filter": "interactive"
+}
+```
+
+Response contains `pageContent` (text tree), `viewport`, and a `refMapCount` summary. Use `chrome_get_interactive_elements` or your own logic to act on returned refs.
+
 ### `search_tabs_content`
 
 AI-powered semantic search across browser tabs.
@@ -308,46 +326,70 @@ Extract HTML or text content from web pages.
 }
 ```
 
-### `chrome_get_interactive_elements`
+### `chrome_get_interactive_elements` (deprecated)
 
-Find clickable and interactive elements on the page.
-
-**Parameters**:
-
-- `tabId` (number, optional): Specific tab ID (default: active tab)
-
-**Response**:
-
-```json
-{
-  "elements": [
-    {
-      "selector": "#submit-button",
-      "type": "button",
-      "text": "Submit",
-      "visible": true,
-      "clickable": true
-    }
-  ]
-}
-```
+Replaced by `chrome_read_page` as the primary discovery tool. The `read_page` implementation will automatically fallback to the interactive-elements logic when the accessibility tree is unavailable or too sparse. This tool is no longer listed via ListTools and is kept only for backward compatibility.
 
 ## 🎯 Interaction
 
+### `chrome_computer`
+
+Unified advanced interaction tool that prioritizes high-level DOM actions with CDP fallback. Supports hover, click, drag, scroll, typing, key chords, fill, wait and screenshot. If a recent screenshot was taken via `chrome_screenshot`, coordinates are auto-scaled from screenshot space to viewport space.
+
+Parameters:
+
+- `action` (string, required): `left_click` | `right_click` | `double_click` | `triple_click` | `left_click_drag` | `scroll` | `type` | `key` | `fill` | `hover` | `wait` | `screenshot`
+- `ref` (string, optional): element ref from `chrome_read_page` (preferred). Used for click/scroll/type/key and as drag end when provided
+- `coordinates` (object, optional): `{ "x": 100, "y": 200 }` for click/scroll or drag end
+- `startRef` (string, optional): element ref for drag start
+- `startCoordinates` (object, optional): for `left_click_drag` when no `startRef`
+- `scrollDirection` (string, optional): `up` | `down` | `left` | `right`
+- `scrollAmount` (number, optional): ticks 1–10 (default 3)
+- `text` (string, optional): for `type` (raw text) or `key` (space-separated chords/keys like `"cmd+a Enter"`)
+- `duration` (number, optional): seconds for `wait` (max 30)
+- `selector` (string, optional): for `fill` when no `ref`
+- `value` (string, optional): for `fill` value
+
+Examples:
+
+```json
+{ "action": "left_click", "coordinates": { "x": 420, "y": 260 } }
+```
+
+```json
+{ "action": "key", "text": "cmd+a Backspace" }
+```
+
+````json
+{ "action": "fill", "ref": "ref_7", "value": "user@example.com" }
+
+```json
+{ "action": "hover", "ref": "ref_12", "duration": 0.6 }
+````
+
+````
+
+```json
+{ "action": "left_click_drag", "startRef": "ref_10", "ref": "ref_15" }
+````
+
 ### `chrome_click_element`
 
-Click elements using CSS selectors.
+Click elements using a ref, selector, or coordinates.
 
 **Parameters**:
 
-- `selector` (string, required): CSS selector for target element
-- `tabId` (number, optional): Specific tab ID (default: active tab)
+- `ref` (string, optional): Element ref from `chrome_read_page` (preferred when available)
+- `selector` (string, optional): CSS selector for target element
+- `coordinates` (object, optional): `{ "x": 120, "y": 240 }` viewport coordinates
+
+At least one of `ref`, `selector`, or `coordinates` must be provided.
 
 **Example**:
 
 ```json
 {
-  "selector": "#submit-button"
+  "ref": "ref_42"
 }
 ```
 
@@ -357,15 +399,17 @@ Fill form fields or select options.
 
 **Parameters**:
 
-- `selector` (string, required): CSS selector for target element
+- `ref` (string, optional): Element ref from `chrome_read_page`
+- `selector` (string, optional): CSS selector for target element
 - `value` (string, required): Value to fill or select
-- `tabId` (number, optional): Specific tab ID (default: active tab)
+
+Provide `ref` or `selector` to identify the element.
 
 **Example**:
 
 ```json
 {
-  "selector": "#email-input",
+  "ref": "ref_7",
   "value": "user@example.com"
 }
 ```

packages/shared/src/tools.ts

@@ -27,6 +27,8 @@ export const TOOL_NAMES = {
     SEND_COMMAND_TO_INJECT_SCRIPT: 'chrome_send_command_to_inject_script',
     CONSOLE: 'chrome_console',
     FILE_UPLOAD: 'chrome_upload_file',
+    READ_PAGE: 'chrome_read_page',
+    COMPUTER: 'chrome_computer',
   },
 };
 
@@ -40,6 +42,90 @@ export const TOOL_SCHEMAS: Tool[] = [
       required: [],
     },
   },
+  {
+    name: TOOL_NAMES.BROWSER.READ_PAGE,
+    description:
+      'Get an accessibility tree representation of visible elements on the page. Only returns elements that are visible in the viewport. Optionally filter for only interactive elements.\nTip: If the returned elements do not include the specific element you need, use the computer tool\'s screenshot (action="screenshot") to capture the element\'s on-screen coordinates, then operate by coordinates.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        filter: {
+          type: 'string',
+          description:
+            'Filter elements: "interactive" for such as  buttons/links/inputs only (default: all visible elements)',
+        },
+      },
+      required: [],
+    },
+  },
+  {
+    name: TOOL_NAMES.BROWSER.COMPUTER,
+    description:
+      "Use a mouse and keyboard to interact with a web browser, and take screenshots.\n* Whenever you intend to click on an element like an icon, you should consult a read_page to determine the ref of the element before moving the cursor.\n* If you tried clicking on a program or link but it failed to load, even after waiting, try screenshot and then adjusting your click location so that the tip of the cursor visually falls on the element that you want to click.\n* Make sure to click any buttons, links, icons, etc with the cursor tip in the center of the element. Don't click boxes on their edges unless asked.",
+    inputSchema: {
+      type: 'object',
+      properties: {
+        action: {
+          type: 'string',
+          description:
+            'Action to perform: left_click | right_click | double_click | triple_click | left_click_drag | scroll | type | key | fill | hover | wait | screenshot',
+        },
+        ref: {
+          type: 'string',
+          description:
+            'Element ref from chrome_read_page. For click/scroll/key/type and drag end when provided; takes precedence over coordinates.',
+        },
+        coordinates: {
+          type: 'object',
+          properties: {
+            x: { type: 'number', description: 'X coordinate' },
+            y: { type: 'number', description: 'Y coordinate' },
+          },
+          description:
+            'Coordinates for actions (in screenshot space if a recent screenshot was taken, otherwise viewport). Required for click/scroll and as end point for drag.',
+        },
+        startCoordinates: {
+          type: 'object',
+          properties: {
+            x: { type: 'number' },
+            y: { type: 'number' },
+          },
+          description: 'Starting coordinates for drag action',
+        },
+        startRef: {
+          type: 'string',
+          description: 'Drag start ref from chrome_read_page (alternative to startCoordinates).',
+        },
+        scrollDirection: {
+          type: 'string',
+          description: 'Scroll direction: up | down | left | right',
+        },
+        scrollAmount: {
+          type: 'number',
+          description: 'Scroll ticks (1-10), default 3',
+        },
+        text: {
+          type: 'string',
+          description:
+            'Text to type (for action=type) or keys/chords separated by space (for action=key, e.g. "Backspace Enter" or "cmd+a")',
+        },
+        // For action=fill
+        selector: {
+          type: 'string',
+          description: 'CSS selector for fill (alternative to ref).',
+        },
+        value: {
+          oneOf: [{ type: 'string' }, { type: 'boolean' }, { type: 'number' }],
+          description: 'Value to set for action=fill (string | boolean | number)',
+        },
+        duration: {
+          type: 'number',
+          description: 'Seconds to wait for action=wait (max 30s)',
+        },
+      },
+      required: ['action'],
+    },
+  },
   {
     name: TOOL_NAMES.BROWSER.NAVIGATE,
     description: 'Navigate to a URL or refresh the current tab',
@@ -65,7 +151,7 @@ export const TOOL_SCHEMAS: Tool[] = [
   {
     name: TOOL_NAMES.BROWSER.SCREENSHOT,
     description:
-      'Take a screenshot of the current page or a specific element(if you want to see the page, recommend to use chrome_get_web_content first)',
+      '[Prefer read_page over taking a screenshot and Prefer chrome_computer] Take a screenshot of the current page or a specific element. For new usage, use chrome_computer with action="screenshot". Use this tool if you need advanced options.',
     inputSchema: {
       type: 'object',
       properties: {
@@ -172,87 +258,6 @@ export const TOOL_SCHEMAS: Tool[] = [
       required: [],
     },
   },
-  {
-    name: TOOL_NAMES.BROWSER.CLICK,
-    description: 'Click on an element in the current page or at specific coordinates',
-    inputSchema: {
-      type: 'object',
-      properties: {
-        selector: {
-          type: 'string',
-          description:
-            'CSS selector for the element to click. Either selector or coordinates must be provided. if coordinates are not provided, the selector must be provided.',
-        },
-        coordinates: {
-          type: 'object',
-          description:
-            'Coordinates to click at (relative to viewport). If provided, takes precedence over selector.',
-          properties: {
-            x: {
-              type: 'number',
-              description: 'X coordinate relative to the viewport',
-            },
-            y: {
-              type: 'number',
-              description: 'Y coordinate relative to the viewport',
-            },
-          },
-          required: ['x', 'y'],
-        },
-        waitForNavigation: {
-          type: 'boolean',
-          description: 'Wait for page navigation to complete after click (default: false)',
-        },
-        timeout: {
-          type: 'number',
-          description:
-            'Timeout in milliseconds for waiting for the element or navigation (default: 5000)',
-        },
-      },
-      required: [],
-    },
-  },
-  {
-    name: TOOL_NAMES.BROWSER.FILL,
-    description: 'Fill a form element or select an option with the specified value',
-    inputSchema: {
-      type: 'object',
-      properties: {
-        selector: {
-          type: 'string',
-          description: 'CSS selector for the input element to fill or select',
-        },
-        value: {
-          type: 'string',
-          description: 'Value to fill or select into the element',
-        },
-      },
-      required: ['selector', 'value'],
-    },
-  },
-  {
-    name: TOOL_NAMES.BROWSER.GET_INTERACTIVE_ELEMENTS,
-    description: 'Get interactive elements from the current page',
-    inputSchema: {
-      type: 'object',
-      properties: {
-        textQuery: {
-          type: 'string',
-          description: 'Text to search for within interactive elements (fuzzy search)',
-        },
-        selector: {
-          type: 'string',
-          description:
-            'CSS selector to filter interactive elements. Takes precedence over textQuery if both are provided.',
-        },
-        includeCoordinates: {
-          type: 'boolean',
-          description: 'Include element coordinates in the response (default: true)',
-        },
-      },
-      required: [],
-    },
-  },
   {
     name: TOOL_NAMES.BROWSER.NETWORK_REQUEST,
     description: 'Send a network request from the browser with cookies and other browser context',
@@ -335,29 +340,6 @@ export const TOOL_SCHEMAS: Tool[] = [
       required: [],
     },
   },
-  {
-    name: TOOL_NAMES.BROWSER.KEYBOARD,
-    description: 'Simulate keyboard events in the browser',
-    inputSchema: {
-      type: 'object',
-      properties: {
-        keys: {
-          type: 'string',
-          description: 'Keys to simulate (e.g., "Enter", "Ctrl+C", "A,B,C" for sequence)',
-        },
-        selector: {
-          type: 'string',
-          description:
-            'CSS selector for the element to send keyboard events to (optional, defaults to active element)',
-        },
-        delay: {
-          type: 'number',
-          description: 'Delay between key sequences in milliseconds (optional, default: 0)',
-        },
-      },
-      required: ['keys'],
-    },
-  },
   {
     name: TOOL_NAMES.BROWSER.HISTORY,
     description: 'Retrieve and search browsing history from Chrome',
@@ -556,7 +538,8 @@ export const TOOL_SCHEMAS: Tool[] = [
   },
   {
     name: TOOL_NAMES.BROWSER.FILE_UPLOAD,
-    description: 'Upload files to web forms with file input elements using Chrome DevTools Protocol',
+    description:
+      'Upload files to web forms with file input elements using Chrome DevTools Protocol',
     inputSchema: {
       type: 'object',
       properties: {

pnpm-lock.yaml

@@ -105,6 +105,9 @@ importers:
       '@modelcontextprotocol/sdk':
         specifier: ^1.11.0
         version: 1.12.1
+      '@types/node-fetch':
+        specifier: '2'
+        version: 2.6.13
       chalk:
         specifier: ^5.4.1
         version: 5.4.1
@@ -120,6 +123,9 @@ importers:
       is-admin:
         specifier: ^4.0.0
         version: 4.0.0
+      node-fetch:
+        specifier: '2'
+        version: 2.7.0
       pino:
         specifier: ^9.6.0
         version: 9.7.0
@@ -906,67 +912,56 @@ packages:
     resolution: {integrity: sha512-UsQD5fyLWm2Fe5CDM7VPYAo+UC7+2Px4Y+N3AcPh/LdZu23YcuGPegQly++XEVaC8XUTFVPscl5y5Cl1twEI4A==}
     cpu: [arm]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-arm-musleabihf@4.42.0':
     resolution: {integrity: sha512-/i8NIrlgc/+4n1lnoWl1zgH7Uo0XK5xK3EDqVTf38KvyYgCU/Rm04+o1VvvzJZnVS5/cWSd07owkzcVasgfIkQ==}
     cpu: [arm]
     os: [linux]
-    libc: [musl]
 
   '@rollup/rollup-linux-arm64-gnu@4.42.0':
     resolution: {integrity: sha512-eoujJFOvoIBjZEi9hJnXAbWg+Vo1Ov8n/0IKZZcPZ7JhBzxh2A+2NFyeMZIRkY9iwBvSjloKgcvnjTbGKHE44Q==}
     cpu: [arm64]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-arm64-musl@4.42.0':
     resolution: {integrity: sha512-/3NrcOWFSR7RQUQIuZQChLND36aTU9IYE4j+TB40VU78S+RA0IiqHR30oSh6P1S9f9/wVOenHQnacs/Byb824g==}
     cpu: [arm64]
     os: [linux]
-    libc: [musl]
 
   '@rollup/rollup-linux-loongarch64-gnu@4.42.0':
     resolution: {integrity: sha512-O8AplvIeavK5ABmZlKBq9/STdZlnQo7Sle0LLhVA7QT+CiGpNVe197/t8Aph9bhJqbDVGCHpY2i7QyfEDDStDg==}
     cpu: [loong64]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-powerpc64le-gnu@4.42.0':
     resolution: {integrity: sha512-6Qb66tbKVN7VyQrekhEzbHRxXXFFD8QKiFAwX5v9Xt6FiJ3BnCVBuyBxa2fkFGqxOCSGGYNejxd8ht+q5SnmtA==}
     cpu: [ppc64]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-riscv64-gnu@4.42.0':
     resolution: {integrity: sha512-KQETDSEBamQFvg/d8jajtRwLNBlGc3aKpaGiP/LvEbnmVUKlFta1vqJqTrvPtsYsfbE/DLg5CC9zyXRX3fnBiA==}
     cpu: [riscv64]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-riscv64-musl@4.42.0':
     resolution: {integrity: sha512-qMvnyjcU37sCo/tuC+JqeDKSuukGAd+pVlRl/oyDbkvPJ3awk6G6ua7tyum02O3lI+fio+eM5wsVd66X0jQtxw==}
     cpu: [riscv64]
     os: [linux]
-    libc: [musl]
 
   '@rollup/rollup-linux-s390x-gnu@4.42.0':
     resolution: {integrity: sha512-I2Y1ZUgTgU2RLddUHXTIgyrdOwljjkmcZ/VilvaEumtS3Fkuhbw4p4hgHc39Ypwvo2o7sBFNl2MquNvGCa55Iw==}
     cpu: [s390x]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-x64-gnu@4.42.0':
     resolution: {integrity: sha512-Gfm6cV6mj3hCUY8TqWa63DB8Mx3NADoFwiJrMpoZ1uESbK8FQV3LXkhfry+8bOniq9pqY1OdsjFWNsSbfjPugw==}
     cpu: [x64]
     os: [linux]
-    libc: [glibc]
 
   '@rollup/rollup-linux-x64-musl@4.42.0':
     resolution: {integrity: sha512-g86PF8YZ9GRqkdi0VoGlcDUb4rYtQKyTD1IVtxxN4Hpe7YqLBShA7oHMKU6oKTCi3uxwW4VkIGnOaH/El8de3w==}
     cpu: [x64]
     os: [linux]
-    libc: [musl]
 
   '@rollup/rollup-win32-arm64-msvc@4.42.0':
     resolution: {integrity: sha512-+axkdyDGSp6hjyzQ5m1pgcvQScfHnMCcsXkx8pTgy/6qBmWVhtRVlgxjWwDp67wEXXUr0x+vD6tp5W4x6V7u1A==}
@@ -1067,6 +1062,9 @@ packages:
   '@types/minimatch@3.0.5':
     resolution: {integrity: sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==}
 
+  '@types/node-fetch@2.6.13':
+    resolution: {integrity: sha512-QGpRVpzSaUs30JBSGPjOg4Uveu384erbHBoT1zeONvyCfwQxIkUshLAOqN/k9EjGviPRmWTTe6aH2qySWKTVSw==}
+
   '@types/node@18.19.111':
     resolution: {integrity: sha512-90sGdgA+QLJr1F9X79tQuEut0gEYIfkX9pydI4XGRgvFo9g2JWswefI+WUSUHPYVBHYSEfTEqBxA5hQvAZB3Mw==}
 
@@ -2302,6 +2300,10 @@ packages:
     resolution: {integrity: sha512-qsITQPfmvMOSAdeyZ+12I1c+CKSstAFAwu+97zrnWAbIr5u8wfsExUzCesVLC8NgHuRUqNN4Zy6UPWUTRGslcA==}
     engines: {node: '>= 6'}
 
+  form-data@4.0.4:
+    resolution: {integrity: sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==}
+    engines: {node: '>= 6'}
+
   formdata-node@6.0.3:
     resolution: {integrity: sha512-8e1++BCiTzUno9v5IZ2J6bv4RU+3UKDmqWUQD0MIMVCd9AdhWkO1gw57oo1mNEX1dMq2EGI+FbWz4B92pscSQg==}
     engines: {node: '>= 18'}
@@ -3259,6 +3261,15 @@ packages:
   node-fetch-native@1.6.6:
     resolution: {integrity: sha512-8Mc2HhqPdlIfedsuZoc3yioPuzp6b+L5jRCRY1QzuWZh2EGJVQrGppC6V6cF0bLdbW0+O2YpqCA25aF/1lvipQ==}
 
+  node-fetch@2.7.0:
+    resolution: {integrity: sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==}
+    engines: {node: 4.x || >=6.0.0}
+    peerDependencies:
+      encoding: ^0.1.0
+    peerDependenciesMeta:
+      encoding:
+        optional: true
+
   node-forge@1.3.1:
     resolution: {integrity: sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==}
     engines: {node: '>= 6.13.0'}
@@ -3927,6 +3938,7 @@ packages:
   source-map@0.8.0-beta.0:
     resolution: {integrity: sha512-2ymg6oRBpebeZi9UUNsgQ89bhx01TcTkmNTGnNO88imTmbSgy4nfujrgVEFKWpMTEGA11EDkTt7mqObTPdigIA==}
     engines: {node: '>= 8'}
+    deprecated: The work that was done in this beta branch won't be included in future versions
 
   spawn-sync@1.0.15:
     resolution: {integrity: sha512-9DWBgrgYZzNghseho0JOuh+5fg9u6QWhAWa51QC7+U5rCheZ/j1DrEZnyE0RBBRqZ9uEXGPgSSM0nky6burpVw==}
@@ -4135,6 +4147,9 @@ packages:
     resolution: {integrity: sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==}
     hasBin: true
 
+  tr46@0.0.3:
+    resolution: {integrity: sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==}
+
   tr46@1.0.1:
     resolution: {integrity: sha512-dTpowEjclQ7Kgx5SdBkqRzVhERQXov8/l9Ft9dVM9fmg0W0KQSVaXX9T4i6twCPNtYiZM53lpSSUAwJbFPOHxA==}
 
@@ -4419,12 +4434,18 @@ packages:
     resolution: {integrity: sha512-u/IiZaZ7dHFqTM1MLF27rBy8mS9fEEsqoOKL0u+kQdOLmEioA/0Szp67ADd3WAJZLd8/hO8cFST1IC/YMXKIjQ==}
     engines: {node: '>=18.0.0', npm: '>=8.0.0'}
 
+  webidl-conversions@3.0.1:
+    resolution: {integrity: sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==}
+
   webidl-conversions@4.0.2:
     resolution: {integrity: sha512-YQ+BmxuTgd6UXZW3+ICGfyqRyHXVlD5GtQr5+qjiNW7bF0cqrzX500HVXPBOvgXb5YnzDd+h0zqyv61KUD7+Sg==}
 
   webpack-virtual-modules@0.6.2:
     resolution: {integrity: sha512-66/V2i5hQanC51vBQKPH4aI8NMAcBW59FVBs+rC7eGHupMyfn34q7rZIE+ETlJ+XTevqfUhVVBgSUNSW2flEUQ==}
 
+  whatwg-url@5.0.0:
+    resolution: {integrity: sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==}
+
   whatwg-url@7.1.0:
     resolution: {integrity: sha512-WUu7Rg1DroM7oQvGWfOiAK21n74Gg+T4elXEQYkOhtyLeWiJFoOGLXPKI/9gzIie9CtwVLm8wtw6YJdKyxSjeg==}
 
@@ -5491,6 +5512,11 @@ snapshots:
 
   '@types/minimatch@3.0.5': {}
 
+  '@types/node-fetch@2.6.13':
+    dependencies:
+      '@types/node': 22.15.30
+      form-data: 4.0.4
+
   '@types/node@18.19.111':
     dependencies:
       undici-types: 5.26.5
@@ -6913,6 +6939,14 @@ snapshots:
       hasown: 2.0.2
       mime-types: 2.1.35
 
+  form-data@4.0.4:
+    dependencies:
+      asynckit: 0.4.0
+      combined-stream: 1.0.8
+      es-set-tostringtag: 2.1.0
+      hasown: 2.0.2
+      mime-types: 2.1.35
+
   formdata-node@6.0.3: {}
 
   formidable@3.5.4:
@@ -7960,6 +7994,10 @@ snapshots:
 
   node-fetch-native@1.6.6: {}
 
+  node-fetch@2.7.0:
+    dependencies:
+      whatwg-url: 5.0.0
+
   node-forge@1.3.1: {}
 
   node-int64@0.4.0: {}
@@ -8965,6 +9003,8 @@ snapshots:
 
   touch@3.1.1: {}
 
+  tr46@0.0.3: {}
+
   tr46@1.0.1:
     dependencies:
       punycode: 2.3.1
@@ -9282,10 +9322,17 @@ snapshots:
       - supports-color
       - utf-8-validate
 
+  webidl-conversions@3.0.1: {}
+
   webidl-conversions@4.0.2: {}
 
   webpack-virtual-modules@0.6.2: {}
 
+  whatwg-url@5.0.0:
+    dependencies:
+      tr46: 0.0.3
+      webidl-conversions: 3.0.1
+
   whatwg-url@7.1.0:
     dependencies:
       lodash.sortby: 4.7.0