From d4e2fb3c8de7784f53a44ffe5125f08782a40d1f Mon Sep 17 00:00:00 2001 From: hangerye Date: Thu, 9 Oct 2025 10:00:36 +0800 Subject: [PATCH] feat: commit for store --- app/chrome-extension/common/message-types.ts | 4 + .../background/tools/base-browser.ts | 3 +- .../background/tools/browser/computer.ts | 179 ++++++++++++++++-- .../background/tools/browser/dialog.ts | 77 ++++++++ .../background/tools/browser/index.ts | 1 + .../inject-scripts/wait-helper.js | 171 +++++++++++++++++ packages/shared/src/tools.ts | 40 ++++ 7 files changed, 461 insertions(+), 14 deletions(-) create mode 100644 app/chrome-extension/entrypoints/background/tools/browser/dialog.ts create mode 100644 app/chrome-extension/inject-scripts/wait-helper.js diff --git a/app/chrome-extension/common/message-types.ts b/app/chrome-extension/common/message-types.ts index 673c3df..38d3097 100644 --- a/app/chrome-extension/common/message-types.ts +++ b/app/chrome-extension/common/message-types.ts @@ -42,6 +42,7 @@ export const CONTENT_MESSAGE_TYPES = { SCREENSHOT_HELPER_PING: 'screenshot_helper_ping', INTERACTIVE_ELEMENTS_HELPER_PING: 'interactive_elements_helper_ping', ACCESSIBILITY_TREE_HELPER_PING: 'chrome_read_page_ping', + WAIT_HELPER_PING: 'wait_helper_ping', } as const; // Tool action message types (for chrome.runtime.sendMessage) @@ -73,6 +74,9 @@ export const TOOL_MESSAGE_TYPES = { // Network requests NETWORK_SEND_REQUEST: 'sendPureNetworkRequest', + // Wait helper + WAIT_FOR_TEXT: 'waitForText', + // Semantic similarity engine SIMILARITY_ENGINE_INIT: 'similarityEngineInit', SIMILARITY_ENGINE_COMPUTE_BATCH: 'similarityEngineComputeBatch', diff --git a/app/chrome-extension/entrypoints/background/tools/base-browser.ts b/app/chrome-extension/entrypoints/background/tools/base-browser.ts index bb77b97..b17bd6e 100644 --- a/app/chrome-extension/entrypoints/background/tools/base-browser.ts +++ b/app/chrome-extension/entrypoints/background/tools/base-browser.ts @@ -19,6 +19,7 @@ export abstract class BaseBrowserToolExecutor implements ToolExecutor { files: string[], injectImmediately = false, world: 'MAIN' | 'ISOLATED' = 'ISOLATED', + allFrames: boolean = false, ): Promise { console.log(`Injecting ${files.join(', ')} into tab ${tabId}`); @@ -50,7 +51,7 @@ export abstract class BaseBrowserToolExecutor implements ToolExecutor { try { await chrome.scripting.executeScript({ - target: { tabId }, + target: { tabId, allFrames }, files, injectImmediately, world, diff --git a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts index fcbd897..17d2c66 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/computer.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/computer.ts @@ -28,6 +28,8 @@ interface ComputerParams { | 'hover' | 'wait' | 'fill' + | 'fill_form' + | 'resize_page' | 'screenshot'; // click/scroll coordinates in screenshot space (if screenshot context exists) or viewport space coordinates?: Coordinates; // for click/scroll; for drag, this is endCoordinates @@ -243,6 +245,58 @@ class ComputerTool extends BaseBrowserToolExecutor { }; switch (params.action) { + case 'resize_page': { + const width = Number((params as any).coordinates?.x || (params as any).text); + const height = Number((params as any).coordinates?.y || (params as any).value); + const w = Number((params as any).width ?? width); + const h = Number((params as any).height ?? height); + if (!Number.isFinite(w) || !Number.isFinite(h) || w <= 0 || h <= 0) { + return createErrorResponse( + 'Provide width and height for resize_page (positive numbers)', + ); + } + try { + // Prefer precise CDP emulation + await CDPHelper.attach(tab.id); + try { + await chrome.debugger.sendCommand( + { tabId: tab.id }, + 'Emulation.setDeviceMetricsOverride', + { + width: Math.round(w), + height: Math.round(h), + deviceScaleFactor: 0, + mobile: false, + screenWidth: Math.round(w), + screenHeight: Math.round(h), + }, + ); + } finally { + await CDPHelper.detach(tab.id); + } + } catch (e) { + // Fallback: window resize + if (tab.windowId !== undefined) { + await chrome.windows.update(tab.windowId, { + width: Math.round(w), + height: Math.round(h), + }); + } else { + return createErrorResponse( + `Failed to resize via CDP and cannot determine windowId: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'resize_page', width: w, height: h }), + }, + ], + isError: false, + }; + } case 'hover': { // Resolve target point from ref | selector | coordinates let coord: Coordinates | undefined = undefined; @@ -782,6 +836,52 @@ class ComputerTool extends BaseBrowserToolExecutor { } as any); return res; } + case 'fill_form': { + const elements = (params as any).elements as Array<{ + ref: string; + value: string | number | boolean; + }>; + if (!Array.isArray(elements) || elements.length === 0) { + return createErrorResponse('elements must be a non-empty array for fill_form'); + } + const results: Array<{ ref: string; ok: boolean; error?: string }> = []; + for (const item of elements) { + if (!item || !item.ref) { + results.push({ ref: String(item?.ref || ''), ok: false, error: 'missing ref' }); + continue; + } + try { + const r = await fillTool.execute({ + ref: item.ref as any, + value: item.value as any, + } as any); + const ok = !r.isError; + results.push({ ref: item.ref, ok, error: ok ? undefined : 'failed' }); + } catch (e) { + results.push({ + ref: item.ref, + ok: false, + error: String(e instanceof Error ? e.message : e), + }); + } + } + const successCount = results.filter((r) => r.ok).length; + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'fill_form', + filled: successCount, + total: results.length, + results, + }), + }, + ], + isError: false, + }; + } case 'key': { if (!params.text) return createErrorResponse( @@ -821,19 +921,72 @@ class ComputerTool extends BaseBrowserToolExecutor { } } case 'wait': { - const seconds = Math.max(0, Math.min(params.duration || 0, 30)); - if (!seconds) - return createErrorResponse('Duration parameter is required and must be > 0'); - await new Promise((r) => setTimeout(r, seconds * 1000)); - return { - content: [ - { - type: 'text', - text: JSON.stringify({ success: true, action: 'wait', duration: seconds }), - }, - ], - isError: false, - }; + const hasTextCondition = + typeof (params as any).text === 'string' && (params as any).text.trim().length > 0; + if (hasTextCondition) { + try { + // Conditional wait for text appearance/disappearance using content script + await this.injectContentScript( + tab.id, + ['inject-scripts/wait-helper.js'], + false, + 'ISOLATED', + true, + ); + const appear = (params as any).appear !== false; // default to true + const timeoutMs = Math.max( + 0, + Math.min(((params as any).timeout as number) || 10000, 120000), + ); + const resp = await this.sendMessageToTab(tab.id, { + action: TOOL_MESSAGE_TYPES.WAIT_FOR_TEXT, + text: (params as any).text, + appear, + timeout: timeoutMs, + }); + if (!resp || resp.success !== true) { + return createErrorResponse( + resp && resp.reason === 'timeout' + ? `wait_for timed out after ${timeoutMs}ms for text: ${(params as any).text}` + : `wait_for failed: ${resp && resp.error ? resp.error : 'unknown error'}`, + ); + } + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'wait_for', + appear, + text: (params as any).text, + matched: resp.matched || null, + tookMs: resp.tookMs, + }), + }, + ], + isError: false, + }; + } catch (e) { + return createErrorResponse( + `wait_for failed: ${e instanceof Error ? e.message : String(e)}`, + ); + } + } else { + const seconds = Math.max(0, Math.min((params as any).duration || 0, 30)); + if (!seconds) + return createErrorResponse('Duration parameter is required and must be > 0'); + await new Promise((r) => setTimeout(r, seconds * 1000)); + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action: 'wait', duration: seconds }), + }, + ], + isError: false, + }; + } } case 'screenshot': { // Reuse existing screenshot tool; it already supports base64 save option diff --git a/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts b/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts new file mode 100644 index 0000000..274562a --- /dev/null +++ b/app/chrome-extension/entrypoints/background/tools/browser/dialog.ts @@ -0,0 +1,77 @@ +import { createErrorResponse, ToolResult } from '@/common/tool-handler'; +import { BaseBrowserToolExecutor } from '../base-browser'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; + +interface HandleDialogParams { + action: 'accept' | 'dismiss'; + promptText?: string; +} + +/** + * Handle JavaScript dialogs (alert/confirm/prompt) via CDP Page.handleJavaScriptDialog + */ +class HandleDialogTool extends BaseBrowserToolExecutor { + name = TOOL_NAMES.BROWSER.HANDLE_DIALOG; + + async execute(args: HandleDialogParams): Promise { + const { action, promptText } = args || ({} as HandleDialogParams); + if (!action || (action !== 'accept' && action !== 'dismiss')) { + return createErrorResponse('action must be "accept" or "dismiss"'); + } + + try { + const [activeTab] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (!activeTab?.id) return createErrorResponse('No active tab found'); + + // Attach debugger and try handling the dialog + try { + await chrome.debugger.attach({ tabId: activeTab.id }, '1.3'); + } catch (e: any) { + if (String(e?.message || '').includes('attached')) { + // If already attached by us, proceed; otherwise fail with clear message + const targets = await chrome.debugger.getTargets(); + const existing = targets.find((t) => t.tabId === activeTab.id && t.attached); + if (!existing || existing.extensionId !== chrome.runtime.id) { + return createErrorResponse( + `Debugger already attached to tab ${activeTab.id} by another client (e.g., DevTools). Close it and retry.`, + ); + } + } else { + throw e; + } + } + + try { + // Enable Page domain to be safe + await chrome.debugger.sendCommand({ tabId: activeTab.id }, 'Page.enable'); + await chrome.debugger.sendCommand({ tabId: activeTab.id }, 'Page.handleJavaScriptDialog', { + accept: action === 'accept', + promptText: action === 'accept' ? promptText : undefined, + }); + } finally { + // Best-effort detach if we were the owners + try { + await chrome.debugger.detach({ tabId: activeTab.id }); + } catch { + // ignore + } + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ success: true, action, promptText: promptText || null }), + }, + ], + isError: false, + }; + } catch (error) { + return createErrorResponse( + `Failed to handle dialog: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } +} + +export const handleDialogTool = new HandleDialogTool(); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/index.ts b/app/chrome-extension/entrypoints/background/tools/browser/index.ts index f4bcc2b..d473e08 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/index.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/index.ts @@ -15,3 +15,4 @@ export { consoleTool } from './console'; export { fileUploadTool } from './file-upload'; export { readPageTool } from './read-page'; export { computerTool } from './computer'; +export { handleDialogTool } from './dialog'; diff --git a/app/chrome-extension/inject-scripts/wait-helper.js b/app/chrome-extension/inject-scripts/wait-helper.js new file mode 100644 index 0000000..d97dde5 --- /dev/null +++ b/app/chrome-extension/inject-scripts/wait-helper.js @@ -0,0 +1,171 @@ +/* eslint-disable */ +// wait-helper.js +// Listen for text appearance/disappearance in the current document using MutationObserver. +// Returns a stable ref (compatible with accessibility-tree-helper) for the first matching element. + +(function () { + if (window.__WAIT_HELPER_INITIALIZED__) return; + window.__WAIT_HELPER_INITIALIZED__ = true; + + // Ensure ref mapping infra exists (compatible with accessibility-tree-helper.js) + if (!window.__claudeElementMap) window.__claudeElementMap = {}; + if (!window.__claudeRefCounter) window.__claudeRefCounter = 0; + + function isVisible(el) { + try { + if (!(el instanceof Element)) return false; + const style = getComputedStyle(el); + if (style.display === 'none' || style.visibility === 'hidden' || style.opacity === '0') + return false; + const rect = el.getBoundingClientRect(); + if (rect.width <= 0 || rect.height <= 0) return false; + return true; + } catch { + return false; + } + } + + function normalize(str) { + return String(str || '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + } + + function matchesText(el, needle) { + const t = normalize(needle); + if (!t) return false; + try { + if (!isVisible(el)) return false; + const aria = el.getAttribute('aria-label'); + if (aria && normalize(aria).includes(t)) return true; + const title = el.getAttribute('title'); + if (title && normalize(title).includes(t)) return true; + const alt = el.getAttribute('alt'); + if (alt && normalize(alt).includes(t)) return true; + const placeholder = el.getAttribute('placeholder'); + if (placeholder && normalize(placeholder).includes(t)) return true; + // input/textarea value + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + const value = el.value || el.getAttribute('value'); + if (value && normalize(value).includes(t)) return true; + } + const text = el.innerText || el.textContent || ''; + if (normalize(text).includes(t)) return true; + } catch {} + return false; + } + + function findElementByText(text) { + // Fast path: query common interactive elements first + const prioritized = Array.from( + document.querySelectorAll('a,button,input,textarea,select,label,summary,[role]'), + ); + for (const el of prioritized) if (matchesText(el, text)) return el; + + // Fallback: broader scan with cap to avoid blocking on huge pages + const walker = document.createTreeWalker( + document.body || document.documentElement, + NodeFilter.SHOW_ELEMENT, + ); + let count = 0; + while (walker.nextNode()) { + const el = /** @type {Element} */ (walker.currentNode); + if (matchesText(el, text)) return el; + if (++count > 5000) break; // Hard cap to avoid long scans + } + return null; + } + + function ensureRefForElement(el) { + // Try to reuse an existing ref + for (const k in window.__claudeElementMap) { + const weak = window.__claudeElementMap[k]; + if (weak && typeof weak.deref === 'function' && weak.deref() === el) return k; + } + const refId = `ref_${++window.__claudeRefCounter}`; + window.__claudeElementMap[refId] = new WeakRef(el); + return refId; + } + + function centerOf(el) { + const r = el.getBoundingClientRect(); + return { x: Math.round(r.left + r.width / 2), y: Math.round(r.top + r.height / 2) }; + } + + function waitFor({ text, appear = true, timeout = 5000 }) { + return new Promise((resolve) => { + const start = Date.now(); + let resolved = false; + + const check = () => { + try { + const match = findElementByText(text); + if (appear) { + if (match) { + const ref = ensureRefForElement(match); + const center = centerOf(match); + done({ success: true, matched: { ref, center }, tookMs: Date.now() - start }); + } + } else { + // wait for disappearance + if (!match) { + done({ success: true, matched: null, tookMs: Date.now() - start }); + } + } + } catch {} + }; + + const done = (result) => { + if (resolved) return; + resolved = true; + obs && obs.disconnect(); + clearTimeout(timer); + resolve(result); + }; + + const obs = new MutationObserver(() => check()); + try { + obs.observe(document.documentElement || document.body, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + }); + } catch {} + + // Initial check + check(); + const timer = setTimeout( + () => { + done({ success: false, reason: 'timeout', tookMs: Date.now() - start }); + }, + Math.max(0, timeout), + ); + }); + } + + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { + try { + if (request && request.action === 'wait_helper_ping') { + sendResponse({ status: 'pong' }); + return false; + } + if (request && request.action === 'waitForText') { + const text = String(request.text || '').trim(); + const appear = request.appear !== false; // default true + const timeout = Number(request.timeout || 5000); + if (!text) { + sendResponse({ success: false, error: 'text is required' }); + return true; + } + waitFor({ text, appear, timeout }).then((res) => sendResponse(res)); + return true; // async + } + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + return false; + }); +})(); diff --git a/packages/shared/src/tools.ts b/packages/shared/src/tools.ts index 068e15b..7bd1b42 100644 --- a/packages/shared/src/tools.ts +++ b/packages/shared/src/tools.ts @@ -29,6 +29,7 @@ export const TOOL_NAMES = { FILE_UPLOAD: 'chrome_upload_file', READ_PAGE: 'chrome_read_page', COMPUTER: 'chrome_computer', + HANDLE_DIALOG: 'chrome_handle_dialog', }, }; @@ -118,6 +119,30 @@ export const TOOL_SCHEMAS: Tool[] = [ oneOf: [{ type: 'string' }, { type: 'boolean' }, { type: 'number' }], description: 'Value to set for action=fill (string | boolean | number)', }, + elements: { + type: 'array', + description: 'For action=fill_form: list of elements to fill (ref + value)', + items: { + type: 'object', + properties: { + ref: { type: 'string', description: 'Element ref from chrome_read_page' }, + value: { type: 'string', description: 'Value to set (stringified if non-string)' }, + }, + required: ['ref', 'value'], + }, + }, + width: { type: 'number', description: 'For action=resize_page: viewport width' }, + height: { type: 'number', description: 'For action=resize_page: viewport height' }, + appear: { + type: 'boolean', + description: + 'For action=wait with text: whether to wait for the text to appear (true, default) or disappear (false)', + }, + timeout: { + type: 'number', + description: + 'For action=wait with text: timeout in milliseconds (default 10000, max 120000)', + }, duration: { type: 'number', description: 'Seconds to wait for action=wait (max 30s)', @@ -571,4 +596,19 @@ export const TOOL_SCHEMAS: Tool[] = [ required: ['selector'], }, }, + { + name: TOOL_NAMES.BROWSER.HANDLE_DIALOG, + description: 'Handle JavaScript dialogs (alert/confirm/prompt) via CDP', + inputSchema: { + type: 'object', + properties: { + action: { type: 'string', description: 'accept | dismiss' }, + promptText: { + type: 'string', + description: 'Optional prompt text when accepting a prompt', + }, + }, + required: ['action'], + }, + }, ];