From 0857ede7b031642e6074b0c6867e303273c8b5e9 Mon Sep 17 00:00:00 2001 From: hangerye Date: Thu, 13 Nov 2025 18:18:27 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BC=98=E5=8C=96=E5=85=83=E7=B4=A0?= =?UTF-8?q?=E6=A0=87=E6=B3=A8=E5=B7=A5=E5=85=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../common/element-marker-types.ts | 2 + app/chrome-extension/common/message-types.ts | 1 + .../element-marker/element-marker-storage.ts | 1 + .../background/element-marker/index.ts | 62 +- .../background/tools/browser/computer.ts | 138 ++++- .../components/ElementMarkerManagement.vue | 48 +- .../accessibility-tree-helper.js | 370 ++++++++++- .../inject-scripts/element-marker.js | 578 ++++++++++++++---- 8 files changed, 1042 insertions(+), 158 deletions(-) diff --git a/app/chrome-extension/common/element-marker-types.ts b/app/chrome-extension/common/element-marker-types.ts index c6f7318..30403fe 100644 --- a/app/chrome-extension/common/element-marker-types.ts +++ b/app/chrome-extension/common/element-marker-types.ts @@ -15,6 +15,7 @@ export interface ElementMarker { name: string; // Human-friendly name, e.g., "Login Button" selector: string; // Selector string selectorType?: 'css' | 'xpath'; // Default: css + listMode?: boolean; // Whether this marker was created in list mode (allows multiple matches) action?: 'click' | 'fill' | 'custom'; // Intended action hint (optional) createdAt: number; @@ -27,6 +28,7 @@ export interface UpsertMarkerRequest { name: string; selector: string; selectorType?: 'css' | 'xpath'; + listMode?: boolean; matchType?: UrlMatchType; action?: 'click' | 'fill' | 'custom'; } diff --git a/app/chrome-extension/common/message-types.ts b/app/chrome-extension/common/message-types.ts index 876381a..2037dc5 100644 --- a/app/chrome-extension/common/message-types.ts +++ b/app/chrome-extension/common/message-types.ts @@ -101,6 +101,7 @@ export const TOOL_MESSAGE_TYPES = { GENERATE_ACCESSIBILITY_TREE: 'generateAccessibilityTree', RESOLVE_REF: 'resolveRef', ENSURE_REF_FOR_SELECTOR: 'ensureRefForSelector', + DISPATCH_HOVER_FOR_REF: 'dispatchHoverForRef', // Network requests NETWORK_SEND_REQUEST: 'sendPureNetworkRequest', diff --git a/app/chrome-extension/entrypoints/background/element-marker/element-marker-storage.ts b/app/chrome-extension/entrypoints/background/element-marker/element-marker-storage.ts index c71c8a2..bd1d348 100644 --- a/app/chrome-extension/entrypoints/background/element-marker/element-marker-storage.ts +++ b/app/chrome-extension/entrypoints/background/element-marker/element-marker-storage.ts @@ -68,6 +68,7 @@ export async function saveMarker(req: UpsertMarkerRequest): Promise { + // Try ref-based approach first (handles iframes correctly) + if (ref) { + try { + const resp = await this.sendMessageToTab(tabId, { + action: TOOL_MESSAGE_TYPES.DISPATCH_HOVER_FOR_REF, + ref, + }); + if (resp?.success) { + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'hover', + resolvedBy: 'ref', + transport: 'dom-ref', + target: resp.target, + }), + }, + ], + isError: false, + }; + } + } catch (error) { + console.warn('[ComputerTool] DOM ref hover failed, falling back to coordinates', error); + } + } + + // Fallback to coordinate-based approach + if (!coord) { + return createErrorResponse('Hover fallback requires coordinates or ref'); + } + + try { + const [injection] = await chrome.scripting.executeScript({ + target: { tabId }, + world: 'MAIN', + func: (point) => { + const target = document.elementFromPoint(point.x, point.y); + if (!target) { + return { success: false, error: 'No element found at coordinates' }; + } + + // Dispatch hover-related events + for (const type of ['mousemove', 'mouseover', 'mouseenter']) { + target.dispatchEvent( + new MouseEvent(type, { + bubbles: true, + cancelable: true, + clientX: point.x, + clientY: point.y, + view: window, + }), + ); + } + + return { + success: true, + target: { + tagName: target.tagName, + id: target.id, + className: target.className, + text: target.textContent?.trim()?.slice(0, 100) || '', + }, + }; + }, + args: [coord], + }); + + const payload = injection?.result; + if (!payload?.success) { + return createErrorResponse(payload?.error || 'DOM hover fallback failed'); + } + + return { + content: [ + { + type: 'text', + text: JSON.stringify({ + success: true, + action: 'hover', + coordinates: coord, + resolvedBy, + transport: 'dom', + target: payload.target, + }), + }, + ], + isError: false, + }; + } catch (error) { + return createErrorResponse( + `DOM hover fallback failed: ${error instanceof Error ? error.message : String(error)}`, + ); + } + } } export const computerTool = new ComputerTool(); diff --git a/app/chrome-extension/entrypoints/popup/components/ElementMarkerManagement.vue b/app/chrome-extension/entrypoints/popup/components/ElementMarkerManagement.vue index 7c20f79..71612bd 100644 --- a/app/chrome-extension/entrypoints/popup/components/ElementMarkerManagement.vue +++ b/app/chrome-extension/entrypoints/popup/components/ElementMarkerManagement.vue @@ -100,7 +100,16 @@ async function load() { } function prefill(m: ElementMarker) { - form.value = { url: m.url, name: m.name, selector: m.selector, matchType: m.matchType, id: m.id }; + form.value = { + url: m.url, + name: m.name, + selector: m.selector, + selectorType: m.selectorType, + listMode: m.listMode, + matchType: m.matchType, + action: m.action, + id: m.id, + }; } async function onAdd() { @@ -134,16 +143,51 @@ async function remove(m: ElementMarker) { async function validate(m: ElementMarker) { try { - await chrome.runtime.sendMessage({ + const res: any = await chrome.runtime.sendMessage({ type: BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_VALIDATE, selector: m.selector, selectorType: m.selectorType || 'css', action: 'hover', + listMode: !!m.listMode, } as any); + + // Trigger highlight in the page only if tool validation succeeded + if (res?.tool?.ok !== false) { + await highlightInTab(m); + } } catch (e) { /* ignore */ } } +async function highlightInTab(m: ElementMarker) { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs[0]?.id; + if (!tabId) return; + + // Ensure element-marker.js is injected + try { + await chrome.scripting.executeScript({ + target: { tabId, allFrames: true }, + files: ['inject-scripts/element-marker.js'], + world: 'ISOLATED', + }); + } catch { + // Already injected, ignore + } + + // Send highlight message to content script + await chrome.tabs.sendMessage(tabId, { + action: 'element_marker_highlight', + selector: m.selector, + selectorType: m.selectorType || 'css', + listMode: !!m.listMode, + }); + } catch (e) { + // Ignore errors (tab might not support content scripts) + } +} + onMounted(load); diff --git a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js index b9eec8f..8bc26cd 100644 --- a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js +++ b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js @@ -300,6 +300,162 @@ return null; } + /** + * Query CSS selector and return match info including uniqueness check. + * @param {string} selector - CSS selector to query + * @param {boolean} allowMultiple - If true, skip uniqueness check and return first match + * @returns {{element: Element | null, matchCount: number, error?: string}} + * Note: matchCount is capped at 2 (where 2 means "2 or more") for performance + */ + function querySelectorWithUniquenessCheck(selector, allowMultiple = false) { + const seen = new Set(); + let firstMatch = null; + let matchCount = 0; + + const recordMatch = (el) => { + if (!(el instanceof Element) || seen.has(el)) return false; + seen.add(el); + matchCount++; + if (!firstMatch) firstMatch = el; + // Short-circuit if: + // - allowMultiple is true and we found first match (no need to continue) + // - allowMultiple is false and we found multiple matches + if (allowMultiple && firstMatch) return true; + if (!allowMultiple && matchCount >= 2) return true; + return false; + }; + + // Query in main document + let selectorError = null; + try { + const directMatches = document.querySelectorAll(selector); + for (let i = 0; i < directMatches.length; i++) { + if (recordMatch(directMatches[i])) { + // Early exit: either found first match (allowMultiple) or found multiple (not allowed) + return { element: firstMatch, matchCount: allowMultiple ? 1 : 2 }; + } + } + } catch (e) { + selectorError = e; + } + + if (selectorError) { + return { + element: null, + matchCount: 0, + error: `Invalid CSS selector "${selector}": ${selectorError.message || selectorError}`, + }; + } + + // If allowMultiple and we already have a match, return immediately + if (allowMultiple && firstMatch) { + return { element: firstMatch, matchCount: 1 }; + } + + // Query in shadow DOMs + const visited = new Set(); + const stack = [document.documentElement]; + while (stack.length) { + const node = stack.pop(); + if (!node || visited.has(node)) continue; + visited.add(node); + + try { + const shadowRoot = /** @type {any} */ (node).shadowRoot; + if (shadowRoot) { + try { + const shadowMatches = shadowRoot.querySelectorAll(selector); + for (let i = 0; i < shadowMatches.length; i++) { + if (recordMatch(shadowMatches[i])) { + // Early exit: either found first match (allowMultiple) or found multiple (not allowed) + return { element: firstMatch, matchCount: allowMultiple ? 1 : 2 }; + } + } + } catch (e) { + return { + element: null, + matchCount: 0, + error: `Invalid CSS selector "${selector}": ${e.message || e}`, + }; + } + + // Add shadow root children to stack + try { + const shadowChildren = shadowRoot.children || []; + for (let i = 0; i < shadowChildren.length; i++) { + stack.push(shadowChildren[i]); + } + } catch (_) {} + } + } catch (_) {} + + // Add regular children to stack + try { + const children = /** @type {Element} */ (node).children || []; + for (let i = 0; i < children.length; i++) { + stack.push(children[i]); + } + } catch (_) {} + } + + return { element: firstMatch, matchCount: Math.min(matchCount, 2) }; + } + + /** + * Query XPath selector and return match info including uniqueness check. + * @param {string} selector - XPath selector to query + * @param {boolean} allowMultiple - If true, skip uniqueness check and return first match + * @returns {{element: Element | null, matchCount: number, error?: string}} + * Note: matchCount is capped at 2 (where 2 means "2 or more") for performance + */ + function queryXPathWithUniquenessCheck(selector, allowMultiple = false) { + if (!selector) { + return { element: null, matchCount: 0 }; + } + + try { + if (allowMultiple) { + // When multiple matches are allowed, use ANY_UNORDERED_NODE_TYPE for performance + // This returns just the first match without evaluating the entire result set + const result = document.evaluate( + selector, + document, + null, + XPathResult.ANY_UNORDERED_NODE_TYPE, + null, + ); + const firstMatch = + result.singleNodeValue instanceof Element + ? /** @type {Element} */ (result.singleNodeValue) + : null; + return { element: firstMatch, matchCount: firstMatch ? 1 : 0 }; + } else { + // When uniqueness is required, use ORDERED_NODE_SNAPSHOT_TYPE to count matches + const snapshot = document.evaluate( + selector, + document, + null, + XPathResult.ORDERED_NODE_SNAPSHOT_TYPE, + null, + ); + const totalMatches = snapshot.snapshotLength; + // Cap at 2 for performance (2 means "2 or more") + const matchCount = Math.min(totalMatches, 2); + const firstMatch = + totalMatches > 0 && snapshot.snapshotItem(0) instanceof Element + ? /** @type {Element} */ (snapshot.snapshotItem(0)) + : null; + return { element: firstMatch, matchCount }; + } + } catch (e) { + return { + element: null, + matchCount: 0, + error: `Invalid XPath "${selector}": ${e.message || e}`, + }; + } + } + /** * Whether to include element in tree under config * @param {Element} el @@ -502,6 +658,81 @@ // Expose API on window window.__generateAccessibilityTree = __generateAccessibilityTree; + // ============================================================================ + // Hover for Ref (DOM Fallback Support) + // ============================================================================ + + async function handleHoverForRef(ref) { + if (!ref) return { success: false, error: 'ref is required' }; + const el = resolveRef(ref); + if (el) { + dispatchHoverEvents(el); + return { success: true, target: summarizeElement(el) }; + } + return await forwardHoverRefToChildren(ref); + } + + function resolveRef(ref) { + const map = window.__claudeElementMap || {}; + const weak = map[ref]; + return weak && typeof weak.deref === 'function' ? weak.deref() : null; + } + + function dispatchHoverEvents(el) { + const rect = el.getBoundingClientRect(); + const center = { + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2), + }; + ['mousemove', 'mouseover', 'mouseenter'].forEach((type) => { + el.dispatchEvent( + new MouseEvent(type, { + bubbles: true, + cancelable: true, + clientX: center.x, + clientY: center.y, + view: window, + }), + ); + }); + } + + function summarizeElement(el) { + return { + tagName: el.tagName, + id: el.id || '', + className: el.className || '', + text: (el.textContent || '').trim().slice(0, 100), + }; + } + + function forwardHoverRefToChildren(ref) { + return new Promise((resolve) => { + const frames = Array.from(document.querySelectorAll('iframe, frame')); + if (!frames.length) { + resolve({ success: false, error: `ref "${ref}" not found` }); + return; + } + const reqId = `hover_ref_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const listener = (ev) => { + const data = ev?.data; + if (!data || data.type !== 'rr-bridge-hover-ref-result' || data.reqId !== reqId) return; + window.removeEventListener('message', listener, true); + resolve(data.result); + }; + window.addEventListener('message', listener, true); + setTimeout(() => { + window.removeEventListener('message', listener, true); + resolve({ success: false, error: `ref "${ref}" not found in child frames` }); + }, 1500); + for (const frame of frames) { + try { + frame.contentWindow?.postMessage({ type: 'rr-bridge-hover-ref', reqId, ref }, '*'); + } catch {} + } + }); + } + // Chrome message bridge for ping and tree generation chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { try { @@ -783,6 +1014,7 @@ try { // Composite selector support: "frameSelector |> innerSelector" const maybeSel = String(request.selector || '').trim(); + const allowMultiple = !!request.allowMultiple; if (maybeSel.includes('|>')) { try { const parts = maybeSel @@ -851,6 +1083,7 @@ useText: !!request.useText, isXPath: !!request.isXPath, tagName: String(request.tagName || ''), + allowMultiple: !!request.allowMultiple, }, '*', ); @@ -949,26 +1182,45 @@ sendResponse({ success: false, error: 'selector is required' }); return true; } - try { - const doc = document; - const xpathResult = doc.evaluate( - sel, - doc, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - null, - ); - el = /** @type {Element|null} */ (xpathResult.singleNodeValue); - } catch (e) { - // fall back to null - el = null; + const result = queryXPathWithUniquenessCheck(sel, allowMultiple); + if (result.error) { + sendResponse({ success: false, error: result.error }); + return true; } + if (result.matchCount === 0) { + sendResponse({ success: false, error: `selector not found: ${sel}` }); + return true; + } + if (!allowMultiple && result.matchCount > 1) { + sendResponse({ + success: false, + error: `Selector "${sel}" matched multiple elements. Please refine the selector to match only one element.`, + }); + return true; + } + el = result.element; } else { if (!sel) { sendResponse({ success: false, error: 'selector is required' }); return true; } - el = document.querySelector(sel) || querySelectorDeepFirst(sel); + const result = querySelectorWithUniquenessCheck(sel, allowMultiple); + if (result.error) { + sendResponse({ success: false, error: result.error }); + return true; + } + if (result.matchCount === 0) { + sendResponse({ success: false, error: `selector not found: ${sel}` }); + return true; + } + if (!allowMultiple && result.matchCount > 1) { + sendResponse({ + success: false, + error: `Selector "${sel}" matched multiple elements. Please refine the selector to match only one element.`, + }); + return true; + } + el = result.element; } if (!el) { sendResponse({ success: false, error: `selector not found: ${sel}` }); @@ -1000,6 +1252,14 @@ return true; } } + if (request && request.action === 'dispatchHoverForRef') { + handleHoverForRef(String(request.ref || '').trim()) + .then((result) => sendResponse(result)) + .catch((error) => + sendResponse({ success: false, error: error?.message || String(error) }), + ); + return true; + } if (request && request.action === 'getAttributeForSelector') { try { const sel = String(request.selector || '').trim(); @@ -1297,6 +1557,27 @@ (ev) => { try { const data = ev && ev.data; + // Handle hover-ref bridge requests from parent frame + if (data && data.type === 'rr-bridge-hover-ref') { + handleHoverForRef(data.ref) + .then((result) => { + ev.source?.postMessage( + { type: 'rr-bridge-hover-ref-result', reqId: data.reqId, result }, + '*', + ); + }) + .catch((error) => { + ev.source?.postMessage( + { + type: 'rr-bridge-hover-ref-result', + reqId: data.reqId, + result: { success: false, error: error?.message || String(error) }, + }, + '*', + ); + }); + return; + } if (!data || data.type !== 'rr-bridge-ensure-ref') return; const { reqId, selector, useText, isXPath, tagName } = data || {}; const respond = (payload) => { @@ -1378,24 +1659,51 @@ } if (!el && best.el) el = best.el; } else if (isXPath) { - try { - const it = document.evaluate( - sel, - document, - null, - XPathResult.FIRST_ORDERED_NODE_TYPE, - null, - ); - el = it.singleNodeValue instanceof Element ? it.singleNodeValue : null; - } catch {} + if (!sel) { + respond({ success: false, error: 'selector is required' }); + return; + } + const allowMultiple = !!data.allowMultiple; + const result = queryXPathWithUniquenessCheck(sel, allowMultiple); + if (result.error) { + respond({ success: false, error: result.error }); + return; + } + if (result.matchCount === 0) { + respond({ success: false, error: `Selector "${sel}" not found in child frame` }); + return; + } + if (!allowMultiple && result.matchCount > 1) { + respond({ + success: false, + error: `Selector "${sel}" matched multiple elements inside frame. Please refine the selector to match only one element.`, + }); + return; + } + el = result.element; } else { - try { - el = - document.querySelector(sel) || - (typeof querySelectorDeepFirst === 'function' - ? querySelectorDeepFirst(sel) - : null); - } catch {} + if (!sel) { + respond({ success: false, error: 'selector is required' }); + return; + } + const allowMultiple = !!data.allowMultiple; + const result = querySelectorWithUniquenessCheck(sel, allowMultiple); + if (result.error) { + respond({ success: false, error: result.error }); + return; + } + if (result.matchCount === 0) { + respond({ success: false, error: `Selector "${sel}" not found in child frame` }); + return; + } + if (!allowMultiple && result.matchCount > 1) { + respond({ + success: false, + error: `Selector "${sel}" matched multiple elements inside frame. Please refine the selector to match only one element.`, + }); + return; + } + el = result.element; } if (!el || !(el instanceof Element)) { respond({ success: false, error: 'Element not found in child frame' }); diff --git a/app/chrome-extension/inject-scripts/element-marker.js b/app/chrome-extension/inject-scripts/element-marker.js index 7f60eeb..0aedf4c 100644 --- a/app/chrome-extension/inject-scripts/element-marker.js +++ b/app/chrome-extension/inject-scripts/element-marker.js @@ -5,6 +5,14 @@ const IS_MAIN = window === window.top; + // ============================================================================ + // Utility Functions + // ============================================================================ + + function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); + } + // ============================================================================ // Constants & Configuration // ============================================================================ @@ -629,7 +637,7 @@ - - - + @@ -690,14 +697,36 @@ +

Selector Preferences

+
+
+ + + +
+
+ +
+ +
+
- -