fix: 优化元素标注工具

This commit is contained in:
hangerye
2025-11-13 18:18:27 +08:00
parent 6b9cf7929f
commit 0857ede7b0
8 changed files with 1042 additions and 158 deletions
@@ -15,6 +15,7 @@ export interface ElementMarker {
name: string; // Human-friendly name, e.g., "Login Button"
selector: string; // Selector string
selectorType?: 'css' | 'xpath'; // Default: css
listMode?: boolean; // Whether this marker was created in list mode (allows multiple matches)
action?: 'click' | 'fill' | 'custom'; // Intended action hint (optional)
createdAt: number;
@@ -27,6 +28,7 @@ export interface UpsertMarkerRequest {
name: string;
selector: string;
selectorType?: 'css' | 'xpath';
listMode?: boolean;
matchType?: UrlMatchType;
action?: 'click' | 'fill' | 'custom';
}
@@ -101,6 +101,7 @@ export const TOOL_MESSAGE_TYPES = {
GENERATE_ACCESSIBILITY_TREE: 'generateAccessibilityTree',
RESOLVE_REF: 'resolveRef',
ENSURE_REF_FOR_SELECTOR: 'ensureRefForSelector',
DISPATCH_HOVER_FOR_REF: 'dispatchHoverForRef',
// Network requests
NETWORK_SEND_REQUEST: 'sendPureNetworkRequest',
@@ -68,6 +68,7 @@ export async function saveMarker(req: UpsertMarkerRequest): Promise<ElementMarke
name: req.name || selector,
selector,
selectorType: req.selectorType || 'css',
listMode: req.listMode || false,
action: req.action || 'custom',
createdAt: ts,
updatedAt: ts,
@@ -18,6 +18,32 @@ import { keyboardTool } from '@/entrypoints/background/tools/browser/keyboard';
const CONTEXT_MENU_ID = 'element_marker_mark';
/**
* Extract error message from MCP tool result
*/
function extractToolError(result: any): string | undefined {
if (!result) return undefined;
// Check for error in result content array
if (Array.isArray(result.content)) {
for (const item of result.content) {
if (item?.text) {
try {
const parsed = JSON.parse(item.text);
if (parsed?.error) return parsed.error;
if (parsed?.message) return parsed.message;
} catch {
// Not JSON, use as-is
return item.text;
}
}
}
}
// Fallback to direct error field
return result.error || (result.isError ? 'unknown tool error' : undefined);
}
async function ensureContextMenu() {
try {
// Guard: contextMenus permission may be missing
@@ -106,6 +132,7 @@ export function initElementMarkerListeners() {
selector: string;
selectorType?: 'css' | 'xpath';
action: MarkerValidationAction;
listMode?: boolean;
text?: string;
keys?: string;
button?: 'left' | 'right' | 'middle';
@@ -146,6 +173,7 @@ export function initElementMarkerListeners() {
action: 'ensureRefForSelector',
selector,
isXPath: selectorType === 'xpath',
allowMultiple: !!req.listMode,
} as any);
} catch (e) {
return sendResponse({
@@ -194,7 +222,8 @@ export function initElementMarkerListeners() {
? { action: 'hover', coordinates: coords }
: ({ action: 'hover', ref: ensured.ref } as any),
);
base.tool = { name: 'computer.hover', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'computer.hover', ok: !r.isError, error };
break;
}
case 'left_click': {
@@ -207,7 +236,8 @@ export function initElementMarkerListeners() {
button: (req.button || 'left') as any,
modifiers: req.modifiers || {},
} as any);
base.tool = { name: 'interaction.click', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'interaction.click', ok: !r.isError, error };
break;
}
case 'double_click': {
@@ -221,7 +251,8 @@ export function initElementMarkerListeners() {
button: (req.button || 'left') as any,
modifiers: req.modifiers || {},
} as any);
base.tool = { name: 'interaction.click(double)', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'interaction.click(double)', ok: !r.isError, error };
break;
}
case 'right_click': {
@@ -234,7 +265,8 @@ export function initElementMarkerListeners() {
button: 'right',
modifiers: req.modifiers || {},
} as any);
base.tool = { name: 'interaction.click(right)', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'interaction.click(right)', ok: !r.isError, error };
break;
}
case 'scroll': {
@@ -256,13 +288,15 @@ export function initElementMarkerListeners() {
ref: ensured.ref,
} as any);
const r = await computerTool.execute(payload as any);
base.tool = { name: 'computer.scroll', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'computer.scroll', ok: !r.isError, error };
break;
}
case 'type_text': {
const text = String(req.text || '');
const r = await computerTool.execute({ action: 'type', ref: ensured.ref, text });
base.tool = { name: 'computer.type', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'computer.type', ok: !r.isError, error };
break;
}
case 'press_keys': {
@@ -276,7 +310,8 @@ export function initElementMarkerListeners() {
});
} catch {}
const r = await keyboardTool.execute({ keys, delay: 0 } as any);
base.tool = { name: 'keyboard.simulate', ok: !r.isError };
const error = r.isError ? extractToolError(r) : undefined;
base.tool = { name: 'keyboard.simulate', ok: !r.isError, error };
break;
}
default: {
@@ -284,12 +319,25 @@ export function initElementMarkerListeners() {
}
}
} catch (e) {
console.warn('[ElementMarker] Validation failed before tool execution', e);
base.tool = {
name: 'unknown',
ok: false,
error: String(e instanceof Error ? e.message : e),
};
}
// Log tool failures for debugging
if (base.tool && base.tool.ok === false) {
console.warn('[ElementMarker] Tool validation failure', {
action,
toolName: base.tool.name,
error: base.tool.error,
selector,
selectorType,
});
}
return sendResponse(base);
})();
return true;
@@ -332,15 +332,18 @@ class ComputerTool extends BaseBrowserToolExecutor {
try {
await CDPHelper.attach(tab.id);
// Move pointer to target. We can dispatch a single mouseMoved; browsers will generate mouseover/mouseenter as needed.
await CDPHelper.dispatchMouseEvent(tab.id, {
type: 'mouseMoved',
x: coord.x,
y: coord.y,
button: 'none',
buttons: 0,
});
await CDPHelper.detach(tab.id);
try {
// Move pointer to target. We can dispatch a single mouseMoved; browsers will generate mouseover/mouseenter as needed.
await CDPHelper.dispatchMouseEvent(tab.id, {
type: 'mouseMoved',
x: coord.x,
y: coord.y,
button: 'none',
buttons: 0,
});
} finally {
await CDPHelper.detach(tab.id);
}
// Optional hold to allow UI (menus/tooltips) to appear
const holdMs = Math.max(
@@ -358,16 +361,15 @@ class ComputerTool extends BaseBrowserToolExecutor {
action: 'hover',
coordinates: coord,
resolvedBy,
transport: 'cdp',
}),
},
],
isError: false,
};
} catch (e) {
await CDPHelper.detach(tab.id);
return createErrorResponse(
`Hover failed: ${e instanceof Error ? e.message : String(e)}`,
);
} catch (error) {
console.warn('[ComputerTool] CDP hover failed, attempting DOM fallback', error);
return await this.domHoverFallback(tab.id, coord, resolvedBy, params.ref);
}
}
case 'left_click':
@@ -1013,6 +1015,114 @@ class ComputerTool extends BaseBrowserToolExecutor {
);
}
}
/**
* DOM-based hover fallback when CDP is unavailable
* Tries ref-based approach first (works with iframes), falls back to coordinates
*/
private async domHoverFallback(
tabId: number,
coord?: Coordinates,
resolvedBy?: 'ref' | 'selector' | 'coordinates',
ref?: string,
): Promise<ToolResult> {
// Try ref-based approach first (handles iframes correctly)
if (ref) {
try {
const resp = await this.sendMessageToTab(tabId, {
action: TOOL_MESSAGE_TYPES.DISPATCH_HOVER_FOR_REF,
ref,
});
if (resp?.success) {
return {
content: [
{
type: 'text',
text: JSON.stringify({
success: true,
action: 'hover',
resolvedBy: 'ref',
transport: 'dom-ref',
target: resp.target,
}),
},
],
isError: false,
};
}
} catch (error) {
console.warn('[ComputerTool] DOM ref hover failed, falling back to coordinates', error);
}
}
// Fallback to coordinate-based approach
if (!coord) {
return createErrorResponse('Hover fallback requires coordinates or ref');
}
try {
const [injection] = await chrome.scripting.executeScript({
target: { tabId },
world: 'MAIN',
func: (point) => {
const target = document.elementFromPoint(point.x, point.y);
if (!target) {
return { success: false, error: 'No element found at coordinates' };
}
// Dispatch hover-related events
for (const type of ['mousemove', 'mouseover', 'mouseenter']) {
target.dispatchEvent(
new MouseEvent(type, {
bubbles: true,
cancelable: true,
clientX: point.x,
clientY: point.y,
view: window,
}),
);
}
return {
success: true,
target: {
tagName: target.tagName,
id: target.id,
className: target.className,
text: target.textContent?.trim()?.slice(0, 100) || '',
},
};
},
args: [coord],
});
const payload = injection?.result;
if (!payload?.success) {
return createErrorResponse(payload?.error || 'DOM hover fallback failed');
}
return {
content: [
{
type: 'text',
text: JSON.stringify({
success: true,
action: 'hover',
coordinates: coord,
resolvedBy,
transport: 'dom',
target: payload.target,
}),
},
],
isError: false,
};
} catch (error) {
return createErrorResponse(
`DOM hover fallback failed: ${error instanceof Error ? error.message : String(error)}`,
);
}
}
}
export const computerTool = new ComputerTool();
@@ -100,7 +100,16 @@ async function load() {
}
function prefill(m: ElementMarker) {
form.value = { url: m.url, name: m.name, selector: m.selector, matchType: m.matchType, id: m.id };
form.value = {
url: m.url,
name: m.name,
selector: m.selector,
selectorType: m.selectorType,
listMode: m.listMode,
matchType: m.matchType,
action: m.action,
id: m.id,
};
}
async function onAdd() {
@@ -134,16 +143,51 @@ async function remove(m: ElementMarker) {
async function validate(m: ElementMarker) {
try {
await chrome.runtime.sendMessage({
const res: any = await chrome.runtime.sendMessage({
type: BACKGROUND_MESSAGE_TYPES.ELEMENT_MARKER_VALIDATE,
selector: m.selector,
selectorType: m.selectorType || 'css',
action: 'hover',
listMode: !!m.listMode,
} as any);
// Trigger highlight in the page only if tool validation succeeded
if (res?.tool?.ok !== false) {
await highlightInTab(m);
}
} catch (e) {
/* ignore */
}
}
async function highlightInTab(m: ElementMarker) {
try {
const tabs = await chrome.tabs.query({ active: true, currentWindow: true });
const tabId = tabs[0]?.id;
if (!tabId) return;
// Ensure element-marker.js is injected
try {
await chrome.scripting.executeScript({
target: { tabId, allFrames: true },
files: ['inject-scripts/element-marker.js'],
world: 'ISOLATED',
});
} catch {
// Already injected, ignore
}
// Send highlight message to content script
await chrome.tabs.sendMessage(tabId, {
action: 'element_marker_highlight',
selector: m.selector,
selectorType: m.selectorType || 'css',
listMode: !!m.listMode,
});
} catch (e) {
// Ignore errors (tab might not support content scripts)
}
}
onMounted(load);
</script>
@@ -300,6 +300,162 @@
return null;
}
/**
* Query CSS selector and return match info including uniqueness check.
* @param {string} selector - CSS selector to query
* @param {boolean} allowMultiple - If true, skip uniqueness check and return first match
* @returns {{element: Element | null, matchCount: number, error?: string}}
* Note: matchCount is capped at 2 (where 2 means "2 or more") for performance
*/
function querySelectorWithUniquenessCheck(selector, allowMultiple = false) {
const seen = new Set();
let firstMatch = null;
let matchCount = 0;
const recordMatch = (el) => {
if (!(el instanceof Element) || seen.has(el)) return false;
seen.add(el);
matchCount++;
if (!firstMatch) firstMatch = el;
// Short-circuit if:
// - allowMultiple is true and we found first match (no need to continue)
// - allowMultiple is false and we found multiple matches
if (allowMultiple && firstMatch) return true;
if (!allowMultiple && matchCount >= 2) return true;
return false;
};
// Query in main document
let selectorError = null;
try {
const directMatches = document.querySelectorAll(selector);
for (let i = 0; i < directMatches.length; i++) {
if (recordMatch(directMatches[i])) {
// Early exit: either found first match (allowMultiple) or found multiple (not allowed)
return { element: firstMatch, matchCount: allowMultiple ? 1 : 2 };
}
}
} catch (e) {
selectorError = e;
}
if (selectorError) {
return {
element: null,
matchCount: 0,
error: `Invalid CSS selector "${selector}": ${selectorError.message || selectorError}`,
};
}
// If allowMultiple and we already have a match, return immediately
if (allowMultiple && firstMatch) {
return { element: firstMatch, matchCount: 1 };
}
// Query in shadow DOMs
const visited = new Set();
const stack = [document.documentElement];
while (stack.length) {
const node = stack.pop();
if (!node || visited.has(node)) continue;
visited.add(node);
try {
const shadowRoot = /** @type {any} */ (node).shadowRoot;
if (shadowRoot) {
try {
const shadowMatches = shadowRoot.querySelectorAll(selector);
for (let i = 0; i < shadowMatches.length; i++) {
if (recordMatch(shadowMatches[i])) {
// Early exit: either found first match (allowMultiple) or found multiple (not allowed)
return { element: firstMatch, matchCount: allowMultiple ? 1 : 2 };
}
}
} catch (e) {
return {
element: null,
matchCount: 0,
error: `Invalid CSS selector "${selector}": ${e.message || e}`,
};
}
// Add shadow root children to stack
try {
const shadowChildren = shadowRoot.children || [];
for (let i = 0; i < shadowChildren.length; i++) {
stack.push(shadowChildren[i]);
}
} catch (_) {}
}
} catch (_) {}
// Add regular children to stack
try {
const children = /** @type {Element} */ (node).children || [];
for (let i = 0; i < children.length; i++) {
stack.push(children[i]);
}
} catch (_) {}
}
return { element: firstMatch, matchCount: Math.min(matchCount, 2) };
}
/**
* Query XPath selector and return match info including uniqueness check.
* @param {string} selector - XPath selector to query
* @param {boolean} allowMultiple - If true, skip uniqueness check and return first match
* @returns {{element: Element | null, matchCount: number, error?: string}}
* Note: matchCount is capped at 2 (where 2 means "2 or more") for performance
*/
function queryXPathWithUniquenessCheck(selector, allowMultiple = false) {
if (!selector) {
return { element: null, matchCount: 0 };
}
try {
if (allowMultiple) {
// When multiple matches are allowed, use ANY_UNORDERED_NODE_TYPE for performance
// This returns just the first match without evaluating the entire result set
const result = document.evaluate(
selector,
document,
null,
XPathResult.ANY_UNORDERED_NODE_TYPE,
null,
);
const firstMatch =
result.singleNodeValue instanceof Element
? /** @type {Element} */ (result.singleNodeValue)
: null;
return { element: firstMatch, matchCount: firstMatch ? 1 : 0 };
} else {
// When uniqueness is required, use ORDERED_NODE_SNAPSHOT_TYPE to count matches
const snapshot = document.evaluate(
selector,
document,
null,
XPathResult.ORDERED_NODE_SNAPSHOT_TYPE,
null,
);
const totalMatches = snapshot.snapshotLength;
// Cap at 2 for performance (2 means "2 or more")
const matchCount = Math.min(totalMatches, 2);
const firstMatch =
totalMatches > 0 && snapshot.snapshotItem(0) instanceof Element
? /** @type {Element} */ (snapshot.snapshotItem(0))
: null;
return { element: firstMatch, matchCount };
}
} catch (e) {
return {
element: null,
matchCount: 0,
error: `Invalid XPath "${selector}": ${e.message || e}`,
};
}
}
/**
* Whether to include element in tree under config
* @param {Element} el
@@ -502,6 +658,81 @@
// Expose API on window
window.__generateAccessibilityTree = __generateAccessibilityTree;
// ============================================================================
// Hover for Ref (DOM Fallback Support)
// ============================================================================
async function handleHoverForRef(ref) {
if (!ref) return { success: false, error: 'ref is required' };
const el = resolveRef(ref);
if (el) {
dispatchHoverEvents(el);
return { success: true, target: summarizeElement(el) };
}
return await forwardHoverRefToChildren(ref);
}
function resolveRef(ref) {
const map = window.__claudeElementMap || {};
const weak = map[ref];
return weak && typeof weak.deref === 'function' ? weak.deref() : null;
}
function dispatchHoverEvents(el) {
const rect = el.getBoundingClientRect();
const center = {
x: Math.round(rect.left + rect.width / 2),
y: Math.round(rect.top + rect.height / 2),
};
['mousemove', 'mouseover', 'mouseenter'].forEach((type) => {
el.dispatchEvent(
new MouseEvent(type, {
bubbles: true,
cancelable: true,
clientX: center.x,
clientY: center.y,
view: window,
}),
);
});
}
function summarizeElement(el) {
return {
tagName: el.tagName,
id: el.id || '',
className: el.className || '',
text: (el.textContent || '').trim().slice(0, 100),
};
}
function forwardHoverRefToChildren(ref) {
return new Promise((resolve) => {
const frames = Array.from(document.querySelectorAll('iframe, frame'));
if (!frames.length) {
resolve({ success: false, error: `ref "${ref}" not found` });
return;
}
const reqId = `hover_ref_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
const listener = (ev) => {
const data = ev?.data;
if (!data || data.type !== 'rr-bridge-hover-ref-result' || data.reqId !== reqId) return;
window.removeEventListener('message', listener, true);
resolve(data.result);
};
window.addEventListener('message', listener, true);
setTimeout(() => {
window.removeEventListener('message', listener, true);
resolve({ success: false, error: `ref "${ref}" not found in child frames` });
}, 1500);
for (const frame of frames) {
try {
frame.contentWindow?.postMessage({ type: 'rr-bridge-hover-ref', reqId, ref }, '*');
} catch {}
}
});
}
// Chrome message bridge for ping and tree generation
chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => {
try {
@@ -783,6 +1014,7 @@
try {
// Composite selector support: "frameSelector |> innerSelector"
const maybeSel = String(request.selector || '').trim();
const allowMultiple = !!request.allowMultiple;
if (maybeSel.includes('|>')) {
try {
const parts = maybeSel
@@ -851,6 +1083,7 @@
useText: !!request.useText,
isXPath: !!request.isXPath,
tagName: String(request.tagName || ''),
allowMultiple: !!request.allowMultiple,
},
'*',
);
@@ -949,26 +1182,45 @@
sendResponse({ success: false, error: 'selector is required' });
return true;
}
try {
const doc = document;
const xpathResult = doc.evaluate(
sel,
doc,
null,
XPathResult.FIRST_ORDERED_NODE_TYPE,
null,
);
el = /** @type {Element|null} */ (xpathResult.singleNodeValue);
} catch (e) {
// fall back to null
el = null;
const result = queryXPathWithUniquenessCheck(sel, allowMultiple);
if (result.error) {
sendResponse({ success: false, error: result.error });
return true;
}
if (result.matchCount === 0) {
sendResponse({ success: false, error: `selector not found: ${sel}` });
return true;
}
if (!allowMultiple && result.matchCount > 1) {
sendResponse({
success: false,
error: `Selector "${sel}" matched multiple elements. Please refine the selector to match only one element.`,
});
return true;
}
el = result.element;
} else {
if (!sel) {
sendResponse({ success: false, error: 'selector is required' });
return true;
}
el = document.querySelector(sel) || querySelectorDeepFirst(sel);
const result = querySelectorWithUniquenessCheck(sel, allowMultiple);
if (result.error) {
sendResponse({ success: false, error: result.error });
return true;
}
if (result.matchCount === 0) {
sendResponse({ success: false, error: `selector not found: ${sel}` });
return true;
}
if (!allowMultiple && result.matchCount > 1) {
sendResponse({
success: false,
error: `Selector "${sel}" matched multiple elements. Please refine the selector to match only one element.`,
});
return true;
}
el = result.element;
}
if (!el) {
sendResponse({ success: false, error: `selector not found: ${sel}` });
@@ -1000,6 +1252,14 @@
return true;
}
}
if (request && request.action === 'dispatchHoverForRef') {
handleHoverForRef(String(request.ref || '').trim())
.then((result) => sendResponse(result))
.catch((error) =>
sendResponse({ success: false, error: error?.message || String(error) }),
);
return true;
}
if (request && request.action === 'getAttributeForSelector') {
try {
const sel = String(request.selector || '').trim();
@@ -1297,6 +1557,27 @@
(ev) => {
try {
const data = ev && ev.data;
// Handle hover-ref bridge requests from parent frame
if (data && data.type === 'rr-bridge-hover-ref') {
handleHoverForRef(data.ref)
.then((result) => {
ev.source?.postMessage(
{ type: 'rr-bridge-hover-ref-result', reqId: data.reqId, result },
'*',
);
})
.catch((error) => {
ev.source?.postMessage(
{
type: 'rr-bridge-hover-ref-result',
reqId: data.reqId,
result: { success: false, error: error?.message || String(error) },
},
'*',
);
});
return;
}
if (!data || data.type !== 'rr-bridge-ensure-ref') return;
const { reqId, selector, useText, isXPath, tagName } = data || {};
const respond = (payload) => {
@@ -1378,24 +1659,51 @@
}
if (!el && best.el) el = best.el;
} else if (isXPath) {
try {
const it = document.evaluate(
sel,
document,
null,
XPathResult.FIRST_ORDERED_NODE_TYPE,
null,
);
el = it.singleNodeValue instanceof Element ? it.singleNodeValue : null;
} catch {}
if (!sel) {
respond({ success: false, error: 'selector is required' });
return;
}
const allowMultiple = !!data.allowMultiple;
const result = queryXPathWithUniquenessCheck(sel, allowMultiple);
if (result.error) {
respond({ success: false, error: result.error });
return;
}
if (result.matchCount === 0) {
respond({ success: false, error: `Selector "${sel}" not found in child frame` });
return;
}
if (!allowMultiple && result.matchCount > 1) {
respond({
success: false,
error: `Selector "${sel}" matched multiple elements inside frame. Please refine the selector to match only one element.`,
});
return;
}
el = result.element;
} else {
try {
el =
document.querySelector(sel) ||
(typeof querySelectorDeepFirst === 'function'
? querySelectorDeepFirst(sel)
: null);
} catch {}
if (!sel) {
respond({ success: false, error: 'selector is required' });
return;
}
const allowMultiple = !!data.allowMultiple;
const result = querySelectorWithUniquenessCheck(sel, allowMultiple);
if (result.error) {
respond({ success: false, error: result.error });
return;
}
if (result.matchCount === 0) {
respond({ success: false, error: `Selector "${sel}" not found in child frame` });
return;
}
if (!allowMultiple && result.matchCount > 1) {
respond({
success: false,
error: `Selector "${sel}" matched multiple elements inside frame. Please refine the selector to match only one element.`,
});
return;
}
el = result.element;
}
if (!el || !(el instanceof Element)) {
respond({ success: false, error: 'Element not found in child frame' });
@@ -5,6 +5,14 @@
const IS_MAIN = window === window.top;
// ============================================================================
// Utility Functions
// ============================================================================
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// ============================================================================
// Constants & Configuration
// ============================================================================
@@ -629,7 +637,7 @@
<path stroke-linecap="round" stroke-linejoin="round" d="M4 6h16M4 12h16M4 18h16"/>
</svg>
</button>
<button class="em-square-btn" id="__em_toggle_tab" title="Toggle settings">
<button class="em-square-btn" id="__em_toggle_tab" title="Toggle Execute tab">
<svg viewBox="0 0 24 24">
<path stroke-linecap="round" stroke-linejoin="round" d="M10.325 4.317c.426-1.756 2.924-1.756 3.35 0a1.724 1.724 0 002.573 1.066c1.543-.94 3.31.826 2.37 2.37a1.724 1.724 0 001.065 2.572c1.756.426 1.756 2.924 0 3.35a1.724 1.724 0 00-1.066 2.573c.94 1.543-.826 3.31-2.37 2.37a1.724 1.724 0 00-2.572 1.065c-.426 1.756-2.924 1.756-3.35 0a1.724 1.724 0 00-2.573-1.066c-1.543.94-3.31-.826-2.37-2.37a1.724 1.724 0 00-1.065-2.572c-1.756-.426-1.756-2.924 0-3.35a1.724 1.724 0 001.066-2.573c-.94-1.543.826-3.31 2.37-2.37.996.608 2.296.07 2.572-1.065z"/>
<path stroke-linecap="round" stroke-linejoin="round" d="M15 12a3 3 0 11-6 0 3 3 0 016 0z"/>
@@ -660,8 +668,7 @@
<!-- Tabs -->
<div class="em-tabs">
<button class="em-tab active" data-tab="attributes">Attributes</button>
<button class="em-tab" data-tab="validation">Validation</button>
<button class="em-tab" data-tab="settings">Options</button>
<button class="em-tab" data-tab="execute">Execute</button>
</div>
<!-- Status -->
@@ -690,14 +697,36 @@
</div>
</div>
<h3 class="em-section-title">Selector Preferences</h3>
<div class="em-settings">
<div class="em-checkbox-group">
<label class="em-checkbox-label">
<input type="checkbox" id="__em_pref_id" checked />
<span>Prefer ID</span>
</label>
<label class="em-checkbox-label">
<input type="checkbox" id="__em_pref_attr" checked />
<span>Prefer stable attributes</span>
</label>
<label class="em-checkbox-label">
<input type="checkbox" id="__em_pref_class" checked />
<span>Prefer class names</span>
</label>
</div>
</div>
<div class="em-actions">
<button class="em-btn em-btn-primary" id="__em_verify">Verify (Highlight Only)</button>
</div>
<div class="em-actions">
<button class="em-btn em-btn-success" id="__em_save">Save</button>
<button class="em-btn em-btn-ghost" id="__em_cancel">Cancel</button>
</div>
</div>
<!-- Content: Validation Tab -->
<div class="em-content" id="__em_tab_validation" style="display: none;">
<!-- Content: Execute Tab -->
<div class="em-content" id="__em_tab_execute" style="display: none;">
<div class="em-settings">
<div class="em-settings-group">
<div class="em-settings-label">Action</div>
@@ -783,40 +812,17 @@
</div>
<div class="em-actions" style="margin-top: 16px;">
<button class="em-btn em-btn-primary" id="__em_validate">Validate</button>
<button class="em-btn em-btn-primary" id="__em_execute">Execute</button>
</div>
<!-- Validation History -->
<div id="__em_validation_history" style="margin-top: 16px; display: none;">
<div class="em-settings-label">Recent Validations</div>
<!-- Execution History -->
<div id="__em_execution_history" style="margin-top: 16px; display: none;">
<div class="em-settings-label">Recent Executions</div>
<div id="__em_history_list" style="font-size: 12px; color: #737373; margin-top: 8px;"></div>
</div>
</div>
</div>
<!-- Content: Settings Tab -->
<div class="em-content" id="__em_tab_settings" style="display: none;">
<div class="em-settings">
<div class="em-settings-group">
<div class="em-settings-label">Selector Preferences</div>
<div class="em-checkbox-group">
<label class="em-checkbox-label">
<input type="checkbox" id="__em_pref_id" checked />
<span>Prefer ID</span>
</label>
<label class="em-checkbox-label">
<input type="checkbox" id="__em_pref_attr" checked />
<span>Prefer stable attributes</span>
</label>
<label class="em-checkbox-label">
<input type="checkbox" id="__em_pref_class" checked />
<span>Prefer class names</span>
</label>
</div>
</div>
</div>
</div>
<!-- Footer -->
<div class="em-footer">
Click or press <kbd>Space</kbd> to select an element
@@ -983,22 +989,22 @@
});
const attrContent = shadow.getElementById('__em_tab_attributes');
const validationContent = shadow.getElementById('__em_tab_validation');
const settingsContent = shadow.getElementById('__em_tab_settings');
const executeContent = shadow.getElementById('__em_tab_execute');
if (attrContent)
attrContent.style.display = state.activeTab === 'attributes' ? 'block' : 'none';
if (validationContent)
validationContent.style.display = state.activeTab === 'validation' ? 'block' : 'none';
if (settingsContent)
settingsContent.style.display = state.activeTab === 'settings' ? 'block' : 'none';
if (executeContent)
executeContent.style.display = state.activeTab === 'execute' ? 'block' : 'none';
// Sync interaction mode when tab changes
syncInteractionMode();
}
function updateValidationHistoryUI() {
const shadow = PanelHost.getShadow();
if (!shadow) return;
const historyContainer = shadow.getElementById('__em_validation_history');
const historyContainer = shadow.getElementById('__em_execution_history');
const historyList = shadow.getElementById('__em_history_list');
if (!historyContainer || !historyList) return;
@@ -1450,6 +1456,11 @@
const node = stack.pop();
if (!node || ++count > MAX) continue;
// Skip overlay elements to prevent panel self-highlighting
if (isOverlayElement(node)) {
continue;
}
yield node;
try {
@@ -1494,7 +1505,10 @@
for (let i = 0; i < res.snapshotLength; i++) {
const n = res.snapshotItem(i);
if (n?.nodeType === 1) arr.push(n);
// Filter out overlay elements to prevent panel self-highlighting
if (n?.nodeType === 1 && !isOverlayElement(n)) {
arr.push(n);
}
}
return arr;
} catch {
@@ -1512,8 +1526,10 @@
selectedEl: null,
box: null,
highlighter: null,
listenersAttached: false,
rectsHost: null,
hoveredList: [],
verifyRectsActive: false, // Track if verify rects are showing
};
function ensureHighlighter() {
@@ -1564,16 +1580,25 @@
}
function clearHighlighter() {
console.log('[clearHighlighter] called, verifyRectsActive:', STATE.verifyRectsActive);
if (STATE.highlighter) STATE.highlighter.style.display = 'none';
clearRects();
// Only clear hover rects, not verify rects
if (!STATE.verifyRectsActive) {
console.log('[clearHighlighter] clearing rects because verifyRectsActive is false');
clearRects();
} else {
console.log('[clearHighlighter] NOT clearing rects because verifyRectsActive is true');
}
}
function clearRects() {
if (STATE.rectsHost) STATE.rectsHost.innerHTML = '';
STATE.verifyRectsActive = false;
}
function drawRects(elements, color = CONFIG.COLORS.HOVER, dashed = true) {
const host = ensureRectsHost();
// Clear previous rects
clearRects();
for (const el of elements) {
@@ -1598,6 +1623,44 @@
// Interaction Logic
// ============================================================================
function isInsidePanel(target) {
const shadow = PanelHost.getShadow();
return !!shadow && target instanceof Node && shadow.contains(target);
}
/**
* Check if a node belongs to the element marker overlay (panel host or its shadow DOM)
* This is used to filter out overlay elements from query results to prevent self-highlighting
*
* @param {Node} node - The node to check
* @returns {boolean} True if the node is part of the overlay
*/
function isOverlayElement(node) {
if (!(node instanceof Node)) return false;
const host = PanelHost.getHost();
if (!host) return false;
// Check if node is the panel host itself
if (node === host) return true;
// Check if node is within the shadow DOM of the panel host
const root = typeof node.getRootNode === 'function' ? node.getRootNode() : null;
return root instanceof ShadowRoot && root.host === host;
}
/**
* Filter out overlay elements from an array of elements
* This ensures that panel components are never included in highlight/verification results
*
* @param {Array} elements - Array of elements to filter
* @returns {Array} Filtered array without overlay elements
*/
function filterOverlayElements(elements) {
if (!Array.isArray(elements)) return [];
return elements.filter((node) => !isOverlayElement(node));
}
function onMouseMove(ev) {
if (!STATE.active) return;
@@ -1608,7 +1671,9 @@
}
const host = PanelHost.getHost();
if (host && (target === host || target.closest?.('#__element_marker_overlay'))) {
// Check if target is the panel host itself or inside the shadow DOM
if ((host && target === host) || isInsidePanel(target)) {
STATE.hoverEl = null; // Reset to prevent keyboard shortcuts from reselecting the last element
clearHighlighter();
return;
}
@@ -1645,13 +1710,59 @@
}
}
// ============================================================================
// Event Listeners Management
// ============================================================================
function attachPointerListeners() {
if (STATE.listenersAttached) return;
window.addEventListener('mousemove', onMouseMove, true);
window.addEventListener('click', onClick, true);
STATE.listenersAttached = true;
}
function detachPointerListeners() {
if (!STATE.listenersAttached) return;
window.removeEventListener('mousemove', onMouseMove, true);
window.removeEventListener('click', onClick, true);
STATE.listenersAttached = false;
}
function attachKeyboardListener() {
window.addEventListener('keydown', onKeyDown, true);
}
function detachKeyboardListener() {
window.removeEventListener('keydown', onKeyDown, true);
}
function syncInteractionMode() {
if (!STATE.active) return;
const activeTab = StateStore.get('activeTab');
if (activeTab === 'execute') {
// In execute mode, detach pointer listeners to allow real interactions
// but keep keyboard listener for Esc key
detachPointerListeners();
// Only clear the hover highlighter, not the verification rects
if (STATE.highlighter) STATE.highlighter.style.display = 'none';
} else {
// In attributes mode, attach all listeners for element selection
attachPointerListeners();
}
}
// ============================================================================
// Event Handlers
// ============================================================================
function onClick(ev) {
if (!STATE.active) return;
const target = ev.target;
const host = PanelHost.getHost();
if (host && (target === host || target.closest?.('#__element_marker_overlay'))) {
// Check if click is on the panel host itself or inside the shadow DOM
if ((host && target === host) || isInsidePanel(target)) {
return;
}
@@ -1683,6 +1794,12 @@
function onKeyDown(e) {
if (!STATE.active) return;
// Check if the focused element is inside the panel - if so, don't handle selection keys
if (isInsidePanel(e.target)) {
// Key event is from panel, don't interfere
if (e.key !== 'Escape') return; // Still allow Escape to close
}
if (e.key === 'Escape') {
e.preventDefault();
stop();
@@ -1733,13 +1850,97 @@
// Validation Logic
// ============================================================================
/**
* Verify selector by highlighting only (non-destructive)
*/
async function verifyHighlightOnly() {
try {
const selector = STATE.box?.querySelector('#__em_selector')?.textContent?.trim();
console.log('[verifyHighlightOnly] selector:', selector);
if (!selector) return;
StateStore.set({
validation: { status: 'running', message: 'Verifying selector...' },
});
const selectorType = StateStore.get('selectorType');
const listMode = StateStore.get('listMode');
const effectiveType = listMode ? 'css' : selectorType;
// Query for matches
const matches =
effectiveType === 'xpath' ? evaluateXPathAll(selector) : queryAllDeep(selector);
// Additional defense: filter out any overlay elements that might have slipped through
const filteredMatches = filterOverlayElements(matches);
console.log('[verifyHighlightOnly] matches:', filteredMatches.length);
if (!filteredMatches || filteredMatches.length === 0) {
StateStore.set({
validation: { status: 'error', message: 'No elements found' },
});
return;
}
// Scroll first match into view
const primaryMatch = filteredMatches[0];
if (primaryMatch) {
primaryMatch.scrollIntoView({
block: 'center',
inline: 'center',
behavior: 'smooth',
});
}
await sleep(200);
// Highlight matches
console.log(
'[verifyHighlightOnly] calling drawRects with',
filteredMatches.length,
'elements',
);
drawRects(filteredMatches, CONFIG.COLORS.VERIFY, false);
STATE.verifyRectsActive = true; // Mark as verify rects to prevent clearing
console.log('[verifyHighlightOnly] STATE.verifyRectsActive set to true');
console.log(
'[verifyHighlightOnly] STATE.rectsHost children:',
STATE.rectsHost?.children.length,
);
StateStore.set({
validation: {
status: 'success',
message: `Found ${filteredMatches.length} element${filteredMatches.length > 1 ? 's' : ''}`,
},
});
// Auto-clear highlight after 2 seconds
setTimeout(() => {
clearRects();
StateStore.set({
validation: { status: 'idle', message: '' },
});
}, 2000);
} catch (error) {
console.error('[verifyHighlightOnly] error:', error);
StateStore.set({
validation: { status: 'error', message: error.message || 'Verification failed' },
});
}
}
/**
* Execute action on selector (destructive)
*/
async function verifySelectorNow() {
try {
const selector = STATE.box?.querySelector('#__em_selector')?.textContent?.trim();
if (!selector) return;
StateStore.set({
validation: { status: 'running', message: '正在验证...' },
validation: { status: 'running', message: 'Executing action...' },
});
const selectorType = StateStore.get('selectorType');
@@ -1750,7 +1951,17 @@
const matches =
effectiveType === 'xpath' ? evaluateXPathAll(selector) : queryAllDeep(selector);
drawRects(matches, CONFIG.COLORS.VERIFY, false);
// Additional defense: filter out any overlay elements that might have slipped through
const filteredMatches = filterOverlayElements(matches);
if (!filteredMatches || filteredMatches.length === 0) {
StateStore.set({
validation: { status: 'error', message: 'No elements found' },
});
return;
}
drawRects(filteredMatches, CONFIG.COLORS.VERIFY, false);
const action = STATE.box?.querySelector('#__em_action')?.value || 'hover';
@@ -1759,6 +1970,7 @@
selector,
selectorType: effectiveType,
action,
listMode,
};
// Action-specific parameters
@@ -1798,7 +2010,7 @@
action,
success,
timestamp: Date.now(),
matchCount: matches.length,
matchCount: filteredMatches.length,
};
const history = [...(StateStore.get('validationHistory') || []), newEntry].slice(-5);
@@ -1806,7 +2018,7 @@
StateStore.set({
validation: {
status: 'success',
message: `✓ 验证成功 (匹配 ${matches.length} 个元素)`,
message: `✓ 验证成功 (匹配 ${filteredMatches.length} 个元素)`,
},
validationHistory: history,
});
@@ -1838,6 +2050,116 @@
}
}
/**
* Highlight selector from external request (popup/background)
* Supports composite iframe selectors: "frameSelector |> innerSelector"
*/
async function highlightSelectorExternal({ selector, selectorType = 'css', listMode = false }) {
const normalized = String(selector || '').trim();
if (!normalized) {
return { success: false, error: 'selector is required' };
}
try {
// Handle composite iframe selector
if (normalized.includes('|>')) {
const parts = normalized
.split('|>')
.map((s) => s.trim())
.filter(Boolean);
if (parts.length >= 2) {
const frameSel = parts[0];
const innerSel = parts.slice(1).join(' |> ');
// Find frame element
let frameEl = null;
try {
frameEl = querySelectorDeepFirst(frameSel) || document.querySelector(frameSel);
} catch {}
if (
!frameEl ||
!(frameEl instanceof HTMLIFrameElement || frameEl instanceof HTMLFrameElement)
) {
return { success: false, error: `Frame element not found: ${frameSel}` };
}
const cw = frameEl.contentWindow;
if (!cw) {
return { success: false, error: 'Unable to access frame contentWindow' };
}
// Forward highlight request to iframe
return new Promise((resolve) => {
const reqId = `em_highlight_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
const listener = (ev) => {
try {
const data = ev?.data;
if (!data || data.type !== 'em-highlight-result' || data.reqId !== reqId) return;
window.removeEventListener('message', listener, true);
resolve(data.result);
} catch {}
};
window.addEventListener('message', listener, true);
setTimeout(() => {
window.removeEventListener('message', listener, true);
resolve({ success: false, error: 'Frame highlight timeout' });
}, 3000);
cw.postMessage(
{
type: 'em-highlight-request',
reqId,
selector: innerSel,
selectorType,
listMode,
},
'*',
);
});
}
}
// Handle normal selector (non-iframe)
const effectiveType = listMode ? 'css' : selectorType;
const matches =
effectiveType === 'xpath' ? evaluateXPathAll(normalized) : queryAllDeep(normalized);
// Additional defense: filter out any overlay elements that might have slipped through
const filteredMatches = filterOverlayElements(matches);
if (!filteredMatches || filteredMatches.length === 0) {
return { success: false, error: 'No elements found for selector' };
}
// Scroll first match into view
const primaryMatch = filteredMatches[0];
if (primaryMatch) {
primaryMatch.scrollIntoView({
block: 'center',
inline: 'center',
behavior: 'smooth',
});
}
await sleep(150);
// Draw highlight rectangles
drawRects(filteredMatches, CONFIG.COLORS.VERIFY, false);
// Auto-clear after 2 seconds
setTimeout(() => {
clearRects();
}, 2000);
return { success: true, count: filteredMatches.length };
} catch (error) {
return { success: false, error: error.message || String(error) };
}
}
function copySelectorNow() {
try {
const sel = STATE.box?.querySelector('#__em_selector')?.textContent?.trim();
@@ -1876,6 +2198,7 @@
name: name || selector,
selector,
selectorType,
listMode,
},
});
} catch {}
@@ -1901,17 +2224,16 @@
ensureHighlighter();
ensureRectsHost();
window.addEventListener('mousemove', onMouseMove, true);
window.addEventListener('click', onClick, true);
window.addEventListener('keydown', onKeyDown, true);
attachPointerListeners();
attachKeyboardListener();
syncInteractionMode();
}
function stop() {
STATE.active = false;
window.removeEventListener('mousemove', onMouseMove, true);
window.removeEventListener('click', onClick, true);
window.removeEventListener('keydown', onKeyDown, true);
detachPointerListeners();
detachKeyboardListener();
try {
STATE.highlighter?.remove();
@@ -1943,8 +2265,11 @@
// Save
host.querySelector('#__em_save')?.addEventListener('click', save);
// Validate & Copy
host.querySelector('#__em_validate')?.addEventListener('click', verifySelectorNow);
// Verify (highlight only) & Execute (real action)
host.querySelector('#__em_verify')?.addEventListener('click', verifyHighlightOnly);
host.querySelector('#__em_execute')?.addEventListener('click', verifySelectorNow);
// Copy
host.querySelector('#__em_copy')?.addEventListener('click', copySelectorNow);
host.querySelector('#__em_copy_selector')?.addEventListener('click', copySelectorNow);
@@ -2005,10 +2330,10 @@
clearHighlighter();
});
// Tab toggle
// Tab toggle (switch between Attributes and Execute)
host.querySelector('#__em_toggle_tab')?.addEventListener('click', () => {
const currentTab = StateStore.get('activeTab');
StateStore.set({ activeTab: currentTab === 'attributes' ? 'settings' : 'attributes' });
StateStore.set({ activeTab: currentTab === 'attributes' ? 'execute' : 'attributes' });
});
// Tab switching
@@ -2125,58 +2450,94 @@
// Cross-Frame Bridge
// ============================================================================
if (IS_MAIN) {
window.addEventListener(
'message',
(ev) => {
try {
const data = ev?.data;
if (!data || !STATE.active) return;
// Register window message listener in all frames (not just main)
// to support cross-frame highlighting from popup validation
window.addEventListener(
'message',
(ev) => {
try {
const data = ev?.data;
if (!data) return;
const iframes = Array.from(document.querySelectorAll('iframe'));
const host = iframes.find((f) => {
try {
return f.contentWindow === ev.source;
} catch {
return false;
}
});
// Handle iframe highlight request (works even when overlay is inactive)
if (data.type === 'em-highlight-request') {
highlightSelectorExternal({
selector: data.selector,
selectorType: data.selectorType || 'css',
listMode: !!data.listMode,
})
.then((result) => {
window.parent.postMessage(
{
type: 'em-highlight-result',
reqId: data.reqId,
result,
},
'*',
);
})
.catch((error) => {
window.parent.postMessage(
{
type: 'em-highlight-result',
reqId: data.reqId,
result: { success: false, error: error?.message || String(error) },
},
'*',
);
});
return;
}
if (!host) return;
// Following messages only relevant when overlay is active
if (!STATE.active) return;
const base = host.getBoundingClientRect();
// Only main frame handles these overlay-related messages
if (!IS_MAIN) return;
if (data.type === 'em_hover' && Array.isArray(data.rects)) {
const overlay = ensureRectsHost();
clearRects();
for (const r of data.rects) {
const box = document.createElement('div');
Object.assign(box.style, {
position: 'fixed',
left: `${base.left + r.x}px`,
top: `${base.top + r.y}px`,
width: `${r.width}px`,
height: `${r.height}px`,
border: `2px dashed ${CONFIG.COLORS.HOVER}`,
borderRadius: '4px',
boxShadow: `0 0 0 2px ${CONFIG.COLORS.HOVER}22`,
});
overlay.appendChild(box);
}
} else if (data.type === 'em_click' && data.innerSel) {
const frameSel = generateSelector(host);
const composite = frameSel ? `${frameSel} |> ${data.innerSel}` : data.innerSel;
const selectorText = STATE.box?.querySelector('#__em_selector');
const selectorDisplay = STATE.box?.querySelector('#__em_selector_text');
if (selectorText) selectorText.textContent = composite;
if (selectorDisplay) selectorDisplay.textContent = composite;
const iframes = Array.from(document.querySelectorAll('iframe'));
const host = iframes.find((f) => {
try {
return f.contentWindow === ev.source;
} catch {
return false;
}
} catch {}
},
true,
);
}
});
if (!host) return;
const base = host.getBoundingClientRect();
if (data.type === 'em_hover' && Array.isArray(data.rects)) {
const overlay = ensureRectsHost();
clearRects();
for (const r of data.rects) {
const box = document.createElement('div');
Object.assign(box.style, {
position: 'fixed',
left: `${base.left + r.x}px`,
top: `${base.top + r.y}px`,
width: `${r.width}px`,
height: `${r.height}px`,
border: `2px dashed ${CONFIG.COLORS.HOVER}`,
borderRadius: '4px',
boxShadow: `0 0 0 2px ${CONFIG.COLORS.HOVER}22`,
});
overlay.appendChild(box);
}
} else if (data.type === 'em_click' && data.innerSel) {
const frameSel = generateSelector(host);
const composite = frameSel ? `${frameSel} |> ${data.innerSel}` : data.innerSel;
const selectorText = STATE.box?.querySelector('#__em_selector');
const selectorDisplay = STATE.box?.querySelector('#__em_selector_text');
if (selectorText) selectorText.textContent = composite;
if (selectorDisplay) selectorDisplay.textContent = composite;
}
} catch {}
},
true,
);
// ============================================================================
// Message Handlers
@@ -2190,6 +2551,15 @@
} else if (request?.action === 'element_marker_ping') {
sendResponse({ status: 'pong' });
return false;
} else if (request?.action === 'element_marker_highlight') {
highlightSelectorExternal({
selector: request.selector,
selectorType: request.selectorType,
listMode: !!request.listMode,
})
.then((result) => sendResponse(result))
.catch((error) => sendResponse({ success: false, error: error?.message || String(error) }));
return true;
}
return false;
});