From 27e78c656173cc2bc66039c19a9968a5efde16f1 Mon Sep 17 00:00:00 2001 From: hangerye Date: Sun, 26 Oct 2025 19:24:17 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E5=BD=95=E5=88=B6=E8=BF=87=E7=A8=8B?= =?UTF-8?q?=E6=80=A7=E8=83=BD=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../background/record-replay/node-registry.ts | 71 +++--- .../record-replay/selector-engine.ts | 105 ++++++--- .../background/tools/base-browser.ts | 7 +- .../background/tools/browser/interaction.ts | 41 ++-- .../accessibility-tree-helper.js | 223 ++++++++++++++++++ .../inject-scripts/recorder.js | 186 ++++++++++++++- 6 files changed, 536 insertions(+), 97 deletions(-) diff --git a/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts b/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts index 9bd49d4..1ee83bf 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts @@ -262,15 +262,17 @@ const registry: Partial>> = { await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); const s: any = expandTemplatesDeep(step as any, ctx.vars); const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; + const frameId = (located as any)?.frameId ?? ctx.frameId; const first = s.target?.candidates?.[0]?.type; const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; if ((located as any)?.ref) { - const resolved = await chrome.tabs.sendMessage( + const resolved: any = (await chrome.tabs.sendMessage( tabId, { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId: ctx.frameId } as any, - ); + { frameId } as any, + )) as any; const rect = resolved?.rect; if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); } @@ -283,6 +285,7 @@ const registry: Partial>> = { : undefined, waitForNavigation: false, timeout: Math.max(1000, Math.min(s.timeoutMs || 10000, 30000)), + frameId, }, }); if ((res as any).isError) throw new Error('click failed'); @@ -310,15 +313,16 @@ const registry: Partial>> = { await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); const s: any = expandTemplatesDeep(step as any, ctx.vars); const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; const first = s.target?.candidates?.[0]?.type; const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; if ((located as any)?.ref) { - const resolved = await chrome.tabs.sendMessage( + const resolved: any = (await chrome.tabs.sendMessage( tabId, { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId: ctx.frameId } as any, - ); + { frameId } as any, + )) as any; const rect = resolved?.rect; if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); } @@ -351,6 +355,7 @@ const registry: Partial>> = { if (!tabId) throw new Error('Active tab not found'); await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; const first = s.target?.candidates?.[0]?.type; const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; @@ -361,11 +366,11 @@ const registry: Partial>> = { : v; const value = interpolate(s.value); if ((located as any)?.ref) { - const resolved = await chrome.tabs.sendMessage( + const resolved: any = (await chrome.tabs.sendMessage( tabId, { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId: ctx.frameId } as any, - ); + { frameId } as any, + )) as any; const rect = resolved?.rect; if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); } @@ -376,15 +381,15 @@ const registry: Partial>> = { : undefined; if (cssSelector) { try { - const attr = await chrome.tabs.sendMessage( + const attr: any = (await chrome.tabs.sendMessage( tabId, { action: 'getAttributeForSelector', selector: cssSelector, name: 'type', } as any, - { frameId: ctx.frameId } as any, - ); + { frameId } as any, + )) as any; const typeName = (attr && attr.value ? String(attr.value) : '').toLowerCase(); if (typeName === 'file') { const uploadRes = await handleCallTool({ @@ -423,7 +428,7 @@ const registry: Partial>> = { await chrome.tabs.sendMessage( tabId, { action: 'focusByRef', ref: (located as any).ref } as any, - { frameId: ctx.frameId } as any, + { frameId } as any, ); else if (cssSelector) await handleCallTool({ @@ -440,6 +445,7 @@ const registry: Partial>> = { ref: (located as any)?.ref || s.target?.ref, selector: cssSelector, value, + frameId, }, }); if ((res as any).isError) throw new Error('fill failed'); @@ -486,7 +492,7 @@ const registry: Partial>> = { files: ['inject-scripts/wait-helper.js'], world: 'ISOLATED', } as any); - const resp = await chrome.tabs.sendMessage( + const resp: any = (await chrome.tabs.sendMessage( tabId, { action: 'waitForText', @@ -495,7 +501,7 @@ const registry: Partial>> = { timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), } as any, { frameId: ctx.frameId } as any, - ); + )) as any; if (!resp || resp.success !== true) throw new Error('wait text failed'); } else if ('networkIdle' in cond) { const total = Math.min(Math.max(1000, (s as any).timeoutMs || 5000), 120000); @@ -515,12 +521,12 @@ const registry: Partial>> = { files: ['inject-scripts/wait-helper.js'], world: 'ISOLATED', } as any); - const resp = await chrome.tabs.sendMessage(tabId, { + const resp: any = (await chrome.tabs.sendMessage(tabId, { action: 'waitForSelector', selector: cond.selector, visible: cond.visible !== false, timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), - } as any); + } as any)) as any; if (!resp || resp.success !== true) throw new Error('wait selector failed'); } }, @@ -562,10 +568,10 @@ const registry: Partial>> = { const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; if (!tabId) return fail('Active tab not found'); await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const ensured = await chrome.tabs.sendMessage(tabId, { + const ensured: any = (await chrome.tabs.sendMessage(tabId, { action: 'ensureRefForSelector', selector, - } as any); + } as any)) as any; if (!ensured || !ensured.success) return fail('assert selector not found'); if ('visible' in s.assert) { const rect = ensured && ensured.center ? ensured.center : null; @@ -578,11 +584,11 @@ const registry: Partial>> = { const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; if (!tabId) return fail('Active tab not found'); await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const resp = await chrome.tabs.sendMessage( + const resp: any = (await chrome.tabs.sendMessage( tabId, { action: 'getAttributeForSelector', selector, name } as any, { frameId: ctx.frameId } as any, - ); + )) as any; if (!resp || !resp.success) return fail('assert attribute: element not found'); const actual: string | null = resp.value ?? null; if (equals !== undefined && equals !== null) { @@ -859,11 +865,11 @@ const registry: Partial>> = { let sel = cssSelector as string | undefined; if (!sel && (located as any)?.ref) { try { - const resolved = await chrome.tabs.sendMessage( + const resolved: any = (await chrome.tabs.sendMessage( tabId, { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId: ctx.frameId } as any, - ); + { frameId } as any, + )) as any; sel = resolved?.selector; } catch {} } @@ -873,10 +879,7 @@ const registry: Partial>> = { const bubbles = s.bubbles !== false; const cancelable = s.cancelable === true; await chrome.scripting.executeScript({ - target: { - tabId, - frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, - } as any, + target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any, world, func: (selector: string, type: string, bubbles: boolean, cancelable: boolean) => { try { @@ -907,17 +910,18 @@ const registry: Partial>> = { if (typeof tabId !== 'number') throw new Error('Active tab not found'); await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; const cssSelector = !(located as any)?.ref ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value : undefined; let sel = cssSelector as string | undefined; if (!sel && (located as any)?.ref) { try { - const resolved = await chrome.tabs.sendMessage( + const resolved: any = (await chrome.tabs.sendMessage( tabId, { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId: ctx.frameId } as any, - ); + { frameId } as any, + )) as any; sel = resolved?.selector; } catch {} } @@ -927,10 +931,7 @@ const registry: Partial>> = { const value = s.value != null ? String(s.value) : null; const remove = s.remove === true || value == null; await chrome.scripting.executeScript({ - target: { - tabId, - frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, - } as any, + target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any, world, func: (selector: string, name: string, value: string | null, remove: boolean) => { try { diff --git a/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts b/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts index 45c0b92..1b3087e 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts @@ -7,6 +7,57 @@ export interface LocatedElement { ref?: string; center?: { x: number; y: number }; resolvedBy?: 'ref' | SelectorCandidate['type']; + frameId?: number; +} + +// Helper: decide whether selector is a composite cross-frame selector +function isCompositeSelector(sel: string): boolean { + return typeof sel === 'string' && sel.includes('|>'); +} + +// Helper: typed wrapper for chrome.tabs.sendMessage with optional frameId +async function sendToTab(tabId: number, message: any, frameId?: number): Promise { + if (typeof frameId === 'number') { + return await chrome.tabs.sendMessage(tabId, message, { frameId }); + } + return await chrome.tabs.sendMessage(tabId, message); +} + +// Helper: ensure ref for a selector, handling composite selectors and mapping frameId +async function ensureRefForSelector( + tabId: number, + selector: string, + frameId?: number, +): Promise<{ ref: string; center: { x: number; y: number }; frameId?: number } | null> { + try { + let ensured: any = null; + if (isCompositeSelector(selector)) { + // Always query top for composite; helper will bridge to child and return href + ensured = await sendToTab(tabId, { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + selector, + }); + } else { + ensured = await sendToTab( + tabId, + { action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector }, + frameId, + ); + } + if (!ensured || !ensured.success || !ensured.ref || !ensured.center) return null; + // Map frameId when composite via returned href + let locFrameId: number | undefined = undefined; + if (isCompositeSelector(selector) && ensured.href) { + try { + const frames = (await chrome.webNavigation.getAllFrames({ tabId })) as any[]; + const match = frames?.find((f) => typeof f.url === 'string' && f.url === ensured.href); + if (match) locFrameId = match.frameId; + } catch {} + } + return { ref: ensured.ref, center: ensured.center, frameId: locFrameId }; + } catch { + return null; + } } /** @@ -20,15 +71,8 @@ export async function locateElement( // 0) Fast path: try primary selector if provided const primarySel = (target as any)?.selector ? String((target as any).selector).trim() : ''; if (primarySel) { - try { - const ensured = await chrome.tabs.sendMessage( - tabId, - { action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector: primarySel } as any, - { frameId } as any, - ); - if (ensured && ensured.success && ensured.ref && ensured.center) - return { ref: ensured.ref, center: ensured.center, resolvedBy: 'css' }; - } catch {} + const ensured = await ensureRefForSelector(tabId, primarySel, frameId); + if (ensured) return { ...ensured, resolvedBy: 'css' }; } // 1) Non-text candidates first for stability (css/attr/aria/xpath) @@ -36,17 +80,8 @@ export async function locateElement( for (const c of nonText) { try { if (c.type === 'css' || c.type === 'attr') { - const ensured = await chrome.tabs.sendMessage( - tabId, - { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - selector: c.value, - } as any, - { frameId } as any, - ); - if (ensured && ensured.success && ensured.ref && ensured.center) { - return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; - } + const ensured = await ensureRefForSelector(tabId, String(c.value || ''), frameId); + if (ensured) return { ...ensured, resolvedBy: c.type }; } else if (c.type === 'aria') { // Minimal ARIA role+name parser like: "button[name=提交]" or "textbox[name=用户名]" const v = String(c.value || '').trim(); @@ -78,31 +113,28 @@ export async function locateElement( ); } for (const sel of ariaSelectors) { - const ensured = await chrome.tabs.sendMessage( + const ensured = await sendToTab( tabId, - { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - selector: sel, - } as any, - { frameId } as any, + { action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector: sel } as any, + frameId, ); if (ensured && ensured.success && ensured.ref && ensured.center) { - return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; + return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type, frameId }; } } } else if (c.type === 'xpath') { // Minimal xpath support via document.evaluate through injected helper - const ensured = await chrome.tabs.sendMessage( + const ensured = await sendToTab( tabId, { action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector: c.value, isXPath: true, } as any, - { frameId } as any, + frameId, ); if (ensured && ensured.success && ensured.ref && ensured.center) { - return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; + return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type, frameId }; } } } catch (e) { @@ -114,7 +146,7 @@ export async function locateElement( const tagName = ((target as any)?.tag || '').toString(); for (const c of textCands) { try { - const ensured = await chrome.tabs.sendMessage( + const ensured = await sendToTab( tabId, { action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, @@ -122,7 +154,7 @@ export async function locateElement( text: c.value, tagName, } as any, - { frameId } as any, + frameId, ); if (ensured && ensured.success && ensured.ref && ensured.center) { return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; @@ -132,13 +164,10 @@ export async function locateElement( // Fallback: try ref (works when ref was produced in the same page lifecycle) if (target.ref) { try { - const res = await chrome.tabs.sendMessage( + const res = await sendToTab( tabId, - { - action: TOOL_MESSAGE_TYPES.RESOLVE_REF, - ref: target.ref, - } as any, - { frameId } as any, + { action: TOOL_MESSAGE_TYPES.RESOLVE_REF, ref: target.ref } as any, + frameId, ); if (res && res.success && res.center) { return { ref: target.ref, center: res.center, resolvedBy: 'ref' }; diff --git a/app/chrome-extension/entrypoints/background/tools/base-browser.ts b/app/chrome-extension/entrypoints/background/tools/base-browser.ts index b17bd6e..fd4be65 100644 --- a/app/chrome-extension/entrypoints/background/tools/base-browser.ts +++ b/app/chrome-extension/entrypoints/background/tools/base-browser.ts @@ -72,9 +72,12 @@ export abstract class BaseBrowserToolExecutor implements ToolExecutor { /** * Send message to tab */ - protected async sendMessageToTab(tabId: number, message: any): Promise { + protected async sendMessageToTab(tabId: number, message: any, frameId?: number): Promise { try { - const response = await chrome.tabs.sendMessage(tabId, message); + const response = + typeof frameId === 'number' + ? await chrome.tabs.sendMessage(tabId, message, { frameId }) + : await chrome.tabs.sendMessage(tabId, message); if (response && response.error) { throw new Error(String(response.error)); diff --git a/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts b/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts index 0086349..5ad8ba4 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts @@ -15,6 +15,7 @@ interface ClickToolParams { coordinates?: Coordinates; // Coordinates to click at (x, y relative to viewport) waitForNavigation?: boolean; // Whether to wait for navigation to complete after click timeout?: number; // Timeout in milliseconds for waiting for the element or navigation + frameId?: number; // Target frame for ref/selector resolution } /** @@ -32,6 +33,7 @@ class ClickTool extends BaseBrowserToolExecutor { coordinates, waitForNavigation = false, timeout = TIMEOUTS.DEFAULT_WAIT * 5, + frameId, } = args; console.log(`Starting click operation with options:`, args); @@ -57,14 +59,18 @@ class ClickTool extends BaseBrowserToolExecutor { await this.injectContentScript(tab.id, ['inject-scripts/click-helper.js']); // Send click message to content script - const result = await this.sendMessageToTab(tab.id, { - action: TOOL_MESSAGE_TYPES.CLICK_ELEMENT, - selector, - coordinates, - ref: args.ref, - waitForNavigation, - timeout, - }); + const result = await this.sendMessageToTab( + tab.id, + { + action: TOOL_MESSAGE_TYPES.CLICK_ELEMENT, + selector, + coordinates, + ref: args.ref, + waitForNavigation, + timeout, + }, + frameId, + ); return { content: [ @@ -97,6 +103,7 @@ interface FillToolParams { ref?: string; // Element ref from accessibility tree // Accept string | number | boolean for broader form input coverage value: string | number | boolean; + frameId?: number; } /** @@ -109,7 +116,7 @@ class FillTool extends BaseBrowserToolExecutor { * Execute fill operation */ async execute(args: FillToolParams): Promise { - const { selector, ref, value } = args; + const { selector, ref, value, frameId } = args; console.log(`Starting fill operation with options:`, args); @@ -136,12 +143,16 @@ class FillTool extends BaseBrowserToolExecutor { await this.injectContentScript(tab.id, ['inject-scripts/fill-helper.js']); // Send fill message to content script - const result = await this.sendMessageToTab(tab.id, { - action: TOOL_MESSAGE_TYPES.FILL_ELEMENT, - selector, - ref, - value, - }); + const result = await this.sendMessageToTab( + tab.id, + { + action: TOOL_MESSAGE_TYPES.FILL_ELEMENT, + selector, + ref, + value, + }, + frameId, + ); if (result && result.error) { return createErrorResponse(result.error); diff --git a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js index 78afd5b..b9eec8f 100644 --- a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js +++ b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js @@ -781,6 +781,86 @@ } if (request && request.action === 'ensureRefForSelector') { try { + // Composite selector support: "frameSelector |> innerSelector" + const maybeSel = String(request.selector || '').trim(); + if (maybeSel.includes('|>')) { + try { + const parts = maybeSel + .split('|>') + .map((s) => s.trim()) + .filter(Boolean); + if (parts.length >= 2) { + const frameSel = parts[0]; + const innerSel = parts.slice(1).join(' |> '); + // Find target frame element in current document + let frameEl = null; + try { + frameEl = querySelectorDeepFirst(frameSel) || document.querySelector(frameSel); + } catch {} + if ( + !frameEl || + !(frameEl instanceof HTMLIFrameElement || frameEl instanceof HTMLFrameElement) + ) { + sendResponse({ + success: false, + error: `Composite frame selector not found: ${frameSel}`, + }); + return true; + } + const cw = frameEl.contentWindow; + if (!cw) { + sendResponse({ + success: false, + error: 'Unable to obtain contentWindow of target frame', + }); + return true; + } + // Bridge to child frame via postMessage + const reqId = `rrc_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const listener = (ev) => { + try { + const data = ev && ev.data; + if ( + !data || + data.type !== 'rr-bridge-ensure-ref-result' || + data.reqId !== reqId + ) + return; + window.removeEventListener('message', listener, true); + if (data.success) { + sendResponse({ + success: true, + ref: data.ref, + center: data.center, + href: data.href, + }); + } else { + sendResponse({ success: false, error: data.error || 'child failed' }); + } + } catch (e) { + window.removeEventListener('message', listener, true); + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + } + }; + window.addEventListener('message', listener, true); + cw.postMessage( + { + type: 'rr-bridge-ensure-ref', + reqId, + selector: innerSel, + useText: !!request.useText, + isXPath: !!request.isXPath, + tagName: String(request.tagName || ''), + }, + '*', + ); + return true; // async response via message bridge + } + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + } // Support CSS selector, XPath, or visible text search const useText = !!request.useText; const textQuery = String(request.text || '').trim(); @@ -1210,4 +1290,147 @@ }); console.log('Accessibility tree helper script loaded'); + // Cross-frame bridge: child listens for ensure-ref requests from parent (composite selector) + try { + window.addEventListener( + 'message', + (ev) => { + try { + const data = ev && ev.data; + if (!data || data.type !== 'rr-bridge-ensure-ref') return; + const { reqId, selector, useText, isXPath, tagName } = data || {}; + const respond = (payload) => { + try { + ev.source && + ev.source.postMessage( + { type: 'rr-bridge-ensure-ref-result', reqId, ...payload }, + '*', + ); + } catch {} + }; + try { + const sel = String(selector || '').trim(); + const limitTag = String(tagName || '') + .trim() + .toUpperCase(); + let el = null; + if (useText && sel) { + const normalize = (s) => + String(s || '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + const query = normalize(sel); + const bigrams = (s) => { + const arr = []; + for (let i = 0; i < s.length - 1; i++) arr.push(s.slice(i, i + 2)); + return arr; + }; + const dice = (a, b) => { + if (!a || !b) return 0; + const A = bigrams(a), + B = bigrams(b); + if (!A.length || !B.length) return 0; + let inter = 0; + const m = new Map(); + for (const t of A) m.set(t, (m.get(t) || 0) + 1); + for (const t of B) { + const c = m.get(t) || 0; + if (c > 0) { + inter++; + m.set(t, c - 1); + } + } + return (2 * inter) / (A.length + B.length); + }; + let best = { el: null, score: 0 }; + const stack = [document.documentElement]; + while (stack.length) { + const node = stack.pop(); + if (!node || !(node instanceof Element)) continue; + try { + if (limitTag && String(node.tagName || '').toUpperCase() !== limitTag) { + } else { + const cs = window.getComputedStyle(node); + if (cs.display !== 'none' && cs.visibility !== 'hidden' && cs.opacity !== '0') { + const rect = node.getBoundingClientRect(); + if (rect.width > 0 && rect.height > 0) { + const txt = normalize(node.textContent || ''); + if (txt) { + if (txt.includes(query)) { + el = node; + break; + } + const sc = dice(txt, query); + if (sc > best.score) best = { el: node, score: sc }; + } + } + } + } + } catch {} + try { + const children = node.children || []; + for (let i = 0; i < children.length; i++) stack.push(children[i]); + const sr = node.shadowRoot; + if (sr && sr.children) + for (let i = 0; i < sr.children.length; i++) stack.push(sr.children[i]); + } catch {} + } + if (!el && best.el) el = best.el; + } else if (isXPath) { + try { + const it = document.evaluate( + sel, + document, + null, + XPathResult.FIRST_ORDERED_NODE_TYPE, + null, + ); + el = it.singleNodeValue instanceof Element ? it.singleNodeValue : null; + } catch {} + } else { + try { + el = + document.querySelector(sel) || + (typeof querySelectorDeepFirst === 'function' + ? querySelectorDeepFirst(sel) + : null); + } catch {} + } + if (!el || !(el instanceof Element)) { + respond({ success: false, error: 'Element not found in child frame' }); + return; + } + if (!window.__claudeElementMap) window.__claudeElementMap = {}; + if (!window.__claudeRefCounter) window.__claudeRefCounter = 0; + let refId = null; + for (const k in window.__claudeElementMap) { + const w = window.__claudeElementMap[k]; + if (w && typeof w.deref === 'function' && w.deref && w.deref() === el) { + refId = k; + break; + } + } + if (!refId) { + refId = `ref_${++window.__claudeRefCounter}`; + window.__claudeElementMap[refId] = new WeakRef(el); + } + const rect = el.getBoundingClientRect(); + respond({ + success: true, + ref: refId, + center: { + x: Math.round(rect.left + rect.width / 2), + y: Math.round(rect.top + rect.height / 2), + }, + href: String(location && location.href ? location.href : ''), + }); + } catch (e) { + respond({ success: false, error: String(e && e.message ? e.message : e) }); + } + } catch {} + }, + true, + ); + } catch {} })(); diff --git a/app/chrome-extension/inject-scripts/recorder.js b/app/chrome-extension/inject-scripts/recorder.js index d2abb10..5f46078 100644 --- a/app/chrome-extension/inject-scripts/recorder.js +++ b/app/chrome-extension/inject-scripts/recorder.js @@ -15,6 +15,8 @@ SENSITIVE_INPUT_TYPES: new Set(['password']), UI_MAX_STEPS: 30, }; + // Cross-frame event channel + const FRAME_EVENT = 'rr_iframe_event'; // Memoization caches for selector computations during recording const __cacheUnique = new WeakMap(); @@ -365,8 +367,21 @@ this._onChange = this._onChange.bind(this); this._onMouseMove = this._onMouseMove.bind(this); this._onScroll = this._onScroll.bind(this); + this._onFocusIn = this._onFocusIn.bind(this); + this._onFocusOut = this._onFocusOut.bind(this); + this._onKeyDown = this._onKeyDown.bind(this); + this._onKeyUp = this._onKeyUp.bind(this); + this._onWindowMessage = this._onWindowMessage.bind(this); this.ui = new UI(this); this._scrollPending = null; + + // Focus tracking for per-element input listening + this._focusedEl = null; + // Keyboard state for combo recording + this._pressed = new Set(); + this._lastKeyTs = 0; + // Map to avoid duplicate switchFrame per iframe source (keyed by frame selector) + this._frameSwitchMap = new Set(); } // Lifecycle @@ -394,8 +409,12 @@ this.isRecording = false; this._detach(); this.ui.remove(); + if (this.batchTimer) clearTimeout(this.batchTimer); + this.batchTimer = null; if (this.scrollTimer) clearTimeout(this.scrollTimer); this.scrollTimer = null; + if (this.hoverRAF) cancelAnimationFrame(this.hoverRAF); + this.hoverRAF = 0; this.lastFill = { step: null, ts: 0 }; const ret = this.sessionBuffer; this.sessionBuffer.steps = []; @@ -419,19 +438,40 @@ // DOM listeners _attach() { document.addEventListener('click', this._onClick, true); - document.addEventListener('input', this._onInput, true); + // Use focusin/out to attach input listener only to focused element + document.addEventListener('focusin', this._onFocusIn, true); + document.addEventListener('focusout', this._onFocusOut, true); document.addEventListener('change', this._onChange, true); // capture-phase scroll to catch non-bubbling events on any container document.addEventListener('scroll', this._onScroll, true); + // Keyboard: record Enter and modifier combos + document.addEventListener('keydown', this._onKeyDown, true); + document.addEventListener('keyup', this._onKeyUp, true); + // Cross-frame: top window aggregates iframe-recorded steps + if (window === window.top) window.addEventListener('message', this._onWindowMessage, true); this._updateHoverListener(); } _detach() { document.removeEventListener('click', this._onClick, true); - document.removeEventListener('input', this._onInput, true); + document.removeEventListener('focusin', this._onFocusIn, true); + document.removeEventListener('focusout', this._onFocusOut, true); document.removeEventListener('change', this._onChange, true); document.removeEventListener('scroll', this._onScroll, true); + document.removeEventListener('keydown', this._onKeyDown, true); + document.removeEventListener('keyup', this._onKeyUp, true); document.removeEventListener('mousemove', this._onMouseMove, true); + if (window === window.top) window.removeEventListener('message', this._onWindowMessage, true); + // Detach per-element input listener if any + if (this._focusedEl) this._focusedEl.removeEventListener('input', this._onInput, true); + this._focusedEl = null; + // Best-effort cleanup for timers/raf when detaching + if (this.batchTimer) clearTimeout(this.batchTimer); + this.batchTimer = null; + if (this.scrollTimer) clearTimeout(this.scrollTimer); + this.scrollTimer = null; + if (this.hoverRAF) cancelAnimationFrame(this.hoverRAF); + this.hoverRAF = 0; } _updateHoverListener() { @@ -470,12 +510,19 @@ _pushStep(step) { step.id = step.id || `step_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`; - if (window !== window.top && !this.frameSwitchPushed) { - const href = String(location && location.href ? location.href : ''); - const frameStep = { type: 'switchFrame', frame: { urlContains: href } }; - this.sessionBuffer.steps.push(frameStep); - this.frameSwitchPushed = true; + // In iframes, forward to top for aggregation (compute frame selector there) + if (window !== window.top) { + try { + const payload = { + kind: 'iframeStep', + href: String(location && location.href ? location.href : ''), + step, + }; + window.top.postMessage({ type: FRAME_EVENT, payload }, '*'); + return; // Do not push locally in subframe + } catch {} } + // Top window: optionally insert a switchFrame if this step originated from an iframe message this.sessionBuffer.steps.push(step); this.sessionBuffer.meta.updatedAt = new Date().toISOString(); this.batch.push(step); @@ -648,6 +695,29 @@ // UI handled by injected UI class + _onFocusIn(e) { + if (!this.isRecording || this.isPaused) return; + const el = e.target; + const isEditable = + el instanceof HTMLInputElement || + el instanceof HTMLTextAreaElement || + (el && el.nodeType === 1 && /** @type {HTMLElement} */ (el).isContentEditable === true); + if (!isEditable) return; + if (this._focusedEl && this._focusedEl !== el) + this._focusedEl.removeEventListener('input', this._onInput, true); + el.addEventListener('input', this._onInput, true); + this._focusedEl = el; + } + + _onFocusOut(e) { + const el = e.target; + if (!el) return; + if (this._focusedEl === el) { + el.removeEventListener('input', this._onInput, true); + this._focusedEl = null; + } + } + _onMouseMove(e) { if (!this.highlightEnabled || !this.ui._box || !this.isRecording || this.isPaused) return; if (this.hoverRAF) return; @@ -740,6 +810,108 @@ } }, CONFIG.SCROLL_DEBOUNCE_MS); } + + // Minimal key recorder: record Enter and modifier combos; avoid plain typing + _onKeyDown(e) { + if (!this.isRecording || this.isPaused) return; + try { + // Ignore autorepeat to prevent spam + if (e.repeat) return; + const key = String(e.key || '').toLowerCase(); + const isModifier = key === 'shift' || key === 'control' || key === 'meta' || key === 'alt'; + const isEditable = + e.target instanceof HTMLInputElement || + e.target instanceof HTMLTextAreaElement || + (e.target && + e.target.nodeType === 1 && + /** @type {HTMLElement} */ (e.target).isContentEditable === true); + const enterKey = key === 'enter'; + + // Track pressed modifiers + if (isModifier) this._pressed.add(key); + + // Handle Enter in editable contexts (including contenteditable) + if (isEditable && enterKey) { + // prevent duplicate with input handler; record explicit key action with target + const target = SelectorEngine.buildTarget(/** @type {Element} */ (e.target)); + const combo = this._formatKeysCombo(e, 'Enter'); + this._pushStep({ type: 'key', keys: combo, target, screenshotOnFail: false }); + this._lastKeyTs = Date.now(); + return; + } + + // For non-text fields: record modifier combos and special keys + const special = enterKey || key === 'escape' || key === 'tab'; + if (special || e.ctrlKey || e.metaKey || e.altKey || e.shiftKey) { + const comboName = this._formatKeysCombo(e, e.key); + this._pushStep({ type: 'key', keys: comboName, screenshotOnFail: false }); + this._lastKeyTs = Date.now(); + } + } catch {} + } + + _onKeyUp(e) { + const key = String(e.key || '').toLowerCase(); + if (key === 'shift' || key === 'control' || key === 'meta' || key === 'alt') + this._pressed.delete(key); + } + + _formatKeysCombo(e, mainKey) { + const parts = []; + if (e.ctrlKey) parts.push('Ctrl'); + if (e.altKey) parts.push('Alt'); + if (e.shiftKey) parts.push('Shift'); + if (e.metaKey) parts.push('Meta'); + const mk = String(mainKey || '').trim(); + // Normalize common names to match keyboard-helper parsing + const norm = (s) => { + const k = s.toLowerCase(); + if (k === 'escape') return 'Esc'; + if (k === ' ') return 'Space'; + if (k.length === 1) return k.toUpperCase(); + return s; + }; + parts.push(norm(mk)); + return parts.join('+'); + } + + // Top-level aggregator: receives iframe events and merges into session + _onWindowMessage(ev) { + try { + const d = ev && ev.data; + if (!d || d.type !== FRAME_EVENT || !d.payload) return; + const { step, href } = d.payload || {}; + if (!step || typeof step !== 'object') return; + + // Identify iframe element by event.source + let frameEl = null; + try { + const frames = document.querySelectorAll('iframe,frame'); + for (let i = 0; i < frames.length; i++) { + const f = frames[i]; + if (f && f.contentWindow === ev.source) { + frameEl = f; + break; + } + } + } catch {} + + // Stateless: compose composite selector and push single step + if (frameEl && step && step.target) { + const frameTarget = SelectorEngine.buildTarget(frameEl); + const frameSel = frameTarget?.selector || ''; + const inner = String(step.target.selector || '').trim(); + if (frameSel && inner) { + const composite = `${frameSel} |> ${inner}`; + step.target.selector = composite; + if (Array.isArray(step.target.candidates)) { + step.target.candidates.unshift({ type: 'css', value: composite }); + } + } + this._pushStep(step); + } + } catch {} + } } // ================================================================