opensource/mcp-chrome
3
0
Fork 0
mirror of https://github.com/hangwin/mcp-chrome.git synced 2026-09-24 23:24:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: 录制过程性能优化

Browse Source
This commit is contained in:
hangerye
2025-10-26 19:24:17 +08:00
parent 2f12cd60c2
commit 27e78c6561
6 changed files with 536 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/chrome-extension/entrypoints/background/record-replay/node-registry.ts
+36 -35
View File
@@ -262,15 +262,17 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} });
const s: any = expandTemplatesDeep(step as any, ctx.vars);
const located = await locateElement(tabId, s.target, ctx.frameId);
const frameId = (located as any)?.frameId ?? ctx.frameId;
const frameId = (located as any)?.frameId ?? ctx.frameId;
const first = s.target?.candidates?.[0]?.type;
const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : '');
const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first;
if ((located as any)?.ref) {
const resolved = await chrome.tabs.sendMessage(
const resolved: any = (await chrome.tabs.sendMessage(
tabId,
{ action: 'resolveRef', ref: (located as any).ref } as any,
{ frameId: ctx.frameId } as any,
);
{ frameId } as any,
)) as any;
const rect = resolved?.rect;
if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible');
}
@@ -283,6 +285,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
: undefined,
waitForNavigation: false,
timeout: Math.max(1000, Math.min(s.timeoutMs || 10000, 30000)),
frameId,
},
});
if ((res as any).isError) throw new Error('click failed');
@@ -310,15 +313,16 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} });
const s: any = expandTemplatesDeep(step as any, ctx.vars);
const located = await locateElement(tabId, s.target, ctx.frameId);
const frameId = (located as any)?.frameId ?? ctx.frameId;
const first = s.target?.candidates?.[0]?.type;
const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : '');
const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first;
if ((located as any)?.ref) {
const resolved = await chrome.tabs.sendMessage(
const resolved: any = (await chrome.tabs.sendMessage(
tabId,
{ action: 'resolveRef', ref: (located as any).ref } as any,
{ frameId: ctx.frameId } as any,
);
{ frameId } as any,
)) as any;
const rect = resolved?.rect;
if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible');
}
@@ -351,6 +355,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
if (!tabId) throw new Error('Active tab not found');
await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} });
const located = await locateElement(tabId, s.target, ctx.frameId);
const frameId = (located as any)?.frameId ?? ctx.frameId;
const first = s.target?.candidates?.[0]?.type;
const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : '');
const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first;
@@ -361,11 +366,11 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
: v;
const value = interpolate(s.value);
if ((located as any)?.ref) {
const resolved = await chrome.tabs.sendMessage(
const resolved: any = (await chrome.tabs.sendMessage(
tabId,
{ action: 'resolveRef', ref: (located as any).ref } as any,
{ frameId: ctx.frameId } as any,
);
{ frameId } as any,
)) as any;
const rect = resolved?.rect;
if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible');
}
@@ -376,15 +381,15 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
: undefined;
if (cssSelector) {
try {
const attr = await chrome.tabs.sendMessage(
const attr: any = (await chrome.tabs.sendMessage(
tabId,
{
action: 'getAttributeForSelector',
selector: cssSelector,
name: 'type',
} as any,
{ frameId: ctx.frameId } as any,
);
{ frameId } as any,
)) as any;
const typeName = (attr && attr.value ? String(attr.value) : '').toLowerCase();
if (typeName === 'file') {
const uploadRes = await handleCallTool({
@@ -423,7 +428,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
await chrome.tabs.sendMessage(
tabId,
{ action: 'focusByRef', ref: (located as any).ref } as any,
{ frameId: ctx.frameId } as any,
{ frameId } as any,
);
else if (cssSelector)
await handleCallTool({
@@ -440,6 +445,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
ref: (located as any)?.ref || s.target?.ref,
selector: cssSelector,
value,
frameId,
},
});
if ((res as any).isError) throw new Error('fill failed');
@@ -486,7 +492,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
files: ['inject-scripts/wait-helper.js'],
world: 'ISOLATED',
} as any);
const resp = await chrome.tabs.sendMessage(
const resp: any = (await chrome.tabs.sendMessage(
tabId,
{
action: 'waitForText',
@@ -495,7 +501,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)),
} as any,
{ frameId: ctx.frameId } as any,
);
)) as any;
if (!resp || resp.success !== true) throw new Error('wait text failed');
} else if ('networkIdle' in cond) {
const total = Math.min(Math.max(1000, (s as any).timeoutMs || 5000), 120000);
@@ -515,12 +521,12 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
files: ['inject-scripts/wait-helper.js'],
world: 'ISOLATED',
} as any);
const resp = await chrome.tabs.sendMessage(tabId, {
const resp: any = (await chrome.tabs.sendMessage(tabId, {
action: 'waitForSelector',
selector: cond.selector,
visible: cond.visible !== false,
timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)),
} as any);
} as any)) as any;
if (!resp || resp.success !== true) throw new Error('wait selector failed');
}
},
@@ -562,10 +568,10 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined;
if (!tabId) return fail('Active tab not found');
await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} });
const ensured = await chrome.tabs.sendMessage(tabId, {
const ensured: any = (await chrome.tabs.sendMessage(tabId, {
action: 'ensureRefForSelector',
selector,
} as any);
} as any)) as any;
if (!ensured || !ensured.success) return fail('assert selector not found');
if ('visible' in s.assert) {
const rect = ensured && ensured.center ? ensured.center : null;
@@ -578,11 +584,11 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined;
if (!tabId) return fail('Active tab not found');
await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} });
const resp = await chrome.tabs.sendMessage(
const resp: any = (await chrome.tabs.sendMessage(
tabId,
{ action: 'getAttributeForSelector', selector, name } as any,
{ frameId: ctx.frameId } as any,
);
)) as any;
if (!resp || !resp.success) return fail('assert attribute: element not found');
const actual: string | null = resp.value ?? null;
if (equals !== undefined && equals !== null) {
@@ -859,11 +865,11 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
let sel = cssSelector as string | undefined;
if (!sel && (located as any)?.ref) {
try {
const resolved = await chrome.tabs.sendMessage(
const resolved: any = (await chrome.tabs.sendMessage(
tabId,
{ action: 'resolveRef', ref: (located as any).ref } as any,
{ frameId: ctx.frameId } as any,
);
{ frameId } as any,
)) as any;
sel = resolved?.selector;
} catch {}
}
@@ -873,10 +879,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
const bubbles = s.bubbles !== false;
const cancelable = s.cancelable === true;
await chrome.scripting.executeScript({
target: {
tabId,
frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined,
} as any,
target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any,
world,
func: (selector: string, type: string, bubbles: boolean, cancelable: boolean) => {
try {
@@ -907,17 +910,18 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
if (typeof tabId !== 'number') throw new Error('Active tab not found');
await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} });
const located = await locateElement(tabId, s.target, ctx.frameId);
const frameId = (located as any)?.frameId ?? ctx.frameId;
const cssSelector = !(located as any)?.ref
? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value
: undefined;
let sel = cssSelector as string | undefined;
if (!sel && (located as any)?.ref) {
try {
const resolved = await chrome.tabs.sendMessage(
const resolved: any = (await chrome.tabs.sendMessage(
tabId,
{ action: 'resolveRef', ref: (located as any).ref } as any,
{ frameId: ctx.frameId } as any,
);
{ frameId } as any,
)) as any;
sel = resolved?.selector;
} catch {}
}
@@ -927,10 +931,7 @@ const registry: Partial<Record<Step['type'], NodeRuntime<any>>> = {
const value = s.value != null ? String(s.value) : null;
const remove = s.remove === true || value == null;
await chrome.scripting.executeScript({
target: {
tabId,
frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined,
} as any,
target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any,
world,
func: (selector: string, name: string, value: string | null, remove: boolean) => {
try {
app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts
+67 -38
View File
@@ -7,6 +7,57 @@ export interface LocatedElement {
ref?: string;
center?: { x: number; y: number };
resolvedBy?: 'ref' | SelectorCandidate['type'];
frameId?: number;
}
// Helper: decide whether selector is a composite cross-frame selector
function isCompositeSelector(sel: string): boolean {
return typeof sel === 'string' && sel.includes('|>');
}
// Helper: typed wrapper for chrome.tabs.sendMessage with optional frameId
async function sendToTab(tabId: number, message: any, frameId?: number): Promise<any> {
if (typeof frameId === 'number') {
return await chrome.tabs.sendMessage(tabId, message, { frameId });
}
return await chrome.tabs.sendMessage(tabId, message);
}
// Helper: ensure ref for a selector, handling composite selectors and mapping frameId
async function ensureRefForSelector(
tabId: number,
selector: string,
frameId?: number,
): Promise<{ ref: string; center: { x: number; y: number }; frameId?: number } | null> {
try {
let ensured: any = null;
if (isCompositeSelector(selector)) {
// Always query top for composite; helper will bridge to child and return href
ensured = await sendToTab(tabId, {
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
selector,
});
} else {
ensured = await sendToTab(
tabId,
{ action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector },
frameId,
);
}
if (!ensured || !ensured.success || !ensured.ref || !ensured.center) return null;
// Map frameId when composite via returned href
let locFrameId: number | undefined = undefined;
if (isCompositeSelector(selector) && ensured.href) {
try {
const frames = (await chrome.webNavigation.getAllFrames({ tabId })) as any[];
const match = frames?.find((f) => typeof f.url === 'string' && f.url === ensured.href);
if (match) locFrameId = match.frameId;
} catch {}
}
return { ref: ensured.ref, center: ensured.center, frameId: locFrameId };
} catch {
return null;
}
}
/**
@@ -20,15 +71,8 @@ export async function locateElement(
// 0) Fast path: try primary selector if provided
const primarySel = (target as any)?.selector ? String((target as any).selector).trim() : '';
if (primarySel) {
try {
const ensured = await chrome.tabs.sendMessage(
tabId,
{ action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector: primarySel } as any,
{ frameId } as any,
);
if (ensured && ensured.success && ensured.ref && ensured.center)
return { ref: ensured.ref, center: ensured.center, resolvedBy: 'css' };
} catch {}
const ensured = await ensureRefForSelector(tabId, primarySel, frameId);
if (ensured) return { ...ensured, resolvedBy: 'css' };
}
// 1) Non-text candidates first for stability (css/attr/aria/xpath)
@@ -36,17 +80,8 @@ export async function locateElement(
for (const c of nonText) {
try {
if (c.type === 'css' || c.type === 'attr') {
const ensured = await chrome.tabs.sendMessage(
tabId,
{
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
selector: c.value,
} as any,
{ frameId } as any,
);
if (ensured && ensured.success && ensured.ref && ensured.center) {
return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type };
}
const ensured = await ensureRefForSelector(tabId, String(c.value || ''), frameId);
if (ensured) return { ...ensured, resolvedBy: c.type };
} else if (c.type === 'aria') {
// Minimal ARIA role+name parser like: "button[name=提交]" or "textbox[name=用户名]"
const v = String(c.value || '').trim();
@@ -78,31 +113,28 @@ export async function locateElement(
);
}
for (const sel of ariaSelectors) {
const ensured = await chrome.tabs.sendMessage(
const ensured = await sendToTab(
tabId,
{
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
selector: sel,
} as any,
{ frameId } as any,
{ action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, selector: sel } as any,
frameId,
);
if (ensured && ensured.success && ensured.ref && ensured.center) {
return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type };
return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type, frameId };
}
}
} else if (c.type === 'xpath') {
// Minimal xpath support via document.evaluate through injected helper
const ensured = await chrome.tabs.sendMessage(
const ensured = await sendToTab(
tabId,
{
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
selector: c.value,
isXPath: true,
} as any,
{ frameId } as any,
frameId,
);
if (ensured && ensured.success && ensured.ref && ensured.center) {
return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type };
return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type, frameId };
}
}
} catch (e) {
@@ -114,7 +146,7 @@ export async function locateElement(
const tagName = ((target as any)?.tag || '').toString();
for (const c of textCands) {
try {
const ensured = await chrome.tabs.sendMessage(
const ensured = await sendToTab(
tabId,
{
action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR,
@@ -122,7 +154,7 @@ export async function locateElement(
text: c.value,
tagName,
} as any,
{ frameId } as any,
frameId,
);
if (ensured && ensured.success && ensured.ref && ensured.center) {
return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type };
@@ -132,13 +164,10 @@ export async function locateElement(
// Fallback: try ref (works when ref was produced in the same page lifecycle)
if (target.ref) {
try {
const res = await chrome.tabs.sendMessage(
const res = await sendToTab(
tabId,
{
action: TOOL_MESSAGE_TYPES.RESOLVE_REF,
ref: target.ref,
} as any,
{ frameId } as any,
{ action: TOOL_MESSAGE_TYPES.RESOLVE_REF, ref: target.ref } as any,
frameId,
);
if (res && res.success && res.center) {
return { ref: target.ref, center: res.center, resolvedBy: 'ref' };
app/chrome-extension/entrypoints/background/tools/base-browser.ts
+5 -2
View File
@@ -72,9 +72,12 @@ export abstract class BaseBrowserToolExecutor implements ToolExecutor {
/**
* Send message to tab
*/
protected async sendMessageToTab(tabId: number, message: any): Promise<any> {
protected async sendMessageToTab(tabId: number, message: any, frameId?: number): Promise<any> {
try {
const response = await chrome.tabs.sendMessage(tabId, message);
const response =
typeof frameId === 'number'
? await chrome.tabs.sendMessage(tabId, message, { frameId })
: await chrome.tabs.sendMessage(tabId, message);
if (response && response.error) {
throw new Error(String(response.error));
app/chrome-extension/entrypoints/background/tools/browser/interaction.ts
+26 -15
View File
@@ -15,6 +15,7 @@ interface ClickToolParams {
coordinates?: Coordinates; // Coordinates to click at (x, y relative to viewport)
waitForNavigation?: boolean; // Whether to wait for navigation to complete after click
timeout?: number; // Timeout in milliseconds for waiting for the element or navigation
frameId?: number; // Target frame for ref/selector resolution
}
/**
@@ -32,6 +33,7 @@ class ClickTool extends BaseBrowserToolExecutor {
coordinates,
waitForNavigation = false,
timeout = TIMEOUTS.DEFAULT_WAIT * 5,
frameId,
} = args;
console.log(`Starting click operation with options:`, args);
@@ -57,14 +59,18 @@ class ClickTool extends BaseBrowserToolExecutor {
await this.injectContentScript(tab.id, ['inject-scripts/click-helper.js']);
// Send click message to content script
const result = await this.sendMessageToTab(tab.id, {
action: TOOL_MESSAGE_TYPES.CLICK_ELEMENT,
selector,
coordinates,
ref: args.ref,
waitForNavigation,
timeout,
});
const result = await this.sendMessageToTab(
tab.id,
{
action: TOOL_MESSAGE_TYPES.CLICK_ELEMENT,
selector,
coordinates,
ref: args.ref,
waitForNavigation,
timeout,
},
frameId,
);
return {
content: [
@@ -97,6 +103,7 @@ interface FillToolParams {
ref?: string; // Element ref from accessibility tree
// Accept string | number | boolean for broader form input coverage
value: string | number | boolean;
frameId?: number;
}
/**
@@ -109,7 +116,7 @@ class FillTool extends BaseBrowserToolExecutor {
* Execute fill operation
*/
async execute(args: FillToolParams): Promise<ToolResult> {
const { selector, ref, value } = args;
const { selector, ref, value, frameId } = args;
console.log(`Starting fill operation with options:`, args);
@@ -136,12 +143,16 @@ class FillTool extends BaseBrowserToolExecutor {
await this.injectContentScript(tab.id, ['inject-scripts/fill-helper.js']);
// Send fill message to content script
const result = await this.sendMessageToTab(tab.id, {
action: TOOL_MESSAGE_TYPES.FILL_ELEMENT,
selector,
ref,
value,
});
const result = await this.sendMessageToTab(
tab.id,
{
action: TOOL_MESSAGE_TYPES.FILL_ELEMENT,
selector,
ref,
value,
},
frameId,
);
if (result && result.error) {
return createErrorResponse(result.error);
app/chrome-extension/inject-scripts/accessibility-tree-helper.js
+223
View File
@@ -781,6 +781,86 @@
}
if (request && request.action === 'ensureRefForSelector') {
try {
// Composite selector support: "frameSelector |> innerSelector"
const maybeSel = String(request.selector || '').trim();
if (maybeSel.includes('|>')) {
try {
const parts = maybeSel
.split('|>')
.map((s) => s.trim())
.filter(Boolean);
if (parts.length >= 2) {
const frameSel = parts[0];
const innerSel = parts.slice(1).join(' |> ');
// Find target frame element in current document
let frameEl = null;
try {
frameEl = querySelectorDeepFirst(frameSel) || document.querySelector(frameSel);
} catch {}
if (
!frameEl ||
!(frameEl instanceof HTMLIFrameElement || frameEl instanceof HTMLFrameElement)
) {
sendResponse({
success: false,
error: `Composite frame selector not found: ${frameSel}`,
});
return true;
}
const cw = frameEl.contentWindow;
if (!cw) {
sendResponse({
success: false,
error: 'Unable to obtain contentWindow of target frame',
});
return true;
}
// Bridge to child frame via postMessage
const reqId = `rrc_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`;
const listener = (ev) => {
try {
const data = ev && ev.data;
if (
!data ||
data.type !== 'rr-bridge-ensure-ref-result' ||
data.reqId !== reqId
)
return;
window.removeEventListener('message', listener, true);
if (data.success) {
sendResponse({
success: true,
ref: data.ref,
center: data.center,
href: data.href,
});
} else {
sendResponse({ success: false, error: data.error || 'child failed' });
}
} catch (e) {
window.removeEventListener('message', listener, true);
sendResponse({ success: false, error: String(e && e.message ? e.message : e) });
}
};
window.addEventListener('message', listener, true);
cw.postMessage(
{
type: 'rr-bridge-ensure-ref',
reqId,
selector: innerSel,
useText: !!request.useText,
isXPath: !!request.isXPath,
tagName: String(request.tagName || ''),
},
'*',
);
return true; // async response via message bridge
}
} catch (e) {
sendResponse({ success: false, error: String(e && e.message ? e.message : e) });
return true;
}
}
// Support CSS selector, XPath, or visible text search
const useText = !!request.useText;
const textQuery = String(request.text || '').trim();
@@ -1210,4 +1290,147 @@
});
console.log('Accessibility tree helper script loaded');
// Cross-frame bridge: child listens for ensure-ref requests from parent (composite selector)
try {
window.addEventListener(
'message',
(ev) => {
try {
const data = ev && ev.data;
if (!data || data.type !== 'rr-bridge-ensure-ref') return;
const { reqId, selector, useText, isXPath, tagName } = data || {};
const respond = (payload) => {
try {
ev.source &&
ev.source.postMessage(
{ type: 'rr-bridge-ensure-ref-result', reqId, ...payload },
'*',
);
} catch {}
};
try {
const sel = String(selector || '').trim();
const limitTag = String(tagName || '')
.trim()
.toUpperCase();
let el = null;
if (useText && sel) {
const normalize = (s) =>
String(s || '')
.replace(/\s+/g, ' ')
.trim()
.toLowerCase();
const query = normalize(sel);
const bigrams = (s) => {
const arr = [];
for (let i = 0; i < s.length - 1; i++) arr.push(s.slice(i, i + 2));
return arr;
};
const dice = (a, b) => {
if (!a || !b) return 0;
const A = bigrams(a),
B = bigrams(b);
if (!A.length || !B.length) return 0;
let inter = 0;
const m = new Map();
for (const t of A) m.set(t, (m.get(t) || 0) + 1);
for (const t of B) {
const c = m.get(t) || 0;
if (c > 0) {
inter++;
m.set(t, c - 1);
}
}
return (2 * inter) / (A.length + B.length);
};
let best = { el: null, score: 0 };
const stack = [document.documentElement];
while (stack.length) {
const node = stack.pop();
if (!node || !(node instanceof Element)) continue;
try {
if (limitTag && String(node.tagName || '').toUpperCase() !== limitTag) {
} else {
const cs = window.getComputedStyle(node);
if (cs.display !== 'none' && cs.visibility !== 'hidden' && cs.opacity !== '0') {
const rect = node.getBoundingClientRect();
if (rect.width > 0 && rect.height > 0) {
const txt = normalize(node.textContent || '');
if (txt) {
if (txt.includes(query)) {
el = node;
break;
}
const sc = dice(txt, query);
if (sc > best.score) best = { el: node, score: sc };
}
}
}
}
} catch {}
try {
const children = node.children || [];
for (let i = 0; i < children.length; i++) stack.push(children[i]);
const sr = node.shadowRoot;
if (sr && sr.children)
for (let i = 0; i < sr.children.length; i++) stack.push(sr.children[i]);
} catch {}
}
if (!el && best.el) el = best.el;
} else if (isXPath) {
try {
const it = document.evaluate(
sel,
document,
null,
XPathResult.FIRST_ORDERED_NODE_TYPE,
null,
);
el = it.singleNodeValue instanceof Element ? it.singleNodeValue : null;
} catch {}
} else {
try {
el =
document.querySelector(sel) ||
(typeof querySelectorDeepFirst === 'function'
? querySelectorDeepFirst(sel)
: null);
} catch {}
}
if (!el || !(el instanceof Element)) {
respond({ success: false, error: 'Element not found in child frame' });
return;
}
if (!window.__claudeElementMap) window.__claudeElementMap = {};
if (!window.__claudeRefCounter) window.__claudeRefCounter = 0;
let refId = null;
for (const k in window.__claudeElementMap) {
const w = window.__claudeElementMap[k];
if (w && typeof w.deref === 'function' && w.deref && w.deref() === el) {
refId = k;
break;
}
}
if (!refId) {
refId = `ref_${++window.__claudeRefCounter}`;
window.__claudeElementMap[refId] = new WeakRef(el);
}
const rect = el.getBoundingClientRect();
respond({
success: true,
ref: refId,
center: {
x: Math.round(rect.left + rect.width / 2),
y: Math.round(rect.top + rect.height / 2),
},
href: String(location && location.href ? location.href : ''),
});
} catch (e) {
respond({ success: false, error: String(e && e.message ? e.message : e) });
}
} catch {}
},
true,
);
} catch {}
})();
app/chrome-extension/inject-scripts/recorder.js
+179 -7
View File
@@ -15,6 +15,8 @@
SENSITIVE_INPUT_TYPES: new Set(['password']),
UI_MAX_STEPS: 30,
};
// Cross-frame event channel
const FRAME_EVENT = 'rr_iframe_event';
// Memoization caches for selector computations during recording
const __cacheUnique = new WeakMap();
@@ -365,8 +367,21 @@
this._onChange = this._onChange.bind(this);
this._onMouseMove = this._onMouseMove.bind(this);
this._onScroll = this._onScroll.bind(this);
this._onFocusIn = this._onFocusIn.bind(this);
this._onFocusOut = this._onFocusOut.bind(this);
this._onKeyDown = this._onKeyDown.bind(this);
this._onKeyUp = this._onKeyUp.bind(this);
this._onWindowMessage = this._onWindowMessage.bind(this);
this.ui = new UI(this);
this._scrollPending = null;
// Focus tracking for per-element input listening
this._focusedEl = null;
// Keyboard state for combo recording
this._pressed = new Set();
this._lastKeyTs = 0;
// Map to avoid duplicate switchFrame per iframe source (keyed by frame selector)
this._frameSwitchMap = new Set();
}
// Lifecycle
@@ -394,8 +409,12 @@
this.isRecording = false;
this._detach();
this.ui.remove();
if (this.batchTimer) clearTimeout(this.batchTimer);
this.batchTimer = null;
if (this.scrollTimer) clearTimeout(this.scrollTimer);
this.scrollTimer = null;
if (this.hoverRAF) cancelAnimationFrame(this.hoverRAF);
this.hoverRAF = 0;
this.lastFill = { step: null, ts: 0 };
const ret = this.sessionBuffer;
this.sessionBuffer.steps = [];
@@ -419,19 +438,40 @@
// DOM listeners
_attach() {
document.addEventListener('click', this._onClick, true);
document.addEventListener('input', this._onInput, true);
// Use focusin/out to attach input listener only to focused element
document.addEventListener('focusin', this._onFocusIn, true);
document.addEventListener('focusout', this._onFocusOut, true);
document.addEventListener('change', this._onChange, true);
// capture-phase scroll to catch non-bubbling events on any container
document.addEventListener('scroll', this._onScroll, true);
// Keyboard: record Enter and modifier combos
document.addEventListener('keydown', this._onKeyDown, true);
document.addEventListener('keyup', this._onKeyUp, true);
// Cross-frame: top window aggregates iframe-recorded steps
if (window === window.top) window.addEventListener('message', this._onWindowMessage, true);
this._updateHoverListener();
}
_detach() {
document.removeEventListener('click', this._onClick, true);
document.removeEventListener('input', this._onInput, true);
document.removeEventListener('focusin', this._onFocusIn, true);
document.removeEventListener('focusout', this._onFocusOut, true);
document.removeEventListener('change', this._onChange, true);
document.removeEventListener('scroll', this._onScroll, true);
document.removeEventListener('keydown', this._onKeyDown, true);
document.removeEventListener('keyup', this._onKeyUp, true);
document.removeEventListener('mousemove', this._onMouseMove, true);
if (window === window.top) window.removeEventListener('message', this._onWindowMessage, true);
// Detach per-element input listener if any
if (this._focusedEl) this._focusedEl.removeEventListener('input', this._onInput, true);
this._focusedEl = null;
// Best-effort cleanup for timers/raf when detaching
if (this.batchTimer) clearTimeout(this.batchTimer);
this.batchTimer = null;
if (this.scrollTimer) clearTimeout(this.scrollTimer);
this.scrollTimer = null;
if (this.hoverRAF) cancelAnimationFrame(this.hoverRAF);
this.hoverRAF = 0;
}
_updateHoverListener() {
@@ -470,12 +510,19 @@
_pushStep(step) {
step.id = step.id || `step_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`;
if (window !== window.top && !this.frameSwitchPushed) {
const href = String(location && location.href ? location.href : '');
const frameStep = { type: 'switchFrame', frame: { urlContains: href } };
this.sessionBuffer.steps.push(frameStep);
this.frameSwitchPushed = true;
// In iframes, forward to top for aggregation (compute frame selector there)
if (window !== window.top) {
try {
const payload = {
kind: 'iframeStep',
href: String(location && location.href ? location.href : ''),
step,
};
window.top.postMessage({ type: FRAME_EVENT, payload }, '*');
return; // Do not push locally in subframe
} catch {}
}
// Top window: optionally insert a switchFrame if this step originated from an iframe message
this.sessionBuffer.steps.push(step);
this.sessionBuffer.meta.updatedAt = new Date().toISOString();
this.batch.push(step);
@@ -648,6 +695,29 @@
// UI handled by injected UI class
_onFocusIn(e) {
if (!this.isRecording || this.isPaused) return;
const el = e.target;
const isEditable =
el instanceof HTMLInputElement ||
el instanceof HTMLTextAreaElement ||
(el && el.nodeType === 1 && /** @type {HTMLElement} */ (el).isContentEditable === true);
if (!isEditable) return;
if (this._focusedEl && this._focusedEl !== el)
this._focusedEl.removeEventListener('input', this._onInput, true);
el.addEventListener('input', this._onInput, true);
this._focusedEl = el;
}
_onFocusOut(e) {
const el = e.target;
if (!el) return;
if (this._focusedEl === el) {
el.removeEventListener('input', this._onInput, true);
this._focusedEl = null;
}
}
_onMouseMove(e) {
if (!this.highlightEnabled || !this.ui._box || !this.isRecording || this.isPaused) return;
if (this.hoverRAF) return;
@@ -740,6 +810,108 @@
}
}, CONFIG.SCROLL_DEBOUNCE_MS);
}
// Minimal key recorder: record Enter and modifier combos; avoid plain typing
_onKeyDown(e) {
if (!this.isRecording || this.isPaused) return;
try {
// Ignore autorepeat to prevent spam
if (e.repeat) return;
const key = String(e.key || '').toLowerCase();
const isModifier = key === 'shift' || key === 'control' || key === 'meta' || key === 'alt';
const isEditable =
e.target instanceof HTMLInputElement ||
e.target instanceof HTMLTextAreaElement ||
(e.target &&
e.target.nodeType === 1 &&
/** @type {HTMLElement} */ (e.target).isContentEditable === true);
const enterKey = key === 'enter';
// Track pressed modifiers
if (isModifier) this._pressed.add(key);
// Handle Enter in editable contexts (including contenteditable)
if (isEditable && enterKey) {
// prevent duplicate with input handler; record explicit key action with target
const target = SelectorEngine.buildTarget(/** @type {Element} */ (e.target));
const combo = this._formatKeysCombo(e, 'Enter');
this._pushStep({ type: 'key', keys: combo, target, screenshotOnFail: false });
this._lastKeyTs = Date.now();
return;
}
// For non-text fields: record modifier combos and special keys
const special = enterKey || key === 'escape' || key === 'tab';
if (special || e.ctrlKey || e.metaKey || e.altKey || e.shiftKey) {
const comboName = this._formatKeysCombo(e, e.key);
this._pushStep({ type: 'key', keys: comboName, screenshotOnFail: false });
this._lastKeyTs = Date.now();
}
} catch {}
}
_onKeyUp(e) {
const key = String(e.key || '').toLowerCase();
if (key === 'shift' || key === 'control' || key === 'meta' || key === 'alt')
this._pressed.delete(key);
}
_formatKeysCombo(e, mainKey) {
const parts = [];
if (e.ctrlKey) parts.push('Ctrl');
if (e.altKey) parts.push('Alt');
if (e.shiftKey) parts.push('Shift');
if (e.metaKey) parts.push('Meta');
const mk = String(mainKey || '').trim();
// Normalize common names to match keyboard-helper parsing
const norm = (s) => {
const k = s.toLowerCase();
if (k === 'escape') return 'Esc';
if (k === ' ') return 'Space';
if (k.length === 1) return k.toUpperCase();
return s;
};
parts.push(norm(mk));
return parts.join('+');
}
// Top-level aggregator: receives iframe events and merges into session
_onWindowMessage(ev) {
try {
const d = ev && ev.data;
if (!d || d.type !== FRAME_EVENT || !d.payload) return;
const { step, href } = d.payload || {};
if (!step || typeof step !== 'object') return;
// Identify iframe element by event.source
let frameEl = null;
try {
const frames = document.querySelectorAll('iframe,frame');
for (let i = 0; i < frames.length; i++) {
const f = frames[i];
if (f && f.contentWindow === ev.source) {
frameEl = f;
break;
}
}
} catch {}
// Stateless: compose composite selector and push single step
if (frameEl && step && step.target) {
const frameTarget = SelectorEngine.buildTarget(frameEl);
const frameSel = frameTarget?.selector || '';
const inner = String(step.target.selector || '').trim();
if (frameSel && inner) {
const composite = `${frameSel} |> ${inner}`;
step.target.selector = composite;
if (Array.isArray(step.target.candidates)) {
step.target.candidates.unshift({ type: 'css', value: composite });
}
}
this._pushStep(step);
}
} catch {}
}
}
// ================================================================