diff --git a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts index 30c711b..daeabdf 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts @@ -47,6 +47,63 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise + (options.args?.[v.key] == null || options.args?.[v.key] === '') && + (v.rules?.required || (v.default ?? '') === ''), + ); + if (needed.length > 0) { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.SEND_COMMAND_TO_INJECT_SCRIPT, + args: { eventName: 'collectVariables', payload: undefined }, + }); + // Fallback: if direct collectVariables without payload not supported, call with explicit variables + let values: Record | null = null; + try { + const t = (res?.content || []).find((c: any) => c.type === 'text')?.text; + const j = t ? JSON.parse(t) : null; + if (j && j.success && j.values) values = j.values; + } catch { + /* ignore */ + } + if (!values) { + const res2 = await chrome.tabs + .query({ active: true, currentWindow: true }) + .then(async (tabs) => { + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') return null; + return await chrome.tabs.sendMessage(tabId, { + action: 'collectVariables', + variables: needed, + } as any); + }); + if (res2 && res2.success && res2.values) values = res2.values; + } + if (values) Object.assign(vars, values); + } + } catch { + // ignore prompt failures + } + + // Init simple overlay for real-time logs + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) { + await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'init' } as any); + } + } catch { + /* ignore */ + } + // Binding enforcement: if bindings exist and no startUrl, verify current tab URL matches try { const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); @@ -302,6 +359,25 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise c.type === 'css' || c.type === 'attr', + )?.value + : undefined; + if (sel) { + await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { + type: 'MAIN', + jsScript: `try{var el=document.querySelector(${JSON.stringify(sel)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, + }, + }); + } + } catch { + /* ignore */ + } const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.CLICK, args: { @@ -343,6 +419,23 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise c.type === 'css' || c.type === 'attr')?.value + : undefined; + if (sel) { + await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { + type: 'MAIN', + jsScript: `try{var el=document.querySelector(${JSON.stringify(sel)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, + }, + }); + } + } catch { + /* ignore */ + } const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.FILL, args: { @@ -508,6 +601,18 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise 0) { while (pendingAfterScripts.length) { @@ -541,6 +646,18 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise ${e?.message || String(e)}`, + } as any); + } + } catch { + /* ignore */ + } if (step.screenshotOnFail !== false) { try { const shot = await handleCallTool({ @@ -584,6 +701,14 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { + try { + if (!currentRecording) return; + const tab = await chrome.tabs.get(activeInfo.tabId); + if (!tab) return; + // 确保该标签页注入并继续录制(不重置流) + try { + await ensureRecorderInjected(activeInfo.tabId); + await chrome.tabs.sendMessage(activeInfo.tabId, { + action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, + cmd: 'resume', + } as any); + } catch { + /* ignore */ + } + if (!currentRecording.flow) return; + const url = typeof tab.url === 'string' ? tab.url : ''; + if (url) { + currentRecording.flow.steps.push({ + id: `step_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`, + type: 'navigate', + url, + } as any); + currentRecording.flow.meta = { + ...(currentRecording.flow.meta || ({} as any)), + updatedAt: new Date().toISOString(), + } as any; + } + } catch { + // ignore + } + }); + + // 全局录制:监听顶级导航提交(含显式地址栏输入/刷新),在非链接点击触发时追加“导航”步骤 + chrome.webNavigation.onCommitted.addListener(async (details) => { + try { + if (!currentRecording) return; + if (details.frameId !== 0) return; // 仅记录顶级 frame + const tabId = details.tabId; + // 对于点击触发的 link 导航,不追加 navigate(已有点击+waitForNavigation 富化) + const t = details.transitionType as string | undefined; + const isLink = t === 'link'; + if (isLink) { + // 同时确保继续录制 + try { + await ensureRecorderInjected(tabId); + await chrome.tabs.sendMessage(tabId, { + action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, + cmd: 'resume', + } as any); + } catch { + /* ignore */ + } + return; + } + // 显式导航/刷新/地址栏输入等 + const shouldRecord = + t === 'reload' || + t === 'typed' || + t === 'generated' || + t === 'auto_bookmark' || + t === 'keyword'; + if (!shouldRecord) { + // 仍确保脚本在该页活跃,用于持续录制 + try { + await ensureRecorderInjected(tabId); + await chrome.tabs.sendMessage(tabId, { + action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, + cmd: 'resume', + } as any); + } catch { + /* ignore */ + } + return; + } + const tab = await chrome.tabs.get(tabId); + const url = typeof tab.url === 'string' ? tab.url : details.url; + if (!url || !currentRecording.flow) return; + currentRecording.flow.steps.push({ + id: `step_${Date.now()}_${Math.random().toString(36).slice(2, 6)}`, + type: 'navigate', + url, + } as any); + currentRecording.flow.meta = { + ...(currentRecording.flow.meta || ({} as any)), + updatedAt: new Date().toISOString(), + } as any; + // 确保继续录制 + try { + await ensureRecorderInjected(tabId); + await chrome.tabs.sendMessage(tabId, { + action: TOOL_MESSAGE_TYPES.RR_RECORDER_CONTROL, + cmd: 'resume', + } as any); + } catch { + /* ignore */ + } + } catch { + // ignore + } + }); } diff --git a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js index d81cdba..ae7eee5 100644 --- a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js +++ b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js @@ -474,6 +474,56 @@ sendResponse({ status: 'pong' }); return false; } + if (request && request.action === 'rr_overlay') { + try { + const cmd = request.cmd || 'init'; + let root = document.getElementById('__rr_overlay_root'); + if (!root) { + root = document.createElement('div'); + root.id = '__rr_overlay_root'; + Object.assign(root.style, { + position: 'fixed', + right: '8px', + bottom: '8px', + zIndex: 2_147_483_647, + maxWidth: '40vw', + maxHeight: '40vh', + overflow: 'auto', + background: 'rgba(0,0,0,0.6)', + color: '#fff', + fontFamily: + 'ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace', + fontSize: '12px', + padding: '8px', + borderRadius: '6px', + boxShadow: '0 2px 8px rgba(0,0,0,0.3)', + }); + const title = document.createElement('div'); + title.textContent = 'Record-Replay 运行日志'; + Object.assign(title.style, { fontWeight: 'bold', marginBottom: '6px' }); + const body = document.createElement('div'); + body.id = '__rr_overlay_body'; + root.appendChild(title); + root.appendChild(body); + document.documentElement.appendChild(root); + } + const body = document.getElementById('__rr_overlay_body'); + if (cmd === 'append' && body) { + const line = document.createElement('div'); + line.textContent = String(request.text || ''); + body.appendChild(line); + body.scrollTop = body.scrollHeight; + } + if (cmd === 'done' && root) { + root.style.opacity = '0.5'; + } + sendResponse({ success: true }); + return true; + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + } if (request && request.action === 'generateAccessibilityTree') { const result = __generateAccessibilityTree(request.filter || null); sendResponse({ success: true, ...result }); @@ -573,6 +623,28 @@ return true; } } + if (request && request.action === 'collectVariables') { + try { + const vars = Array.isArray(request.variables) ? request.variables : []; + const values = {}; + for (const v of vars) { + const key = String(v && v.key ? v.key : ''); + if (!key) continue; + const label = v.label || key; + const def = v.default || ''; + const promptText = `请输入参数 ${label} (${key})`; + // Note: prompt in page context; in some sites may be blocked by CSP + let val = window.prompt(promptText, def); + if (typeof val !== 'string') val = def; + values[key] = val; + } + sendResponse({ success: true, values }); + return true; + } catch (e) { + sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); + return true; + } + } if (request && request.action === 'resolveRef') { const ref = request.ref; try { diff --git a/app/chrome-extension/inject-scripts/recorder.js b/app/chrome-extension/inject-scripts/recorder.js index 9c814ca..e0a10f9 100644 --- a/app/chrome-extension/inject-scripts/recorder.js +++ b/app/chrome-extension/inject-scripts/recorder.js @@ -157,6 +157,28 @@ pushStep({ type: 'key', keys, screenshotOnFail: false }); } + // Composition IME events (record markers for analysis; playback is no-op via script step) + function onCompositionStart() { + if (!isRecording) return; + pushStep({ + type: 'script', + world: 'ISOLATED', + when: 'before', + code: '/* compositionstart */', + screenshotOnFail: false, + }); + } + function onCompositionEnd() { + if (!isRecording) return; + pushStep({ + type: 'script', + world: 'ISOLATED', + when: 'before', + code: '/* compositionend */', + screenshotOnFail: false, + }); + } + let lastScrollAt = 0; function onScroll(e) { if (!isRecording) return; @@ -206,6 +228,8 @@ document.addEventListener('input', onInput, true); document.addEventListener('keydown', onKeydown, true); document.addEventListener('keyup', onKeyup, true); + document.addEventListener('compositionstart', onCompositionStart, true); + document.addEventListener('compositionend', onCompositionEnd, true); window.addEventListener('scroll', onScroll, { passive: true }); document.addEventListener('mousedown', onMouseDown, true); document.addEventListener('mousemove', onMouseMove, true); @@ -218,6 +242,8 @@ document.removeEventListener('input', onInput, true); document.removeEventListener('keydown', onKeydown, true); document.removeEventListener('keyup', onKeyup, true); + document.removeEventListener('compositionstart', onCompositionStart, true); + document.removeEventListener('compositionend', onCompositionEnd, true); window.removeEventListener('scroll', onScroll, { passive: true }); document.removeEventListener('mousedown', onMouseDown, true); document.removeEventListener('mousemove', onMouseMove, true); @@ -268,6 +294,12 @@ start(request.meta || {}); sendResponse({ success: true }); return true; + } else if (cmd === 'resume') { + // Attach without resetting flow or sending start event + isRecording = true; + attach(); + sendResponse({ success: true }); + return true; } else if (cmd === 'stop') { const flow = stop(); sendResponse({ success: true, flow });