From 4436996c9915924a6ce36c57691dfe2a55613235 Mon Sep 17 00:00:00 2001 From: hangerye Date: Thu, 16 Oct 2025 10:10:06 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=8D=A1=E9=A1=BF=E9=97=AE=E9=A2=98?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../background/record-replay/index.ts | 126 +++++--- .../background/record-replay/rr-utils.ts | 43 ++- .../background/record-replay/runner.ts | 55 +++- .../record-replay/selector-engine.ts | 39 +-- .../entrypoints/popup/App.vue | 17 ++ .../components/builder/components/Canvas.vue | 6 + .../components/builder/components/Sidebar.vue | 6 + .../builder/components/nodes/node-util.ts | 8 + .../components/properties/PropertyDrag.vue | 24 ++ .../components/properties/PropertyScroll.vue | 99 ++++++ .../components/properties/SelectorEditor.vue | 24 +- .../components/builder/model/transforms.ts | 18 ++ .../components/builder/model/ui-nodes.ts | 18 ++ .../builder/store/useBuilderStore.ts | 2 + .../inject-scripts/recorder.js | 288 ++++++++++++++++-- packages/shared/src/rr-graph.ts | 30 ++ 16 files changed, 688 insertions(+), 115 deletions(-) create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyDrag.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyScroll.vue diff --git a/app/chrome-extension/entrypoints/background/record-replay/index.ts b/app/chrome-extension/entrypoints/background/record-replay/index.ts index 98fa2b3..407a8a9 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/index.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/index.ts @@ -97,17 +97,57 @@ async function rescheduleAlarms() { } async function ensureRecorderInjected(tabId: number): Promise { - // Inject helper and recorder scripts into all frames to aggregate same-origin iframes - await chrome.scripting.executeScript({ - target: { tabId, allFrames: true }, - files: ['inject-scripts/accessibility-tree-helper.js'], - world: 'ISOLATED', - } as any); - await chrome.scripting.executeScript({ - target: { tabId, allFrames: true }, - files: ['inject-scripts/recorder.js'], - world: 'ISOLATED', - } as any); + // Determine frames and selectively inject only where missing to minimize overhead. + let frames: Array<{ frameId: number } & Record> = []; + try { + frames = await chrome.webNavigation.getAllFrames({ tabId }); + } catch {} + if (!Array.isArray(frames) || frames.length === 0) frames = [{ frameId: 0 } as any]; + + const needHelper: number[] = []; + const needRecorder: number[] = []; + await Promise.all( + frames.map(async (f) => { + const frameId = (f as any).frameId ?? 0; + // ping helper + try { + const res = await chrome.tabs.sendMessage( + tabId, + { action: 'chrome_read_page_ping' } as any, + { frameId } as any, + ); + if (!res) needHelper.push(frameId); + } catch { + needHelper.push(frameId); + } + // ping recorder + try { + const res = await chrome.tabs.sendMessage( + tabId, + { action: 'rr_recorder_ping' } as any, + { frameId } as any, + ); + if (!res) needRecorder.push(frameId); + } catch { + needRecorder.push(frameId); + } + }), + ); + + if (needHelper.length > 0) { + await chrome.scripting.executeScript({ + target: { tabId, frameIds: needHelper }, + files: ['inject-scripts/accessibility-tree-helper.js'], + world: 'ISOLATED', + } as any); + } + if (needRecorder.length > 0) { + await chrome.scripting.executeScript({ + target: { tabId, frameIds: needRecorder }, + files: ['inject-scripts/recorder.js'], + world: 'ISOLATED', + } as any); + } } async function broadcastRecorderCommandToAllTabs(cmd: 'start' | 'stop' | 'resume' | 'pause') { @@ -286,43 +326,47 @@ export function initRecordReplayListeners() { } } catch {} } - } else if (message.payload?.kind === 'step') { + } else if (message.payload?.kind === 'step' || message.payload?.kind === 'steps') { // Guard: only aggregate when an active recording session is ongoing if (currentRecording && recordingActive) { - // 记录最近一次 click/dblclick 的索引和时间,用于后续 tabs.onUpdated 导航富化 - const step = message.payload.step as any; - if (step && (step.type === 'click' || step.type === 'dblclick')) { + const stepsArr: any[] = Array.isArray(message.payload.steps) + ? message.payload.steps + : [message.payload.step]; + for (const step of stepsArr) { + // 记录最近一次 click/dblclick 的索引和时间,用于后续 tabs.onUpdated 导航富化 + if (step && (step.type === 'click' || step.type === 'dblclick')) { + try { + const idx = currentRecording.flow?.steps?.length ?? 0; + lastClickIdx = idx; + lastClickTime = Date.now(); + } catch { + // ignore + } + } + // 统一在后台累积步骤,便于跨标签页聚合 try { - const idx = currentRecording.flow?.steps?.length ?? 0; - lastClickIdx = idx; - lastClickTime = Date.now(); + if (!currentRecording.flow) { + currentRecording.flow = { + id: `flow_${Date.now()}`, + name: '未命名录制', + version: 1, + steps: [], + variables: [], + meta: { + createdAt: new Date().toISOString(), + updatedAt: new Date().toISOString(), + }, + } as Flow; + } + currentRecording.flow.steps.push(step); + currentRecording.flow.meta = { + ...(currentRecording.flow.meta || ({} as any)), + updatedAt: new Date().toISOString(), + } as any; } catch { // ignore } } - // 统一在后台累积步骤,便于跨标签页聚合 - try { - if (!currentRecording.flow) { - currentRecording.flow = { - id: `flow_${Date.now()}`, - name: '未命名录制', - version: 1, - steps: [], - variables: [], - meta: { - createdAt: new Date().toISOString(), - updatedAt: new Date().toISOString(), - }, - } as Flow; - } - currentRecording.flow.steps.push(step); - currentRecording.flow.meta = { - ...(currentRecording.flow.meta || ({} as any)), - updatedAt: new Date().toISOString(), - } as any; - } catch { - // ignore - } } } else if (message.payload?.kind === 'stop') { // User clicked stop inside page overlay: finalize and persist diff --git a/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts b/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts index 4469217..a132353 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/rr-utils.ts @@ -56,21 +56,46 @@ export async function ensureTab(options: { tabTarget?: 'current' | 'new'; startUrl?: string; refresh?: boolean; -}) { +}): Promise<{ tabId: number; url?: string }> { const target = options.tabTarget || 'current'; const startUrl = options.startUrl; - const [active] = await chrome.tabs.query({ active: true, currentWindow: true }); + const isWebUrl = (u?: string | null) => !!u && /^(https?:|file:)/i.test(u); + + const tabs = await chrome.tabs.query({ currentWindow: true }); + const [active] = tabs.filter((t) => t.active); + if (target === 'new') { let urlToOpen = startUrl; - if (!urlToOpen) urlToOpen = active?.url || 'about:blank'; - await chrome.tabs.create({ url: urlToOpen, active: true }); - await new Promise((r) => setTimeout(r, 500)); - } else { - if (startUrl) - await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url: startUrl } }); - else if (options.refresh) + if (!urlToOpen) urlToOpen = isWebUrl(active?.url) ? active!.url! : 'about:blank'; + const created = await chrome.tabs.create({ url: urlToOpen, active: true }); + await new Promise((r) => setTimeout(r, 300)); + return { tabId: created.id!, url: created.url }; + } + + // current tab target + if (startUrl) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url: startUrl } }); + } else if (options.refresh) { + // only refresh if current tab is a web page + if (isWebUrl(active?.url)) await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { refresh: true } }); } + + // Re-evaluate active after potential navigation + const cur = (await chrome.tabs.query({ active: true, currentWindow: true }))[0]; + let tabId = cur?.id; + let url = cur?.url; + + // If still on extension/internal page and no startUrl, try switch to an existing web tab + if (!isWebUrl(url) && !startUrl) { + const candidate = tabs.find((t) => isWebUrl(t.url)); + if (candidate?.id) { + await chrome.tabs.update(candidate.id, { active: true }); + tabId = candidate.id; + url = candidate.url; + } + } + return { tabId: tabId!, url }; } export async function waitForNetworkIdle(totalTimeoutMs: number, idleThresholdMs: number) { diff --git a/app/chrome-extension/entrypoints/background/record-replay/runner.ts b/app/chrome-extension/entrypoints/background/record-replay/runner.ts index 38b7b92..d764935 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/runner.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/runner.ts @@ -49,16 +49,33 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise 0; + const nodes0 = hasDag0 ? (((flow as any).nodes || []) as any[]) : []; + const edges0 = hasDag0 ? (((flow as any).edges || []) as any[]) : []; + const defaultEdges0 = hasDag0 ? defaultEdgesOnly(edges0 as any) : []; + const order0 = hasDag0 ? topoOrder(nodes0 as any, defaultEdges0 as any) : []; + const steps0: Step[] = hasDag0 + ? order0.map((n) => mapDagNodeToStep(n as any)) + : ((flow.steps || []) as Step[]); + const nav = steps0.find((s: any) => s && (s as any).type === 'navigate') as any; + if (nav && typeof nav.url === 'string') derivedStartUrl = expandTemplatesDeep(nav.url, {}); + } catch {} + const ensured = await ensureTab({ tabTarget: options.tabTarget, - startUrl: options.startUrl, + startUrl: options.startUrl || derivedStartUrl, refresh: options.refresh, }); - // pre-load read_page to init bridges + // pre-load read_page to init bridges only when on a web page (avoid builder.html) try { - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const u = ensured?.url || ''; + if (/^(https?:|file:)/i.test(u)) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } } catch {} // collect required variables via overlay prompt @@ -267,6 +284,12 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { await waitForNavigation(timeoutMs, prevUrl); }; + const isWebUrl = (u?: string | null) => !!u && /^(https?:|file:)/i.test(String(u || '')); + const ensureReadPageIfWeb = async () => { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const url = tabs?.[0]?.url || ''; + if (isWebUrl(url)) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } + } catch {} + }; try { if (!hasDag) { @@ -346,6 +379,12 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { - // Try ref first - if (target.ref) { - try { - const res = await chrome.tabs.sendMessage( - tabId, - { - action: TOOL_MESSAGE_TYPES.RESOLVE_REF, - ref: target.ref, - } as any, - { frameId } as any, - ); - if (res && res.success && res.center) { - return { ref: target.ref, center: res.center, resolvedBy: 'ref' }; - } - } catch (e) { - // ignore and fallback - } - } - // Try candidates in order + // Prefer stable selectors (css/attr/aria/text/xpath); only use ref as a last resort. + // Rationale: ref is ephemeral across navigations/renders and should not be prioritized in replay. for (const c of target.candidates || []) { try { if (c.type === 'css' || c.type === 'attr') { @@ -126,6 +109,24 @@ export async function locateElement( // continue to next candidate } } + // Fallback: try ref (works when ref was produced in the same page lifecycle) + if (target.ref) { + try { + const res = await chrome.tabs.sendMessage( + tabId, + { + action: TOOL_MESSAGE_TYPES.RESOLVE_REF, + ref: target.ref, + } as any, + { frameId } as any, + ); + if (res && res.success && res.center) { + return { ref: target.ref, center: res.center, resolvedBy: 'ref' }; + } + } catch (e) { + // ignore + } + } return null; } diff --git a/app/chrome-extension/entrypoints/popup/App.vue b/app/chrome-extension/entrypoints/popup/App.vue index 05e6ed7..2f51c1f 100644 --- a/app/chrome-extension/entrypoints/popup/App.vue +++ b/app/chrome-extension/entrypoints/popup/App.vue @@ -1377,11 +1377,28 @@ onMounted(async () => { await checkSemanticEngineStatus(); setupServerStatusListener(); + // Auto-refresh workflows list when storage rr_flows changes + try { + const onChanged = (changes: any, area: string) => { + try { + if (area !== 'local') return; + if (Object.prototype.hasOwnProperty.call(changes || {}, 'rr_flows')) loadFlows(); + } catch {} + }; + chrome.storage.onChanged.addListener(onChanged); + (window as any).__rr_popup_onChanged = onChanged; + } catch {} }); onUnmounted(() => { stopModelStatusMonitoring(); stopSemanticEngineStatusPolling(); + try { + const fn = (window as any).__rr_popup_onChanged; + if (fn && chrome?.storage?.onChanged?.removeListener) { + chrome.storage.onChanged.removeListener(fn); + } + } catch {} }); diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue index d065672..7b3ff26 100644 --- a/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue @@ -410,6 +410,12 @@ defineExpose({ zoomIn, zoomOut, fitAll }); :deep(.icon-dblclick) { background: #fe5196; } +:deep(.icon-drag) { + background: #f97316; +} +:deep(.icon-scroll) { + background: #0ea5e9; +} :deep(.icon-openTab), :deep(.icon-switchTab), :deep(.icon-closeTab) { diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue index 1ebbb7c..b466628 100644 --- a/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue @@ -302,6 +302,12 @@ const filtered = computed(() => { background: #8ec5fc; color: #111; } +.icon-scroll { + background: #0ea5e9; +} +.icon-drag { + background: #f97316; +} .icon-assert { background: #16a34a; } diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/nodes/node-util.ts b/app/chrome-extension/entrypoints/popup/components/builder/components/nodes/node-util.ts index e51c323..2ee5252 100644 --- a/app/chrome-extension/entrypoints/popup/components/builder/components/nodes/node-util.ts +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/nodes/node-util.ts @@ -24,6 +24,8 @@ import ILucideBell from '~icons/lucide/bell'; import ILucideWrench from '~icons/lucide/wrench'; import ILucideFrame from '~icons/lucide/frame'; import ILucideDownload from '~icons/lucide/download'; +import ILucideArrowUpDown from '~icons/lucide/arrow-up-down'; +import ILucideMoveVertical from '~icons/lucide/move-vertical'; export function iconComp(t?: string) { switch (t) { @@ -34,6 +36,10 @@ export function iconComp(t?: string) { return ILucideMousePointerClick; case 'fill': return ILucideEdit3; + case 'drag': + return ILucideArrowUpDown; + case 'scroll': + return ILucideMoveVertical; case 'key': return ILucideKeyboard; case 'navigate': @@ -93,11 +99,13 @@ export function getTypeLabel(type?: string) { foreach: '循环', assert: '断言', key: '键盘', + drag: '拖拽', dblclick: '双击', openTab: '打开标签', switchTab: '切换标签', closeTab: '关闭标签', delay: '延迟', + scroll: '滚动', while: '循环', }; return labels[String(type || '')] || type || ''; diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyDrag.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyDrag.vue new file mode 100644 index 0000000..a8cf035 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyDrag.vue @@ -0,0 +1,24 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyScroll.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyScroll.vue new file mode 100644 index 0000000..529eda9 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/properties/PropertyScroll.vue @@ -0,0 +1,99 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/properties/SelectorEditor.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/properties/SelectorEditor.vue index 765d61a..f988a8c 100644 --- a/app/chrome-extension/entrypoints/popup/components/builder/components/properties/SelectorEditor.vue +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/properties/SelectorEditor.vue @@ -1,7 +1,7 @@