From ce7a5e5bb99d4553deaa01abd03f2ea22bdf18c7 Mon Sep 17 00:00:00 2001 From: hangerye Date: Fri, 10 Oct 2025 20:38:43 +0800 Subject: [PATCH] feat: add builder editor --- app/chrome-extension/common/constants.ts | 1 + app/chrome-extension/common/message-types.ts | 4 + .../background/record-replay/flow-runner.ts | 449 +++++++++++++++--- .../background/record-replay/flow-store.ts | 32 ++ .../background/record-replay/index.ts | 90 ++++ .../record-replay/selector-engine.ts | 44 +- .../background/record-replay/types.ts | 87 +++- .../entrypoints/popup/App.vue | 123 +++++ .../popup/components/BuilderEditor.vue | 416 ++++++++++++++++ .../popup/components/FlowEditor.vue | 363 ++++++++++++++ .../popup/components/ScheduleDialog.vue | 233 +++++++++ .../components/builder/components/Canvas.vue | 202 ++++++++ .../builder/components/KeyValueEditor.vue | 80 ++++ .../builder/components/PropertyPanel.vue | 442 +++++++++++++++++ .../components/builder/components/Sidebar.vue | 70 +++ .../components/builder/model/transforms.ts | 182 +++++++ .../components/builder/model/validation.ts | 86 ++++ .../builder/store/useBuilderStore.ts | 251 ++++++++++ .../accessibility-tree-helper.js | 195 +++++++- .../inject-scripts/recorder.js | 141 +++++- .../inject-scripts/wait-helper.js | 63 +++ app/chrome-extension/package.json | 4 + builder.design.md | 199 ++++++++ builder.prd.md | 146 ++++++ pnpm-lock.yaml | 186 ++++++++ task.md | 67 +++ 26 files changed, 4043 insertions(+), 113 deletions(-) create mode 100644 app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/FlowEditor.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts create mode 100644 app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts create mode 100644 builder.design.md create mode 100644 builder.prd.md create mode 100644 task.md diff --git a/app/chrome-extension/common/constants.ts b/app/chrome-extension/common/constants.ts index b4292f9..c2f39e1 100644 --- a/app/chrome-extension/common/constants.ts +++ b/app/chrome-extension/common/constants.ts @@ -108,6 +108,7 @@ export const STORAGE_KEYS = { RR_FLOWS: 'rr_flows', RR_RUNS: 'rr_runs', RR_PUBLISHED: 'rr_published_flows', + RR_SCHEDULES: 'rr_schedules', } as const; // Notification Configuration diff --git a/app/chrome-extension/common/message-types.ts b/app/chrome-extension/common/message-types.ts index 01b00e6..03ac06f 100644 --- a/app/chrome-extension/common/message-types.ts +++ b/app/chrome-extension/common/message-types.ts @@ -34,6 +34,10 @@ export const BACKGROUND_MESSAGE_TYPES = { RR_EXPORT_FLOW: 'rr_export_flow', RR_EXPORT_ALL: 'rr_export_all', RR_IMPORT_FLOW: 'rr_import_flow', + // Scheduling + RR_SCHEDULE_FLOW: 'rr_schedule_flow', + RR_UNSCHEDULE_FLOW: 'rr_unschedule_flow', + RR_LIST_SCHEDULES: 'rr_list_schedules', } as const; // Offscreen message types diff --git a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts index b1d5e9e..01bb425 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts @@ -1,6 +1,6 @@ import { TOOL_NAMES } from 'chrome-mcp-shared'; import { handleCallTool } from '../tools'; -import { +import type { Flow, RunLogEntry, RunRecord, @@ -13,6 +13,8 @@ import { StepDrag, StepWait, StepScript, + NodeBase as DagNode, + Edge as DagEdge, } from './types'; import { appendRun } from './flow-store'; import { locateElement } from './selector-engine'; @@ -27,6 +29,7 @@ export interface RunOptions { timeoutMs?: number; startUrl?: string; args?: Record; + startNodeId?: string; // start executing from this node/step id if present } export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { @@ -39,12 +42,52 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { + const target = options.tabTarget || 'current'; + const startUrl = options.startUrl; + const [active] = await chrome.tabs.query({ active: true, currentWindow: true }); + if (target === 'new') { + let urlToOpen = startUrl; + if (!urlToOpen) { + // duplicate current active tab's URL when startUrl not provided + urlToOpen = active?.url || 'about:blank'; + } + const created = await chrome.tabs.create({ url: urlToOpen, active: true }); + // Best-effort wait for loading to begin and settle a bit + await new Promise((r) => setTimeout(r, 500)); + } else { + // current tab target + if (startUrl) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url: startUrl } }); + } else if (options.refresh) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { refresh: true } }); + } + } + }; + await ensureTab(); + + // helper to apply assign mapping: { varName: 'a.b[0].c' } + function applyAssign(target: Record, source: any, assign: Record) { + const getByPath = (obj: any, path: string) => { + try { + const parts = path + .replace(/\[(\d+)\]/g, '.$1') + .split('.') + .filter(Boolean); + let cur = obj; + for (const p of parts) { + if (cur == null) return undefined; + cur = cur[p as any]; + } + return cur; + } catch { + return undefined; + } + }; + for (const [k, v] of Object.entries(assign || {})) { + target[k] = getByPath(source, String(v)); + } } // Ensure helper scripts are present for overlay/collectVariables @@ -64,7 +107,10 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise 0) { const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.SEND_COMMAND_TO_INJECT_SCRIPT, - args: { eventName: 'collectVariables', payload: undefined }, + args: { + eventName: 'collectVariables', + payload: JSON.stringify({ variables: needed, useOverlay: true }), + }, }); // Fallback: if direct collectVariables without payload not supported, call with explicit variables let values: Record | null = null; @@ -84,6 +130,7 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { + try { + if (Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0) { + const nodes = ((flow as any).nodes || []) as DagNode[]; + const edges = (((flow as any).edges || []) as DagEdge[]).filter( + (e) => !e.label || e.label === 'default', + ); + const order = topoOrder(nodes, edges); + return order.map((n) => mapDagNodeToStep(n)); + } + } catch { + // ignore and fallback + } + return flow.steps || []; + })(); + + // If a startNodeId is provided, slice the plan to start from that node/step id + const startIdx = options.startNodeId + ? stepsToRun.findIndex((s) => s?.id === options.startNodeId) + : -1; + const steps = startIdx >= 0 ? stepsToRun.slice(startIdx) : stepsToRun.slice(); + try { const pendingAfterScripts: StepScript[] = []; - for (const step of flow.steps) { + for (const step of steps) { const t0 = Date.now(); const maxRetries = Math.max(0, step.retry?.count ?? 0); const baseInterval = Math.max(0, step.retry?.intervalMs ?? 0); @@ -266,7 +336,138 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tab = tabs[0]; + return { url: tab?.url || '', status: (tab as any)?.status || '' }; + }; + // Wait for navigation completion or readiness + const waitForNavigation = async (prevUrl: string, timeoutMs: number) => { + const deadline = Date.now() + Math.max(1000, Math.min(timeoutMs || 15000, 30000)); + let sawLoading = false; + while (Date.now() < deadline) { + const { url, status } = await getActiveTabInfo(); + if (url && url !== prevUrl) return true; + if (status === 'loading') sawLoading = true; + if (sawLoading && status === 'complete') return true; + await new Promise((r) => setTimeout(r, 200)); + } + // as a last attempt, try a brief network idle wait + try { + await waitForNetworkIdle(2000, 800); + return true; + } catch (e) { + // noop + void 0; + } + throw new Error('navigation timeout'); + }; + switch (step.type) { + case 'http': { + const s = step as any; + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_REQUEST, + args: { + url: s.url, + method: s.method || 'GET', + headers: s.headers || {}, + body: s.body, + }, + }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload !== undefined) vars[s.saveAs] = payload; + if (s.assign && payload !== undefined) applyAssign(vars, payload, s.assign); + } catch { + // ignore parse error + } + break; + } + case 'extract': { + const s = step as any; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + let value: any = null; + if (s.js && String(s.js).trim()) { + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (code: string) => { + try { + return (0, eval)(code); + } catch (e) { + return null; + } + }, + args: [String(s.js)], + } as any); + value = result; + } else if (s.selector) { + const attr = String(s.attr || 'text'); + const sel = String(s.selector); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (selector: string, attr: string) => { + try { + const el = document.querySelector(selector) as any; + if (!el) return null; + if (attr === 'text' || attr === 'textContent') + return (el.textContent || '').trim(); + return el.getAttribute ? el.getAttribute(attr) : null; + } catch { + return null; + } + }, + args: [sel, attr], + } as any); + value = result; + } + if (s.saveAs) vars[s.saveAs] = value; + break; + } + case 'openTab': { + const s = step as any; + if (s.newWindow) { + await chrome.windows.create({ url: s.url || undefined, focused: true }); + } else { + await chrome.tabs.create({ url: s.url || undefined, active: true }); + } + break; + } + case 'switchTab': { + const s = step as any; + let targetTabId: number | undefined = s.tabId; + if (!targetTabId) { + const tabs = await chrome.tabs.query({}); + const hit = tabs.find( + (t) => + (s.urlContains && (t.url || '').includes(String(s.urlContains))) || + (s.titleContains && (t.title || '').includes(String(s.titleContains))), + ); + targetTabId = (hit && hit.id) as number | undefined; + } + if (targetTabId) { + await handleCallTool({ + name: TOOL_NAMES.BROWSER.SWITCH_TAB, + args: { tabId: targetTabId }, + }); + } else { + throw new Error('switchTab: no matching tab'); + } + break; + } + case 'closeTab': { + const s = step as any; + const args: any = {}; + if (Array.isArray(s.tabIds) && s.tabIds.length) args.tabIds = s.tabIds; + if (s.url) args.url = s.url; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.CLOSE_TABS, args }); + if ((res as any).isError) throw new Error('closeTab failed'); + break; + } case 'scroll': { const s = step as StepScroll; const top = s.offset?.y ?? undefined; @@ -390,20 +591,34 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise c.type === 'css' || c.type === 'attr', - )?.value - : undefined, - waitForNavigation: (step as any).after?.waitForNavigation || false, - timeout: Math.max(1000, Math.min(step.timeoutMs || 10000, 30000)), - }, - }); + const prevInfo = await getActiveTabInfo(); + let res: any; + if (step.type === 'dblclick') { + // Use precise CDP-based double click for robustness + res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'double_click', ref: located?.ref || (step as any).target?.ref }, + }); + } else { + res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.CLICK, + args: { + ref: located?.ref || (step as any).target?.ref, + selector: !located?.ref + ? (step as any).target?.candidates?.find( + (c: any) => c.type === 'css' || c.type === 'attr', + )?.value + : undefined, + waitForNavigation: false, // we handle navigation explicitly below + timeout: Math.max(1000, Math.min(step.timeoutMs || 10000, 30000)), + }, + }); + } if ((res as any).isError) throw new Error('click failed'); + // If navigation requested, wait explicitly with retries handled by outer loop + if ((step as any).after?.waitForNavigation) { + await waitForNavigation(prevInfo.url, Math.max(step.timeoutMs || 15000, 3000)); + } if (fallbackUsed) { logs.push({ stepId: step.id, @@ -521,16 +736,32 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise setTimeout(r, delay)); } else if ('selector' in s.condition) { - // best-effort: simple text wait with selector string as text - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { - action: 'wait', - text: s.condition.selector, - appear: s.condition.visible !== false, - timeout: Math.max(0, Math.min(step.timeoutMs || 10000, 120000)), - }, - }); - if ((res as any).isError) throw new Error('wait selector failed'); + // Use wait-helper to wait for selector visibility + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await chrome.scripting.executeScript({ + target: { tabId }, + files: ['inject-scripts/wait-helper.js'], + world: 'ISOLATED', + } as any); + const resp = await chrome.tabs.sendMessage(tabId, { + action: 'waitForSelector', + selector: (s.condition as any).selector, + visible: (s.condition as any).visible !== false, + timeout: Math.max(0, Math.min(step.timeoutMs || 10000, 120000)), + } as any); + if (!resp || resp.success !== true) throw new Error('wait selector failed'); } break; } @@ -709,15 +945,28 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { try { ${code} } catch (e) { console.error('flow script error:', e); } })();`; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { type: world, jsScript: wrapped }, - }); - if ((res as any).isError) throw new Error('script execution failed'); + // Prefer executeScript to capture return value for saveAs/assign + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch (e) { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if (s.saveAs) vars[s.saveAs] = result; + if (s.assign && typeof s.assign === 'object') applyAssign(vars, result, s.assign); break; } case 'navigate': { @@ -757,12 +1006,24 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { try { ${code} } catch (e) { console.error('flow script error:', e); } })();`; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { type: world, jsScript: wrapped }, - }); - if ((res as any).isError) throw new Error('script(after) execution failed'); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if ((s as any).saveAs) vars[(s as any).saveAs] = result; + if ((s as any).assign && typeof (s as any).assign === 'object') + applyAssign(vars, result, (s as any).assign); } logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); } @@ -859,8 +1120,8 @@ export async function runFlow(flow: Flow, options: RunOptions = {}): Promise l.status === 'failed')?.screenshotBase64 }, }; } + +// --- DAG helpers (M1: default-edge serial) --- +function topoOrder(nodes: DagNode[], edges: DagEdge[]): DagNode[] { + const id2n = new Map(nodes.map((n) => [n.id, n] as const)); + const indeg = new Map(nodes.map((n) => [n.id, 0] as const)); + for (const e of edges) indeg.set(e.to, (indeg.get(e.to) || 0) + 1); + const nexts = new Map(nodes.map((n) => [n.id, [] as string[]] as const)); + for (const e of edges) nexts.get(e.from)!.push(e.to); + const q: string[] = nodes.filter((n) => (indeg.get(n.id) || 0) === 0).map((n) => n.id); + const out: DagNode[] = []; + while (q.length) { + const id = q.shift()!; + const n = id2n.get(id); + if (!n) continue; + out.push(n); + for (const v of nexts.get(id)!) { + indeg.set(v, (indeg.get(v) || 0) - 1); + if ((indeg.get(v) || 0) === 0) q.push(v); + } + } + return out.length === nodes.length ? out : nodes.slice(); +} + +function mapDagNodeToStep(n: DagNode): Step { + const c: any = n.config || {}; + const base = { id: n.id } as any; + if (n.type === 'click' || n.type === 'dblclick') + return { + ...base, + type: n.type, + target: c.target || { candidates: [] }, + before: c.before, + after: c.after, + } as any; + if (n.type === 'fill') + return { + ...base, + type: 'fill', + target: c.target || { candidates: [] }, + value: c.value || '', + } as any; + if (n.type === 'key') return { ...base, type: 'key', keys: c.keys || '' } as any; + if (n.type === 'wait') + return { ...base, type: 'wait', condition: c.condition || { text: '', appear: true } } as any; + if (n.type === 'assert') + return { + ...base, + type: 'assert', + assert: c.assert || { exists: '' }, + failStrategy: c.failStrategy, + } as any; + if (n.type === 'navigate') return { ...base, type: 'navigate', url: c.url || '' } as any; + if (n.type === 'script') + return { + ...base, + type: 'script', + world: c.world || 'ISOLATED', + code: c.code || '', + when: c.when, + } as any; + if (n.type === 'delay') + return { + ...base, + type: 'wait', + timeoutMs: Math.max(0, Number(c.ms ?? 1000)), + condition: { navigation: true }, + } as any; + // Fallback: no-op script + return { ...base, type: 'script', world: 'ISOLATED', code: '' } as any; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts b/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts index bbf0004..4896497 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/flow-store.ts @@ -108,3 +108,35 @@ export async function importFlowFromJson(json: string): Promise { } return flowsToImport; } + +// Scheduling support +export type ScheduleType = 'once' | 'interval' | 'daily'; +export interface FlowSchedule { + id: string; // schedule id + flowId: string; + type: ScheduleType; + enabled: boolean; + // when: ISO string for 'once'; HH:mm for 'daily'; minutes for 'interval' + when: string; + // optional variables to pass when running + args?: Record; +} + +export async function listSchedules(): Promise { + const res = await chrome.storage.local.get([STORAGE_KEYS.RR_SCHEDULES]); + return (res[STORAGE_KEYS.RR_SCHEDULES] as FlowSchedule[]) || []; +} + +export async function saveSchedule(s: FlowSchedule): Promise { + const list = await listSchedules(); + const idx = list.findIndex((x) => x.id === s.id); + if (idx >= 0) list[idx] = s; + else list.push(s); + await chrome.storage.local.set({ [STORAGE_KEYS.RR_SCHEDULES]: list }); +} + +export async function removeSchedule(scheduleId: string): Promise { + const list = await listSchedules(); + const filtered = list.filter((s) => s.id !== scheduleId); + await chrome.storage.local.set({ [STORAGE_KEYS.RR_SCHEDULES]: filtered }); +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/index.ts b/app/chrome-extension/entrypoints/background/record-replay/index.ts index 5fe1db7..dff9164 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/index.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/index.ts @@ -10,6 +10,10 @@ import { exportFlow, exportAllFlows, importFlowFromJson, + listSchedules, + saveSchedule, + removeSchedule, + type FlowSchedule, } from './flow-store'; import { runFlow } from './flow-runner'; @@ -21,6 +25,39 @@ let lastClickIdx: number | null = null; let lastClickTime = 0; let lastNavTaggedAt = 0; +// Alarm helpers for schedules +async function rescheduleAlarms() { + const schedules = await listSchedules(); + // Clear existing rr_schedule_* alarms + const alarms = await chrome.alarms.getAll(); + await Promise.all( + alarms + .filter((a) => a.name && a.name.startsWith('rr_schedule_')) + .map((a) => chrome.alarms.clear(a.name)), + ); + for (const s of schedules) { + if (!s.enabled) continue; + const name = `rr_schedule_${s.id}`; + if (s.type === 'interval') { + const minutes = Math.max(1, Math.floor(Number(s.when) || 0)); + await chrome.alarms.create(name, { periodInMinutes: minutes }); + } else if (s.type === 'once') { + const whenMs = Date.parse(s.when); + if (Number.isFinite(whenMs)) await chrome.alarms.create(name, { when: whenMs }); + } else if (s.type === 'daily') { + // daily HH:mm local time + const [hh, mm] = String(s.when || '00:00') + .split(':') + .map((x) => Number(x)); + const now = new Date(); + const next = new Date(); + next.setHours(hh || 0, mm || 0, 0, 0); + if (next.getTime() <= now.getTime()) next.setDate(next.getDate() + 1); + await chrome.alarms.create(name, { when: next.getTime(), periodInMinutes: 24 * 60 }); + } + } +} + async function ensureRecorderInjected(tabId: number): Promise { // Inject helper and recorder scripts await chrome.scripting.executeScript({ @@ -96,6 +133,9 @@ async function stopRecording(): Promise<{ success: boolean; flow?: Flow; error?: } export function initRecordReplayListeners() { + // On startup, re-schedule alarms + rescheduleAlarms().catch(() => {}); + chrome.runtime.onMessage.addListener((message, sender, sendResponse) => { try { if (message && message.type === 'rr_recorder_event') { @@ -231,6 +271,40 @@ export function initRecordReplayListeners() { .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); return true; } + case BACKGROUND_MESSAGE_TYPES.RR_LIST_SCHEDULES: { + listSchedules() + .then((s) => sendResponse({ success: true, schedules: s })) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_SCHEDULE_FLOW: { + const s = message.schedule as FlowSchedule; + if (!s || !s.id || !s.flowId) { + sendResponse({ success: false, error: 'invalid schedule' }); + return true; + } + saveSchedule(s) + .then(async () => { + await rescheduleAlarms(); + sendResponse({ success: true }); + }) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } + case BACKGROUND_MESSAGE_TYPES.RR_UNSCHEDULE_FLOW: { + const scheduleId = String(message.scheduleId || ''); + if (!scheduleId) { + sendResponse({ success: false, error: 'invalid scheduleId' }); + return true; + } + removeSchedule(scheduleId) + .then(async () => { + await rescheduleAlarms(); + sendResponse({ success: true }); + }) + .catch((e) => sendResponse({ success: false, error: e?.message || String(e) })); + return true; + } } } catch (err) { sendResponse({ success: false, error: (err as any)?.message || String(err) }); @@ -365,3 +439,19 @@ export function initRecordReplayListeners() { } }); } + +// Alarm listener executes scheduled flows +chrome.alarms.onAlarm.addListener(async (alarm) => { + try { + if (!alarm?.name || !alarm.name.startsWith('rr_schedule_')) return; + const id = alarm.name.slice('rr_schedule_'.length); + const schedules = await listSchedules(); + const s = schedules.find((x) => x.id === id && x.enabled); + if (!s) return; + const flow = await getFlow(s.flowId); + if (!flow) return; + await runFlow(flow, { args: s.args || {}, returnLogs: false }); + } catch (e) { + // swallow to not spam logs + } +}); diff --git a/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts b/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts index 1e6557f..4333990 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/selector-engine.ts @@ -52,13 +52,43 @@ export async function locateElement( return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; } } else if (c.type === 'aria') { - // Best-effort: try as CSS first, otherwise ignore in M2 - const ensured = await chrome.tabs.sendMessage(tabId, { - action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, - selector: c.value, - }); - if (ensured && ensured.success && ensured.ref && ensured.center) { - return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; + // Minimal ARIA role+name parser like: "button[name=提交]" or "textbox[name=用户名]" + const v = String(c.value || '').trim(); + const m = v.match(/^(\w+)\s*\[\s*name\s*=\s*([^\]]+)\]$/); + const role = m ? m[1] : ''; + const name = m ? m[2] : ''; + const cleanName = name.replace(/^['"]|['"]$/g, ''); + const ariaSelectors: string[] = []; + if (role === 'textbox') { + ariaSelectors.push( + `[role="textbox"][aria-label=${JSON.stringify(cleanName)}]`, + `input[aria-label=${JSON.stringify(cleanName)}]`, + `textarea[aria-label=${JSON.stringify(cleanName)}]`, + ); + } else if (role === 'button') { + ariaSelectors.push( + `[role="button"][aria-label=${JSON.stringify(cleanName)}]`, + `button[aria-label=${JSON.stringify(cleanName)}]`, + ); + } else if (role === 'link') { + ariaSelectors.push( + `[role="link"][aria-label=${JSON.stringify(cleanName)}]`, + `a[aria-label=${JSON.stringify(cleanName)}]`, + ); + } + if (!ariaSelectors.length && role) { + ariaSelectors.push( + `[role=${JSON.stringify(role)}][aria-label=${JSON.stringify(cleanName)}]`, + ); + } + for (const sel of ariaSelectors) { + const ensured = await chrome.tabs.sendMessage(tabId, { + action: TOOL_MESSAGE_TYPES.ENSURE_REF_FOR_SELECTOR, + selector: sel, + }); + if (ensured && ensured.success && ensured.ref && ensured.center) { + return { ref: ensured.ref, center: ensured.center, resolvedBy: c.type }; + } } } else if (c.type === 'xpath') { // Minimal xpath support via document.evaluate through injected helper diff --git a/app/chrome-extension/entrypoints/background/record-replay/types.ts b/app/chrome-extension/entrypoints/background/record-replay/types.ts index feb7c35..7aaad0c 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/types.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/types.ts @@ -23,7 +23,12 @@ export type StepType = | 'wait' | 'assert' | 'script' - | 'navigate'; + | 'navigate' + | 'http' + | 'extract' + | 'openTab' + | 'switchTab' + | 'closeTab'; export interface StepBase { id: string; @@ -102,7 +107,49 @@ export type Step = | StepWait | StepAssert | StepScript - | (StepBase & { type: 'navigate'; url: string }); + | (StepBase & { type: 'navigate'; url: string }) + | StepHttp + | StepExtract + | StepOpenTab + | StepSwitchTab + | StepCloseTab; + +export interface StepHttp extends StepBase { + type: 'http'; + method?: string; + url: string; + headers?: Record; + body?: any; + saveAs?: string; + assign?: Record; +} + +export interface StepExtract extends StepBase { + type: 'extract'; + selector?: string; + attr?: string; // 'text'|'textContent' to read text + js?: string; // custom JS that returns value + saveAs: string; +} + +export interface StepOpenTab extends StepBase { + type: 'openTab'; + url?: string; + newWindow?: boolean; +} + +export interface StepSwitchTab extends StepBase { + type: 'switchTab'; + tabId?: number; + urlContains?: string; + titleContains?: string; +} + +export interface StepCloseTab extends StepBase { + type: 'closeTab'; + tabIds?: number[]; + url?: string; +} export interface VariableDef { key: string; @@ -112,6 +159,38 @@ export interface VariableDef { rules?: { required?: boolean; pattern?: string }; } +export type NodeType = + | 'click' + | 'dblclick' + | 'fill' + | 'key' + | 'wait' + | 'assert' + | 'script' + | 'navigate' + | 'openTab' + | 'switchTab' + | 'closeTab' + | 'http' + | 'extract' + | 'delay'; + +export interface NodeBase { + id: string; + type: NodeType; + name?: string; + disabled?: boolean; + config?: any; + ui?: { x: number; y: number }; +} + +export interface Edge { + id: string; + from: string; + to: string; + label?: 'default' | 'true' | 'false' | 'onError'; +} + export interface Flow { id: string; name: string; @@ -128,6 +207,10 @@ export interface Flow { }; variables?: VariableDef[]; steps: Step[]; + // Flow V2（可选）：画布编排 + nodes?: NodeBase[]; + edges?: Edge[]; + subflows?: Record; } export interface RunLogEntry { diff --git a/app/chrome-extension/entrypoints/popup/App.vue b/app/chrome-extension/entrypoints/popup/App.vue index 9177544..7a63e8f 100644 --- a/app/chrome-extension/entrypoints/popup/App.vue +++ b/app/chrome-extension/entrypoints/popup/App.vue @@ -252,7 +252,10 @@
+ + +
@@ -291,6 +294,27 @@ @confirm="confirmClearAllData" @cancel="hideClearDataConfirmation" /> + + + + @@ -310,6 +334,9 @@ import { getMessage } from '@/utils/i18n'; import ConfirmDialog from './components/ConfirmDialog.vue'; import ProgressIndicator from './components/ProgressIndicator.vue'; import ModelCacheManagement from './components/ModelCacheManagement.vue'; +import FlowEditor from './components/FlowEditor.vue'; +import BuilderEditor from './components/BuilderEditor.vue'; +import ScheduleDialog from './components/ScheduleDialog.vue'; import { DocumentIcon, DatabaseIcon, @@ -329,6 +356,15 @@ const filteredRrFlows = computed(() => rrOnlyBound.value ? rrFlows.value.filter(isFlowBoundToCurrent) : rrFlows.value, ); +// Flow editor state +const showFlowEditor = ref(false); +const editingFlow = ref(null); +const showBuilderEditor = ref(false); +const editingFlowBuilder = ref(null); +const showSchedule = ref(false); +const schedulingFlowId = ref(null); +const schedules = ref([]); + const loadFlows = async () => { try { const res = await chrome.runtime.sendMessage({ type: BACKGROUND_MESSAGE_TYPES.RR_LIST_FLOWS }); @@ -420,6 +456,93 @@ const deleteFlow = async (flowId: string) => { } }; +function editFlow(flow: any) { + editingFlow.value = flow; + showFlowEditor.value = true; +} + +function openBuilder(flow: any) { + editingFlowBuilder.value = flow; + showBuilderEditor.value = true; +} + +async function saveEditedFlow(f: any) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_SAVE_FLOW, + flow: f, + }); + if (res && res.success) { + showFlowEditor.value = false; + editingFlow.value = null; + await loadFlows(); + } + } catch (e) { + console.error('保存失败:', e); + } +} + +async function saveEditedFlowFromBuilder(f: any) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_SAVE_FLOW, + flow: f, + }); + if (res && res.success) { + showBuilderEditor.value = false; + editingFlowBuilder.value = null; + await loadFlows(); + } + } catch (e) { + console.error('保存失败:', e); + } +} + +async function openSchedule(flowId: string) { + schedulingFlowId.value = flowId; + await loadSchedules(); + showSchedule.value = true; +} + +async function loadSchedules() { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_LIST_SCHEDULES, + }); + if (res && res.success) schedules.value = res.schedules || []; + } catch (e) { + console.error('加载定时失败:', e); + } +} + +async function saveSchedule(schedule: any) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_SCHEDULE_FLOW, + schedule, + }); + if (res && res.success) { + await loadSchedules(); + showSchedule.value = false; + schedulingFlowId.value = null; + } + } catch (e) { + console.error('保存定时失败:', e); + } +} + +async function removeSchedule(id: string) { + try { + const res = await chrome.runtime.sendMessage({ + type: BACKGROUND_MESSAGE_TYPES.RR_UNSCHEDULE_FLOW, + scheduleId: id, + }); + if (res && res.success) await loadSchedules(); + } catch (e) { + console.error('删除计划失败:', e); + } +} + const nativeConnectionStatus = ref<'unknown' | 'connected' | 'disconnected'>('unknown'); const isConnecting = ref(false); const nativeServerPort = ref(12306); diff --git a/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue b/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue new file mode 100644 index 0000000..c8b76f9 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/BuilderEditor.vue @@ -0,0 +1,416 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue b/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue new file mode 100644 index 0000000..0a78b67 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/FlowEditor.vue @@ -0,0 +1,363 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue b/app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue new file mode 100644 index 0000000..0d4a0fa --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/ScheduleDialog.vue @@ -0,0 +1,233 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue new file mode 100644 index 0000000..78e683e --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Canvas.vue @@ -0,0 +1,202 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue new file mode 100644 index 0000000..d3b3982 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/KeyValueEditor.vue @@ -0,0 +1,80 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue new file mode 100644 index 0000000..14d2b3b --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/PropertyPanel.vue @@ -0,0 +1,442 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue b/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue new file mode 100644 index 0000000..499bb88 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/components/Sidebar.vue @@ -0,0 +1,70 @@ + + + + + diff --git a/app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts b/app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts new file mode 100644 index 0000000..8cbb620 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/model/transforms.ts @@ -0,0 +1,182 @@ +import type { + Flow as FlowV2, + NodeBase, + Edge as EdgeV2, +} from '@/entrypoints/background/record-replay/types'; + +export function newId(prefix: string) { + return `${prefix}_${Math.random().toString(36).slice(2, 8)}`; +} + +export type NodeType = NodeBase['type']; + +export function defaultConfigFor(t: NodeType): any { + if (t === 'click' || t === 'fill') + return { target: { candidates: [] }, value: t === 'fill' ? '' : undefined }; + if (t === 'navigate') return { url: '' }; + if (t === 'wait') return { condition: { text: '', appear: true } }; + if (t === 'assert') return { assert: { exists: '' } }; + if (t === 'key') return { keys: '' }; + if (t === 'delay') return { ms: 1000 }; + if (t === 'http') return { method: 'GET', url: '', headers: {}, body: null, saveAs: '' }; + if (t === 'extract') return { selector: '', attr: 'text', js: '', saveAs: '' }; + if (t === 'openTab') return { url: '', newWindow: false }; + if (t === 'switchTab') return { tabId: null, urlContains: '', titleContains: '' }; + if (t === 'closeTab') return { tabIds: [], url: '' }; + if (t === 'script') return { world: 'ISOLATED', code: '', saveAs: '', assign: {} }; + return {}; +} + +export function stepsToNodes(steps: any[]): NodeBase[] { + const arr: NodeBase[] = []; + steps.forEach((s, i) => { + const id = s.id || newId(String(s.type || 'step')); + const node: NodeBase = { + id, + type: (s.type || 'script') as NodeType, + name: '', + disabled: false, + ui: { x: 200, y: 120 + i * 120 }, + config: mapStepToConfig(s), + }; + arr.push(node); + }); + return arr; +} + +export function mapStepToConfig(s: any) { + const t = s.type; + if (t === 'click' || t === 'dblclick') + return { target: s.target || { candidates: [] }, after: s.after, before: s.before }; + if (t === 'fill') return { target: s.target || { candidates: [] }, value: s.value || '' }; + if (t === 'wait') return { condition: s.condition || { text: '', appear: true } }; + if (t === 'assert') return { assert: s.assert || { exists: '' }, failStrategy: s.failStrategy }; + if (t === 'navigate') return { url: s.url || '' }; + if (t === 'script') return { world: s.world || 'ISOLATED', code: s.code || '' }; + return { ...s }; +} + +export function mapConfigToStep(n: NodeBase) { + const base = { id: n.id, type: n.type } as any; + const c = n.config || {}; + if (n.type === 'click' || n.type === 'dblclick') + return { ...base, target: c.target || { candidates: [] }, after: c.after, before: c.before }; + if (n.type === 'fill') + return { ...base, target: c.target || { candidates: [] }, value: c.value || '' }; + if (n.type === 'key') return { ...base, keys: c.keys || '' }; + if (n.type === 'wait') return { ...base, condition: c.condition || { text: '', appear: true } }; + if (n.type === 'assert') + return { ...base, assert: c.assert || { exists: '' }, failStrategy: c.failStrategy }; + if (n.type === 'navigate') return { ...base, url: c.url || '' }; + if (n.type === 'delay') + return { + ...base, + type: 'wait', + timeoutMs: Math.max(0, Number(c.ms ?? 1000)), + condition: { navigation: true }, + }; + if (n.type === 'http') + return { + ...base, + type: 'http', + method: c.method || 'GET', + url: c.url || '', + headers: c.headers || {}, + body: c.body, + saveAs: c.saveAs || '', + } as any; + if (n.type === 'extract') + return { + ...base, + type: 'extract', + selector: c.selector || '', + attr: c.attr || 'text', + js: c.js || '', + saveAs: c.saveAs || '', + } as any; + if (n.type === 'openTab') + return { ...base, type: 'openTab', url: c.url || '', newWindow: !!c.newWindow } as any; + if (n.type === 'switchTab') + return { + ...base, + type: 'switchTab', + tabId: c.tabId || undefined, + urlContains: c.urlContains || '', + titleContains: c.titleContains || '', + } as any; + if (n.type === 'closeTab') + return { + ...base, + type: 'closeTab', + tabIds: Array.isArray(c.tabIds) ? c.tabIds : undefined, + url: c.url || '', + } as any; + if (n.type === 'script') + return { + ...base, + world: c.world || 'ISOLATED', + code: c.code || '', + when: c.when, + saveAs: c.saveAs || '', + assign: c.assign || {}, + } as any; + return { ...base }; +} + +export function topoOrder(nodes: NodeBase[], edges: EdgeV2[]): NodeBase[] { + const id2n = new Map(nodes.map((n) => [n.id, n] as const)); + const indeg = new Map(nodes.map((n) => [n.id, 0] as const)); + for (const e of edges) + if (!e.label || e.label === 'default') indeg.set(e.to, (indeg.get(e.to) || 0) + 1); + const q: string[] = nodes.filter((n) => (indeg.get(n.id) || 0) === 0).map((n) => n.id); + const out: NodeBase[] = []; + const nexts = new Map(nodes.map((n) => [n.id, [] as string[]] as const)); + for (const e of edges) if (!e.label || e.label === 'default') nexts.get(e.from)!.push(e.to); + while (q.length) { + const id = q.shift()!; + const n = id2n.get(id); + if (!n) continue; + out.push(n); + for (const v of nexts.get(id)!) { + indeg.set(v, (indeg.get(v) || 0) - 1); + if ((indeg.get(v) || 0) === 0) q.push(v); + } + } + if (out.length === nodes.length) return out; + return nodes.slice(); +} + +export function nodesToSteps(nodes: NodeBase[], edges: EdgeV2[]): any[] { + const order = edges.length ? topoOrder(nodes, edges) : nodes.slice(); + return order.map((n) => mapConfigToStep(n)); +} + +export function autoChainEdges(nodes: NodeBase[]): EdgeV2[] { + const arr: EdgeV2[] = []; + for (let i = 0; i < nodes.length - 1; i++) + arr.push({ id: newId('e'), from: nodes[i].id, to: nodes[i + 1].id, label: 'default' }); + return arr; +} + +export function summarizeNode(n?: NodeBase | null): string { + if (!n) return ''; + if (n.type === 'click' || n.type === 'fill') + return n.config?.target?.candidates?.[0]?.value || '未配置选择器'; + if (n.type === 'navigate') return n.config?.url || ''; + if (n.type === 'key') return n.config?.keys || ''; + if (n.type === 'delay') return `${Number(n.config?.ms || 0)}ms`; + if (n.type === 'http') return `${n.config?.method || 'GET'} ${n.config?.url || ''}`; + if (n.type === 'extract') return `${n.config?.selector || ''} -> ${n.config?.saveAs || ''}`; + if (n.type === 'openTab') return `open ${n.config?.url || ''}`; + if (n.type === 'switchTab') + return `switch ${n.config?.tabId || n.config?.urlContains || n.config?.titleContains || ''}`; + if (n.type === 'closeTab') return `close ${n.config?.url || ''}`; + if (n.type === 'wait') return JSON.stringify(n.config?.condition || {}); + if (n.type === 'assert') return JSON.stringify(n.config?.assert || {}); + if (n.type === 'script') return (n.config?.code || '').slice(0, 30); + return ''; +} + +export function cloneFlow(flow: FlowV2): FlowV2 { + return JSON.parse(JSON.stringify(flow)); +} diff --git a/app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts b/app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts new file mode 100644 index 0000000..cb24f37 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/model/validation.ts @@ -0,0 +1,86 @@ +import type { NodeBase } from '@/entrypoints/background/record-replay/types'; + +export function validateNode(n: NodeBase): string[] { + const errs: string[] = []; + if (n.disabled) return errs; // 忽略禁用节点 + const c: any = n.config || {}; + + switch (n.type) { + case 'click': + case 'dblclick': + case 'fill': { + const hasCandidate = !!c?.target?.candidates?.length; + if (!hasCandidate) errs.push('缺少目标选择器候选'); + if (n.type === 'fill' && (!('value' in c) || c.value === undefined)) errs.push('缺少输入值'); + break; + } + case 'wait': { + if (!c?.condition) errs.push('缺少等待条件'); + break; + } + case 'assert': { + if (!c?.assert) errs.push('缺少断言条件'); + break; + } + case 'navigate': { + if (!c?.url) errs.push('缺少 URL'); + break; + } + case 'http': { + if (!c?.url) errs.push('HTTP: 缺少 URL'); + if (c?.assign && typeof c.assign === 'object') { + const pathRe = /^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+|\[\d+\])*$/; + for (const v of Object.values(c.assign)) { + const s = String(v); + if (!pathRe.test(s)) errs.push(`Assign: 路径非法 ${s}`); + } + } + break; + } + case 'extract': { + if (!c?.saveAs) errs.push('Extract: 需填写保存变量名'); + if (!c?.selector && !c?.js) errs.push('Extract: 需提供 selector 或 js'); + break; + } + case 'switchTab': { + if (!c?.tabId && !c?.urlContains && !c?.titleContains) + errs.push('SwitchTab: 需提供 tabId 或 URL/标题包含'); + break; + } + case 'closeTab': { + // 允许空（关闭当前标签页），不强制 + break; + } + case 'script': { + // 若配置了 saveAs/assign，应提供 code + const hasAssign = c?.assign && Object.keys(c.assign).length > 0; + if ((c?.saveAs || hasAssign) && !String(c?.code || '').trim()) + errs.push('Script: 配置了保存/映射但缺少代码'); + if (hasAssign) { + const pathRe = /^[A-Za-z0-9_]+(?:\.[A-Za-z0-9_]+|\[\d+\])*$/; + for (const v of Object.values(c.assign || {})) { + const s = String(v); + if (!pathRe.test(s)) errs.push(`Assign: 路径非法 ${s}`); + } + } + break; + } + } + return errs; +} + +export function validateFlow(nodes: NodeBase[]): { + totalErrors: number; + nodeErrors: Record; +} { + const nodeErrors: Record = {}; + let totalErrors = 0; + for (const n of nodes) { + const e = validateNode(n); + if (e.length) { + nodeErrors[n.id] = e; + totalErrors += e.length; + } + } + return { totalErrors, nodeErrors }; +} diff --git a/app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts b/app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts new file mode 100644 index 0000000..87f6b71 --- /dev/null +++ b/app/chrome-extension/entrypoints/popup/components/builder/store/useBuilderStore.ts @@ -0,0 +1,251 @@ +import { reactive, ref } from 'vue'; +import type { + Flow as FlowV2, + NodeBase, + Edge as EdgeV2, +} from '@/entrypoints/background/record-replay/types'; +import { + autoChainEdges, + cloneFlow, + defaultConfigFor, + newId, + nodesToSteps, + stepsToNodes, + summarizeNode, + topoOrder, +} from '../model/transforms'; + +export function useBuilderStore(initial?: FlowV2 | null) { + const flowLocal = reactive({ id: '', name: '', version: 1, steps: [], variables: [] }); + const nodes = reactive([]); + const edges = reactive([]); + const activeNodeId = ref(null); + const pendingFrom = ref(null); + const paletteTypes = [ + 'click', + 'fill', + 'key', + 'wait', + 'assert', + 'navigate', + 'script', + 'delay', + 'http', + 'extract', + 'openTab', + 'switchTab', + 'closeTab', + ] as NodeBase['type'][]; + + // --- history (undo/redo) --- + type Snapshot = { + flow: Pick; + nodes: NodeBase[]; + edges: EdgeV2[]; + }; + const HISTORY_MAX = 50; + const past: Snapshot[] = []; + const future: Snapshot[] = []; + function takeSnapshot(): Snapshot { + return { + flow: { name: flowLocal.name, description: flowLocal.description } as any, + nodes: JSON.parse(JSON.stringify(nodes)), + edges: JSON.parse(JSON.stringify(edges)), + }; + } + function applySnapshot(s: Snapshot) { + flowLocal.name = (s.flow as any).name || ''; + (flowLocal as any).description = (s.flow as any).description || ''; + nodes.splice(0, nodes.length, ...JSON.parse(JSON.stringify(s.nodes))); + edges.splice(0, edges.length, ...JSON.parse(JSON.stringify(s.edges))); + } + function recordChange() { + past.push(takeSnapshot()); + // clear redo stack on new change + future.length = 0; + if (past.length > HISTORY_MAX) past.splice(0, past.length - HISTORY_MAX); + } + function undo() { + if (past.length === 0) return; + const current = takeSnapshot(); + const prev = past.pop()!; + future.push(current); + applySnapshot(prev); + } + function redo() { + if (future.length === 0) return; + const current = takeSnapshot(); + const next = future.pop()!; + past.push(current); + applySnapshot(next); + } + + function layoutIfNeeded() { + const startX = 120, + startY = 80, + gapY = 120; + nodes.forEach((n, i) => { + if (!n.ui || isNaN(n.ui.x) || isNaN(n.ui.y)) n.ui = { x: startX, y: startY + i * gapY }; + }); + } + + function initFromFlow(flow: FlowV2) { + const deep = cloneFlow(flow); + Object.assign(flowLocal, deep); + nodes.splice( + 0, + nodes.length, + ...(Array.isArray(deep.nodes) && deep.nodes.length + ? deep.nodes + : stepsToNodes(deep.steps || [])), + ); + edges.splice( + 0, + edges.length, + ...(Array.isArray(deep.edges) && deep.edges.length ? deep.edges : autoChainEdges(nodes)), + ); + layoutIfNeeded(); + activeNodeId.value = nodes[0]?.id || null; + // reset history + past.length = 0; + future.length = 0; + past.push(takeSnapshot()); + } + + function selectNode(id: string) { + if (pendingFrom.value && pendingFrom.value !== id) { + onConnect(pendingFrom.value, id); + pendingFrom.value = null; + } + activeNodeId.value = id; + } + + function addNode(t: NodeBase['type']) { + const id = newId(t); + const n: NodeBase = { + id, + type: t, + name: '', + disabled: false, + config: defaultConfigFor(t), + ui: { x: 200 + nodes.length * 24, y: 120 + nodes.length * 96 }, + }; + nodes.push(n); + if (nodes.length > 1) { + const prev = nodes[nodes.length - 2]; + edges.push({ id: newId('e'), from: prev.id, to: id, label: 'default' }); + } + activeNodeId.value = id; + recordChange(); + } + + function duplicateNode(id: string) { + const src = nodes.find((n) => n.id === id); + if (!src) return; + const cp: NodeBase = JSON.parse(JSON.stringify(src)); + cp.id = newId(src.type); + cp.name = src.name ? `${src.name} Copy` : ''; + const baseX = cp.ui && typeof cp.ui.x === 'number' ? cp.ui.x : 200; + const baseY = cp.ui && typeof cp.ui.y === 'number' ? cp.ui.y : 120; + cp.ui = { x: baseX + 40, y: baseY + 40 }; + nodes.push(cp); + activeNodeId.value = cp.id; + recordChange(); + } + + function removeNode(id: string) { + const idx = nodes.findIndex((n) => n.id === id); + if (idx < 0) return; + nodes.splice(idx, 1); + for (let i = edges.length - 1; i >= 0; i--) { + const e = edges[i]; + if (e.from === id || e.to === id) edges.splice(i, 1); + } + activeNodeId.value = nodes[Math.min(idx, nodes.length - 1)]?.id || null; + recordChange(); + } + + function setNodePosition(id: string, x: number, y: number) { + const n = nodes.find((n) => n.id === id); + if (!n) return; + n.ui = { x: Math.round(x), y: Math.round(y) }; + // 不计入历史栈，避免频繁记录；由用户触发操作（连接/新增/删除等）记录。 + } + + function connectFrom(id: string) { + pendingFrom.value = id; + } + + function onConnect(sourceId: string, targetId: string) { + // 单一默认出边：删除同源 default 出边 + for (let i = edges.length - 1; i >= 0; i--) { + const e = edges[i]; + if (e.from === sourceId && (!e.label || e.label === 'default')) edges.splice(i, 1); + } + edges.push({ id: newId('e'), from: sourceId, to: targetId, label: 'default' }); + recordChange(); + } + + function importFromSteps() { + const arr = stepsToNodes(flowLocal.steps || []); + nodes.splice(0, nodes.length, ...arr); + edges.splice(0, edges.length, ...autoChainEdges(arr)); + layoutIfNeeded(); + recordChange(); + } + + function exportSteps() { + return nodesToSteps(nodes, edges); + } + + function summarize(id?: string) { + const n = nodes.find((x) => x.id === id); + return summarizeNode(n || null); + } + + // 自动排版：根据拓扑顺序纵向排列，列宽 300、行高 120；若存在分叉，简单按顺序换行 + function layoutAuto() { + const order = topoOrder(nodes, edges); + const startX = 120, + startY = 80, + stepY = 120, + stepX = 300, + maxPerCol = Math.max(6, Math.ceil(order.length / 3)); + let col = 0, + row = 0; + for (const n of order) { + n.ui = { x: startX + col * stepX, y: startY + row * stepY } as any; + row++; + if (row >= maxPerCol) { + row = 0; + col++; + } + } + recordChange(); + } + + if (initial) initFromFlow(initial); + + return { + flowLocal, + nodes, + edges, + activeNodeId, + pendingFrom, + paletteTypes, + undo, + redo, + initFromFlow, + selectNode, + addNode, + duplicateNode, + removeNode, + setNodePosition, + connectFrom, + onConnect, + importFromSteps, + exportSteps, + summarize, + layoutAuto, + }; +} diff --git a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js index ae7eee5..7fa4fa0 100644 --- a/app/chrome-extension/inject-scripts/accessibility-tree-helper.js +++ b/app/chrome-extension/inject-scripts/accessibility-tree-helper.js @@ -537,23 +537,61 @@ const sel = String(request.selector || '').trim(); let el = null; if (useText && textQuery) { - const all = Array.from(document.querySelectorAll('body *')); - for (const node of all) { + const normalize = (s) => + String(s || '') + .replace(/\s+/g, ' ') + .trim() + .toLowerCase(); + const query = normalize(textQuery); + const bigrams = (s) => { + const arr = []; + for (let i = 0; i < s.length - 1; i++) arr.push(s.slice(i, i + 2)); + return arr; + }; + const dice = (a, b) => { + if (!a || !b) return 0; + const A = bigrams(a); + const B = bigrams(b); + if (A.length === 0 || B.length === 0) return 0; + let inter = 0; + const map = new Map(); + for (const t of A) map.set(t, (map.get(t) || 0) + 1); + for (const t of B) { + const c = map.get(t) || 0; + if (c > 0) { + inter++; + map.set(t, c - 1); + } + } + return (2 * inter) / (A.length + B.length); + }; + let best = { el: null, score: 0 }; + const walker = document.createTreeWalker( + document.body || document.documentElement, + NodeFilter.SHOW_ELEMENT, + ); + let visited = 0; + while (walker.nextNode()) { + const node = /** @type {Element} */ (walker.currentNode); try { const cs = window.getComputedStyle(node); if (cs.display === 'none' || cs.visibility === 'hidden' || cs.opacity === '0') continue; const rect = /** @type {HTMLElement} */ (node).getBoundingClientRect(); if (rect.width <= 0 || rect.height <= 0) continue; - const txt = (node.textContent || '').trim(); - if (txt && txt.includes(textQuery)) { + const txt = normalize(node.textContent || ''); + if (!txt) continue; + // quick path: substring contains + if (txt.includes(query)) { el = node; break; } - } catch (_) { - /* ignore */ - } + const sc = dice(txt, query); + if (sc > best.score) best = { el: node, score: sc }; + } catch {} + if (++visited > 5000) break; } + if (!el && best.el && best.score >= 0.6) el = best.el; } else { if (!sel) { sendResponse({ success: false, error: 'selector is required' }); @@ -625,21 +663,136 @@ } if (request && request.action === 'collectVariables') { try { - const vars = Array.isArray(request.variables) ? request.variables : []; - const values = {}; - for (const v of vars) { - const key = String(v && v.key ? v.key : ''); - if (!key) continue; - const label = v.label || key; - const def = v.default || ''; - const promptText = `请输入参数 ${label} (${key})`; - // Note: prompt in page context; in some sites may be blocked by CSP - let val = window.prompt(promptText, def); - if (typeof val !== 'string') val = def; - values[key] = val; + let vars = Array.isArray(request.variables) ? request.variables : []; + if ((!vars || vars.length === 0) && request.payload) { + try { + const p = JSON.parse(String(request.payload || '{}')); + if (Array.isArray(p.variables)) vars = p.variables; + } catch {} } - sendResponse({ success: true, values }); - return true; + const useOverlay = request.useOverlay !== false; // default true + const values = {}; + if (!useOverlay) { + for (const v of vars) { + const key = String(v && v.key ? v.key : ''); + if (!key) continue; + const label = v.label || key; + const def = v.default || ''; + const promptText = `请输入参数 ${label} (${key})`; + let val = window.prompt(promptText, def); + if (typeof val !== 'string') val = def; + values[key] = val; + } + sendResponse({ success: true, values }); + return true; + } + // Build overlay form + const hostId = '__rr_var_overlay__'; + let host = document.getElementById(hostId); + if (host) host.remove(); + host = document.createElement('div'); + host.id = hostId; + Object.assign(host.style, { + position: 'fixed', + inset: '0', + background: 'rgba(0,0,0,0.35)', + zIndex: 2147483646, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + }); + const panel = document.createElement('div'); + Object.assign(panel.style, { + background: '#fff', + borderRadius: '8px', + width: 'min(520px, 96vw)', + maxHeight: '80vh', + overflow: 'auto', + boxShadow: '0 8px 24px rgba(0,0,0,0.2)', + padding: '16px', + fontFamily: 'system-ui, -apple-system, Segoe UI, Roboto, Arial, sans-serif', + }); + const title = document.createElement('div'); + title.textContent = '请输入回放参数'; + Object.assign(title.style, { fontSize: '16px', fontWeight: '600', marginBottom: '12px' }); + const form = document.createElement('form'); + for (const v of vars) { + const row = document.createElement('div'); + Object.assign(row.style, { marginBottom: '10px' }); + const label = document.createElement('label'); + label.textContent = `${v.label || v.key}${v.sensitive ? ' (敏感)' : ''}`; + Object.assign(label.style, { + display: 'block', + marginBottom: '6px', + fontWeight: '500', + }); + const input = document.createElement('input'); + input.type = v.sensitive ? 'password' : 'text'; + input.name = String(v.key); + input.value = String(v.default || ''); + Object.assign(input.style, { + width: '100%', + boxSizing: 'border-box', + padding: '8px 10px', + border: '1px solid #d0d7de', + borderRadius: '6px', + outline: 'none', + }); + row.appendChild(label); + row.appendChild(input); + form.appendChild(row); + } + const actions = document.createElement('div'); + Object.assign(actions.style, { display: 'flex', gap: '8px', marginTop: '12px' }); + const ok = document.createElement('button'); + ok.type = 'submit'; + ok.textContent = '确定'; + Object.assign(ok.style, { + background: '#0969da', + color: '#fff', + border: 'none', + padding: '8px 16px', + borderRadius: '6px', + cursor: 'pointer', + }); + const cancel = document.createElement('button'); + cancel.type = 'button'; + cancel.textContent = '取消'; + Object.assign(cancel.style, { + background: '#f3f4f6', + color: '#111', + border: '1px solid #d0d7de', + padding: '8px 16px', + borderRadius: '6px', + cursor: 'pointer', + }); + actions.appendChild(ok); + actions.appendChild(cancel); + panel.appendChild(title); + panel.appendChild(form); + panel.appendChild(actions); + host.appendChild(panel); + document.documentElement.appendChild(host); + + const cleanup = () => { + try { + host.remove(); + } catch {} + }; + cancel.onclick = () => { + cleanup(); + sendResponse({ success: false, cancelled: true }); + }; + form.onsubmit = (e) => { + e.preventDefault(); + for (const v of vars) { + const el = form.querySelector(`input[name="${CSS.escape(String(v.key))}"]`); + if (el) values[v.key] = /** @type {HTMLInputElement} */ (el).value; + } + cleanup(); + sendResponse({ success: true, values }); + }; + return true; // async } catch (e) { sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); return true; diff --git a/app/chrome-extension/inject-scripts/recorder.js b/app/chrome-extension/inject-scripts/recorder.js index e0a10f9..89cbb0b 100644 --- a/app/chrome-extension/inject-scripts/recorder.js +++ b/app/chrome-extension/inject-scripts/recorder.js @@ -11,6 +11,9 @@ const sampledDrag = []; let isRecording = false; + let isPaused = false; + let hideInputValues = false; + let highlightBox = null; let pendingFlow = { id: `flow_${Date.now()}`, name: '未命名录制', @@ -104,7 +107,7 @@ } function onClick(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; const el = e.target instanceof Element ? e.target : null; if (!el) return; const target = buildTarget(el); @@ -112,14 +115,15 @@ } function onInput(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; const el = e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement ? e.target : null; if (!el) return; const target = buildTarget(el); - const isSensitive = SENSITIVE_INPUT_TYPES.has((el.getAttribute('type') || '').toLowerCase()); + const isSensitive = + hideInputValues || SENSITIVE_INPUT_TYPES.has((el.getAttribute('type') || '').toLowerCase()); let value = el.value || ''; if (isSensitive) { const varKey = el.name ? el.name : `var_${Math.random().toString(36).slice(2, 6)}`; @@ -130,7 +134,7 @@ } function onKeydown(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; // modifier+key or Enter/Backspace etc const mods = []; if (e.ctrlKey) mods.push('ctrl'); @@ -144,22 +148,11 @@ pushStep({ type: 'key', keys, screenshotOnFail: false }); } - function onKeyup(e) { - if (!isRecording) return; - const mods = []; - if (e.ctrlKey) mods.push('ctrl'); - if (e.metaKey) mods.push('cmd'); - if (e.altKey) mods.push('alt'); - if (e.shiftKey) mods.push('shift'); - let keyToken = e.key || ''; - keyToken = keyToken.length === 1 ? keyToken.toLowerCase() : keyToken.toLowerCase(); - const keys = mods.length ? `${mods.join('+')}+${keyToken}` : keyToken; - pushStep({ type: 'key', keys, screenshotOnFail: false }); - } + // keyup 不再记录，避免重复噪声 // Composition IME events (record markers for analysis; playback is no-op via script step) function onCompositionStart() { - if (!isRecording) return; + if (!isRecording || isPaused) return; pushStep({ type: 'script', world: 'ISOLATED', @@ -169,7 +162,7 @@ }); } function onCompositionEnd() { - if (!isRecording) return; + if (!isRecording || isPaused) return; pushStep({ type: 'script', world: 'ISOLATED', @@ -181,7 +174,7 @@ let lastScrollAt = 0; function onScroll(e) { - if (!isRecording) return; + if (!isRecording || isPaused) return; const nowTs = now(); if (nowTs - lastScrollAt < THROTTLE_SCROLL_MS) return; lastScrollAt = nowTs; @@ -227,7 +220,7 @@ document.addEventListener('change', onInput, true); document.addEventListener('input', onInput, true); document.addEventListener('keydown', onKeydown, true); - document.addEventListener('keyup', onKeyup, true); + // document.addEventListener('keyup', onKeyup, true); document.addEventListener('compositionstart', onCompositionStart, true); document.addEventListener('compositionend', onCompositionEnd, true); window.addEventListener('scroll', onScroll, { passive: true }); @@ -241,7 +234,7 @@ document.removeEventListener('change', onInput, true); document.removeEventListener('input', onInput, true); document.removeEventListener('keydown', onKeydown, true); - document.removeEventListener('keyup', onKeyup, true); + // document.removeEventListener('keyup', onKeyup, true); document.removeEventListener('compositionstart', onCompositionStart, true); document.removeEventListener('compositionend', onCompositionEnd, true); window.removeEventListener('scroll', onScroll, { passive: true }); @@ -269,7 +262,9 @@ function start(flowMeta) { reset(flowMeta || {}); isRecording = true; + isPaused = false; attach(); + ensureOverlay(); chrome.runtime.sendMessage({ type: 'rr_recorder_event', payload: { kind: 'start', flow: pendingFlow }, @@ -279,6 +274,7 @@ function stop() { isRecording = false; detach(); + removeOverlay(); chrome.runtime.sendMessage({ type: 'rr_recorder_event', payload: { kind: 'stop', flow: pendingFlow }, @@ -286,6 +282,101 @@ return pendingFlow; } + function pause() { + isPaused = true; + updateOverlayStatus(); + } + + function resume() { + isRecording = true; + isPaused = false; + attach(); + ensureOverlay(); + updateOverlayStatus(); + } + + function ensureOverlay() { + let root = document.getElementById('__rr_rec_overlay'); + if (root) return; + root = document.createElement('div'); + root.id = '__rr_rec_overlay'; + Object.assign(root.style, { + position: 'fixed', + top: '10px', + right: '10px', + zIndex: 2147483646, + fontFamily: 'system-ui,-apple-system,Segoe UI,Roboto,Arial', + }); + root.innerHTML = ` +
+ 录制中 + + + +
+ `; + document.documentElement.appendChild(root); + const btnPause = root.querySelector('#__rr_pause'); + const btnStop = root.querySelector('#__rr_stop'); + const hideChk = root.querySelector('#__rr_hide_values'); + hideChk.checked = hideInputValues; + hideChk.addEventListener('change', () => (hideInputValues = hideChk.checked)); + btnPause.addEventListener('click', () => { + if (!isPaused) pause(); + else resume(); + }); + btnStop.addEventListener('click', () => { + stop(); + }); + updateOverlayStatus(); + // element highlight box + highlightBox = document.createElement('div'); + Object.assign(highlightBox.style, { + position: 'fixed', + border: '2px solid rgba(59,130,246,0.9)', + borderRadius: '4px', + background: 'rgba(59,130,246,0.15)', + pointerEvents: 'none', + zIndex: 2147483645, + }); + document.documentElement.appendChild(highlightBox); + document.addEventListener('mousemove', onHoverMove, true); + } + + function removeOverlay() { + try { + const root = document.getElementById('__rr_rec_overlay'); + if (root) root.remove(); + if (highlightBox) highlightBox.remove(); + document.removeEventListener('mousemove', onHoverMove, true); + } catch {} + } + + function updateOverlayStatus() { + const badge = document.getElementById('__rr_badge'); + const pauseBtn = document.getElementById('__rr_pause'); + if (badge) badge.textContent = isPaused ? '已暂停' : '录制中'; + if (pauseBtn) pauseBtn.textContent = isPaused ? '继续' : '暂停'; + } + + function onHoverMove(e) { + if (!highlightBox || !isRecording || isPaused) return; + const el = e.target instanceof Element ? e.target : null; + if (!el) return; + try { + const r = el.getBoundingClientRect(); + Object.assign(highlightBox.style, { + left: `${Math.round(r.left)}px`, + top: `${Math.round(r.top)}px`, + width: `${Math.round(Math.max(0, r.width))}px`, + height: `${Math.round(Math.max(0, r.height))}px`, + display: r.width > 0 && r.height > 0 ? 'block' : 'none', + }); + } catch {} + } + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { try { if (request && request.action === 'rr_recorder_control') { @@ -294,10 +385,12 @@ start(request.meta || {}); sendResponse({ success: true }); return true; + } else if (cmd === 'pause') { + pause(); + sendResponse({ success: true }); + return true; } else if (cmd === 'resume') { - // Attach without resetting flow or sending start event - isRecording = true; - attach(); + resume(); sendResponse({ success: true }); return true; } else if (cmd === 'stop') { diff --git a/app/chrome-extension/inject-scripts/wait-helper.js b/app/chrome-extension/inject-scripts/wait-helper.js index d97dde5..77ffb11 100644 --- a/app/chrome-extension/inject-scripts/wait-helper.js +++ b/app/chrome-extension/inject-scripts/wait-helper.js @@ -145,6 +145,58 @@ }); } + function waitForSelector({ selector, visible = true, timeout = 5000 }) { + return new Promise((resolve) => { + const start = Date.now(); + let resolved = false; + + const isMatch = () => { + try { + const el = document.querySelector(selector); + if (!el) return null; + if (!visible) return el; + return isVisible(el) ? el : null; + } catch { + return null; + } + }; + + const done = (result) => { + if (resolved) return; + resolved = true; + obs && obs.disconnect(); + clearTimeout(timer); + resolve(result); + }; + + const check = () => { + const el = isMatch(); + if (el) { + const ref = ensureRefForElement(el); + const center = centerOf(el); + done({ success: true, matched: { ref, center }, tookMs: Date.now() - start }); + } + }; + + const obs = new MutationObserver(check); + try { + obs.observe(document.documentElement || document.body, { + subtree: true, + childList: true, + characterData: true, + attributes: true, + }); + } catch {} + + // initial check + check(); + const timer = setTimeout( + () => done({ success: false, reason: 'timeout', tookMs: Date.now() - start }), + Math.max(0, timeout), + ); + }); + } + chrome.runtime.onMessage.addListener((request, _sender, sendResponse) => { try { if (request && request.action === 'wait_helper_ping') { @@ -162,6 +214,17 @@ waitFor({ text, appear, timeout }).then((res) => sendResponse(res)); return true; // async } + if (request && request.action === 'waitForSelector') { + const selector = String(request.selector || '').trim(); + const visible = request.visible !== false; // default true + const timeout = Number(request.timeout || 5000); + if (!selector) { + sendResponse({ success: false, error: 'selector is required' }); + return true; + } + waitForSelector({ selector, visible, timeout }).then((res) => sendResponse(res)); + return true; // async + } } catch (e) { sendResponse({ success: false, error: String(e && e.message ? e.message : e) }); return true; diff --git a/app/chrome-extension/package.json b/app/chrome-extension/package.json index bbb9f26..fb862ce 100644 --- a/app/chrome-extension/package.json +++ b/app/chrome-extension/package.json @@ -21,6 +21,10 @@ }, "dependencies": { "@modelcontextprotocol/sdk": "^1.11.0", + "@vue-flow/background": "^1.3.2", + "@vue-flow/controls": "^1.1.3", + "@vue-flow/core": "^1.47.0", + "@vue-flow/minimap": "^1.5.4", "@xenova/transformers": "^2.17.2", "chrome-mcp-shared": "workspace:*", "date-fns": "^4.1.0", diff --git a/builder.design.md b/builder.design.md new file mode 100644 index 0000000..2c0cc4f --- /dev/null +++ b/builder.design.md @@ -0,0 +1,199 @@ +# 模块 C — 编排画布（Builder）技术设计（builder.design.md） + +版本：v1.0（基于 builder.prd.md） +目标读者：前端架构/扩展工程/MCP 工具维护者 + +## 1. 架构概览 + +- 编辑端（Popup 内） + - 画布 UI：Vue + 画布库（建议 VueFlow），包含节点库、画布、迷你地图、属性面板、搜索/对齐、撤销/重做、自动保存。 + - 状态管理：本地 store（可用组合式 API/Pinia），维护 `nodes/edges/variables/meta` 与 UI 状态。 + - 序列化：与 `FlowV2` 对齐，保存至 `chrome.storage.local`，并确保与 `Flow` 线性模式互转。 +- 执行端（Background） + - Runner 增强：在存在 `nodes/edges` 时按 DAG 模式执行；缺省时沿用 steps[] 线性执行。 + - Node Registry：节点注册表（type→validate/run 映射），各节点内部复用 MCP 工具（chrome\_\*）。 +- 消息/发布 + - 导入/导出/发布工具：沿用现有 message types 与 native host 动态工具注册流程。 + +## 2. 数据模型 + +### 2.1 Flow V2 接口（与 design.md 保持一致） + +```ts +export type NodeType = + | 'click' + | 'fill' + | 'key' + | 'wait' + | 'assert' + | 'script' + | 'navigate' + | 'openTab' + | 'switchTab' + | 'closeTab' + | 'http' + | 'extract' + | 'delay'; + +export interface NodeBase { + id: string; + type: NodeType; + name?: string; + disabled?: boolean; + config?: any; // 每个节点的专有配置（见 2.2） + ui?: { x: number; y: number }; +} +export interface Edge { + id: string; + from: string; + to: string; + label?: 'default' | 'true' | 'false' | 'onError'; // M1 仅 default +} +export interface FlowV2 extends Flow { + // 兼容 V1 + nodes?: NodeBase[]; + edges?: Edge[]; + subflows?: Record; +} +``` + +### 2.2 节点配置（config）约定 + +- 通用：`timeoutMs?`、`retry? { count; intervalMs; backoff? }`、`screenshotOnFail?: boolean`、`saveAs?: string`（extract/http/script 可用）。 +- click/fill：`target: TargetLocator`（含 ref/candidates），fill 另有 `value: string`（支持 `{var}`）。 +- key：`keys: string`（e.g. `Backspace Enter` / `cmd+a`）。 +- wait：`condition: { text | selector | navigation | networkIdle }`。 +- assert：`assert: { exists | visible | textPresent | attribute{ selector; name; equals? | matches? } }`。 +- script：`world?: 'MAIN'|'ISOLATED'`，`code: string`，`assign?: Record`（把返回对象字段映射到 vars）。 +- http：`method; url; headers?; body?; assign?: Record`（JSONPath → vars）。 +- extract：`selector/attr/text/regex/js`（至少一种），`saveAs: string`。 +- navigate/openTab/switchTab/closeTab：与现有浏览器工具参数一致（tabTarget/startUrl/refresh 保持在运行选项层）。 + +## 3. 画布 UI 设计 + +- 组件构成 + - 左侧：节点库（基础动作分组）、搜索。 + - 中间：画布区（VueFlow），支持缩放/平移/网格吸附/多选/框选/快捷键（Del/⌘C/⌘V/⌘Z/⌘Shift+Z）。 + - 右侧：属性面板（分组：基本、目标/选择器、等待/断言、变量/映射、重试/超时、备注）。 + - 底部：日志区（运行时使用，可隐藏）。 +- 交互细节 + - 新建：拖入节点自动放置，连接时高亮可落点；禁止自环；断开连线回收端点。 + - 校验：必填项红框/提示；不合法连线阻止；保存前校验。 + - 兼容：从 steps[] 打开时自动串联节点；保存时可选择“覆盖 steps[]”（线性模式）或“仅保存 nodes/edges”。 +- 性能与体验 + - 大图优化：虚拟化/分层渲染；节点模板缓存；平移/缩放节流；100~300 节点流畅。 + +## 4. Runner（DAG 模式） + +### 4.1 执行语义（M1） + +- 当 `nodes/edges` 存在且不为空：按拓扑排序获得可执行序列；仅处理 label=default 的边。 +- 逐节点执行： + 1. 变量展开：把 config 中字符串字段里的 `{var}` 替换为 ctx.vars 的值； + 2. validate：节点注册表校验 config； + 3. run：映射到 MCP 工具（见 5），拿到结果； + 4. saveAs/assign：把产出写入 ctx.vars（或 ctx.outputs[节点ID]）。 + 5. 日志：推送节点级 RunLogEntry；失败按 `failStrategy`/`retry` 处理。 +- 退出条件：到达尾部或遇到不可恢复错误。 + +### 4.2 伪代码 + +```ts +async function runDag(flow: FlowV2, options): Promise { + const ctx = { vars: resolveVars(flow.variables, options.args), outputs: {}, runId }; + const order = topoSort(flow.nodes, flow.edges); // label=default only + for (const nodeId of order) { + const node = getNode(nodeId); + if (node.disabled) continue; + const runtime = registry[node.type]; + const conf = expandTemplates(node.config, ctx.vars); + try { + runtime.validate(conf); + const out = await runtime.run({ tabId, ctx, logger }, conf); + if (conf.saveAs) ctx.vars[conf.saveAs] = out?.value ?? out; + if (conf.assign) applyAssign(ctx.vars, out, conf.assign); // JSONPath 支持留到 M2 + logSuccess(nodeId); + } catch (e) { + const retryOk = await maybeRetry(runtime, conf, e); + if (!retryOk) handleFail(nodeId, e); // stop/continue + } + } + return summarize(ctx); +} +``` + +## 5. 节点注册表与工具映射 + +- 注册表结构 + +```ts +export interface NodeRuntime { + validate(config: T): { ok: boolean; errors?: string[] }; + run(ctx: NodeContext, config: T): Promise; +} +export interface NodeContext { + tabId: number; + vars: Record; + outputs: Record; + runId: string; + logger: (e: RunLogEntry) => void; +} +``` + +- 工具映射（与现有一致）： + - click → `chrome_click_element`（双击用 `chrome_computer.double_click`）。 + - fill → `chrome_fill_or_select`；key → `chrome_keyboard`。 + - wait/assert → `wait-helper` + `chrome_read_page`；navigate/open/switch/close → `chrome_navigate`/窗口工具。 + - script → `chrome_inject_script`（MAIN/ISOLATED）。 + - http → `chrome_network_request`（已有则复用）。 + - extract → `chrome_read_page` + 内容脚本聚合。 + +## 6. 存储/导入/导出/发布 + +- 存储：沿用 `chrome.storage.local`，key 不变（rr_flows）；在 Flow 实体上新增 `nodes/edges` 字段。 +- 导入导出：与线性一致，JSON 中可同时含 steps/nodes/edges；导入时做版本迁移。 +- 发布为 MCP 动态工具：工具名 `flow.`；inputSchema 基于 variables + 运行选项生成；调用时走通用 `record_replay_flow_run`。 + +## 7. 校验与错误处理 + +- 编辑期校验：必填/格式/选择器空值；运行期校验：validate + 容错（默认 stop）。 +- 失败截图：统一由 runner 在 catch 路径触发 screenshot 工具；日志中标出节点 ID 与错误原因。 +- 选择器回退：沿用 selector-engine 策略；fallback 用信息写入日志，并在编辑器弹出更新提示。 + +## 8. 安全与隐私 + +- 敏感变量：不落盘，不导出；运行时仅从 args 注入；Overlay 表单/提示敏感字段。 +- 注入世界：ISOLATED 为默认；MAIN 仅在用户明确选择时使用。 +- 权限最小化：不新增超出现有扩展范围的权限。 + +## 9. 性能策略 + +- 编辑端：虚拟化/节流；避免频繁全图重绘；自动保存去抖（≥500ms）。 +- 执行端：节点超时/重试/退避；M2 后引入并发与限流（全局/域级）。 + +## 10. 迁移与兼容 + +- 打开旧 Flow（仅 steps[]）→ 自动生成链式 DAG（nodes/edges）并允许切换“线性/画布”视图； +- 只要 nodes/edges 存在，优先 DAG 执行；否则走 steps[]；导出时兼容两者。 + +## 11. 开发分期（落地建议） + +- M1（2~3 周）： + - 选型并接入 VueFlow；完成画布基础、节点库、连线、属性面板、撤销/重做、自动保存； + - 支持基础节点与 DAG 串行执行；线性兼容；导入导出/发布； + - 日志/失败截图在节点上联动高亮（可先在列表展示）。 +- M2： + - If/Else/While/ForEach；OnError 分支；从节点开始调试；open/switch/close 完善；分组/折叠。 +- M3： + - 并发/限流；表达式/JSONPath 映射器；数据集/凭据；高级调试。 + +## 12. 开放问题 + +- JSONPath 与表达式语言的边界与安全沙箱如何定义？ +- 节点产出统一结构与 assign/saveAs 的歧义如何避免？ +- 画布超大规模（500+ 节点）时的严重退化处理策略？ +- 团队协作与冲突解决是否需要引入（ID 锁/合并策略）？ + +--- + +说明：本设计围绕“高度复用现有 MCP 工具、最小化新增复杂度”的原则，画布仅作为编排与可视化层；执行统一收口到背景 Runner 与工具层，便于稳定落地与运维。 diff --git a/builder.prd.md b/builder.prd.md new file mode 100644 index 0000000..489f3a2 --- /dev/null +++ b/builder.prd.md @@ -0,0 +1,146 @@ +# 模块 C — 编排画布（Builder）PRD v1.0 + +版本：v1.0（参考 Automa，兼容现有 Record & Replay） +状态：Ready for Design +负责人：产品/前端架构 + +## 0. 现状与定位（与录制回放的关系） + +- 现状：已具备“录制 → 线性步骤（steps[]）→ 回放/发布”的闭环；回放统一复用 MCP 工具（chrome\_\*），并支持变量、失败截图、网络片段等。 +- 定位：本编排画布是“录制回放模块的可视化编辑层”。 + - 录制完成的工作流可“导入画布”进行二次编排与参数化； + - 也支持在画布里“从零拖拽节点”新建工作流； + - 保存后仍与已有回放/发布/定时/导入导出机制完全打通（同一存储与执行通道）。 +- 目标用户路径（高频）：录制草稿 → 打开画布调整/补空 → 保存 → 回放验证 → 发布为 MCP 动态工具/配置定时 → 运营。 + +## 1. 背景与目标 + +- 背景：现有线性 steps[] 能覆盖大多数串行场景，但难以表达条件、循环、并发、多标签切换等复杂流程；编辑体验也不利于可视化理解与协作。 +- 目标：提供“节点 + 连线”的可视化编排画布（DAG），在保持与线性模式完全兼容的前提下，一步到位地支撑复杂业务流程的创建、调试与运行。 +- 价值： + - 降低复杂自动化的心智负担（所见即所得）。 + - 增强流程表达力（条件/循环/分支/子流程/并发）。 + - 与 MCP 工具层打通，形成可沉淀、可分享、可重放的企业级资产。 + +## 2. 范围与非范围 + +- 范围（MVP/M1）： + - 画布编辑：节点拖拽、连线、选择、移动、缩放、对齐、撤销/重做、自动保存。 + - 节点类型（基础动作）：click/fill/key/wait/assert/navigate/script/openTab/switchTab/closeTab/delay/extract/http。 + - 属性面板：每个节点的配置编辑（选择器候选、变量/占位符、等待/断言、超时/重试、保存为变量等）。 + - 变量系统：全局 variables 与节点产出保存（saveAs/assign），字符串字段支持占位符 `{var}`。 + - 执行：无条件边时按拓扑串行执行；步骤失败截图与日志；OnError 可选（stop/continue/retry）。 + - 兼容：线性 steps[] 自动映射为链式 DAG；DAG 缺省时沿用线性模式。 + - 导入/导出/发布为 MCP 动态工具（沿用现有机制，Schema 由变量推导）。 + - 来源与入口： + - 来源：① 录制得到的 steps 一键转化为 DAG（链式）；② 画布新建（拖拽节点）；③ JSON 导入； + - 入口：Popup 的“录制与回放”列表进入“编辑”；录制完成弹出“前往画布编辑”。 +- 非范围（M1 之外）： + - 高级控制流：If/Else、While/Until、ForEach（并发度控制）、OnError 分支（M2）。 + - 数据集（Dataset）/可视化 JSON 映射器/凭据库/表达式引擎（M2/M3）。 + - 团队协作、多用户并发编辑与权限（后续版本）。 + +## 3. 用户与关键场景 + +- 用户：运营/测试/开发/数据标注人员。 +- 关键场景： + 1. 登录并进入后台 → 根据条件决定分支 → 下载报告（click/fill/wait/assert/navigate/http）。 + 2. 批量表单填充 → 提交失败重试 → 记录成功项（foreach/delay/assert/saveAs）。 + 3. 爬取分页数据 → 提取字段到变量/数据集 → 导出 JSON/CSV（extract/http/脚本处理）。 + +## 4. 端到端流程（闭环） + +- 从录制到编排： + + 1. 开始录制（Popup）→ 页面内浮层反馈与事件捕获 → 停止录制； + 2. 生成 Flow 草稿（steps[]）→ 提示“前往画布编辑”； + 3. 画布自动将 steps 链式映射为 nodes/edges → 在属性面板完善：选择器候选优先级、等待/断言、变量占位、saveAs/assign； + 4. 保存 Flow（同一存储键），可导出 JSON。 + +- 从零拖拽到回放： + + 1. 画布中新建 → 从节点库拖入基础节点，连线构建链路； + 2. 在属性面板配置选择器/变量/等待/断言/脚本等 → 保存； + 3. 回放验证（可选择 “当前/新标签”、“起始 URL”）→ 查看日志与失败截图 → 调整后再次保存。 + +- 发布与定时： + 1. Flow 发布为 MCP 动态工具（flow.），输入 Schema 由 variables + 运行选项生成； + 2. 可配置定时执行（interval/daily/once），执行结果写入运行记录； + 3. 运行失败时截图与错误节点高亮，支持导出/导入迁移。 + +## 5. 功能需求（FR） + +- 画布交互（FR-BLD-001~010） + + - FR-BLD-001 画布基础：缩放/平移、对齐网格、吸附对齐、迷你地图。 + - FR-BLD-002 节点库：从侧边栏拖入节点、复制/粘贴、批量选择、删除。 + - FR-BLD-003 连线：连接/断开、自动修复、避免自环；边标签（默认/true/false/onError 预留）。 + - FR-BLD-004 属性面板：点选节点后右侧编辑配置；校验并提示错误。 + - FR-BLD-005 撤销/重做/自动保存；版本与恢复点（简版）。 + - FR-BLD-006 搜索与定位：按节点名/类型/变量引用查找并高亮。 + - FR-BLD-007 从节点开始调试（M2）；单步/断点（M3）。 + - FR-BLD-008 兼容线性：steps↔DAG 双向转换（线性自动生成链式图）。 + - FR-BLD-009 运行入口：整流回放/从选中开始回放；日志与失败截图联动高亮。 + - FR-BLD-010 导入导出：JSON（带 nodes/edges/variables/meta）。 + +- 节点与属性（FR-BLD-011~030） + + - FR-BLD-011 click/fill/key/wait/assert/navigate/script/delay：与线性动作参数一致。 + - FR-BLD-012 openTab/switchTab/closeTab：多标签编排基础动作。 + - FR-BLD-013 http（GET/POST/...）：可保存响应字段到变量（M1 可选若已有工具）。 + - FR-BLD-014 extract：按 selector/属性/文本/正则/JS 抽取，saveAs 到变量。 + - FR-BLD-015 重试策略：count/intervalMs/backoff。 + - FR-BLD-016 超时/失败策略：stop/continue/retry；失败截图开关。 + - FR-BLD-017 选择器候选与优先级：ref/css/attr/aria/text/xpath；回退记录与提示。 + - FR-BLD-018 变量系统：全局 variables；字符串字段支持 `{var}`。脚本/HTTP 支持 assign。 + - FR-BLD-019 运行选项：tabTarget/startUrl/refresh/captureNetwork/returnLogs/timeoutMs。 + - FR-BLD-020 节点命名/备注/标签；节点分组与折叠（M2）。 + +- 执行与日志（FR-BLD-031~040） + - FR-BLD-031 DAG 执行：无条件边时顺序拓扑执行；失败按策略处理。 + - FR-BLD-032 日志：节点级耗时/状态/错误信息；失败截图；网络片段（可选）。 + - FR-BLD-033 上下文：vars/outputs；saveAs/assign 写入；回放后返回 summary/outputs/logs。 + - FR-BLD-034 绑定校验：不匹配时拒绝或需 startUrl；与当前绑定规则一致。 + +## 6. 数据模型（导出/存储） + +- 兼容 Flow V1：`steps[]` 不变。 +- 新增 Flow V2： + - `nodes: NodeBase[]`、`edges: Edge[]`、`subflows?: Record`。 + - NodeBase：`{ id,type,name?,disabled?,config?,ui? }`；Edge：`{ id,from,to,label? }`（label 预留 default/true/false/onError）。 + - 变量：与现有 `variables[]` 统一；字符串字段支持 `{var}`；脚本/HTTP 支持 assign 映射到 vars。 + +## 7. 验收标准 + +- 线性流程自动转 DAG，回放结果与线性一致（10 次成功率 ≥ 95%）。 +- 画布可稳定编辑 200+ 节点；撤销/重做可靠；保存/导出/导入无损。 +- 失败报告包含失败截图与错误节点高亮；用户 1 分钟内定位问题。 +- 节点回退/选择器提示清晰，编辑器可直接调整优先级并保存。 + +## 8. 性能与非功能需求（NFR） + +- 画布交互：常见 200 节点保持流畅（60fps 优先级次之，确保不卡顿）。 +- 执行性能：单节点默认等待 ≤ 10s；并发/循环在 M2 引入；全局并发上限与限流可配置。 +- 稳定性：10 步基准用例回放成功率 ≥ 95%。 +- 安全与隐私：敏感变量不落盘；导出不含敏感值；CSP 兼容；权限最小化。 + +## 9. 里程碑 + +- M1（画布基础 + 串行 DAG） + - 画布/节点库/连线/属性面板/撤销重做/自动保存；基础节点（click/fill/key/wait/assert/navigate/script/delay/extract/http\*）； + - DAG 执行（无条件边，串行拓扑）；线性兼容；日志/截图/变量；导入导出/发布。 +- M2（控制流与多标签） + - If/Else/While/ForEach；OnError 分支；openTab/switchTab/closeTab 完善；从节点开始调试；节点分组/折叠；数据集/凭据（可选）。 +- M3（并发与限流） + - foreach 并发、全局并发上限/域级限流；表达式/JSONPath 映射器；高级调试与监控。 + +--- + +注：节点与工具的映射严格复用现有 MCP 工具（chrome\_\*），减少新增复杂度并保证稳定性。 + +## 10. 成功指标（KPI） + +- 20 分钟内：从录制到画布编排再到首次成功回放（≥ 80% 用户）。 +- 稳定性：基准 10 步流程回放成功率 ≥ 95%，并在页面小改动（文案/结构微调）下仍 ≥ 90%。 +- 工具化：发布为 MCP 工具后，十次调用成功率 ≥ 95%，参数缺失可被 Schema 友好提示。 +- 编辑效率：200+ 节点画布交互不卡顿（≥ 30 fps）；撤销/重做成功率 100%。 diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ed89a4a..d165c7d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -59,6 +59,18 @@ importers: '@modelcontextprotocol/sdk': specifier: ^1.11.0 version: 1.12.1 + '@vue-flow/background': + specifier: ^1.3.2 + version: 1.3.2(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3)) + '@vue-flow/controls': + specifier: ^1.1.3 + version: 1.1.3(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3)) + '@vue-flow/core': + specifier: ^1.47.0 + version: 1.47.0(vue@3.5.16(typescript@5.8.3)) + '@vue-flow/minimap': + specifier: ^1.5.4 + version: 1.5.4(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3)) '@xenova/transformers': specifier: ^2.17.2 version: 2.17.2 @@ -1083,6 +1095,9 @@ packages: '@types/supertest@6.0.3': resolution: {integrity: sha512-8WzXq62EXFhJ7QsH3Ocb/iKQ/Ty9ZVWnVzoTKc9tyyFRRF3a74Tk2+TLFgaFFw364Ere+npzHKEJ6ga2LzIL7w==} + '@types/web-bluetooth@0.0.20': + resolution: {integrity: sha512-g9gZnnXVq7gM7v3tJCWV/qw7w+KeOlSHAhgF9RytFyifW6AF61hdT2ucrYhPq9hLs5JIryeupHV3qGk95dH9ow==} + '@types/yargs-parser@21.0.3': resolution: {integrity: sha512-I4q9QU9MQv4oEOz4tAHJtNz1cwuLxn2F3xcc2iV5WdqLPpUnj30aUuxt1mAxYTG+oe8CZMV/+6rU4S4gRDzqtQ==} @@ -1167,6 +1182,29 @@ packages: '@volar/typescript@2.4.14': resolution: {integrity: sha512-p8Z6f/bZM3/HyCdRNFZOEEzts51uV8WHeN8Tnfnm2EBv6FDB2TQLzfVx7aJvnl8ofKAOnS64B2O8bImBFaauRw==} + '@vue-flow/background@1.3.2': + resolution: {integrity: sha512-eJPhDcLj1wEo45bBoqTXw1uhl0yK2RaQGnEINqvvBsAFKh/camHJd5NPmOdS1w+M9lggc9igUewxaEd3iCQX2w==} + peerDependencies: + '@vue-flow/core': ^1.23.0 + vue: ^3.3.0 + + '@vue-flow/controls@1.1.3': + resolution: {integrity: sha512-XCf+G+jCvaWURdFlZmOjifZGw3XMhN5hHlfMGkWh9xot+9nH9gdTZtn+ldIJKtarg3B21iyHU8JjKDhYcB6JMw==} + peerDependencies: + '@vue-flow/core': ^1.23.0 + vue: ^3.3.0 + + '@vue-flow/core@1.47.0': + resolution: {integrity: sha512-w+qrm/xjQP5NUeKUOMIbQvpOeivTbGZtY2lGffK5kHiN3ZLyEazhESc8OeIV9NZkK2T5DIeyX/nhHxCC45HLiw==} + peerDependencies: + vue: ^3.3.0 + + '@vue-flow/minimap@1.5.4': + resolution: {integrity: sha512-l4C+XTAXnRxsRpUdN7cAVFBennC1sVRzq4bDSpVK+ag7tdMczAnhFYGgbLkUw3v3sY6gokyWwMl8CDonp8eB2g==} + peerDependencies: + '@vue-flow/core': ^1.23.0 + vue: ^3.3.0 + '@vue/compiler-core@3.5.16': resolution: {integrity: sha512-AOQS2eaQOaaZQoL1u+2rCJIKDruNXVBZSiUD3chnUrsoX5ZTQMaCvXlWNIfxBJuU15r1o7+mpo5223KVtIhAgQ==} @@ -1207,6 +1245,15 @@ packages: '@vue/shared@3.5.16': resolution: {integrity: sha512-c/0fWy3Jw6Z8L9FmTyYfkpM5zklnqqa9+a6dz3DvONRKW2NEbh46BP0FHuLFSWi2TnQEtp91Z6zOWNrU6QiyPg==} + '@vueuse/core@10.11.1': + resolution: {integrity: sha512-guoy26JQktXPcz+0n3GukWIy/JDNKti9v6VEMu6kV2sYBsWuGiTU8OWdg+ADfUbHg3/3DlqySDe7JmdHrktiww==} + + '@vueuse/metadata@10.11.1': + resolution: {integrity: sha512-IGa5FXd003Ug1qAZmyE8wF3sJ81xGLSqTqtQ6jaVfkeZ4i5kS2mwQF61yhVqojRnenVew5PldLyRgvdl4YYuSw==} + + '@vueuse/shared@10.11.1': + resolution: {integrity: sha512-LHpC8711VFZlDaYUXEBbFBCQ7GS3dVU9mjOhhMhXP6txTV4EhYQg/KGnQuvt/sPAtoUKq7VVUnL6mVtFoL42sA==} + '@webext-core/fake-browser@1.3.2': resolution: {integrity: sha512-jFyPWWz+VkHAC9DRIiIPOyu6X/KlC8dYqSKweHz6tsDb86QawtVgZSpYcM+GOQBlZc5DHFo92jJ7cIq4uBnU0A==} @@ -1796,6 +1843,44 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + d3-color@3.1.0: + resolution: {integrity: sha512-zg/chbXyeBtMQ1LbD/WSoW2DpC3I0mpmPdW+ynRTj/x2DAWYrIY7qeZIHidozwV24m4iavr15lNwIwLxRmOxhA==} + engines: {node: '>=12'} + + d3-dispatch@3.0.1: + resolution: {integrity: sha512-rzUyPU/S7rwUflMyLc1ETDeBj0NRuHKKAcvukozwhshr6g6c5d8zh4c2gQjY2bZ0dXeGLWc1PF174P2tVvKhfg==} + engines: {node: '>=12'} + + d3-drag@3.0.0: + resolution: {integrity: sha512-pWbUJLdETVA8lQNJecMxoXfH6x+mO2UQo8rSmZ+QqxcbyA3hfeprFgIT//HW2nlHChWeIIMwS2Fq+gEARkhTkg==} + engines: {node: '>=12'} + + d3-ease@3.0.1: + resolution: {integrity: sha512-wR/XK3D3XcLIZwpbvQwQ5fK+8Ykds1ip7A2Txe0yxncXSdq1L9skcG7blcedkOX+ZcgxGAmLX1FrRGbADwzi0w==} + engines: {node: '>=12'} + + d3-interpolate@3.0.1: + resolution: {integrity: sha512-3bYs1rOD33uo8aqJfKP3JWPAibgw8Zm2+L9vBKEHJ2Rg+viTR7o5Mmv5mZcieN+FRYaAOWX5SJATX6k1PWz72g==} + engines: {node: '>=12'} + + d3-selection@3.0.0: + resolution: {integrity: sha512-fmTRWbNMmsmWq6xJV8D19U/gw/bwrHfNXxrIN+HfZgnzqTHp9jOmKMhsTUjXOJnZOdZY9Q28y4yebKzqDKlxlQ==} + engines: {node: '>=12'} + + d3-timer@3.0.1: + resolution: {integrity: sha512-ndfJ/JxxMd3nw31uyKoY2naivF+r29V+Lc0svZxe1JvvIRmi8hUsrMvdOwgS1o6uBHmiz91geQ0ylPP0aj1VUA==} + engines: {node: '>=12'} + + d3-transition@3.0.1: + resolution: {integrity: sha512-ApKvfjsSR6tg06xrL434C0WydLr7JewBB3V+/39RMHsaXTOG0zmt/OAXeng5M5LBm0ojmxJrpomQVZ1aPvBL4w==} + engines: {node: '>=12'} + peerDependencies: + d3-selection: 2 - 3 + + d3-zoom@3.0.0: + resolution: {integrity: sha512-b8AmV3kfQaqWAuacbPuNbL6vahnOJflOhexLzMMNLga62+/nh0JzvJ0aO/5a5MVgUFGS7Hu1P9P03o3fJkDCyw==} + engines: {node: '>=12'} + dargs@8.1.0: resolution: {integrity: sha512-wAV9QHOsNbwnWdNW2FYvE1P56wtgSbM+3SZcdGiWQILwVjACCXDCI3Ai8QlCjMDB8YK5zySiXZYBiwGmNY3lnw==} engines: {node: '>=12'} @@ -4062,6 +4147,7 @@ packages: supertest@7.1.1: resolution: {integrity: sha512-aI59HBTlG9e2wTjxGJV+DygfNLgnWbGdZxiA/sgrnNNikIW8lbDvCtF6RnhZoJ82nU7qv7ZLjrvWqCEm52fAmw==} engines: {node: '>=14.18.0'} + deprecated: Please upgrade to supertest v7.1.3+, see release notes at https://github.com/forwardemail/supertest/releases/tag/v7.1.3 - maintenance is supported by Forward Email @ https://forwardemail.net supports-color@5.5.0: resolution: {integrity: sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==} @@ -4406,6 +4492,17 @@ packages: vscode-uri@3.1.0: resolution: {integrity: sha512-/BpdSx+yCQGnCvecbyXdxHDkuk55/G3xwnC0GqY4gmQ3j+A+g8kzzgB4Nk/SINjqn6+waqw3EgbVF2QKExkRxQ==} + vue-demi@0.14.10: + resolution: {integrity: sha512-nMZBOwuzabUO0nLgIcc6rycZEebF6eeUfaiQx9+WSk8e29IbLvPU9feI6tqW4kTo3hvoYAJkMh8n8D0fuISphg==} + engines: {node: '>=12'} + hasBin: true + peerDependencies: + '@vue/composition-api': ^1.0.0-rc.1 + vue: ^3.0.0-0 || ^2.6.0 + peerDependenciesMeta: + '@vue/composition-api': + optional: true + vue-eslint-parser@10.1.3: resolution: {integrity: sha512-dbCBnd2e02dYWsXoqX5yKUZlOt+ExIpq7hmHKPb5ZqKcjf++Eo0hMseFTZMLKThrUk61m+Uv6A2YSBve6ZvuDQ==} engines: {node: ^18.18.0 || ^20.9.0 || >=21.1.0} @@ -5545,6 +5642,8 @@ snapshots: '@types/methods': 1.1.4 '@types/superagent': 8.1.9 + '@types/web-bluetooth@0.0.20': {} + '@types/yargs-parser@21.0.3': {} '@types/yargs@17.0.33': @@ -5665,6 +5764,34 @@ snapshots: path-browserify: 1.0.1 vscode-uri: 3.1.0 + '@vue-flow/background@1.3.2(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vue-flow/core': 1.47.0(vue@3.5.16(typescript@5.8.3)) + vue: 3.5.16(typescript@5.8.3) + + '@vue-flow/controls@1.1.3(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vue-flow/core': 1.47.0(vue@3.5.16(typescript@5.8.3)) + vue: 3.5.16(typescript@5.8.3) + + '@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vueuse/core': 10.11.1(vue@3.5.16(typescript@5.8.3)) + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + vue: 3.5.16(typescript@5.8.3) + transitivePeerDependencies: + - '@vue/composition-api' + + '@vue-flow/minimap@1.5.4(@vue-flow/core@1.47.0(vue@3.5.16(typescript@5.8.3)))(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@vue-flow/core': 1.47.0(vue@3.5.16(typescript@5.8.3)) + d3-selection: 3.0.0 + d3-zoom: 3.0.0 + vue: 3.5.16(typescript@5.8.3) + '@vue/compiler-core@3.5.16': dependencies: '@babel/parser': 7.27.5 @@ -5737,6 +5864,25 @@ snapshots: '@vue/shared@3.5.16': {} + '@vueuse/core@10.11.1(vue@3.5.16(typescript@5.8.3))': + dependencies: + '@types/web-bluetooth': 0.0.20 + '@vueuse/metadata': 10.11.1 + '@vueuse/shared': 10.11.1(vue@3.5.16(typescript@5.8.3)) + vue-demi: 0.14.10(vue@3.5.16(typescript@5.8.3)) + transitivePeerDependencies: + - '@vue/composition-api' + - vue + + '@vueuse/metadata@10.11.1': {} + + '@vueuse/shared@10.11.1(vue@3.5.16(typescript@5.8.3))': + dependencies: + vue-demi: 0.14.10(vue@3.5.16(typescript@5.8.3)) + transitivePeerDependencies: + - '@vue/composition-api' + - vue + '@webext-core/fake-browser@1.3.2': dependencies: lodash.merge: 4.6.2 @@ -6372,6 +6518,42 @@ snapshots: csstype@3.1.3: {} + d3-color@3.1.0: {} + + d3-dispatch@3.0.1: {} + + d3-drag@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-selection: 3.0.0 + + d3-ease@3.0.1: {} + + d3-interpolate@3.0.1: + dependencies: + d3-color: 3.1.0 + + d3-selection@3.0.0: {} + + d3-timer@3.0.1: {} + + d3-transition@3.0.1(d3-selection@3.0.0): + dependencies: + d3-color: 3.1.0 + d3-dispatch: 3.0.1 + d3-ease: 3.0.1 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-timer: 3.0.1 + + d3-zoom@3.0.0: + dependencies: + d3-dispatch: 3.0.1 + d3-drag: 3.0.0 + d3-interpolate: 3.0.1 + d3-selection: 3.0.0 + d3-transition: 3.0.1(d3-selection@3.0.0) + dargs@8.1.0: {} date-fns@4.1.0: {} @@ -9260,6 +9442,10 @@ snapshots: vscode-uri@3.1.0: {} + vue-demi@0.14.10(vue@3.5.16(typescript@5.8.3)): + dependencies: + vue: 3.5.16(typescript@5.8.3) + vue-eslint-parser@10.1.3(eslint@9.28.0(jiti@2.4.2)): dependencies: debug: 4.4.1(supports-color@5.5.0) diff --git a/task.md b/task.md new file mode 100644 index 0000000..278b400 --- /dev/null +++ b/task.md @@ -0,0 +1,67 @@ +## 录制回放 · 编排画布（Builder）落地进度 + +更新时间：2025-10-10 + +### 已完成（本次） + +- 数据模型：在 `record-replay/types.ts` 扩展 `Flow`，新增可选 `nodes`/`edges`（Flow V2 结构），兼容线性 `steps[]`。 +- 画布编辑器（M1 骨架）：新增 `popup/components/BuilderEditor.vue`，使用 VueFlow 渲染 DAG。 + - 节点库：click/fill/key/wait/assert/navigate/script/delay。 + - 画布：缩放/平移（VueFlow 内置）、网格吸附、拖拽、连线（默认单一出边）。 + - 属性面板：按节点类型编辑（选择器候选、fill 值、wait/assert/navigate/script）。 + - 互转：支持 steps→nodes（链式）与 nodes→steps（按 default 边拓扑），保存时同步覆盖 `steps[]` 以保证可立即回放。 +- 集成入口：在 Popup 的“录制与回放”列表加入“画布编辑”按钮；保存沿用 `RR_SAVE_FLOW`。 +- Runner 增强：`flow-runner.ts` 在检测到 `nodes/edges` 时，运行期进行 DAG→steps 线性化（按 default 边拓扑），复用现有线性执行与日志/截图机制。 +- Builder 小增强： + - 节点类型补充 key/delay 的默认配置、属性面板与摘要展示。 + - 快捷键：Delete/Backspace 删除选中；Cmd/Ctrl+D 复制；Cmd/Ctrl+S 保存。 + - 自动保存：节点/连线/名称变化 800ms 去抖自动保存，状态提示（保存中/已保存）。 + - 搜索定位：顶栏输入命中节点名或首个选择器，回车自动聚焦到节点并选中。 + - 新增节点类型与执行： + - http（method/url/headers/body/saveAs），Runner 调用 NETWORK_REQUEST 并可保存 JSON 响应到变量。 + - extract（selector/attr/js/saveAs），Runner 在页面执行提取并保存变量。 + - openTab/switchTab/closeTab，Runner 分别创建新标签/切换标签/关闭标签（支持按 url/title 匹配）。 + - script 支持 saveAs/assign：执行返回值支持保存到变量；assign 支持点路径（a.b[0].c）映射多个变量。 + - 校验与提示： + - 节点级校验（http/extract/switchTab/script 等必填/组合约束）与 UI 提示（字段红框+错误列表）。 + - 顶栏显示错误计数，便于定位问题。 + - 映射编辑器：KeyValueEditor 组件，用于 script/http 的 assign 键值映射编辑。 + - 从选中节点回放：Builder 顶栏支持从当前选中节点启动回放（传入 startNodeId）。 + - 错误列表面板：可展开全局错误列表，点击条目定位并聚焦到对应节点。 + - 快捷键补充：⌘/Ctrl+Z 撤销；⌘/Ctrl+Shift+Z 重做。 + - 自动排版与视图：一键自动排版（简单拓扑布局），自适应视图（fit view）。 + - 导出：Builder 顶栏直接导出 Flow JSON（保持与后台导出一致）。 + - 历史栈优化：限制最多 50 个快照，避免内存无限增长。 + - 字段级高亮：点击错误项可高亮并滚动到对应属性字段（PropertyPanel）。 + +### 目录结构与模块拆分 + +- 遵循组件聚合原则，将编辑器完整聚合在一个目录下：`popup/components/builder/` + - `model/transforms.ts`：DAG/steps 互转、ID 生成、默认配置、拓扑排序、摘要。 + - `store/useBuilderStore.ts`：编辑器状态与操作（选择/新增/删除/连线/布局/导入导出）。 + - `components/{Canvas,Sidebar,PropertyPanel}.vue`：画布/节点库/属性面板子组件。 + +### 待安装依赖 + +- 画布基于 VueFlow：需要安装 + - `@vue-flow/core` + - `@vue-flow/controls` + - `@vue-flow/minimap` + +### 下一步规划（短期） + +1. DAG 执行路径进行更细粒度控制（后续支持 true/false/onError 边），并补齐 http/extract/openTab/switchTab/closeTab 映射。 +2. 画布体验升级：撤销/重做历史上限与压缩策略、MiniMap/Controls 配置、节点模板与样式。 +3. 属性面板补齐更多节点（http/extract/openTab/switchTab/closeTab），并完善字段校验/提示。 +4. 自动保存（去抖 ≥500ms）与版本快照；画布搜索与定位。 + +### 里程碑对齐（builder.prd.md） + +- M1：画布基础 + 串行 DAG（当前已具备可用骨架，待 Runner DAG） +- M2：控制流（If/Else/While/ForEach）、OnError 分支、多标签完善（后续迭代） +- M3：并发与限流（后续迭代） + +### 影响范围与兼容性 + +- 不破坏原有线性流程与回放；保存时同步 `steps[]`，可立即回放。 +- 类型扩展仅新增可选字段，旧数据不受影响；导入导出将兼容两种结构。