From e460ebdc0381b5b5e6edd485ce500f7a8fb2352f Mon Sep 17 00:00:00 2001 From: hangerye Date: Mon, 27 Oct 2025 10:05:44 +0800 Subject: [PATCH] =?UTF-8?q?chore:=20=E9=87=8D=E6=9E=84=E5=9B=9E=E6=94=BE?= =?UTF-8?q?=E5=BC=95=E6=93=8E?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../engine/logging/run-logger.ts | 69 ++ .../engine/plugins/breakpoint.ts | 19 + .../record-replay/engine/plugins/manager.ts | 74 ++ .../record-replay/engine/plugins/types.ts | 56 + .../record-replay/engine/policies/retry.ts | 31 + .../record-replay/engine/policies/wait.ts | 94 ++ .../record-replay/engine/scheduler.ts | 538 +++++++++ .../record-replay/engine/state-manager.ts | 87 ++ .../background/record-replay/flow-runner.ts | 4 +- .../background/record-replay/index.ts | 59 +- .../background/record-replay/node-registry.ts | 1070 ----------------- .../background/record-replay/nodes/assert.ts | 87 ++ .../background/record-replay/nodes/click.ts | 108 ++ .../record-replay/nodes/conditional.ts | 55 + ...wnload-screenshot-attr-event-frame-loop.ts | 252 ++++ .../background/record-replay/nodes/drag.ts | 42 + .../record-replay/nodes/execute-flow.ts | 88 ++ .../background/record-replay/nodes/extract.ts | 47 + .../background/record-replay/nodes/fill.ts | 116 ++ .../background/record-replay/nodes/http.ts | 28 + .../background/record-replay/nodes/index.ts | 64 + .../background/record-replay/nodes/key.ts | 17 + .../background/record-replay/nodes/loops.ts | 42 + .../record-replay/nodes/navigate.ts | 17 + .../background/record-replay/nodes/script.ts | 31 + .../background/record-replay/nodes/scroll.ts | 45 + .../background/record-replay/nodes/tabs.ts | 49 + .../background/record-replay/nodes/types.ts | 21 + .../background/record-replay/nodes/wait.ts | 67 ++ .../background/record-replay/runner.ts | 718 ----------- .../background/record-replay/types.ts | 1 + .../background/tools/browser/interaction.ts | 2 + .../inject-scripts/click-helper.js | 57 +- automa.md | 2 +- record-refact.md | 17 + 35 files changed, 2280 insertions(+), 1794 deletions(-) create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/logging/run-logger.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/plugins/breakpoint.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/plugins/manager.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/plugins/types.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/policies/retry.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/policies/wait.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/scheduler.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/engine/state-manager.ts delete mode 100644 app/chrome-extension/entrypoints/background/record-replay/node-registry.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/assert.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/click.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/conditional.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/download-screenshot-attr-event-frame-loop.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/drag.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/execute-flow.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/extract.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/fill.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/http.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/index.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/key.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/loops.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/navigate.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/script.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/scroll.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/tabs.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/types.ts create mode 100644 app/chrome-extension/entrypoints/background/record-replay/nodes/wait.ts delete mode 100644 app/chrome-extension/entrypoints/background/record-replay/runner.ts diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/logging/run-logger.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/logging/run-logger.ts new file mode 100644 index 0000000..419799b --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/logging/run-logger.ts @@ -0,0 +1,69 @@ +// engine/logging/run-logger.ts — run logs, overlay and persistence +import type { RunLogEntry, RunRecord, Flow } from '../../types'; +import { appendRun } from '../../flow-store'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; + +export class RunLogger { + private logs: RunLogEntry[] = []; + constructor(private runId: string) {} + + push(e: RunLogEntry) { + this.logs.push(e); + } + + getLogs() { + return this.logs; + } + + async overlayInit() { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) + await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'init' } as any); + } catch {} + } + + async overlayAppend(text: string) { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) + await chrome.tabs.sendMessage(tabs[0].id, { + action: 'rr_overlay', + cmd: 'append', + text, + } as any); + } catch {} + } + + async overlayDone() { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + if (tabs[0]?.id) + await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'done' } as any); + } catch {} + } + + async screenshotOnFailure() { + try { + const shot = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'screenshot' }, + }); + const img = (shot?.content?.find((c: any) => c.type === 'image') as any)?.data as string; + if (img) this.logs[this.logs.length - 1].screenshotBase64 = img; + } catch {} + } + + async persist(flow: Flow, startedAt: number, success: boolean) { + const record: RunRecord = { + id: this.runId, + flowId: flow.id, + startedAt: new Date(startedAt).toISOString(), + finishedAt: new Date().toISOString(), + success, + entries: this.logs, + }; + await appendRun(record); + } +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/breakpoint.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/breakpoint.ts new file mode 100644 index 0000000..0d668a1 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/breakpoint.ts @@ -0,0 +1,19 @@ +import type { RunPlugin, StepContext } from './types'; +import { runState } from '../state-manager'; + +export function breakpointPlugin(): RunPlugin { + return { + name: 'breakpoint', + async onBeforeStep(ctx: StepContext) { + try { + const step: any = ctx.step as any; + const hasBreakpoint = step?.$breakpoint === true || step?.breakpoint === true; + if (!hasBreakpoint) return; + // mark run paused for external UI to resume + await runState.update(ctx.runId, { status: 'stopped', updatedAt: Date.now() } as any); + return { pause: true }; + } catch {} + return; + }, + }; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/manager.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/manager.ts new file mode 100644 index 0000000..509a463 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/manager.ts @@ -0,0 +1,74 @@ +import type { + RunPlugin, + HookControl, + RunContext, + StepContext, + StepAfterContext, + StepErrorContext, + StepRetryContext, + RunEndContext, + SubflowContext, +} from './types'; + +export class PluginManager { + constructor(private plugins: RunPlugin[]) {} + + async runStart(ctx: RunContext) { + for (const p of this.plugins) await safeCall(p, 'onRunStart', ctx); + } + + async beforeStep(ctx: StepContext): Promise { + for (const p of this.plugins) { + const out = await safeCall(p, 'onBeforeStep', ctx); + if (out && (out.pause || out.nextLabel)) return out; + } + return undefined; + } + + async afterStep(ctx: StepAfterContext) { + for (const p of this.plugins) await safeCall(p, 'onAfterStep', ctx); + } + + async onError(ctx: StepErrorContext): Promise { + for (const p of this.plugins) { + const out = await safeCall(p, 'onStepError', ctx); + if (out && (out.pause || out.nextLabel)) return out; + } + return undefined; + } + + async onRetry(ctx: StepRetryContext) { + for (const p of this.plugins) await safeCall(p, 'onRetry', ctx); + } + + async onChooseNextLabel(ctx: StepContext & { suggested?: string }): Promise { + for (const p of this.plugins) { + const out = await safeCall(p, 'onChooseNextLabel', ctx); + if (out && out.nextLabel) return String(out.nextLabel); + } + return undefined; + } + + async subflowStart(ctx: SubflowContext) { + for (const p of this.plugins) await safeCall(p, 'onSubflowStart', ctx); + } + + async subflowEnd(ctx: SubflowContext) { + for (const p of this.plugins) await safeCall(p, 'onSubflowEnd', ctx); + } + + async runEnd(ctx: RunEndContext) { + for (const p of this.plugins) await safeCall(p, 'onRunEnd', ctx); + } +} + +async function safeCall(plugin: RunPlugin, key: T, arg: any) { + try { + const fn = plugin[key] as any; + if (typeof fn === 'function') return await fn.call(plugin, arg); + } catch (e) { + // swallow plugin errors to keep core stable + // console.warn(`[plugin:${plugin.name}] ${String(key)} error:`, e); + } + return undefined; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/types.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/types.ts new file mode 100644 index 0000000..b27cac4 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/plugins/types.ts @@ -0,0 +1,56 @@ +// Plugin system for record-replay engine +// Inspired by webpack-like lifecycle hooks, to avoid touching core for extensibility + +import type { Flow, Step } from '../../types'; +import type { ExecResult } from '../../nodes'; + +export interface RunContext { + runId: string; + flow: Flow; + vars: Record; +} + +export interface StepContext extends RunContext { + step: Step; +} + +export interface StepErrorContext extends StepContext { + error: any; +} + +export interface StepRetryContext extends StepErrorContext { + attempt: number; +} + +export interface StepAfterContext extends StepContext { + result?: ExecResult; +} + +export interface SubflowContext extends RunContext { + subflowId: string; +} + +export interface RunEndContext extends RunContext { + success: boolean; + failed: number; +} + +export interface HookControl { + pause?: boolean; // request scheduler to pause run (e.g., breakpoint) + nextLabel?: string; // override next edge label +} + +export interface RunPlugin { + name: string; + onRunStart?(ctx: RunContext): Promise | void; + onBeforeStep?(ctx: StepContext): Promise | HookControl | void; + onAfterStep?(ctx: StepAfterContext): Promise | void; + onStepError?(ctx: StepErrorContext): Promise | HookControl | void; + onRetry?(ctx: StepRetryContext): Promise | void; + onChooseNextLabel?( + ctx: StepContext & { suggested?: string }, + ): Promise | HookControl | void; + onSubflowStart?(ctx: SubflowContext): Promise | void; + onSubflowEnd?(ctx: SubflowContext): Promise | void; + onRunEnd?(ctx: RunEndContext): Promise | void; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/policies/retry.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/policies/retry.ts new file mode 100644 index 0000000..3b0cb4a --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/policies/retry.ts @@ -0,0 +1,31 @@ +// engine/policies/retry.ts — unified retry/backoff policy + +export type BackoffKind = 'none' | 'exp'; + +export interface RetryOptions { + count?: number; // max attempts beyond the first run + intervalMs?: number; + backoff?: BackoffKind; +} + +export async function withRetry( + run: () => Promise, + onRetry?: (attempt: number, err: any) => Promise | void, + opts?: RetryOptions, +): Promise { + const max = Math.max(0, Number(opts?.count ?? 0)); + const base = Math.max(0, Number(opts?.intervalMs ?? 0)); + const backoff = (opts?.backoff || 'none') as BackoffKind; + let attempt = 0; + while (true) { + try { + return await run(); + } catch (e) { + if (attempt >= max) throw e; + if (onRetry) await onRetry(attempt, e); + const delay = base > 0 ? (backoff === 'exp' ? base * Math.pow(2, attempt) : base) : 0; + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + attempt += 1; + } + } +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/policies/wait.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/policies/wait.ts new file mode 100644 index 0000000..67f26fe --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/policies/wait.ts @@ -0,0 +1,94 @@ +// engine/policies/wait.ts — wrappers around rr-utils navigation/network waits +// Keep logic centralized to avoid duplication in schedulers and nodes + +import { handleCallTool } from '@/entrypoints/background/tools'; +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { waitForNavigation as rrWaitForNavigation, waitForNetworkIdle } from '../../rr-utils'; + +export async function waitForNavigationDone(prevUrl: string, timeoutMs?: number) { + await rrWaitForNavigation(timeoutMs, prevUrl); +} + +export async function ensureReadPageIfWeb() { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const url = tabs?.[0]?.url || ''; + if (/^(https?:|file:)/i.test(url)) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } + } catch {} +} + +export async function maybeQuickWaitForNav(prevUrl: string, timeoutMs?: number) { + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') return; + const sniffMs = 350; + const startedAt = Date.now(); + let seen = false; + await new Promise((resolve) => { + let timer: any = null; + const cleanup = () => { + try { + chrome.webNavigation.onCommitted.removeListener(onCommitted); + } catch {} + try { + chrome.webNavigation.onCompleted.removeListener(onCompleted); + } catch {} + try { + (chrome.webNavigation as any).onHistoryStateUpdated?.removeListener?.( + onHistoryStateUpdated, + ); + } catch {} + try { + chrome.tabs.onUpdated.removeListener(onUpdated); + } catch {} + if (timer) { + try { + clearTimeout(timer); + } catch {} + } + }; + const finish = async () => { + cleanup(); + if (seen) { + try { + await rrWaitForNavigation( + prevUrl ? Math.min(timeoutMs || 15000, 30000) : undefined, + prevUrl, + ); + } catch {} + } + resolve(); + }; + const mark = () => { + seen = true; + }; + const onCommitted = (d: any) => { + if (d.tabId === tabId && d.frameId === 0 && d.timeStamp >= startedAt) mark(); + }; + const onCompleted = (d: any) => { + if (d.tabId === tabId && d.frameId === 0 && d.timeStamp >= startedAt) mark(); + }; + const onHistoryStateUpdated = (d: any) => { + if (d.tabId === tabId && d.frameId === 0 && d.timeStamp >= startedAt) mark(); + }; + const onUpdated = (updatedId: number, change: chrome.tabs.TabChangeInfo) => { + if (updatedId !== tabId) return; + if (change.status === 'loading') mark(); + if (typeof change.url === 'string' && (!prevUrl || change.url !== prevUrl)) mark(); + }; + + chrome.webNavigation.onCommitted.addListener(onCommitted); + chrome.webNavigation.onCompleted.addListener(onCompleted); + try { + (chrome.webNavigation as any).onHistoryStateUpdated?.addListener?.(onHistoryStateUpdated); + } catch {} + chrome.tabs.onUpdated.addListener(onUpdated); + timer = setTimeout(finish, sniffMs); + }); + } catch {} +} + +export { waitForNetworkIdle }; diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/scheduler.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/scheduler.ts new file mode 100644 index 0000000..a44fbf2 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/scheduler.ts @@ -0,0 +1,538 @@ +// engine/scheduler.ts — DAG-only orchestrator for record-replay +// Note: consolidates wait/retry/logging and delegates node execution to nodes/* registry + +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { Flow, RunLogEntry, RunResult, Step, StepScript } from '../types'; +import { + mapDagNodeToStep, + topoOrder, + ensureTab, + expandTemplatesDeep, + waitForNetworkIdle, + applyAssign, + defaultEdgesOnly, +} from '../rr-utils'; +import { executeStep, type ExecCtx } from '../nodes'; +import { RunLogger } from './logging/run-logger'; +import { PluginManager } from './plugins/manager'; +import type { RunPlugin } from './plugins/types'; +import { breakpointPlugin } from './plugins/breakpoint'; +import { waitForNavigationDone, maybeQuickWaitForNav, ensureReadPageIfWeb } from './policies/wait'; +import { withRetry } from './policies/retry'; +import { runState } from './state-manager'; + +export interface RunOptions { + tabTarget?: 'current' | 'new'; + refresh?: boolean; + captureNetwork?: boolean; + returnLogs?: boolean; + timeoutMs?: number; + startUrl?: string; + args?: Record; + startNodeId?: string; + plugins?: RunPlugin[]; +} + +export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { + const runId = `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; + const startAt = Date.now(); + const logger = new RunLogger(runId); + const pluginManager = new PluginManager( + options.plugins && options.plugins.length ? options.plugins : [breakpointPlugin()], + ); + + // Global deadline for run (optional) + const globalTimeout = Math.max(0, Number(options.timeoutMs || 0)); + const deadline = globalTimeout > 0 ? startAt + globalTimeout : 0; + const ensureWithinDeadline = () => { + if (deadline > 0 && Date.now() > deadline) { + const err = new Error('Global timeout reached'); + logger.push({ + stepId: 'global-timeout', + status: 'failed', + message: 'Global timeout reached', + }); + throw err; + } + }; + + // prepare variables + const vars: Record = Object.create(null); + for (const v of flow.variables || []) if (v.default !== undefined) vars[v.key] = v.default; + if (options.args) Object.assign(vars, options.args); + + // Derive a default startUrl when not provided: prefer first navigate step + let derivedStartUrl: string | undefined = undefined; + try { + const hasDag0 = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; + const nodes0 = hasDag0 ? (((flow as any).nodes || []) as any[]) : []; + const edges0 = hasDag0 ? (((flow as any).edges || []) as any[]) : []; + const defaultEdges0 = hasDag0 ? defaultEdgesOnly(edges0 as any) : []; + const order0 = hasDag0 ? topoOrder(nodes0 as any, defaultEdges0 as any) : []; + const steps0: Step[] = hasDag0 ? order0.map((n) => mapDagNodeToStep(n as any)) : []; + const nav = steps0.find((s: any) => s && (s as any).type === 'navigate') as any; + if (nav && typeof nav.url === 'string') derivedStartUrl = expandTemplatesDeep(nav.url, {}); + } catch {} + + const ensured = await ensureTab({ + tabTarget: options.tabTarget, + startUrl: options.startUrl || derivedStartUrl, + refresh: options.refresh, + }); + + // register run state + try { + await runState.restore(); + await runState.add(runId, { + id: runId, + flowId: flow.id, + name: flow.name, + status: 'running', + startedAt: startAt, + updatedAt: startAt, + }); + } catch {} + + // plugins: run start + await pluginManager.runStart({ runId, flow, vars }); + + // pre-load read_page to init bridges only when on a web page + try { + const u = ensured?.url || ''; + if (/^(https?:|file:)/i.test(u)) { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } + } catch {} + + // collect required variables via overlay prompt + try { + const needed = (flow.variables || []).filter( + (v) => + (options.args?.[v.key] == null || options.args?.[v.key] === '') && + (v.rules?.required || (v.default ?? '') === ''), + ); + if (needed.length) { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.SEND_COMMAND_TO_INJECT_SCRIPT, + args: { + eventName: 'collectVariables', + payload: JSON.stringify({ variables: needed, useOverlay: true }), + }, + }); + let values: Record | null = null; + try { + const t = (res?.content || []).find((c: any) => c.type === 'text')?.text; + const j = t ? JSON.parse(t) : null; + if (j && j.success && j.values) values = j.values; + } catch {} + if (!values) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId === 'number') { + const res2 = await chrome.tabs.sendMessage(tabId, { + action: 'collectVariables', + variables: needed, + useOverlay: true, + } as any); + if (res2 && res2.success && res2.values) values = res2.values; + } + } + if (values) Object.assign(vars, values); + } + } catch {} + + await logger.overlayInit(); + + // binding enforcement + try { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const currentUrl = tabs?.[0]?.url || ''; + const bindings = flow.meta?.bindings || []; + if (!options.startUrl && bindings.length > 0) { + const ok = bindings.some((b) => { + try { + if (b.type === 'domain') return new URL(currentUrl).hostname.includes(b.value); + if (b.type === 'path') return new URL(currentUrl).pathname.startsWith(b.value); + if (b.type === 'url') return currentUrl.startsWith(b.value); + } catch {} + return false; + }); + if (!ok) { + return { + runId, + success: false, + summary: { total: 0, success: 0, failed: 0, tookMs: 0 }, + url: currentUrl, + outputs: null, + logs: [ + { + stepId: 'binding-check', + status: 'failed', + message: + 'Flow binding mismatch. Provide startUrl or open a page matching flow.meta.bindings.', + }, + ], + screenshots: { onFailure: null }, + } as RunResult; + } + } + } catch {} + + // long-running network capture (debugger) if requested + let networkCaptureStarted = false; + const stopAndSummarizeNetwork = async () => { + try { + const stopRes = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_STOP, + args: {}, + }); + const text = (stopRes?.content || []).find((c: any) => c.type === 'text')?.text; + if (!text) return; + const data = JSON.parse(text); + const requests: any[] = Array.isArray(data?.requests) ? data.requests : []; + const snippets = requests + .filter((r) => ['XHR', 'Fetch'].includes(String(r.type))) + .slice(0, 10) + .map((r) => ({ + method: String(r.method || 'GET'), + url: String(r.url || ''), + status: r.statusCode || r.status, + ms: Math.max(0, (r.responseTime || 0) - (r.requestTime || 0)), + })); + logger.push({ + stepId: 'network-capture', + status: 'success', + message: `Captured ${Number(data?.requestCount || 0)} requests` as any, + networkSnippets: snippets, + } as any); + } catch {} + }; + if (options.captureNetwork) { + try { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_START, + args: { includeStatic: false, maxCaptureTime: 3 * 60_000, inactivityTimeout: 0 }, + }); + if (!(res as any)?.isError) networkCaptureStarted = true; + } catch {} + } + + // DAG required + const hasDag = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; + if (!hasDag) { + await logger.overlayDone(); + return { + runId, + success: false, + summary: { total: 0, success: 0, failed: 0, tookMs: 0 }, + url: null, + outputs: null, + logs: [ + { + stepId: 'dag-required', + status: 'failed', + message: + 'Flow has no DAG nodes. Linear steps are no longer supported. Please migrate this flow to nodes/edges.', + }, + ], + screenshots: { onFailure: null }, + } as RunResult; + } + + const nodes = ((flow as any).nodes || []) as any[]; + const edges = ((flow as any).edges || []) as any[]; + const defaultEdges = defaultEdgesOnly(edges as any); + const order = topoOrder(nodes as any, defaultEdges as any); + const stepsToRun: Step[] = order.map((n) => mapDagNodeToStep(n as any)); + const steps = stepsToRun; + + let failed = 0; + let paused = false; + const ctx: ExecCtx = { vars, logger: (e: RunLogEntry) => logger.push(e) }; + const pendingAfterScripts: StepScript[] = []; + + const appendOverlayOk = (s: Step) => logger.overlayAppend(`✔ ${s.type} (${s.id})`); + const appendOverlayFail = (s: Step, err: any) => + logger.overlayAppend(`✘ ${s.type} (${s.id}) -> ${err?.message || String(err)}`); + + const evalCondition = (cond: any): boolean => { + try { + if (cond && typeof cond.expression === 'string' && cond.expression.trim()) { + const fn = new Function( + 'vars', + `try { return !!(${cond.expression}); } catch (e) { return false; }`, + ); + return !!fn(vars); + } + if (cond && typeof cond.var === 'string') { + const v = vars[cond.var]; + if ('equals' in cond) return String(v) === String(cond.equals); + return !!v; + } + } catch {} + return false; + }; + + const runSubflowById = async (subflowId: string) => { + const sub = (flow.subflows || {})[subflowId]; + if (!sub || !Array.isArray(sub.nodes) || sub.nodes.length === 0) return; + await pluginManager.subflowStart({ runId, flow, vars, subflowId }); + const sNodes: any[] = sub.nodes; + const sEdges: any[] = defaultEdgesOnly((sub.edges || []) as any) as any[]; + const sOrder = topoOrder(sNodes as any, sEdges as any); + const sSteps: Step[] = sOrder.map((n) => mapDagNodeToStep(n as any)) as any; + for (const step of sSteps) { + const t0 = Date.now(); + ensureWithinDeadline(); + const ctrl = await pluginManager.beforeStep({ runId, flow, vars, step }); + if (ctrl?.pause) { + paused = true; + break; + } + await withRetry( + async () => { + const beforeInfo = await getActiveTabInfo(); + const result = await executeStep(ctx, step); + if (step.type === 'click' || step.type === 'dblclick') { + const after = ((step as any).after || {}) as any; + if (after.waitForNavigation) + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); + else await maybeQuickWaitForNav(beforeInfo.url, (step as any).timeoutMs); + } + if (step.type === 'navigate' || step.type === 'openTab') { + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + await ensureReadPageIfWeb(); + } else if (step.type === 'switchTab') { + await ensureReadPageIfWeb(); + } + if (!result?.alreadyLogged) + logger.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); + await pluginManager.afterStep({ runId, flow, vars, step, result }); + await appendOverlayOk(step); + if (result?.deferAfterScript) pendingAfterScripts.push(result.deferAfterScript); + await flushAfterScripts(ctx, pendingAfterScripts, vars, logger); + }, + async (attempt, e) => { + logger.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); + await pluginManager.onRetry({ runId, flow, vars, step, error: e, attempt }); + }, + { + count: Math.max(0, (step as any).retry?.count ?? 0), + intervalMs: Math.max(0, (step as any).retry?.intervalMs ?? 0), + backoff: (step as any).retry?.backoff || 'none', + }, + ); + if (paused) break; + } + await pluginManager.subflowEnd({ runId, flow, vars, subflowId }); + }; + + try { + // DAG traversal (single-branch by nextLabel) + const id2node = new Map(nodes.map((n: any) => [n.id, n] as const)); + const outEdges = new Map>(); + for (const e of edges) { + if (!outEdges.has(e.from)) outEdges.set(e.from, []); + outEdges.get(e.from)!.push(e); + } + const indeg = new Map(nodes.map((n: any) => [n.id, 0] as const)); + for (const e of edges) indeg.set(e.to, (indeg.get(e.to) || 0) + 1); + let currentId = + options.startNodeId && id2node.has(options.startNodeId) + ? options.startNodeId + : nodes.find((n: any) => (indeg.get(n.id) || 0) === 0)?.id || nodes[0]?.id; + let guard = 0; + while (currentId && guard++ < 10000) { + ensureWithinDeadline(); + const node = id2node.get(currentId); + if (!node) break; + const step: any = mapDagNodeToStep(node as any); + const t0 = Date.now(); + const beforeInfo = await getActiveTabInfo(); + let jumpedOnError = false; + let stepNextLabel: string | undefined; + const ctrlStart = await pluginManager.beforeStep({ runId, flow, vars, step }); + if (ctrlStart?.pause) { + paused = true; + break; + } + try { + await withRetry( + async () => { + const result = await executeStep(ctx, step); + if (step.type === 'click' || step.type === 'dblclick') { + const after = ((step as any).after || {}) as any; + if (after.waitForNavigation) + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); + else await maybeQuickWaitForNav(beforeInfo.url, (step as any).timeoutMs); + } + if (step.type === 'navigate' || step.type === 'openTab') { + await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); + await ensureReadPageIfWeb(); + } else if (step.type === 'switchTab') { + await ensureReadPageIfWeb(); + } + if (!result?.alreadyLogged) + logger.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); + await pluginManager.afterStep({ runId, flow, vars, step, result }); + await appendOverlayOk(step); + if (result?.nextLabel) stepNextLabel = String(result.nextLabel); + if (result?.control) { + if (result.control.kind === 'foreach') { + const list = Array.isArray(vars[result.control.listVar]) + ? (vars[result.control.listVar] as any[]) + : []; + for (const it of list) { + vars[result.control.itemVar] = it; + await runSubflowById(result.control.subflowId); + } + } else if (result.control.kind === 'while') { + let i = 0; + while ( + i < result.control.maxIterations && + evalCondition(result.control.condition) + ) { + await runSubflowById(result.control.subflowId); + i++; + } + } + } + if (result?.deferAfterScript) pendingAfterScripts.push(result.deferAfterScript); + await flushAfterScripts(ctx, pendingAfterScripts, vars, logger); + }, + async (attempt, e) => { + logger.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); + await pluginManager.onRetry({ runId, flow, vars, step, error: e, attempt }); + }, + { + count: Math.max(0, (step as any).retry?.count ?? 0), + intervalMs: Math.max(0, (step as any).retry?.intervalMs ?? 0), + backoff: (step as any).retry?.backoff || 'none', + }, + ); + } catch (e: any) { + failed++; + logger.push({ + stepId: step.id, + status: 'failed', + message: e?.message || String(e), + tookMs: Date.now() - t0, + }); + await appendOverlayFail(step, e); + if ((step as any).screenshotOnFail !== false) await logger.screenshotOnFailure(); + const hook = await pluginManager.onError({ runId, flow, vars, step, error: e }); + if (hook?.pause) { + paused = true; + break; + } + const oes = (outEdges.get(currentId) || []) as any[]; + const errEdge = oes.find((edg) => edg.label === 'onError'); + if (errEdge) { + currentId = errEdge.to; + jumpedOnError = true; + } else { + throw e; + } + } + if (paused) break; + if (!jumpedOnError) { + // choose next by label from success path + let nextLabel: string = stepNextLabel ? String(stepNextLabel) : 'default'; + const override = await pluginManager.onChooseNextLabel({ + runId, + flow, + vars, + step, + suggested: nextLabel, + }); + if (override) nextLabel = String(override); + const oes = (outEdges.get(currentId) || []) as any[]; + const edge = + oes.find((e) => String(e.label || 'default') === nextLabel) || + oes.find((e) => !e.label || e.label === 'default'); + currentId = edge ? edge.to : undefined; + } + } + } finally { + if (networkCaptureStarted) await stopAndSummarizeNetwork(); + } + + await logger.overlayDone(); + const tookMs = Date.now() - startAt; + + // outputs: filter sensitive variables + const sensitiveKeys = new Set( + (flow.variables || []).filter((v) => v.sensitive).map((v) => v.key), + ); + const outputs: Record = {}; + for (const [k, v] of Object.entries(vars)) if (!sensitiveKeys.has(k)) outputs[k] = v; + + await pluginManager.runEnd({ runId, flow, vars, success: failed === 0 && !paused, failed }); + if (!paused) await logger.persist(flow, startAt, failed === 0); + try { + await runState.update(runId, { + status: paused ? 'stopped' : failed === 0 ? 'completed' : 'failed', + updatedAt: Date.now(), + } as any); + if (!paused) await runState.delete(runId); + } catch {} + return { + runId, + success: !paused && failed === 0, + summary: { total: steps.length, success: steps.length - failed, failed, tookMs }, + url: null, + outputs, + logs: options.returnLogs ? logger.getLogs() : undefined, + screenshots: { + onFailure: logger.getLogs().find((l) => l.status === 'failed')?.screenshotBase64, + }, + paused, + }; +} + +async function getActiveTabInfo() { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tab = tabs[0]; + return { url: tab?.url || '', status: (tab as any)?.status || '' }; +} + +async function flushAfterScripts( + ctx: ExecCtx, + pendingAfterScripts: StepScript[], + vars: Record, + logger: RunLogger, +) { + if (pendingAfterScripts.length === 0) return; + while (pendingAfterScripts.length) { + const s = pendingAfterScripts.shift()!; + const tScript = Date.now(); + const world = (s as any).world || 'ISOLATED'; + const code = String((s as any).code || ''); + if (code.trim()) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if ((s as any).saveAs) vars[(s as any).saveAs] = result; + if ((s as any).assign && typeof (s as any).assign === 'object') + applyAssign(vars, result, (s as any).assign); + } + logger.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); + } +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/engine/state-manager.ts b/app/chrome-extension/entrypoints/background/record-replay/engine/state-manager.ts new file mode 100644 index 0000000..036e47d --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/engine/state-manager.ts @@ -0,0 +1,87 @@ +// engine/state-manager.ts — lightweight run state store with events and persistence + +type Listener = (payload: T) => void; + +export interface RunState { + id: string; + flowId: string; + name?: string; + status: 'running' | 'completed' | 'failed' | 'stopped'; + startedAt: number; + updatedAt: number; +} + +export class StateManager { + private key: string; + private states = new Map(); + private listeners: Record[]> = Object.create(null); + + constructor(storageKey: string) { + this.key = storageKey; + } + + on(name: string, listener: Listener) { + (this.listeners[name] = this.listeners[name] || []).push(listener); + } + + off(name: string, listener: Listener) { + const arr = this.listeners[name]; + if (!arr) return; + const i = arr.indexOf(listener as any); + if (i >= 0) arr.splice(i, 1); + } + + private emit(name: string, payload: E) { + const arr = this.listeners[name] || []; + for (const fn of arr) + try { + fn(payload); + } catch {} + } + + getAll(): Map { + return this.states; + } + + get(id: string): T | undefined { + return this.states.get(id); + } + + async add(id: string, data: T): Promise { + this.states.set(id, data); + this.emit('add', { id, data }); + await this.persist(); + } + + async update(id: string, patch: Partial): Promise { + const cur = this.states.get(id); + if (!cur) return; + const next = Object.assign({}, cur, patch); + this.states.set(id, next); + this.emit('update', { id, data: next }); + await this.persist(); + } + + async delete(id: string): Promise { + this.states.delete(id); + this.emit('delete', { id }); + await this.persist(); + } + + private async persist(): Promise { + try { + const obj = Object.fromEntries(this.states.entries()); + await chrome.storage.local.set({ [this.key]: obj }); + } catch {} + } + + async restore(): Promise { + try { + const res = await chrome.storage.local.get(this.key); + const obj = (res && res[this.key]) || {}; + this.states = new Map(Object.entries(obj) as any); + } catch {} + } +} + +export const runState = new StateManager('rr_run_states'); diff --git a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts index 75ec618..2682e3a 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/flow-runner.ts @@ -1,3 +1,3 @@ // thin re-export for backward compatibility -export { runFlow } from './runner'; -export type { RunOptions } from './runner'; +export { runFlow } from './engine/scheduler'; +export type { RunOptions } from './engine/scheduler'; diff --git a/app/chrome-extension/entrypoints/background/record-replay/index.ts b/app/chrome-extension/entrypoints/background/record-replay/index.ts index 3f2095b..8e9350f 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/index.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/index.ts @@ -1,4 +1,4 @@ -import { BACKGROUND_MESSAGE_TYPES } from '@/common/message-types'; +import { BACKGROUND_MESSAGE_TYPES, CONTENT_MESSAGE_TYPES } from '@/common/message-types'; import { Flow } from './types'; import { listFlows, @@ -281,6 +281,36 @@ export function initRecordReplayListeners() { try { if (details.frameId !== 0) return; const url = details.url || ''; + // Ensure core content scripts are injected for this tab (pre-heat for replay) + await ensureCoreInjected(details.tabId); + // Ensure DOM observer is active on this tab (if triggers exist) + try { + const { [STORAGE_KEYS.RR_TRIGGERS]: stored } = + (await chrome.storage.local.get(STORAGE_KEYS.RR_TRIGGERS)) || {}; + const triggers: any[] = Array.isArray(stored) ? stored : []; + const domTriggers = triggers + .filter((x) => x.type === 'dom' && x.enabled !== false) + .map((x: any) => ({ + id: x.id, + selector: x.selector, + appear: x.appear !== false, + once: x.once !== false, + debounceMs: x.debounceMs ?? 800, + })); + if (typeof details.tabId === 'number') { + try { + await chrome.scripting.executeScript({ + target: { tabId: details.tabId, allFrames: true }, + files: ['inject-scripts/dom-observer.js'], + world: 'ISOLATED', + } as any); + await chrome.tabs.sendMessage(details.tabId, { + action: 'set_dom_triggers', + triggers: domTriggers, + } as any); + } catch {} + } + } catch {} const triggers = await listTriggers(); const list = triggers.filter((x) => x.type === 'url' && x.enabled !== false) as any[]; for (const t of list) { @@ -380,6 +410,33 @@ async function initTriggerEngine() { await refreshTriggers(); } +// Ensure core content scripts are present for a tab after navigation +async function ensureCoreInjected(tabId?: number) { + try { + if (typeof tabId !== 'number') return; + // Ping accessibility helper + const ok = await pingTab(tabId, CONTENT_MESSAGE_TYPES.ACCESSIBILITY_TREE_HELPER_PING); + if (!ok) { + await chrome.scripting.executeScript({ + target: { tabId, allFrames: true }, + files: ['inject-scripts/inject-bridge.js', 'inject-scripts/accessibility-tree-helper.js'], + world: 'ISOLATED', + } as any); + } + } catch {} +} + +async function pingTab(tabId: number, action: string): Promise { + try { + const resp: any = await chrome.tabs.sendMessage(tabId, { action } as any); + if (!resp) return false; + // Helpers generally respond { status: 'pong' } or { ok: true } + return resp.status === 'pong' || resp.ok === true; + } catch { + return false; + } +} + // Alarm listener executes scheduled flows chrome.alarms.onAlarm.addListener(async (alarm) => { try { diff --git a/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts b/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts deleted file mode 100644 index 612fb8a..0000000 --- a/app/chrome-extension/entrypoints/background/record-replay/node-registry.ts +++ /dev/null @@ -1,1070 +0,0 @@ -// node-registry.ts — execute a single step -// Note: keep side-effects minimal; use provided helpers and ctx.logger for logs - -import { TOOL_NAMES } from 'chrome-mcp-shared'; -import { handleCallTool } from '../tools'; -import type { - RunLogEntry, - Step, - StepAssert, - StepDrag, - StepFill, - StepKey, - StepScroll, - StepScript, - StepWait, -} from './types'; -import { locateElement } from './selector-engine'; -import { - applyAssign, - expandTemplatesDeep, - waitForNetworkIdle, - waitForNavigation, -} from './rr-utils'; - -export interface ExecCtx { - vars: Record; - logger: (e: RunLogEntry) => void; - // Current frame context for same-origin iframe operations; undefined means top frame - frameId?: number; -} - -export interface ExecResult { - alreadyLogged?: boolean; - deferAfterScript?: StepScript | null; - // next edge label to follow; supports 'true'/'false' (legacy) and - // arbitrary labels like 'case:' / 'case:else' for conditional branches - nextLabel?: string; - control?: - | { kind: 'foreach'; listVar: string; itemVar: string; subflowId: string } - | { kind: 'while'; condition: any; subflowId: string; maxIterations: number }; -} - -// NodeRuntime registry scaffolding (incremental adoption) -export interface NodeRuntime { - validate?: (step: S) => { ok: boolean; errors?: string[] }; - run: (ctx: ExecCtx, step: S) => Promise; -} - -const registry: Partial>> = { - http: { - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_REQUEST, - args: { - url: s.url, - method: s.method || 'GET', - headers: s.headers || {}, - body: s.body, - formData: s.formData, - }, - }); - const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; - try { - const payload = text ? JSON.parse(text) : null; - if (s.saveAs && payload !== undefined) ctx.vars[s.saveAs] = payload; - if (s.assign && payload !== undefined) applyAssign(ctx.vars, payload, s.assign); - } catch {} - }, - }, - extract: { - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - let value: any = null; - if (s.js && String(s.js).trim()) { - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (code: string) => { - try { - return (0, eval)(code); - } catch (e) { - return null; - } - }, - args: [String(s.js)], - } as any); - value = result; - } else if (s.selector) { - const attr = String(s.attr || 'text'); - const sel = String(s.selector); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (selector: string, attr: string) => { - try { - const el = document.querySelector(selector) as any; - if (!el) return null; - if (attr === 'text' || attr === 'textContent') return (el.textContent || '').trim(); - return el.getAttribute ? el.getAttribute(attr) : null; - } catch { - return null; - } - }, - args: [sel, attr], - } as any); - value = result; - } - if (s.saveAs) ctx.vars[s.saveAs] = value; - }, - }, - script: { - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - if (s.when === 'after') return { deferAfterScript: s }; - const world = s.world || 'ISOLATED'; - const code = String(s.code || ''); - if (!code.trim()) return {}; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (userCode: string) => { - try { - return (0, eval)(userCode); - } catch { - return null; - } - }, - args: [code], - world: world as any, - } as any); - if (s.saveAs) ctx.vars[s.saveAs] = result; - if (s.assign && typeof s.assign === 'object') applyAssign(ctx.vars, result, s.assign); - return {}; - }, - }, - openTab: { - run: async (_ctx, step) => { - const s: any = expandTemplatesDeep(step as any, {}); - if (s.newWindow) await chrome.windows.create({ url: s.url || undefined, focused: true }); - else await chrome.tabs.create({ url: s.url || undefined, active: true }); - }, - }, - switchTab: { - run: async (_ctx, step) => { - const s: any = expandTemplatesDeep(step as any, {}); - let targetTabId: number | undefined = s.tabId; - if (!targetTabId) { - const tabs = await chrome.tabs.query({}); - const hit = tabs.find( - (t) => - (s.urlContains && (t.url || '').includes(String(s.urlContains))) || - (s.titleContains && (t.title || '').includes(String(s.titleContains))), - ); - targetTabId = (hit && hit.id) as number | undefined; - } - if (!targetTabId) throw new Error('switchTab: no matching tab'); - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.SWITCH_TAB, - args: { tabId: targetTabId }, - }); - if ((res as any).isError) throw new Error('switchTab failed'); - }, - }, - closeTab: { - run: async (_ctx, step) => { - const s: any = expandTemplatesDeep(step as any, {}); - const args: any = {}; - if (Array.isArray(s.tabIds) && s.tabIds.length) args.tabIds = s.tabIds; - if (s.url) args.url = s.url; - const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.CLOSE_TABS, args }); - if ((res as any).isError) throw new Error('closeTab failed'); - }, - }, - scroll: { - run: async (_ctx, step: StepScroll) => { - const s = step as StepScroll; - const top = s.offset?.y ?? undefined; - const left = s.offset?.x ?? undefined; - const selectorFromTarget = (s.target?.candidates || []).find( - (c) => c.type === 'css' || c.type === 'attr', - )?.value; - let code = ''; - if (s.mode === 'offset' && !s.target) { - const t = top != null ? Number(top) : 'undefined'; - const l = left != null ? Number(left) : 'undefined'; - code = `try { window.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {}`; - } else if (s.mode === 'element' && selectorFromTarget) { - code = `(() => { try { const el = document.querySelector(${JSON.stringify(selectorFromTarget)}); if (el) el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }); } catch (e) {} })();`; - } else if (s.mode === 'container' && selectorFromTarget) { - const t = top != null ? Number(top) : 'undefined'; - const l = left != null ? Number(left) : 'undefined'; - code = `(() => { try { const el = document.querySelector(${JSON.stringify(selectorFromTarget)}); if (el && typeof el.scrollTo === 'function') el.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {} })();`; - } else { - const direction = top != null && Number(top) < 0 ? 'up' : 'down'; - const amount = 3; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'scroll', scrollDirection: direction, scrollAmount: amount }, - }); - if ((res as any).isError) throw new Error('scroll failed'); - return {}; - } - if (code) { - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { type: 'MAIN', jsScript: code }, - }); - if ((res as any).isError) throw new Error('scroll failed'); - } - return {}; - }, - }, - drag: { - run: async (_ctx, step: StepDrag) => { - const s = step as StepDrag; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - let startRef: string | undefined; - let endRef: string | undefined; - try { - if (typeof tabId === 'number') { - const locatedStart = await locateElement(tabId, s.start); - const locatedEnd = await locateElement(tabId, s.end); - startRef = (locatedStart as any)?.ref || s.start.ref; - endRef = (locatedEnd as any)?.ref || s.end.ref; - } - } catch {} - let startCoordinates: { x: number; y: number } | undefined; - let endCoordinates: { x: number; y: number } | undefined; - if ((!startRef || !endRef) && Array.isArray(s.path) && s.path.length >= 2) { - startCoordinates = { x: Number(s.path[0].x), y: Number(s.path[0].y) }; - const last = s.path[s.path.length - 1]; - endCoordinates = { x: Number(last.x), y: Number(last.y) }; - } - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { - action: 'left_click_drag', - startRef, - ref: endRef, - startCoordinates, - coordinates: endCoordinates, - }, - }); - if ((res as any).isError) throw new Error('drag failed'); - }, - }, - click: { - validate: (step) => { - const ok = !!(step as any).target?.candidates?.length; - return ok ? { ok } : { ok, errors: ['缺少目标选择器候选'] }; - }, - run: async (ctx, step) => { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const located = await locateElement(tabId, s.target, ctx.frameId); - const frameId = (located as any)?.frameId ?? ctx.frameId; - const first = s.target?.candidates?.[0]?.type; - const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); - const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; - if ((located as any)?.ref) { - const resolved: any = (await chrome.tabs.sendMessage( - tabId, - { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId } as any, - )) as any; - const rect = resolved?.rect; - if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); - } - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.CLICK, - args: { - ref: (located as any)?.ref || s.target?.ref, - selector: !(located as any)?.ref - ? s.target?.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value - : undefined, - waitForNavigation: false, - timeout: Math.max(1000, Math.min(s.timeoutMs || 10000, 30000)), - frameId, - }, - }); - if ((res as any).isError) throw new Error('click failed'); - if (fallbackUsed) - ctx.logger({ - stepId: step.id, - status: 'success', - message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, - fallbackUsed: true, - fallbackFrom: String(first), - fallbackTo: String(resolvedBy), - } as any); - }, - }, - dblclick: { - validate: (step) => { - const ok = !!(step as any).target?.candidates?.length; - return ok ? { ok } : { ok, errors: ['缺少目标选择器候选'] }; - }, - run: async (ctx, step) => { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const located = await locateElement(tabId, s.target, ctx.frameId); - const frameId = (located as any)?.frameId ?? ctx.frameId; - const first = s.target?.candidates?.[0]?.type; - const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); - const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; - if ((located as any)?.ref) { - const resolved: any = (await chrome.tabs.sendMessage( - tabId, - { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId } as any, - )) as any; - const rect = resolved?.rect; - if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); - } - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'double_click', ref: (located as any)?.ref || (step as any).target?.ref }, - }); - if ((res as any).isError) throw new Error('dblclick failed'); - if (fallbackUsed) - ctx.logger({ - stepId: step.id, - status: 'success', - message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, - fallbackUsed: true, - fallbackFrom: String(first), - fallbackTo: String(resolvedBy), - } as any); - }, - }, - fill: { - validate: (step) => { - const ok = !!(step as any).target?.candidates?.length && 'value' in (step as any); - return ok ? { ok } : { ok, errors: ['缺少目标选择器候选或输入值'] }; - }, - run: async (ctx, step: StepFill) => { - const s = step as any; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const located = await locateElement(tabId, s.target, ctx.frameId); - const frameId = (located as any)?.frameId ?? ctx.frameId; - const first = s.target?.candidates?.[0]?.type; - const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); - const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; - // Interpolate only when string; allow boolean/number for checkbox/range/number - const interpolate = (v: any) => - typeof v === 'string' - ? v.replace(/\{([^}]+)\}/g, (_m, k) => (ctx.vars[k] ?? '').toString()) - : v; - const value = interpolate(s.value); - if ((located as any)?.ref) { - const resolved: any = (await chrome.tabs.sendMessage( - tabId, - { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId } as any, - )) as any; - const rect = resolved?.rect; - if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); - } - // Special-case: file inputs must use CDP setInputFiles (file upload tool) instead of value assignment - // Prefer CSS/attr selector when available - const cssSelector = !(located as any)?.ref - ? s.target.candidates?.find((c) => c.type === 'css' || c.type === 'attr')?.value - : undefined; - if (cssSelector) { - try { - const attr: any = (await chrome.tabs.sendMessage( - tabId, - { - action: 'getAttributeForSelector', - selector: cssSelector, - name: 'type', - } as any, - { frameId } as any, - )) as any; - const typeName = (attr && attr.value ? String(attr.value) : '').toLowerCase(); - if (typeName === 'file') { - const uploadRes = await handleCallTool({ - name: TOOL_NAMES.BROWSER.FILE_UPLOAD, - args: { selector: cssSelector, filePath: String(value ?? '') }, - }); - if ((uploadRes as any).isError) throw new Error('file upload failed'); - if (fallbackUsed) - ctx.logger({ - stepId: step.id, - status: 'success', - message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, - fallbackUsed: true, - fallbackFrom: String(first), - fallbackTo: String(resolvedBy), - } as any); - return {} as any; - } - } catch { - // continue to normal fill on errors - } - } - try { - // Scroll into view then focus before filling - if (cssSelector) - await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { - type: 'MAIN', - jsScript: `try{var el=document.querySelector(${JSON.stringify(cssSelector)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, - }, - }); - } catch {} - try { - if ((located as any)?.ref) - await chrome.tabs.sendMessage( - tabId, - { action: 'focusByRef', ref: (located as any).ref } as any, - { frameId } as any, - ); - else if (cssSelector) - await handleCallTool({ - name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, - args: { - type: 'MAIN', - jsScript: `try{var el=document.querySelector(${JSON.stringify(cssSelector)});if(el&&el.focus){el.focus();}}catch(e){}`, - }, - }); - } catch {} - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.FILL, - args: { - ref: (located as any)?.ref || s.target?.ref, - selector: cssSelector, - value, - frameId, - }, - }); - if ((res as any).isError) throw new Error('fill failed'); - if (fallbackUsed) - ctx.logger({ - stepId: step.id, - status: 'success', - message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, - fallbackUsed: true, - fallbackFrom: String(first), - fallbackTo: String(resolvedBy), - } as any); - }, - }, - key: { - run: async (_ctx, step: StepKey) => { - const s = expandTemplatesDeep(step as StepKey, {}); - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.KEYBOARD, - args: { keys: (s as StepKey).keys }, - }); - if ((res as any).isError) throw new Error('key failed'); - }, - }, - wait: { - validate: (step) => { - const ok = !!(step as any).condition; - return ok ? { ok } : { ok, errors: ['缺少等待条件'] }; - }, - run: async (_ctx, step: StepWait) => { - const s = expandTemplatesDeep(step as StepWait, {}); - const cond = (s as StepWait).condition as - | { selector: string; visible?: boolean } - | { text: string; appear?: boolean } - | { navigation: true } - | { networkIdle: true } - | { sleep: number }; - if ('text' in cond) { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - await chrome.scripting.executeScript({ - target: { tabId }, - files: ['inject-scripts/wait-helper.js'], - world: 'ISOLATED', - } as any); - const resp: any = (await chrome.tabs.sendMessage( - tabId, - { - action: 'waitForText', - text: cond.text, - appear: cond.appear !== false, - timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), - } as any, - { frameId: ctx.frameId } as any, - )) as any; - if (!resp || resp.success !== true) throw new Error('wait text failed'); - } else if ('networkIdle' in cond) { - const total = Math.min(Math.max(1000, (s as any).timeoutMs || 5000), 120000); - const idle = Math.min(1500, Math.max(500, Math.floor(total / 3))); - await waitForNetworkIdle(total, idle); - } else if ('navigation' in cond) { - await waitForNavigation((s as any).timeoutMs); - } else if ('sleep' in cond) { - const ms = Math.max(0, Number(cond.sleep ?? 0)); - await new Promise((r) => setTimeout(r, ms)); - } else if ('selector' in cond) { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - await chrome.scripting.executeScript({ - target: { tabId }, - files: ['inject-scripts/wait-helper.js'], - world: 'ISOLATED', - } as any); - const resp: any = (await chrome.tabs.sendMessage(tabId, { - action: 'waitForSelector', - selector: cond.selector, - visible: cond.visible !== false, - timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), - } as any)) as any; - if (!resp || resp.success !== true) throw new Error('wait selector failed'); - } - }, - }, - assert: { - validate: (step) => { - const s = step as any; - const ok = !!s.assert; - // basic shape checks for attribute - if (ok && s.assert && 'attribute' in s.assert) { - const a = s.assert.attribute || {}; - if (!a.selector || !a.name) - return { ok: false, errors: ['assert.attribute: 需提供 selector 与 name'] }; - } - return ok ? { ok } : { ok, errors: ['缺少断言条件'] }; - }, - run: async (ctx, step: StepAssert) => { - const s = expandTemplatesDeep(step as StepAssert, {}) as StepAssert; - const failStrategy = (s as any).failStrategy || 'stop'; - const fail = (msg: string) => { - if (failStrategy === 'warn') { - ctx.logger({ stepId: step.id, status: 'warning', message: msg }); - return { alreadyLogged: true } as any; - } - // retry/stop -> throw to let runner decide by step.retry - throw new Error(msg); - }; - if ('textPresent' in s.assert) { - const text = (s.assert as any).textPresent; - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'wait', text, appear: true, timeout: (step as any).timeoutMs || 5000 }, - }); - if ((res as any).isError) return fail('assert text failed'); - } else if ('exists' in s.assert || 'visible' in s.assert) { - const selector = (s.assert as any).exists || (s.assert as any).visible; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) return fail('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const ensured: any = (await chrome.tabs.sendMessage(tabId, { - action: 'ensureRefForSelector', - selector, - } as any)) as any; - if (!ensured || !ensured.success) return fail('assert selector not found'); - if ('visible' in s.assert) { - const rect = ensured && ensured.center ? ensured.center : null; - if (!rect) return fail('assert visible failed'); - } - } else if ('attribute' in s.assert) { - const { selector, name, equals, matches } = (s.assert as any).attribute || {}; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const firstTab = tabs && tabs[0]; - const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; - if (!tabId) return fail('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const resp: any = (await chrome.tabs.sendMessage( - tabId, - { action: 'getAttributeForSelector', selector, name } as any, - { frameId: ctx.frameId } as any, - )) as any; - if (!resp || !resp.success) return fail('assert attribute: element not found'); - const actual: string | null = resp.value ?? null; - if (equals !== undefined && equals !== null) { - const expected = String(equals); - if (String(actual) !== String(expected)) - return fail( - `assert attribute equals failed: ${name} actual=${String(actual)} expected=${String(expected)}`, - ); - } else if (matches !== undefined && matches !== null) { - try { - const re = new RegExp(String(matches)); - if (!re.test(String(actual))) - return fail( - `assert attribute matches failed: ${name} actual=${String(actual)} regex=${String(matches)}`, - ); - } catch { - return fail(`invalid regex for attribute matches: ${String(matches)}`); - } - } else { - if (actual == null) return fail(`assert attribute failed: ${name} missing`); - } - } - return {} as any; - }, - }, - navigate: { - validate: (step) => { - const ok = !!(step as any).url; - return ok ? { ok } : { ok, errors: ['缺少 URL'] }; - }, - run: async (_ctx, step) => { - const url = (step as any).url; - const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url } }); - if ((res as any).isError) throw new Error('navigate failed'); - }, - }, - if: { - validate: (step) => { - const s = step as any; - const hasBranches = Array.isArray(s.branches) && s.branches.length > 0; - const ok = hasBranches || !!s.condition; - return ok ? { ok } : { ok, errors: ['缺少条件或分支'] }; - }, - run: async (ctx, step) => { - const s: any = step; - // Branch-first evaluation when branches[] provided; fallback to legacy condition - if (Array.isArray(s.branches) && s.branches.length > 0) { - // evaluate in order; first matched wins - const evalExpr = (expr: string): boolean => { - const code = String(expr || '').trim(); - if (!code) return false; - try { - // Note: basic eval with limited scope. Support both `vars` and legacy `workflow` identifiers. - // This preserves backward compatibility with older expressions written as `workflow.xxx`. - const fn = new Function( - 'vars', - 'workflow', - `try { return !!(${code}); } catch (e) { return false; }`, - ); - return !!fn(ctx.vars, ctx.vars); - } catch { - return false; - } - }; - for (const b of s.branches) { - if (evalExpr(b.expr)) return { nextLabel: `case:${String(b.id)}` } as ExecResult; - } - // none matched - return { nextLabel: s.else === false ? 'default' : 'case:else' } as ExecResult; - } else { - // legacy single condition -> true/false - const cond = s.condition || {}; - let result = false; - try { - if (typeof cond.expression === 'string' && cond.expression.trim()) { - const fn = new Function( - 'vars', - `try { return !!(${cond.expression}); } catch (e) { return false; }`, - ); - result = !!fn(ctx.vars); - } else if (typeof cond.var === 'string') { - const v = ctx.vars[cond.var]; - if ('equals' in cond) result = String(v) === String(cond.equals); - else result = !!v; - } - } catch { - result = false; - } - return { nextLabel: result ? 'true' : 'false' } as ExecResult; - } - }, - }, - foreach: { - validate: (step) => { - const s = step as any; - const ok = - typeof s.listVar === 'string' && - s.listVar && - typeof s.subflowId === 'string' && - s.subflowId; - return ok ? { ok } : { ok, errors: ['foreach: 需提供 listVar 与 subflowId'] }; - }, - run: async (_ctx, step) => { - const s: any = step; - const itemVar = typeof s.itemVar === 'string' && s.itemVar ? s.itemVar : 'item'; - return { - control: { - kind: 'foreach', - listVar: String(s.listVar), - itemVar, - subflowId: String(s.subflowId), - }, - } as ExecResult; - }, - }, - while: { - validate: (step) => { - const s = step as any; - const ok = !!s.condition && typeof s.subflowId === 'string' && s.subflowId; - return ok ? { ok } : { ok, errors: ['while: 需提供 condition 与 subflowId'] }; - }, - run: async (_ctx, step) => { - const s: any = step; - const max = Math.max(1, Math.min(10000, Number(s.maxIterations ?? 100))); - return { - control: { - kind: 'while', - condition: s.condition, - subflowId: String(s.subflowId), - maxIterations: max, - }, - } as ExecResult; - }, - }, - executeFlow: { - validate: (step) => { - const s: any = step; - const ok = typeof s.flowId === 'string' && !!s.flowId; - return ok ? { ok } : { ok, errors: ['需提供 flowId'] }; - }, - run: async (ctx, step) => { - const s: any = step; - const { getFlow } = await import('./flow-store'); - const flow = await getFlow(String(s.flowId)); - if (!flow) throw new Error('referenced flow not found'); - const inline = s.inline !== false; // default inline - if (!inline) { - const { runFlow } = await import('./flow-runner'); - await runFlow(flow, { args: s.args || {}, returnLogs: false }); - return; - } - // Inline: execute referenced flow's steps with current ctx/vars - const { - defaultEdgesOnly, - topoOrder, - mapDagNodeToStep, - waitForNetworkIdle, - waitForNavigation, - } = await import('./rr-utils'); - const vars = ctx.vars; - if (s.args && typeof s.args === 'object') Object.assign(vars, s.args); - const hasDag = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; - const nodes = hasDag ? (((flow as any).nodes || []) as any[]) : []; - const edges = hasDag ? (((flow as any).edges || []) as any[]) : []; - const defaultEdges = hasDag ? defaultEdgesOnly(edges as any) : []; - const order = hasDag ? topoOrder(nodes as any, defaultEdges as any) : []; - const stepsToRun: any[] = hasDag - ? order.map((n) => mapDagNodeToStep(n as any)) - : ((flow.steps || []) as any[]); - for (const st of stepsToRun) { - const t0 = Date.now(); - const maxRetries = Math.max(0, (st as any).retry?.count ?? 0); - const baseInterval = Math.max(0, (st as any).retry?.intervalMs ?? 0); - let attempt = 0; - const doDelay = async (i: number) => { - const delay = - baseInterval > 0 - ? (st as any).retry?.backoff === 'exp' - ? baseInterval * Math.pow(2, i) - : baseInterval - : 0; - if (delay > 0) await new Promise((r) => setTimeout(r, delay)); - }; - while (true) { - try { - const beforeInfo = await (async () => { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tab = tabs[0]; - return { url: tab?.url || '', status: (tab as any)?.status || '' }; - })(); - const result = await executeStep(ctx, st as any); - if ((st.type === 'click' || st.type === 'dblclick') && (st as any).after) { - const after = (st as any).after as any; - if (after.waitForNavigation) - await waitForNavigation((st as any).timeoutMs, beforeInfo.url); - else if (after.waitForNetworkIdle) - await waitForNetworkIdle(Math.min((st as any).timeoutMs || 5000, 120000), 1200); - } - if (!result?.alreadyLogged) - ctx.logger({ stepId: st.id, status: 'success', tookMs: Date.now() - t0 } as any); - break; - } catch (e: any) { - if (attempt < maxRetries) { - ctx.logger({ - stepId: st.id, - status: 'retrying', - message: e?.message || String(e), - } as any); - await doDelay(attempt); - attempt += 1; - continue; - } - ctx.logger({ - stepId: st.id, - status: 'failed', - message: e?.message || String(e), - tookMs: Date.now() - t0, - } as any); - throw e; - } - } - } - }, - }, - handleDownload: { - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const args: any = { - filenameContains: s.filenameContains || undefined, - timeoutMs: Math.max(1000, Math.min(Number(s.timeoutMs ?? 60000), 300000)), - waitForComplete: s.waitForComplete !== false, - }; - const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.HANDLE_DOWNLOAD, args }); - const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; - try { - const payload = text ? JSON.parse(text) : null; - if (s.saveAs && payload && payload.download) ctx.vars[s.saveAs] = payload.download; - } catch {} - }, - }, - // P0: screenshot node (wrapper around screenshot tool) - screenshot: { - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const args: any = { name: 'workflow', storeBase64: true }; - if (s.fullPage) args.fullPage = true; - if (s.selector && typeof s.selector === 'string' && s.selector.trim()) - args.selector = s.selector; - const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.SCREENSHOT, args }); - const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; - try { - const payload = text ? JSON.parse(text) : null; - if (s.saveAs && payload && payload.base64Data) ctx.vars[s.saveAs] = payload.base64Data; - } catch {} - }, - }, - // P0: trigger custom DOM event - triggerEvent: { - validate: (step) => { - const s: any = step; - const ok = !!s?.target?.candidates?.length && typeof s?.event === 'string' && s.event; - return ok ? { ok } : { ok, errors: ['缺少目标选择器或事件类型'] }; - }, - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const located = await locateElement(tabId, s.target, ctx.frameId); - const cssSelector = !(located as any)?.ref - ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value - : undefined; - let sel = cssSelector as string | undefined; - if (!sel && (located as any)?.ref) { - try { - const resolved: any = (await chrome.tabs.sendMessage( - tabId, - { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId } as any, - )) as any; - sel = resolved?.selector; - } catch {} - } - if (!sel) throw new Error('triggerEvent: selector not resolved'); - const world: any = 'MAIN'; - const ev = String(s.event || '').trim(); - const bubbles = s.bubbles !== false; - const cancelable = s.cancelable === true; - await chrome.scripting.executeScript({ - target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any, - world, - func: (selector: string, type: string, bubbles: boolean, cancelable: boolean) => { - try { - const el = document.querySelector(selector); - if (!el) return false; - const e = new Event(type, { bubbles, cancelable }); - el.dispatchEvent(e); - return true; - } catch (e) { - return false; - } - }, - args: [sel, ev, !!bubbles, !!cancelable], - } as any); - }, - }, - // P0: set attribute node - setAttribute: { - validate: (step) => { - const s: any = step; - const ok = !!s?.target?.candidates?.length && typeof s?.name === 'string' && s.name; - return ok ? { ok } : { ok, errors: ['需提供目标选择器与属性名'] }; - }, - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - const located = await locateElement(tabId, s.target, ctx.frameId); - const frameId = (located as any)?.frameId ?? ctx.frameId; - const cssSelector = !(located as any)?.ref - ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value - : undefined; - let sel = cssSelector as string | undefined; - if (!sel && (located as any)?.ref) { - try { - const resolved: any = (await chrome.tabs.sendMessage( - tabId, - { action: 'resolveRef', ref: (located as any).ref } as any, - { frameId } as any, - )) as any; - sel = resolved?.selector; - } catch {} - } - if (!sel) throw new Error('setAttribute: selector not resolved'); - const world: any = 'MAIN'; - const name = String(s.name || ''); - const value = s.value != null ? String(s.value) : null; - const remove = s.remove === true || value == null; - await chrome.scripting.executeScript({ - target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any, - world, - func: (selector: string, name: string, value: string | null, remove: boolean) => { - try { - const el = document.querySelector(selector) as any; - if (!el) return false; - if (remove) { - el.removeAttribute(name); - return true; - } - el.setAttribute(name, String(value ?? '')); - return true; - } catch { - return false; - } - }, - args: [sel, name, value, remove], - } as any); - }, - }, - // P0: switch to a same-origin iframe by index or url substring - switchFrame: { - run: async (ctx, step) => { - const s: any = step; - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - // discover frames via webNavigation API - const frames = await chrome.webNavigation.getAllFrames({ tabId }); - if (!Array.isArray(frames) || frames.length === 0) { - ctx.frameId = undefined; - return; - } - // choose by index (excluding 0 which is top frame) - let target: any | undefined; - const idx = Number(s?.frame?.index ?? NaN); - if (Number.isFinite(idx)) { - const list = frames.filter((f) => f.frameId !== 0); - target = list[Math.max(0, Math.min(list.length - 1, idx))]; - } - // choose by urlContains if provided - const urlContains = String(s?.frame?.urlContains || '').trim(); - if (!target && urlContains) - target = frames.find((f) => typeof f.url === 'string' && f.url.includes(urlContains)); - // fallback to top (clear) - if (!target) ctx.frameId = undefined; - else ctx.frameId = target.frameId; - // ensure helper injected into all frames for subsequent operations - try { - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - } catch {} - ctx.logger({ - stepId: step.id, - status: 'success', - message: `frameId=${String(ctx.frameId ?? 'top')}`, - } as any); - }, - }, - // P0: loop over elements matching a selector and branch into subflow - loopElements: { - validate: (step) => { - const s: any = step; - const ok = - typeof s?.selector === 'string' && - s.selector && - typeof s?.subflowId === 'string' && - s.subflowId; - return ok ? { ok } : { ok, errors: ['需提供 selector 与 subflowId'] }; - }, - run: async (ctx, step) => { - const s: any = expandTemplatesDeep(step as any, ctx.vars); - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - const world: any = 'MAIN'; - const selector = String(s.selector || ''); - const res = await chrome.scripting.executeScript({ - target: { - tabId, - frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, - } as any, - world, - func: (sel: string) => { - try { - const list = Array.from(document.querySelectorAll(sel)); - const toCss = (node: Element) => { - try { - if ((node as HTMLElement).id) { - const idSel = `#${CSS.escape((node as HTMLElement).id)}`; - if (document.querySelectorAll(idSel).length === 1) return idSel; - } - } catch {} - let path = ''; - let current: Element | null = node; - while (current && current.tagName !== 'BODY') { - let part = current.tagName.toLowerCase(); - const parent = current.parentElement; - if (parent) { - const siblings = Array.from(parent.children).filter( - (c) => (c as any).tagName === current!.tagName, - ); - if (siblings.length > 1) { - const idx = siblings.indexOf(current) + 1; - part += `:nth-of-type(${idx})`; - } - } - path = path ? `${part} > ${path}` : part; - current = parent; - } - return path ? `body > ${path}` : 'body'; - }; - return list.map(toCss); - } catch (e) { - return []; - } - }, - args: [selector], - } as any); - const arr: string[] = (res && Array.isArray(res[0]?.result) ? res[0].result : []) as any; - const listVar = String(s.saveAs || 'elements'); - const itemVar = String(s.itemVar || 'item'); - ctx.vars[listVar] = arr; - return { - control: { kind: 'foreach', listVar, itemVar, subflowId: String(s.subflowId) }, - } as any; - }, - }, -}; - -// New unified executeStep using registry only -export async function executeStep(ctx: ExecCtx, step: Step): Promise { - const runtime = (registry as any)[step.type] as NodeRuntime | undefined; - if (!runtime) throw new Error(`unsupported step type: ${String(step.type)}`); - const v = runtime.validate ? runtime.validate(step as any) : { ok: true }; - if (!(v as any).ok) throw new Error(((v as any).errors || []).join(', ') || 'validation failed'); - const out = await runtime.run(ctx, step as any); - return (out || {}) as ExecResult; -} diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/assert.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/assert.ts new file mode 100644 index 0000000..488c0ae --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/assert.ts @@ -0,0 +1,87 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepAssert } from '../types'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const assertNode: NodeRuntime = { + validate: (step) => { + const s = step as any; + const ok = !!s.assert; + if (ok && s.assert && 'attribute' in s.assert) { + const a = s.assert.attribute || {}; + if (!a.selector || !a.name) + return { ok: false, errors: ['assert.attribute: 需提供 selector 与 name'] }; + } + return ok ? { ok } : { ok, errors: ['缺少断言条件'] }; + }, + run: async (ctx: ExecCtx, step: StepAssert) => { + const s = expandTemplatesDeep(step as StepAssert, {}) as any; + const failStrategy = (s as any).failStrategy || 'stop'; + const fail = (msg: string) => { + if (failStrategy === 'warn') { + ctx.logger({ stepId: (step as any).id, status: 'warning', message: msg }); + return { alreadyLogged: true } as any; + } + throw new Error(msg); + }; + if ('textPresent' in s.assert) { + const text = (s.assert as any).textPresent; + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'wait', text, appear: true, timeout: (step as any).timeoutMs || 5000 }, + }); + if ((res as any).isError) return fail('assert text failed'); + } else if ('exists' in s.assert || 'visible' in s.assert) { + const selector = (s.assert as any).exists || (s.assert as any).visible; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) return fail('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const ensured: any = (await chrome.tabs.sendMessage(tabId, { + action: 'ensureRefForSelector', + selector, + } as any)) as any; + if (!ensured || !ensured.success) return fail('assert selector not found'); + if ('visible' in s.assert) { + const rect = ensured && ensured.center ? ensured.center : null; + if (!rect) return fail('assert visible failed'); + } + } else if ('attribute' in s.assert) { + const { selector, name, equals, matches } = (s.assert as any).attribute || {}; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) return fail('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const resp: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'getAttributeForSelector', selector, name } as any, + { frameId: ctx.frameId } as any, + )) as any; + if (!resp || !resp.success) return fail('assert attribute: element not found'); + const actual: string | null = resp.value ?? null; + if (equals !== undefined && equals !== null) { + const expected = String(equals); + if (String(actual) !== String(expected)) + return fail( + `assert attribute equals failed: ${name} actual=${String(actual)} expected=${String(expected)}`, + ); + } else if (matches !== undefined && matches !== null) { + try { + const re = new RegExp(String(matches)); + if (!re.test(String(actual))) + return fail( + `assert attribute matches failed: ${name} actual=${String(actual)} regex=${String(matches)}`, + ); + } catch { + return fail(`invalid regex for attribute matches: ${String(matches)}`); + } + } else { + if (actual == null) return fail(`assert attribute failed: ${name} missing`); + } + } + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/click.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/click.ts new file mode 100644 index 0000000..6a4757d --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/click.ts @@ -0,0 +1,108 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { Step } from '../types'; +import { locateElement } from '../selector-engine'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const clickNode: NodeRuntime = { + validate: (step) => { + const ok = !!(step as any).target?.candidates?.length; + return ok ? { ok } : { ok, errors: ['缺少目标选择器候选'] }; + }, + run: async (ctx: ExecCtx, step: Step) => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; + const first = s.target?.candidates?.[0]?.type; + const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); + const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; + if ((located as any)?.ref) { + const resolved: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId } as any, + )) as any; + const rect = resolved?.rect; + if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); + } + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.CLICK, + args: { + ref: (located as any)?.ref || (step as any).target?.ref, + selector: !(located as any)?.ref + ? s.target?.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined, + waitForNavigation: false, + timeout: Math.max(1000, Math.min(s.timeoutMs || 10000, 30000)), + frameId, + }, + }); + if ((res as any).isError) throw new Error('click failed'); + if (fallbackUsed) + ctx.logger({ + stepId: step.id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + return {} as ExecResult; + }, +}; + +export const dblclickNode: NodeRuntime = { + validate: clickNode.validate, + run: async (ctx: ExecCtx, step: Step) => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; + const first = s.target?.candidates?.[0]?.type; + const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); + const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; + if ((located as any)?.ref) { + const resolved: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId } as any, + )) as any; + const rect = resolved?.rect; + if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); + } + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.CLICK, + args: { + ref: (located as any)?.ref || (step as any).target?.ref, + selector: !(located as any)?.ref + ? s.target?.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined, + waitForNavigation: false, + timeout: Math.max(1000, Math.min(s.timeoutMs || 10000, 30000)), + frameId, + double: true, + }, + }); + if ((res as any).isError) throw new Error('dblclick failed'); + if (fallbackUsed) + ctx.logger({ + stepId: step.id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/conditional.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/conditional.ts new file mode 100644 index 0000000..7d08013 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/conditional.ts @@ -0,0 +1,55 @@ +import type { Step } from '../types'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const ifNode: NodeRuntime = { + validate: (step) => { + const s = step as any; + const hasBranches = Array.isArray(s.branches) && s.branches.length > 0; + const ok = hasBranches || !!s.condition; + return ok ? { ok } : { ok, errors: ['缺少条件或分支'] }; + }, + run: async (ctx: ExecCtx, step: Step) => { + const s: any = step; + if (Array.isArray(s.branches) && s.branches.length > 0) { + const evalExpr = (expr: string): boolean => { + const code = String(expr || '').trim(); + if (!code) return false; + try { + const fn = new Function( + 'vars', + 'workflow', + `try { return !!(${code}); } catch (e) { return false; }`, + ); + return !!fn(ctx.vars, ctx.vars); + } catch { + return false; + } + }; + for (const br of s.branches) { + if (br?.expr && evalExpr(String(br.expr))) + return { nextLabel: String(br.label || `case:${br.id || 'match'}`) } as ExecResult; + } + if ('else' in s) return { nextLabel: String(s.else || 'default') } as ExecResult; + return { nextLabel: 'default' } as ExecResult; + } + // legacy condition: { var/equals | expression } + try { + let result = false; + const cond = s.condition; + if (cond && typeof cond.expression === 'string' && cond.expression.trim()) { + const fn = new Function( + 'vars', + `try { return !!(${cond.expression}); } catch (e) { return false; }`, + ); + result = !!fn(ctx.vars); + } else if (cond && typeof cond.var === 'string') { + const v = ctx.vars[cond.var]; + if ('equals' in cond) result = String(v) === String(cond.equals); + else result = !!v; + } + return { nextLabel: result ? 'true' : 'false' } as ExecResult; + } catch { + return { nextLabel: 'false' } as ExecResult; + } + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/download-screenshot-attr-event-frame-loop.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/download-screenshot-attr-event-frame-loop.ts new file mode 100644 index 0000000..f679c6d --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/download-screenshot-attr-event-frame-loop.ts @@ -0,0 +1,252 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { Step } from '../types'; +import { locateElement } from '../selector-engine'; + +export const handleDownloadNode: NodeRuntime = { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const args: any = { + filenameContains: s.filenameContains || undefined, + timeoutMs: Math.max(1000, Math.min(Number(s.timeoutMs ?? 60000), 300000)), + waitForComplete: s.waitForComplete !== false, + }; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.HANDLE_DOWNLOAD, args }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload && payload.download) ctx.vars[s.saveAs] = payload.download; + } catch {} + return {} as ExecResult; + }, +}; + +export const screenshotNode: NodeRuntime = { + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const args: any = { name: 'workflow', storeBase64: true }; + if (s.fullPage) args.fullPage = true; + if (s.selector && typeof s.selector === 'string' && s.selector.trim()) + args.selector = s.selector; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.SCREENSHOT, args }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload && payload.base64Data) ctx.vars[s.saveAs] = payload.base64Data; + } catch {} + return {} as ExecResult; + }, +}; + +export const triggerEventNode: NodeRuntime = { + validate: (step) => { + const s: any = step; + const ok = !!s?.target?.candidates?.length && typeof s?.event === 'string' && s.event; + return ok ? { ok } : { ok, errors: ['缺少目标选择器或事件类型'] }; + }, + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const located = await locateElement(tabId, s.target, ctx.frameId); + const cssSelector = !(located as any)?.ref + ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined; + let sel = cssSelector as string | undefined; + if (!sel && (located as any)?.ref) { + try { + const resolved: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId: ctx.frameId } as any, + )) as any; + sel = resolved?.selector; + } catch {} + } + if (!sel) throw new Error('triggerEvent: selector not resolved'); + const world: any = 'MAIN'; + const ev = String(s.event || '').trim(); + const bubbles = s.bubbles !== false; + const cancelable = s.cancelable === true; + await chrome.scripting.executeScript({ + target: { + tabId, + frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, + } as any, + world, + func: (selector: string, type: string, bubbles: boolean, cancelable: boolean) => { + try { + const el = document.querySelector(selector); + if (!el) return false; + const e = new Event(type, { bubbles, cancelable }); + (el as any).dispatchEvent(e); + return true; + } catch (e) { + return false; + } + }, + args: [sel, ev, !!bubbles, !!cancelable], + } as any); + return {} as ExecResult; + }, +}; + +export const setAttributeNode: NodeRuntime = { + validate: (step) => { + const s: any = step; + const ok = !!s?.target?.candidates?.length && typeof s?.name === 'string' && s.name; + return ok ? { ok } : { ok, errors: ['需提供目标选择器与属性名'] }; + }, + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; + const cssSelector = !(located as any)?.ref + ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined; + let sel = cssSelector as string | undefined; + if (!sel && (located as any)?.ref) { + try { + const resolved: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId } as any, + )) as any; + sel = resolved?.selector; + } catch {} + } + if (!sel) throw new Error('setAttribute: selector not resolved'); + const world: any = 'MAIN'; + const name = String(s.name || ''); + const value = s.value; + const remove = s.remove === true; + await chrome.scripting.executeScript({ + target: { tabId, frameIds: typeof frameId === 'number' ? [frameId] : undefined } as any, + world, + func: (selector: string, name: string, value: any, remove: boolean) => { + try { + const el = document.querySelector(selector) as any; + if (!el) return false; + if (remove) el.removeAttribute(name); + else el.setAttribute(name, String(value ?? '')); + return true; + } catch { + return false; + } + }, + args: [sel, name, value, remove], + } as any); + return {} as ExecResult; + }, +}; + +export const switchFrameNode: NodeRuntime = { + run: async (ctx, step) => { + const s: any = step; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const frames = await chrome.webNavigation.getAllFrames({ tabId }); + if (!Array.isArray(frames) || frames.length === 0) { + ctx.frameId = undefined; + return {} as ExecResult; + } + let target: any | undefined; + const idx = Number(s?.frame?.index ?? NaN); + if (Number.isFinite(idx)) { + const list = frames.filter((f) => f.frameId !== 0); + target = list[Math.max(0, Math.min(list.length - 1, idx))]; + } + const urlContains = String(s?.frame?.urlContains || '').trim(); + if (!target && urlContains) + target = frames.find((f) => typeof f.url === 'string' && f.url.includes(urlContains)); + if (!target) ctx.frameId = undefined; + else ctx.frameId = target.frameId; + try { + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + } catch {} + ctx.logger({ + stepId: (step as any).id, + status: 'success', + message: `frameId=${String(ctx.frameId ?? 'top')}`, + } as any); + return {} as ExecResult; + }, +}; + +export const loopElementsNode: NodeRuntime = { + validate: (step) => { + const s: any = step; + const ok = + typeof s?.selector === 'string' && + s.selector && + typeof s?.subflowId === 'string' && + s.subflowId; + return ok ? { ok } : { ok, errors: ['需提供 selector 与 subflowId'] }; + }, + run: async (ctx, step) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const world: any = 'MAIN'; + const selector = String(s.selector || ''); + const res = await chrome.scripting.executeScript({ + target: { + tabId, + frameIds: typeof ctx.frameId === 'number' ? [ctx.frameId] : undefined, + } as any, + world, + func: (sel: string) => { + try { + const list = Array.from(document.querySelectorAll(sel)); + const toCss = (node: Element) => { + try { + if ((node as HTMLElement).id) { + const idSel = `#${CSS.escape((node as HTMLElement).id)}`; + if (document.querySelectorAll(idSel).length === 1) return idSel; + } + } catch {} + let path = ''; + let current: Element | null = node; + while (current && current.tagName !== 'BODY') { + let part = current.tagName.toLowerCase(); + const parent = current.parentElement; + if (parent) { + const siblings = Array.from(parent.children).filter( + (c) => (c as any).tagName === current!.tagName, + ); + if (siblings.length > 1) { + const idx = siblings.indexOf(current) + 1; + part += `:nth-of-type(${idx})`; + } + } + path = path ? `${part} > ${path}` : part; + current = parent; + } + return path ? `body > ${path}` : 'body'; + }; + return list.map(toCss); + } catch (e) { + return []; + } + }, + args: [selector], + } as any); + const arr: string[] = (res && Array.isArray(res[0]?.result) ? res[0].result : []) as any; + const listVar = String(s.saveAs || 'elements'); + const itemVar = String(s.itemVar || 'item'); + ctx.vars[listVar] = arr; + return { + control: { kind: 'foreach', listVar, itemVar, subflowId: String(s.subflowId) }, + } as any; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/drag.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/drag.ts new file mode 100644 index 0000000..6d35c50 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/drag.ts @@ -0,0 +1,42 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepDrag } from '../types'; +import { locateElement } from '../selector-engine'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const dragNode: NodeRuntime = { + run: async (_ctx, step: StepDrag) => { + const s = step as StepDrag; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + let startRef: string | undefined; + let endRef: string | undefined; + try { + if (typeof tabId === 'number') { + const locatedStart = await locateElement(tabId, (s as any).start); + const locatedEnd = await locateElement(tabId, (s as any).end); + startRef = (locatedStart as any)?.ref || (s as any).start.ref; + endRef = (locatedEnd as any)?.ref || (s as any).end.ref; + } + } catch {} + let startCoordinates: { x: number; y: number } | undefined; + let endCoordinates: { x: number; y: number } | undefined; + if ((!startRef || !endRef) && Array.isArray((s as any).path) && (s as any).path.length >= 2) { + startCoordinates = { x: Number((s as any).path[0].x), y: Number((s as any).path[0].y) }; + const last = (s as any).path[(s as any).path.length - 1]; + endCoordinates = { x: Number(last.x), y: Number(last.y) }; + } + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { + action: 'left_click_drag', + startRef, + ref: endRef, + startCoordinates, + coordinates: endCoordinates, + }, + }); + if ((res as any).isError) throw new Error('drag failed'); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/execute-flow.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/execute-flow.ts new file mode 100644 index 0000000..ae57b5e --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/execute-flow.ts @@ -0,0 +1,88 @@ +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const executeFlowNode: NodeRuntime = { + validate: (step) => { + const s: any = step; + const ok = typeof s.flowId === 'string' && !!s.flowId; + return ok ? { ok } : { ok, errors: ['需提供 flowId'] }; + }, + run: async (ctx: ExecCtx, step) => { + const s: any = step; + const { getFlow } = await import('../flow-store'); + const flow = await getFlow(String(s.flowId)); + if (!flow) throw new Error('referenced flow not found'); + const inline = s.inline !== false; // default inline + if (!inline) { + const { runFlow } = await import('../flow-runner'); + await runFlow(flow, { args: s.args || {}, returnLogs: false }); + return {} as ExecResult; + } + const { defaultEdgesOnly, topoOrder, mapDagNodeToStep, waitForNetworkIdle, waitForNavigation } = + await import('../rr-utils'); + const vars = ctx.vars; + if (s.args && typeof s.args === 'object') Object.assign(vars, s.args); + const hasDag = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; + const nodes = hasDag ? (((flow as any).nodes || []) as any[]) : []; + const edges = hasDag ? (((flow as any).edges || []) as any[]) : []; + const defaultEdges = hasDag ? defaultEdgesOnly(edges as any) : []; + const order = hasDag ? topoOrder(nodes as any, defaultEdges as any) : []; + const stepsToRun: any[] = hasDag + ? order.map((n) => mapDagNodeToStep(n as any)) + : (((flow as any).steps || []) as any[]); + for (const st of stepsToRun) { + const t0 = Date.now(); + const maxRetries = Math.max(0, (st as any).retry?.count ?? 0); + const baseInterval = Math.max(0, (st as any).retry?.intervalMs ?? 0); + let attempt = 0; + const doDelay = async (i: number) => { + const delay = + baseInterval > 0 + ? (st as any).retry?.backoff === 'exp' + ? baseInterval * Math.pow(2, i) + : baseInterval + : 0; + if (delay > 0) await new Promise((r) => setTimeout(r, delay)); + }; + while (true) { + try { + const beforeInfo = await (async () => { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tab = tabs[0]; + return { url: tab?.url || '', status: (tab as any)?.status || '' }; + })(); + const { executeStep } = await import('../nodes'); + const result = await executeStep(ctx as any, st as any); + if ((st.type === 'click' || st.type === 'dblclick') && (st as any).after) { + const after = (st as any).after as any; + if (after.waitForNavigation) + await waitForNavigation((st as any).timeoutMs, beforeInfo.url); + else if (after.waitForNetworkIdle) + await waitForNetworkIdle(Math.min((st as any).timeoutMs || 5000, 120000), 1200); + } + if (!result?.alreadyLogged) + ctx.logger({ stepId: st.id, status: 'success', tookMs: Date.now() - t0 } as any); + break; + } catch (e: any) { + if (attempt < maxRetries) { + ctx.logger({ + stepId: st.id, + status: 'retrying', + message: e?.message || String(e), + } as any); + await doDelay(attempt); + attempt += 1; + continue; + } + ctx.logger({ + stepId: st.id, + status: 'failed', + message: e?.message || String(e), + tookMs: Date.now() - t0, + } as any); + throw e; + } + } + } + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/extract.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/extract.ts new file mode 100644 index 0000000..b277542 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/extract.ts @@ -0,0 +1,47 @@ +import type { StepExtract } from '../types'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const extractNode: NodeRuntime = { + run: async (ctx: ExecCtx, step: StepExtract) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + let value: any = null; + if (s.js && String(s.js).trim()) { + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (code: string) => { + try { + return (0, eval)(code); + } catch (e) { + return null; + } + }, + args: [String(s.js)], + } as any); + value = result; + } else if (s.selector) { + const attr = String(s.attr || 'text'); + const sel = String(s.selector); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (selector: string, attr: string) => { + try { + const el = document.querySelector(selector) as any; + if (!el) return null; + if (attr === 'text' || attr === 'textContent') return (el.textContent || '').trim(); + return el.getAttribute ? el.getAttribute(attr) : null; + } catch { + return null; + } + }, + args: [sel, attr], + } as any); + value = result; + } + if (s.saveAs) ctx.vars[s.saveAs] = value; + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/fill.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/fill.ts new file mode 100644 index 0000000..843b88b --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/fill.ts @@ -0,0 +1,116 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepFill } from '../types'; +import { locateElement } from '../selector-engine'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const fillNode: NodeRuntime = { + validate: (step) => { + const ok = !!(step as any).target?.candidates?.length && 'value' in (step as any); + return ok ? { ok } : { ok, errors: ['缺少目标选择器候选或输入值'] }; + }, + run: async (ctx: ExecCtx, step: StepFill) => { + const s: any = step; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const firstTab = tabs && tabs[0]; + const tabId = firstTab && typeof firstTab.id === 'number' ? firstTab.id : undefined; + if (!tabId) throw new Error('Active tab not found'); + await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); + const located = await locateElement(tabId, s.target, ctx.frameId); + const frameId = (located as any)?.frameId ?? ctx.frameId; + const first = s.target?.candidates?.[0]?.type; + const resolvedBy = (located as any)?.resolvedBy || ((located as any)?.ref ? 'ref' : ''); + const fallbackUsed = resolvedBy && first && resolvedBy !== 'ref' && resolvedBy !== first; + const interpolate = (v: any) => + typeof v === 'string' + ? v.replace(/\{([^}]+)\}/g, (_m, k) => (ctx.vars[k] ?? '').toString()) + : v; + const value = interpolate(s.value); + if ((located as any)?.ref) { + const resolved: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'resolveRef', ref: (located as any).ref } as any, + { frameId } as any, + )) as any; + const rect = resolved?.rect; + if (!rect || rect.width <= 0 || rect.height <= 0) throw new Error('element not visible'); + } + const cssSelector = !(located as any)?.ref + ? s.target.candidates?.find((c: any) => c.type === 'css' || c.type === 'attr')?.value + : undefined; + if (cssSelector) { + try { + const attr: any = (await chrome.tabs.sendMessage( + tabId, + { action: 'getAttributeForSelector', selector: cssSelector, name: 'type' } as any, + { frameId } as any, + )) as any; + const typeName = (attr && attr.value ? String(attr.value) : '').toLowerCase(); + if (typeName === 'file') { + const uploadRes = await handleCallTool({ + name: TOOL_NAMES.BROWSER.FILE_UPLOAD, + args: { selector: cssSelector, filePath: String(value ?? '') }, + }); + if ((uploadRes as any).isError) throw new Error('file upload failed'); + if (fallbackUsed) + ctx.logger({ + stepId: (step as any).id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + return {} as ExecResult; + } + } catch {} + } + try { + if (cssSelector) + await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { + type: 'MAIN', + jsScript: `try{var el=document.querySelector(${JSON.stringify(cssSelector)});if(el){el.scrollIntoView({behavior:'instant',block:'center',inline:'nearest'});} }catch(e){}`, + }, + }); + } catch {} + try { + if ((located as any)?.ref) + await chrome.tabs.sendMessage( + tabId, + { action: 'focusByRef', ref: (located as any).ref } as any, + { frameId } as any, + ); + else if (cssSelector) + await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { + type: 'MAIN', + jsScript: `try{var el=document.querySelector(${JSON.stringify(cssSelector)});if(el&&el.focus){el.focus();}}catch(e){}`, + }, + }); + } catch {} + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.FILL, + args: { + ref: (located as any)?.ref || (s as any).target?.ref, + selector: cssSelector, + value, + frameId, + }, + }); + if ((res as any).isError) throw new Error('fill failed'); + if (fallbackUsed) + ctx.logger({ + stepId: (step as any).id, + status: 'success', + message: `Selector fallback used (${String(first)} -> ${String(resolvedBy)})`, + fallbackUsed: true, + fallbackFrom: String(first), + fallbackTo: String(resolvedBy), + } as any); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/http.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/http.ts new file mode 100644 index 0000000..3a6b1bf --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/http.ts @@ -0,0 +1,28 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepHttp } from '../types'; +import { applyAssign, expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const httpNode: NodeRuntime = { + run: async (ctx: ExecCtx, step: StepHttp) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.NETWORK_REQUEST, + args: { + url: s.url, + method: s.method || 'GET', + headers: s.headers || {}, + body: s.body, + formData: s.formData, + }, + }); + const text = (res as any)?.content?.find((c: any) => c.type === 'text')?.text; + try { + const payload = text ? JSON.parse(text) : null; + if (s.saveAs && payload !== undefined) ctx.vars[s.saveAs] = payload; + if (s.assign && payload !== undefined) applyAssign(ctx.vars, payload, s.assign); + } catch {} + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/index.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/index.ts new file mode 100644 index 0000000..9562f2f --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/index.ts @@ -0,0 +1,64 @@ +import type { Step } from '../types'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; +import { clickNode, dblclickNode } from './click'; +import { fillNode } from './fill'; +import { httpNode } from './http'; +import { extractNode } from './extract'; +import { scriptNode } from './script'; +import { openTabNode, switchTabNode, closeTabNode } from './tabs'; +import { scrollNode } from './scroll'; +import { dragNode } from './drag'; +import { keyNode } from './key'; +import { waitNode } from './wait'; +import { assertNode } from './assert'; +import { navigateNode } from './navigate'; +import { ifNode } from './conditional'; +import { foreachNode, whileNode } from './loops'; +import { executeFlowNode } from './execute-flow'; +import { + handleDownloadNode, + screenshotNode, + triggerEventNode, + setAttributeNode, + switchFrameNode, + loopElementsNode, +} from './download-screenshot-attr-event-frame-loop'; + +const registry = new Map>([ + ['click', clickNode], + ['dblclick', dblclickNode], + ['fill', fillNode], + ['http', httpNode], + ['extract', extractNode], + ['script', scriptNode], + ['openTab', openTabNode], + ['switchTab', switchTabNode], + ['closeTab', closeTabNode], + ['scroll', scrollNode], + ['drag', dragNode], + ['key', keyNode], + ['wait', waitNode], + ['assert', assertNode], + ['navigate', navigateNode], + ['if', ifNode], + ['foreach', foreachNode], + ['while', whileNode], + ['executeFlow', executeFlowNode], + ['handleDownload', handleDownloadNode], + ['screenshot', screenshotNode], + ['triggerEvent', triggerEventNode], + ['setAttribute', setAttributeNode], + ['switchFrame', switchFrameNode], + ['loopElements', loopElementsNode], +]); + +export async function executeStep(ctx: ExecCtx, step: Step): Promise { + const rt = registry.get((step as any).type); + if (!rt) throw new Error(`unsupported step type: ${String((step as any).type)}`); + const v = rt.validate ? rt.validate(step as any) : { ok: true }; + if (!(v as any).ok) throw new Error(((v as any).errors || []).join(', ') || 'validation failed'); + const out = await rt.run(ctx as any, step as any); + return (out || {}) as ExecResult; +} + +export type { ExecCtx, ExecResult, NodeRuntime } from './types'; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/key.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/key.ts new file mode 100644 index 0000000..7fe0349 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/key.ts @@ -0,0 +1,17 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepKey } from '../types'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const keyNode: NodeRuntime = { + run: async (_ctx, step: StepKey) => { + const s = expandTemplatesDeep(step as StepKey, {}); + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.KEYBOARD, + args: { keys: (s as StepKey).keys }, + }); + if ((res as any).isError) throw new Error('key failed'); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/loops.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/loops.ts new file mode 100644 index 0000000..4a0bd43 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/loops.ts @@ -0,0 +1,42 @@ +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const foreachNode: NodeRuntime = { + validate: (step) => { + const s = step as any; + const ok = + typeof s.listVar === 'string' && s.listVar && typeof s.subflowId === 'string' && s.subflowId; + return ok ? { ok } : { ok, errors: ['foreach: 需提供 listVar 与 subflowId'] }; + }, + run: async (_ctx: ExecCtx, step) => { + const s: any = step; + const itemVar = typeof s.itemVar === 'string' && s.itemVar ? s.itemVar : 'item'; + return { + control: { + kind: 'foreach', + listVar: String(s.listVar), + itemVar, + subflowId: String(s.subflowId), + }, + } as ExecResult; + }, +}; + +export const whileNode: NodeRuntime = { + validate: (step) => { + const s = step as any; + const ok = !!s.condition && typeof s.subflowId === 'string' && s.subflowId; + return ok ? { ok } : { ok, errors: ['while: 需提供 condition 与 subflowId'] }; + }, + run: async (_ctx: ExecCtx, step) => { + const s: any = step; + const max = Math.max(1, Math.min(10000, Number(s.maxIterations ?? 100))); + return { + control: { + kind: 'while', + condition: s.condition, + subflowId: String(s.subflowId), + maxIterations: max, + }, + } as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/navigate.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/navigate.ts new file mode 100644 index 0000000..dc6e182 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/navigate.ts @@ -0,0 +1,17 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { Step } from '../types'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const navigateNode: NodeRuntime = { + validate: (step) => { + const ok = !!(step as any).url; + return ok ? { ok } : { ok, errors: ['缺少 URL'] }; + }, + run: async (_ctx: ExecCtx, step: Step) => { + const url = (step as any).url; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.NAVIGATE, args: { url } }); + if ((res as any).isError) throw new Error('navigate failed'); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/script.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/script.ts new file mode 100644 index 0000000..ad98de2 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/script.ts @@ -0,0 +1,31 @@ +import type { StepScript } from '../types'; +import { expandTemplatesDeep, applyAssign } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const scriptNode: NodeRuntime = { + run: async (ctx: ExecCtx, step: StepScript) => { + const s: any = expandTemplatesDeep(step as any, ctx.vars); + if (s.when === 'after') return { deferAfterScript: s } as ExecResult; + const world = s.world || 'ISOLATED'; + const code = String(s.code || ''); + if (!code.trim()) return {} as ExecResult; + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + const [{ result }] = await chrome.scripting.executeScript({ + target: { tabId }, + func: (userCode: string) => { + try { + return (0, eval)(userCode); + } catch { + return null; + } + }, + args: [code], + world: world as any, + } as any); + if (s.saveAs) ctx.vars[s.saveAs] = result; + if (s.assign && typeof s.assign === 'object') applyAssign(ctx.vars, result, s.assign); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/scroll.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/scroll.ts new file mode 100644 index 0000000..eb24551 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/scroll.ts @@ -0,0 +1,45 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepScroll } from '../types'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const scrollNode: NodeRuntime = { + run: async (_ctx, step: StepScroll) => { + const s = expandTemplatesDeep(step as StepScroll, {}); + const top = s.offset?.y ?? undefined; + const left = s.offset?.x ?? undefined; + const selectorFromTarget = (s as any).target?.candidates?.find( + (c: any) => c.type === 'css' || c.type === 'attr', + )?.value; + let code = ''; + if (s.mode === 'offset' && !(s as any).target) { + const t = top != null ? Number(top) : 'undefined'; + const l = left != null ? Number(left) : 'undefined'; + code = `try { window.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {}`; + } else if (s.mode === 'element' && selectorFromTarget) { + code = `(() => { try { const el = document.querySelector(${JSON.stringify(selectorFromTarget)}); if (el) el.scrollIntoView({ behavior: 'instant', block: 'center', inline: 'nearest' }); } catch (e) {} })();`; + } else if (s.mode === 'container' && selectorFromTarget) { + const t = top != null ? Number(top) : 'undefined'; + const l = left != null ? Number(left) : 'undefined'; + code = `(() => { try { const el = document.querySelector(${JSON.stringify(selectorFromTarget)}); if (el && typeof el.scrollTo === 'function') el.scrollTo({ top: ${t}, left: ${l}, behavior: 'instant' }); } catch (e) {} })();`; + } else { + const direction = top != null && Number(top) < 0 ? 'up' : 'down'; + const amount = 3; + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.COMPUTER, + args: { action: 'scroll', scrollDirection: direction, scrollAmount: amount }, + }); + if ((res as any).isError) throw new Error('scroll failed'); + return {} as ExecResult; + } + if (code) { + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.INJECT_SCRIPT, + args: { type: 'MAIN', jsScript: code }, + }); + if ((res as any).isError) throw new Error('scroll failed'); + } + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/tabs.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/tabs.ts new file mode 100644 index 0000000..dece0ab --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/tabs.ts @@ -0,0 +1,49 @@ +import { TOOL_NAMES } from 'chrome-mcp-shared'; +import { handleCallTool } from '@/entrypoints/background/tools'; +import type { StepOpenTab, StepSwitchTab, StepCloseTab } from '../types'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const openTabNode: NodeRuntime = { + run: async (_ctx, step) => { + const s: any = expandTemplatesDeep(step as any, {}); + if (s.newWindow) await chrome.windows.create({ url: s.url || undefined, focused: true }); + else await chrome.tabs.create({ url: s.url || undefined, active: true }); + return {} as ExecResult; + }, +}; + +export const switchTabNode: NodeRuntime = { + run: async (_ctx, step) => { + const s: any = expandTemplatesDeep(step as any, {}); + let targetTabId: number | undefined = s.tabId; + if (!targetTabId) { + const tabs = await chrome.tabs.query({}); + const hit = tabs.find( + (t) => + (s.urlContains && (t.url || '').includes(String(s.urlContains))) || + (s.titleContains && (t.title || '').includes(String(s.titleContains))), + ); + targetTabId = (hit && hit.id) as number | undefined; + } + if (!targetTabId) throw new Error('switchTab: no matching tab'); + const res = await handleCallTool({ + name: TOOL_NAMES.BROWSER.SWITCH_TAB, + args: { tabId: targetTabId }, + }); + if ((res as any).isError) throw new Error('switchTab failed'); + return {} as ExecResult; + }, +}; + +export const closeTabNode: NodeRuntime = { + run: async (_ctx, step) => { + const s: any = expandTemplatesDeep(step as any, {}); + const args: any = {}; + if (Array.isArray(s.tabIds) && s.tabIds.length) args.tabIds = s.tabIds; + if (s.url) args.url = s.url; + const res = await handleCallTool({ name: TOOL_NAMES.BROWSER.CLOSE_TABS, args }); + if ((res as any).isError) throw new Error('closeTab failed'); + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/types.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/types.ts new file mode 100644 index 0000000..c8918c6 --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/types.ts @@ -0,0 +1,21 @@ +import type { RunLogEntry, Step, StepScript } from '../types'; + +export interface ExecCtx { + vars: Record; + logger: (e: RunLogEntry) => void; + frameId?: number; +} + +export interface ExecResult { + alreadyLogged?: boolean; + deferAfterScript?: StepScript | null; + nextLabel?: string; + control?: + | { kind: 'foreach'; listVar: string; itemVar: string; subflowId: string } + | { kind: 'while'; condition: any; subflowId: string; maxIterations: number }; +} + +export interface NodeRuntime { + validate?: (step: S) => { ok: boolean; errors?: string[] }; + run: (ctx: ExecCtx, step: S) => Promise; +} diff --git a/app/chrome-extension/entrypoints/background/record-replay/nodes/wait.ts b/app/chrome-extension/entrypoints/background/record-replay/nodes/wait.ts new file mode 100644 index 0000000..16e65fc --- /dev/null +++ b/app/chrome-extension/entrypoints/background/record-replay/nodes/wait.ts @@ -0,0 +1,67 @@ +import type { StepWait } from '../types'; +import { waitForNetworkIdle, waitForNavigation } from '../rr-utils'; +import { expandTemplatesDeep } from '../rr-utils'; +import type { ExecCtx, ExecResult, NodeRuntime } from './types'; + +export const waitNode: NodeRuntime = { + validate: (step) => { + const ok = !!(step as any).condition; + return ok ? { ok } : { ok, errors: ['缺少等待条件'] }; + }, + run: async (ctx: ExecCtx, step: StepWait) => { + const s = expandTemplatesDeep(step as StepWait, {}); + const cond = (s as StepWait).condition as + | { selector: string; visible?: boolean } + | { text: string; appear?: boolean } + | { navigation: true } + | { networkIdle: true } + | { sleep: number }; + if ('text' in cond) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await chrome.scripting.executeScript({ + target: { tabId }, + files: ['inject-scripts/wait-helper.js'], + world: 'ISOLATED', + } as any); + const resp: any = (await chrome.tabs.sendMessage( + tabId, + { + action: 'waitForText', + text: cond.text, + appear: (cond as any).appear !== false, + timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), + } as any, + { frameId: ctx.frameId } as any, + )) as any; + if (!resp || resp.success !== true) throw new Error('wait text failed'); + } else if ('networkIdle' in cond) { + const total = Math.min(Math.max(1000, (s as any).timeoutMs || 5000), 120000); + const idle = Math.min(1500, Math.max(500, Math.floor(total / 3))); + await waitForNetworkIdle(total, idle); + } else if ('navigation' in cond) { + await waitForNavigation((s as any).timeoutMs); + } else if ('sleep' in cond) { + const ms = Math.max(0, Number(cond.sleep ?? 0)); + await new Promise((r) => setTimeout(r, ms)); + } else if ('selector' in cond) { + const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); + const tabId = tabs?.[0]?.id; + if (typeof tabId !== 'number') throw new Error('Active tab not found'); + await chrome.scripting.executeScript({ + target: { tabId }, + files: ['inject-scripts/wait-helper.js'], + world: 'ISOLATED', + } as any); + const resp: any = (await chrome.tabs.sendMessage(tabId, { + action: 'waitForSelector', + selector: (cond as any).selector, + visible: (cond as any).visible !== false, + timeout: Math.max(0, Math.min((s as any).timeoutMs || 10000, 120000)), + } as any)) as any; + if (!resp || resp.success !== true) throw new Error('wait selector failed'); + } + return {} as ExecResult; + }, +}; diff --git a/app/chrome-extension/entrypoints/background/record-replay/runner.ts b/app/chrome-extension/entrypoints/background/record-replay/runner.ts deleted file mode 100644 index 9807fb8..0000000 --- a/app/chrome-extension/entrypoints/background/record-replay/runner.ts +++ /dev/null @@ -1,718 +0,0 @@ -// runner.ts — orchestrates record-replay flow execution using registry + utils -// Note: comments in English - -import { TOOL_NAMES } from 'chrome-mcp-shared'; -import { handleCallTool } from '../tools'; -import type { Flow, RunLogEntry, RunRecord, RunResult, Step, StepScript } from './types'; -import { appendRun } from './flow-store'; -import { - mapDagNodeToStep, - topoOrder, - ensureTab, - expandTemplatesDeep, - waitForNetworkIdle, - applyAssign, - defaultEdgesOnly, - waitForNavigation, -} from './rr-utils'; -import { executeStep } from './node-registry'; - -export interface RunOptions { - tabTarget?: 'current' | 'new'; - refresh?: boolean; - captureNetwork?: boolean; - returnLogs?: boolean; - timeoutMs?: number; - startUrl?: string; - args?: Record; - startNodeId?: string; -} - -export async function runFlow(flow: Flow, options: RunOptions = {}): Promise { - const runId = `run_${Date.now()}_${Math.random().toString(36).slice(2, 8)}`; - const startAt = Date.now(); - const logs: RunLogEntry[] = []; - // Global deadline for run (optional) - const globalTimeout = Math.max(0, Number(options.timeoutMs || 0)); - const deadline = globalTimeout > 0 ? startAt + globalTimeout : 0; - const ensureWithinDeadline = () => { - if (deadline > 0 && Date.now() > deadline) { - const err = new Error('Global timeout reached'); - // mark a synthetic log entry for visibility - logs.push({ stepId: 'global-timeout', status: 'failed', message: 'Global timeout reached' }); - throw err; - } - }; - - // prepare variables - const vars: Record = Object.create(null); - for (const v of flow.variables || []) if (v.default !== undefined) vars[v.key] = v.default; - if (options.args) Object.assign(vars, options.args); - - // Derive a default startUrl when not provided: prefer first navigate step - let derivedStartUrl: string | undefined = undefined; - try { - // We haven't computed stepsToRun yet; compute minimal set from flow for derive - const hasDag0 = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; - const nodes0 = hasDag0 ? (((flow as any).nodes || []) as any[]) : []; - const edges0 = hasDag0 ? (((flow as any).edges || []) as any[]) : []; - const defaultEdges0 = hasDag0 ? defaultEdgesOnly(edges0 as any) : []; - const order0 = hasDag0 ? topoOrder(nodes0 as any, defaultEdges0 as any) : []; - const steps0: Step[] = hasDag0 - ? order0.map((n) => mapDagNodeToStep(n as any)) - : ((flow.steps || []) as Step[]); - const nav = steps0.find((s: any) => s && (s as any).type === 'navigate') as any; - if (nav && typeof nav.url === 'string') derivedStartUrl = expandTemplatesDeep(nav.url, {}); - } catch {} - const ensured = await ensureTab({ - tabTarget: options.tabTarget, - startUrl: options.startUrl || derivedStartUrl, - refresh: options.refresh, - }); - - // pre-load read_page to init bridges only when on a web page (avoid builder.html) - try { - const u = ensured?.url || ''; - if (/^(https?:|file:)/i.test(u)) { - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - } - } catch {} - - // collect required variables via overlay prompt - try { - const needed = (flow.variables || []).filter( - (v) => - (options.args?.[v.key] == null || options.args?.[v.key] === '') && - (v.rules?.required || (v.default ?? '') === ''), - ); - if (needed.length) { - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.SEND_COMMAND_TO_INJECT_SCRIPT, - args: { - eventName: 'collectVariables', - payload: JSON.stringify({ variables: needed, useOverlay: true }), - }, - }); - let values: Record | null = null; - try { - const t = (res?.content || []).find((c: any) => c.type === 'text')?.text; - const j = t ? JSON.parse(t) : null; - if (j && j.success && j.values) values = j.values; - } catch {} - if (!values) { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId === 'number') { - const res2 = await chrome.tabs.sendMessage(tabId, { - action: 'collectVariables', - variables: needed, - useOverlay: true, - } as any); - if (res2 && res2.success && res2.values) values = res2.values; - } - } - if (values) Object.assign(vars, values); - } - } catch {} - - // init overlay for on-screen log - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) - await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'init' } as any); - } catch {} - - // binding enforcement - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const currentUrl = tabs?.[0]?.url || ''; - const bindings = flow.meta?.bindings || []; - if (!options.startUrl && bindings.length > 0) { - const ok = bindings.some((b) => { - try { - if (b.type === 'domain') return new URL(currentUrl).hostname.includes(b.value); - if (b.type === 'path') return new URL(currentUrl).pathname.startsWith(b.value); - if (b.type === 'url') return currentUrl.startsWith(b.value); - } catch {} - return false; - }); - if (!ok) { - return { - runId, - success: false, - summary: { total: 0, success: 0, failed: 0, tookMs: 0 }, - url: currentUrl, - outputs: null, - logs: [ - { - stepId: 'binding-check', - status: 'failed', - message: - 'Flow binding mismatch. Provide startUrl or open a page matching flow.meta.bindings.', - }, - ], - screenshots: { onFailure: null }, - } as RunResult; - } - } - } catch {} - - // long-running network capture (debugger) if requested - let networkCaptureStarted = false; - const stopAndSummarizeNetwork = async () => { - try { - const stopRes = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_STOP, - args: {}, - }); - const text = (stopRes?.content || []).find((c: any) => c.type === 'text')?.text; - if (!text) return; - const data = JSON.parse(text); - const requests: any[] = Array.isArray(data?.requests) ? data.requests : []; - const snippets = requests - .filter((r) => ['XHR', 'Fetch'].includes(String(r.type))) - .slice(0, 10) - .map((r) => ({ - method: String(r.method || 'GET'), - url: String(r.url || ''), - status: r.statusCode || r.status, - ms: Math.max(0, (r.responseTime || 0) - (r.requestTime || 0)), - })); - logs.push({ - stepId: 'network-capture', - status: 'success', - message: `Captured ${Number(data?.requestCount || 0)} requests` as any, - networkSnippets: snippets, - } as any); - } catch {} - }; - if (options.captureNetwork) { - try { - const res = await handleCallTool({ - name: TOOL_NAMES.BROWSER.NETWORK_DEBUGGER_START, - args: { includeStatic: false, maxCaptureTime: 3 * 60_000, inactivityTimeout: 0 }, - }); - if (!(res as any)?.isError) networkCaptureStarted = true; - } catch {} - } - - const hasDag = Array.isArray((flow as any).nodes) && (flow as any).nodes.length > 0; - const nodes = hasDag ? (((flow as any).nodes || []) as any[]) : []; - const edges = hasDag ? (((flow as any).edges || []) as any[]) : []; - const defaultEdges = hasDag ? defaultEdgesOnly(edges as any) : []; - const order = hasDag ? topoOrder(nodes as any, defaultEdges as any) : []; - const stepsToRun: Step[] = hasDag - ? order.map((n) => mapDagNodeToStep(n as any)) - : ((flow.steps || []) as Step[]); - const startIdx = - !hasDag && options.startNodeId - ? stepsToRun.findIndex((s) => s?.id === options.startNodeId) - : -1; - const steps = !hasDag - ? startIdx >= 0 - ? stepsToRun.slice(startIdx) - : stepsToRun.slice() - : stepsToRun; - - let failed = 0; - const logger = (e: RunLogEntry) => logs.push(e); - const ctx = { vars, logger }; - - // deferred after-scripts - const pendingAfterScripts: StepScript[] = []; - - // small helpers - const appendOverlay = async (text: string) => { - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) - await chrome.tabs.sendMessage(tabs[0].id, { - action: 'rr_overlay', - cmd: 'append', - text, - } as any); - } catch {} - }; - const evalCondition = (cond: any): boolean => { - try { - if (cond && typeof cond.expression === 'string' && cond.expression.trim()) { - const fn = new Function( - 'vars', - `try { return !!(${cond.expression}); } catch (e) { return false; }`, - ); - return !!fn(vars); - } - if (cond && typeof cond.var === 'string') { - const v = vars[cond.var]; - if ('equals' in cond) return String(v) === String(cond.equals); - return !!v; - } - } catch {} - return false; - }; - - // execute a subflow by id (default edges only) - const runSubflowById = async (subflowId: string) => { - const sub = (flow.subflows || {})[subflowId]; - if (!sub || !Array.isArray(sub.nodes) || sub.nodes.length === 0) return; - const sNodes: any[] = sub.nodes; - const sEdges: any[] = defaultEdgesOnly((sub.edges || []) as any) as any[]; - const sOrder = topoOrder(sNodes as any, sEdges as any); - const sSteps: Step[] = sOrder.map((n) => mapDagNodeToStep(n as any)) as any; - for (const step of sSteps) { - const t0 = Date.now(); - const maxRetries = Math.max(0, (step as any).retry?.count ?? 0); - const baseInterval = Math.max(0, (step as any).retry?.intervalMs ?? 0); - let attempt = 0; - const doDelay = async (i: number) => { - const delay = - baseInterval > 0 - ? (step as any).retry?.backoff === 'exp' - ? baseInterval * Math.pow(2, i) - : baseInterval - : 0; - if (delay > 0) await new Promise((r) => setTimeout(r, delay)); - }; - while (true) { - try { - const beforeInfo = await getActiveTabInfo(); - const result = await executeStep(ctx, step); - if (step.type === 'click' || step.type === 'dblclick') { - const after = ((step as any).after || {}) as any; - if (after.waitForNavigation) - await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); - else if (after.waitForNetworkIdle) - await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); - else await maybeQuickWaitForNav(beforeInfo.url, (step as any).timeoutMs); - } - if (step.type === 'navigate' || step.type === 'openTab') { - await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); - await ensureReadPageIfWeb(); - } else if (step.type === 'switchTab') { - await ensureReadPageIfWeb(); - } - if (!result?.alreadyLogged) - logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); - await appendOverlay(`✔ ${step.type} (${step.id})`); - break; - } catch (e: any) { - if (attempt < maxRetries) { - logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); - await doDelay(attempt); - attempt += 1; - continue; - } - logs.push({ - stepId: step.id, - status: 'failed', - message: e?.message || String(e), - tookMs: Date.now() - t0, - }); - await appendOverlay(`✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`); - if ((step as any).screenshotOnFail !== false) { - try { - const shot = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'screenshot' }, - }); - const img = (shot?.content?.find((c: any) => c.type === 'image') as any) - ?.data as string; - if (img) logs[logs.length - 1].screenshotBase64 = img; - } catch {} - } - throw e; - } - } - } - }; - const getActiveTabInfo = async () => { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tab = tabs[0]; - return { url: tab?.url || '', status: (tab as any)?.status || '' }; - }; - const waitForNavigationDone = async (prevUrl: string, timeoutMs?: number) => { - await waitForNavigation(timeoutMs, prevUrl); - }; - const isWebUrl = (u?: string | null) => !!u && /^(https?:|file:)/i.test(String(u || '')); - const ensureReadPageIfWeb = async () => { - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const url = tabs?.[0]?.url || ''; - if (isWebUrl(url)) { - await handleCallTool({ name: TOOL_NAMES.BROWSER.READ_PAGE, args: {} }); - } - } catch {} - }; - - // Opportunistic short-window navigation wait after clicks when not explicitly requested - const maybeQuickWaitForNav = async (prevUrl: string, timeoutMs?: number) => { - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') return; - const sniffMs = 350; - const startedAt = Date.now(); - let seen = false; - await new Promise((resolve) => { - let timer: any = null; - const cleanup = () => { - try { - chrome.webNavigation.onCommitted.removeListener(onCommitted); - } catch {} - try { - chrome.webNavigation.onCompleted.removeListener(onCompleted); - } catch {} - try { - (chrome.webNavigation as any).onHistoryStateUpdated?.removeListener?.( - onHistoryStateUpdated, - ); - } catch {} - try { - chrome.tabs.onUpdated.removeListener(onUpdated); - } catch {} - if (timer) { - try { - clearTimeout(timer); - } catch {} - } - }; - const finish = async () => { - cleanup(); - if (seen) { - try { - await waitForNavigation( - prevUrl ? Math.min(timeoutMs || 15000, 30000) : undefined, - prevUrl, - ); - } catch {} - } - resolve(); - }; - const mark = () => { - seen = true; - }; - const onCommitted = (d: any) => { - if (d.tabId === tabId && d.frameId === 0 && d.timeStamp >= startedAt) mark(); - }; - const onCompleted = (d: any) => { - if (d.tabId === tabId && d.frameId === 0 && d.timeStamp >= startedAt) mark(); - }; - const onHistoryStateUpdated = (d: any) => { - if (d.tabId === tabId && d.frameId === 0 && d.timeStamp >= startedAt) mark(); - }; - const onUpdated = (updatedId: number, change: chrome.tabs.TabChangeInfo) => { - if (updatedId !== tabId) return; - if (change.status === 'loading') mark(); - if (typeof change.url === 'string' && (!prevUrl || change.url !== prevUrl)) mark(); - }; - - chrome.webNavigation.onCommitted.addListener(onCommitted); - chrome.webNavigation.onCompleted.addListener(onCompleted); - try { - (chrome.webNavigation as any).onHistoryStateUpdated?.addListener?.(onHistoryStateUpdated); - } catch {} - chrome.tabs.onUpdated.addListener(onUpdated); - timer = setTimeout(finish, sniffMs); - }); - } catch {} - }; - - try { - if (!hasDag) { - // Linear execution (legacy steps) - for (const step of steps) { - const t0 = Date.now(); - ensureWithinDeadline(); - const maxRetries = Math.max(0, (step as any).retry?.count ?? 0); - const baseInterval = Math.max(0, (step as any).retry?.intervalMs ?? 0); - let attempt = 0; - const doDelay = async (i: number) => { - const delay = - baseInterval > 0 - ? (step as any).retry?.backoff === 'exp' - ? baseInterval * Math.pow(2, i) - : baseInterval - : 0; - if (delay > 0) await new Promise((r) => setTimeout(r, delay)); - }; - while (true) { - try { - const beforeInfo = await getActiveTabInfo(); - // special handling for script when=after: defer - if (step.type === 'script' && (step as any).when === 'after') { - pendingAfterScripts.push(step as any); - logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); - break; - } - const result = await executeStep(ctx, step); - // handle click/dblclick navigation/network-idle waits - if (step.type === 'click' || step.type === 'dblclick') { - const after = ((step as any).after || {}) as any; - if (after.waitForNavigation) - await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); - else if (after.waitForNetworkIdle) - await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); - else await maybeQuickWaitForNav(beforeInfo.url, (step as any).timeoutMs); - } - if (step.type === 'navigate' || step.type === 'openTab') { - await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); - await ensureReadPageIfWeb(); - } else if (step.type === 'switchTab') { - await ensureReadPageIfWeb(); - } - if (!result?.alreadyLogged) { - logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); - } - await appendOverlay(`✔ ${step.type} (${step.id})`); - // control flows - if (result?.control) { - if (result.control.kind === 'foreach') { - const list = Array.isArray(vars[result.control.listVar]) - ? (vars[result.control.listVar] as any[]) - : []; - for (const it of list) { - vars[result.control.itemVar] = it; - await runSubflowById(result.control.subflowId); - } - } else if (result.control.kind === 'while') { - let i = 0; - while ( - i < result.control.maxIterations && - evalCondition(result.control.condition) - ) { - await runSubflowById(result.control.subflowId); - i++; - } - } - } - if (result?.deferAfterScript) pendingAfterScripts.push(result.deferAfterScript); - // run any deferred after-scripts - if (pendingAfterScripts.length > 0) { - while (pendingAfterScripts.length) { - const s = pendingAfterScripts.shift()!; - const tScript = Date.now(); - const world = (s as any).world || 'ISOLATED'; - const code = String((s as any).code || ''); - if (code.trim()) { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (userCode: string) => { - try { - return (0, eval)(userCode); - } catch { - return null; - } - }, - args: [code], - world: world as any, - } as any); - if ((s as any).saveAs) vars[(s as any).saveAs] = result; - if ((s as any).assign && typeof (s as any).assign === 'object') - applyAssign(vars, result, (s as any).assign); - } - logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); - } - } - break; // success - } catch (e: any) { - if (attempt < maxRetries) { - logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); - await doDelay(attempt); - attempt += 1; - continue; - } - failed++; - logs.push({ - stepId: step.id, - status: 'failed', - message: e?.message || String(e), - tookMs: Date.now() - t0, - }); - await appendOverlay(`✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`); - if ((step as any).screenshotOnFail !== false) { - try { - const shot = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'screenshot' }, - }); - const img = (shot?.content?.find((c: any) => c.type === 'image') as any) - ?.data as string; - if (img) logs[logs.length - 1].screenshotBase64 = img; - } catch {} - } - throw e; - } - } - } - } else { - // Graph traversal execution (DAG) - const id2node = new Map(nodes.map((n: any) => [n.id, n] as const)); - const outEdges = new Map>(); - for (const e of edges) { - if (!outEdges.has(e.from)) outEdges.set(e.from, []); - outEdges.get(e.from)!.push(e); - } - const indeg = new Map(nodes.map((n: any) => [n.id, 0] as const)); - for (const e of edges) indeg.set(e.to, (indeg.get(e.to) || 0) + 1); - let currentId = - options.startNodeId && id2node.has(options.startNodeId) - ? options.startNodeId - : nodes.find((n: any) => (indeg.get(n.id) || 0) === 0)?.id || nodes[0]?.id; - let guard = 0; - while (currentId && guard++ < 10000) { - ensureWithinDeadline(); - const node = id2node.get(currentId); - if (!node) break; - const step: any = mapDagNodeToStep(node as any); - const t0 = Date.now(); - const maxRetries = Math.max(0, (step as any).retry?.count ?? 0); - const baseInterval = Math.max(0, (step as any).retry?.intervalMs ?? 0); - let attempt = 0; - const doDelay = async (i: number) => { - const delay = - baseInterval > 0 - ? (step as any).retry?.backoff === 'exp' - ? baseInterval * Math.pow(2, i) - : baseInterval - : 0; - if (delay > 0) await new Promise((r) => setTimeout(r, delay)); - }; - const beforeInfo = await getActiveTabInfo(); - let jumpedOnError = false; - try { - const result = await executeStep(ctx, step); - if (step.type === 'click' || step.type === 'dblclick') { - const after = ((step as any).after || {}) as any; - if (after.waitForNavigation) - await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); - else if (after.waitForNetworkIdle) - await waitForNetworkIdle(Math.min((step as any).timeoutMs || 5000, 120000), 1200); - else await maybeQuickWaitForNav(beforeInfo.url, (step as any).timeoutMs); - } - if (step.type === 'navigate' || step.type === 'openTab') { - await waitForNavigationDone(beforeInfo.url, (step as any).timeoutMs); - await ensureReadPageIfWeb(); - } else if (step.type === 'switchTab') { - await ensureReadPageIfWeb(); - } - if (!result?.alreadyLogged) - logs.push({ stepId: step.id, status: 'success', tookMs: Date.now() - t0 }); - await appendOverlay(`✔ ${step.type} (${step.id})`); - // choose next by label - let nextLabel: string = 'default'; - if (result?.nextLabel) nextLabel = String(result.nextLabel); - const oes = (outEdges.get(currentId) || []) as any[]; - const edge = - oes.find((e) => String(e.label || 'default') === nextLabel) || - oes.find((e) => !e.label || e.label === 'default'); - currentId = edge ? edge.to : undefined; - } catch (e: any) { - if (attempt < maxRetries) { - logs.push({ stepId: step.id, status: 'retrying', message: e?.message || String(e) }); - await doDelay(attempt); - attempt += 1; - continue; - } - logs.push({ - stepId: step.id, - status: 'failed', - message: e?.message || String(e), - tookMs: Date.now() - t0, - }); - await appendOverlay(`✘ ${step.type} (${step.id}) -> ${e?.message || String(e)}`); - if ((step as any).screenshotOnFail !== false) { - try { - const shot = await handleCallTool({ - name: TOOL_NAMES.BROWSER.COMPUTER, - args: { action: 'screenshot' }, - }); - const img = (shot?.content?.find((c: any) => c.type === 'image') as any) - ?.data as string; - if (img) logs[logs.length - 1].screenshotBase64 = img; - } catch {} - } - // onError jump - const oes = (outEdges.get(currentId) || []) as any[]; - const errEdge = oes.find((e) => e.label === 'onError'); - if (errEdge) { - currentId = errEdge.to; - jumpedOnError = true; - } else { - throw e; - } - } - if (!jumpedOnError) { - // flush deferred after-scripts - if (pendingAfterScripts.length > 0) { - while (pendingAfterScripts.length) { - const s = pendingAfterScripts.shift()!; - const tScript = Date.now(); - const world = (s as any).world || 'ISOLATED'; - const code = String((s as any).code || ''); - if (code.trim()) { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - const tabId = tabs?.[0]?.id; - if (typeof tabId !== 'number') throw new Error('Active tab not found'); - const [{ result }] = await chrome.scripting.executeScript({ - target: { tabId }, - func: (userCode: string) => { - try { - return (0, eval)(userCode); - } catch { - return null; - } - }, - args: [code], - world: world as any, - } as any); - if ((s as any).saveAs) vars[(s as any).saveAs] = result; - if ((s as any).assign && typeof (s as any).assign === 'object') - applyAssign(vars, result, (s as any).assign); - } - logs.push({ stepId: s.id, status: 'success', tookMs: Date.now() - tScript }); - } - } - } - } - } - } finally { - if (networkCaptureStarted) await stopAndSummarizeNetwork(); - } - - const tookMs = Date.now() - startAt; - try { - const tabs = await chrome.tabs.query({ active: true, currentWindow: true }); - if (tabs[0]?.id) - await chrome.tabs.sendMessage(tabs[0].id, { action: 'rr_overlay', cmd: 'done' } as any); - } catch {} - - const record: RunRecord = { - id: runId, - flowId: flow.id, - startedAt: new Date(startAt).toISOString(), - finishedAt: new Date().toISOString(), - success: failed === 0, - entries: logs, - }; - await appendRun(record); - - // outputs: filter sensitive variables - const sensitiveKeys = new Set( - (flow.variables || []).filter((v) => v.sensitive).map((v) => v.key), - ); - const outputs: Record = {}; - for (const [k, v] of Object.entries(vars)) if (!sensitiveKeys.has(k)) outputs[k] = v; - - return { - runId, - success: failed === 0, - summary: { total: steps.length, success: steps.length - failed, failed, tookMs }, - url: null, - outputs, - logs: options.returnLogs ? logs : undefined, - screenshots: { onFailure: logs.find((l) => l.status === 'failed')?.screenshotBase64 }, - }; -} diff --git a/app/chrome-extension/entrypoints/background/record-replay/types.ts b/app/chrome-extension/entrypoints/background/record-replay/types.ts index 123c454..a178ace 100644 --- a/app/chrome-extension/entrypoints/background/record-replay/types.ts +++ b/app/chrome-extension/entrypoints/background/record-replay/types.ts @@ -301,4 +301,5 @@ export interface RunResult { outputs?: Record | null; logs?: RunLogEntry[]; screenshots?: { onFailure?: string | null }; + paused?: boolean; // when true, the run was intentionally paused (e.g., breakpoint) } diff --git a/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts b/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts index 5ad8ba4..ccc5c89 100644 --- a/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts +++ b/app/chrome-extension/entrypoints/background/tools/browser/interaction.ts @@ -16,6 +16,7 @@ interface ClickToolParams { waitForNavigation?: boolean; // Whether to wait for navigation to complete after click timeout?: number; // Timeout in milliseconds for waiting for the element or navigation frameId?: number; // Target frame for ref/selector resolution + double?: boolean; // Perform double click when true } /** @@ -68,6 +69,7 @@ class ClickTool extends BaseBrowserToolExecutor { ref: args.ref, waitForNavigation, timeout, + double: args.double === true, }, frameId, ); diff --git a/app/chrome-extension/inject-scripts/click-helper.js b/app/chrome-extension/inject-scripts/click-helper.js index bad7f83..6bb2ee7 100644 --- a/app/chrome-extension/inject-scripts/click-helper.js +++ b/app/chrome-extension/inject-scripts/click-helper.js @@ -22,6 +22,7 @@ if (window.__CLICK_HELPER_INITIALIZED__) { timeout = 5000, coordinates = null, ref = null, + double = false, ) { try { let element = null; @@ -179,9 +180,14 @@ if (window.__CLICK_HELPER_INITIALIZED__) { element && (elementInfo.clickMethod === 'selector' || elementInfo.clickMethod === 'ref') ) { - element.click(); + if (double) { + simulateDomDoubleClick(element, clickX, clickY); + } else { + element.click(); + } } else { - simulateClick(clickX, clickY); + if (double) simulateDoubleClick(clickX, clickY); + else simulateClick(clickX, clickY); } // Wait for navigation if needed @@ -226,6 +232,52 @@ if (window.__CLICK_HELPER_INITIALIZED__) { } } + /** + * Simulate a double click sequence at specific coordinates + */ + function simulateDoubleClick(x, y) { + simulateClick(x, y); + setTimeout(() => { + simulateClick(x, y); + const dbl = new MouseEvent('dblclick', { + view: window, + bubbles: true, + cancelable: true, + clientX: x, + clientY: y, + }); + const el = document.elementFromPoint(x, y); + if (el) el.dispatchEvent(dbl); + else document.dispatchEvent(dbl); + }, 30); + } + + /** + * Simulate double click using element when available + */ + function simulateDomDoubleClick(element, x, y) { + try { + element.click(); + setTimeout(() => { + element.click(); + const rect = element.getBoundingClientRect(); + const cx = x ?? rect.left + rect.width / 2; + const cy = y ?? rect.top + rect.height / 2; + const dbl = new MouseEvent('dblclick', { + view: window, + bubbles: true, + cancelable: true, + clientX: cx, + clientY: cy, + }); + element.dispatchEvent(dbl); + }, 30); + } catch (e) { + // fallback to coordinates + simulateDoubleClick(x, y); + } + } + /** * Check if an element is visible * @param {Element} element - The element to check @@ -271,6 +323,7 @@ if (window.__CLICK_HELPER_INITIALIZED__) { request.timeout, request.coordinates, request.ref, + !!request.double, ) .then(sendResponse) .catch((error) => { diff --git a/automa.md b/automa.md index defc325..6d86521 100644 --- a/automa.md +++ b/automa.md @@ -210,7 +210,7 @@ - Automa 块清单：`other/automa/src/utils/shared.js` - Automa 典型 Handler：`other/automa/src/content/blocksHandler/*`、`other/automa/src/workflowEngine/blocksHandler/*` -- 本实现执行层：`app/chrome-extension/entrypoints/background/record-replay/*`（`runner.ts`、`node-registry.ts`、`selector-engine.ts`、`rr-utils.ts`） +- 本实现执行层：`app/chrome-extension/entrypoints/background/record-replay/*`（`engine/*`、`nodes/*`、`selector-engine.ts`、`rr-utils.ts`） - 浏览器工具层：`app/chrome-extension/entrypoints/background/tools/browser/*` - 录制/注入：`app/chrome-extension/inject-scripts/*` - Builder 相关：`app/chrome-extension/entrypoints/popup/components/builder/*` diff --git a/record-refact.md b/record-refact.md index 8b6d03f..7e5901d 100644 --- a/record-refact.md +++ b/record-refact.md @@ -1679,3 +1679,20 @@ class RecordingStateManager { clear(): Promise; } ``` + +--- + +## 回放重构变更（DAG-only） + +本次回放执行改为仅支持 DAG（nodes + edges + subflows），弃用线性 `steps[]` 路径： + +- 破坏性变更：当 Flow 不含 `nodes` 时，运行器返回错误 `dag-required`，提示迁移。 +- 双击修复：`dblclick` 统一通过 CLICK 工具（内容脚本），支持 `frameId`，iframe 内双击准确。 +- 控制流： + - 条件节点通过 `nextLabel` 选择分支。 + - foreach/while 节点通过返回 `control` 调用 `flow.subflows[subflowId]` 作为循环体（DAG 路径已实现）。 + +迁移建议： + +- 使用构建器的节点画布保存 `nodes/edges`（现有 UI 已支持）。 +- 旧 Flow 若仅存 `steps[]`：通过现有 `stepsToNodes()` 转换为 `nodes/edges` 并保存。涉及循环的步骤需迁移为节点，并在 `flow.subflows` 内定义子流体。