From 027826fd138e8188b9c91ff8d26be8be50116e09 Mon Sep 17 00:00:00 2001 From: musistudio Date: Tue, 7 Jul 2026 18:56:28 +0800 Subject: [PATCH] Refactor router context handling and update integrations --- .../content/docs/en/configuration/toolhub.md | 60 +- .../content/docs/zh/configuration/toolhub.md | 60 +- extension/chrome/README.md | 23 + extension/chrome/background.js | 262 ++ extension/chrome/confirm.js | 46 + extension/chrome/manifest.json | 35 + extension/chrome/popup.html | 110 + extension/chrome/popup.js | 297 ++ packages/core/src/config/config.ts | 4 + packages/core/src/config/default-config.ts | 1 + packages/core/src/contracts/app.ts | 80 + packages/core/src/contracts/ipc-channels.ts | 3 + .../src/gateway/claude-code-router-plugin.ts | 36 +- packages/core/src/gateway/service.ts | 200 +- .../core/src/mcp/fusion-tool-fallback-mcp.ts | 7 +- packages/core/src/mcp/toolhub-config.ts | 98 +- packages/core/src/mcp/toolhub-mcp.ts | 245 +- .../src/main/browser-automation-mcp.ts | 3517 +++++++++++++++++ packages/electron/src/main/browser-preload.ts | 5 +- .../electron/src/main/built-in-browser.ts | 510 ++- .../electron/src/main/chrome-login-import.ts | 593 +++ .../src/main/electron-web-search-mcp.ts | 73 +- packages/electron/src/main/ipc.ts | 153 - packages/electron/src/main/main-app.ts | 2 + packages/ui/src/pages/browser/main.tsx | 250 +- .../ui/src/pages/home/components/settings.tsx | 9 +- packages/ui/src/pages/home/shared/config.ts | 1 + packages/ui/src/pages/home/shared/i18n.tsx | 6 + tests/main/gateway-virtual-models.test.mjs | 36 + tests/main/router-builtins.test.mjs | 12 +- ...toolhub-browser-automation-config.test.mjs | 72 + tests/main/toolhub-mcp-runtime.test.mjs | 389 ++ 32 files changed, 6889 insertions(+), 306 deletions(-) create mode 100644 extension/chrome/README.md create mode 100644 extension/chrome/background.js create mode 100644 extension/chrome/confirm.js create mode 100644 extension/chrome/manifest.json create mode 100644 extension/chrome/popup.html create mode 100644 extension/chrome/popup.js create mode 100644 packages/electron/src/main/browser-automation-mcp.ts create mode 100644 packages/electron/src/main/chrome-login-import.ts create mode 100644 tests/main/toolhub-browser-automation-config.test.mjs diff --git a/docs/src/content/docs/en/configuration/toolhub.md b/docs/src/content/docs/en/configuration/toolhub.md index 1e3ab0d5..c0984563 100644 --- a/docs/src/content/docs/en/configuration/toolhub.md +++ b/docs/src/content/docs/en/configuration/toolhub.md @@ -2,7 +2,7 @@ title: ToolHub pageTitle: ToolHub eyebrow: Detailed Configuration -lead: Collapse many MCP servers into one compact entry point, then let the agent resolve the tools it needs for each task before invoking the real backend tool. +lead: Collapse many MCP servers into one compact entry point so agents lazy-load task-specific tools and save context. --- ## When To Use It @@ -12,26 +12,71 @@ As your MCP setup grows, exposing every tool directly to an agent makes the eage - `tool_hub.resolve`: searches the available MCP tool catalog for the current task. - `tool_hub.invoke`: calls a real MCP tool that was selected for this task. -Use ToolHub for internal systems, business APIs, browser automation, orders, accounts, stores, coupons, and other external capabilities. Simple local code, file, or conversation tasks usually do not need ToolHub. +Use ToolHub for tools that are not needed often but are still useful occasionally. It lazy-loads them only when a task actually needs them. The main value is saving context: large low-frequency tool catalogs do not have to stay in the agent's eager tool list, which reduces context use and the chance of selecting the wrong tool. Simple local code, file, or conversation tasks usually do not need ToolHub. ## How It Works 1. Enable ToolHub in **Settings → ToolHub**. -2. Select a configured model as the **Resolver model**. It reads the MCP tool catalog and chooses the tools needed for the task. +2. Select a configured model as the **Resolver model**. It reads the MCP tool catalog and chooses the tools needed for the task. Prefer `deepseek-v4-flash`, or another stable lightweight model in a similar flash-price tier. 3. Add or import backend MCP servers. ToolHub supports `stdio`, `streamable-http`, and `sse`. 4. Open Claude Code or Codex from CCR. CCR writes the `ccr-toolhub` MCP server into that agent config. 5. When the agent receives a request about external services, installed MCP capabilities, or business APIs, it calls `tool_hub.resolve` first, then uses `tool_hub.invoke` to run the selected tools. ToolHub combines MCP servers configured on the ToolHub page with compatible global Agent MCP servers from older configs, and excludes `ccr-toolhub` itself to avoid recursive calls. +## Built-In Browser Automation + +When ToolHub is enabled in CCR Desktop and **Built-in browser automation** is turned on, agents can use the desktop built-in browser for web tasks. You do not need to add a browser backend on the ToolHub page, and you do not need a separate API key; CCR connects to it through the local gateway authentication path. + +To enable it: + +1. Open **Settings → ToolHub** and turn on **Enable ToolHub**. +2. Turn on **Built-in browser automation** on the same page. This switch is shown only after ToolHub is enabled. +3. After saving settings, reopen Claude Code or Codex from CCR so the new agent instance loads the latest configuration. + +> Already-running agent instances usually do not pick up this switch immediately. Restart the agent instance, or use the agent's own controls to restart ToolHub. + +Built-in browser automation is useful for tasks that need real browser state, such as opening sites, reading pages, filling forms, clicking buttons, scrolling, or completing web flows like ordering, booking, lookup, and checkout when no domain-specific capability exists. After it is enabled, the agent can: + +- Open or attach built-in browser tabs, then navigate to URLs or search queries. +- Read page content and find buttons, links, form fields, and other page elements. +- Click, type, select, press keys, and scroll page elements. +- Wait for page loads, navigation, dialogs, or human handoff results before continuing. +- Request human help for login, verification codes, CAPTCHA, human checks, or manual confirmation. + +When a web flow needs login, verification codes, CAPTCHA, a human check, or manual confirmation, CCR shows the built-in browser window and displays the requested action in the top toolbar. After the user clicks **Done** or **Hide**, the agent receives the result and continues. Handoff waits support up to 10 minutes. + +### Chrome Login Import Extension + +Built-in browser automation can also import login state for selected domains from system Chrome into CCR's in-app browser. This lets the agent reuse sites where you are already signed in to Chrome. It requires the unpacked Chrome extension in this repository: `extension/chrome`. + +Install it: + +1. Open `chrome://extensions` in Chrome. +2. Enable **Developer mode**. +3. Click **Load unpacked**. +4. Select the repository's `extension/chrome` directory. + +Import flow: + +1. When a task needs existing Chrome login state, the agent can request an import; the user can also click the key button in CCR's in-app browser toolbar. +2. CCR creates a one-time import job and opens a confirmation page. If your default browser is not Chrome, copy the confirmation URL into Chrome with the extension installed. +3. Review the requested domains on the confirmation page, then click **Confirm and Import**. +4. The Chrome extension reads cookies and localStorage for those domains and submits them to CCR. After it completes, the agent can continue the task in the built-in browser. + +The extension reads only the domains listed in the CCR import job. It does not enumerate every Chrome cookie. For localStorage, the extension temporarily opens non-active tabs for the selected origins, reads `localStorage`, and closes those tabs. If the confirmation page says the extension does not have site access, allow the extension to access the target domains in Chrome extension settings, reload the unpacked extension, and try again. + +> Note: Built-in browser automation depends on CCR Desktop's built-in browser and is only available in the desktop app. CLI, server deployments, and pure web environments do not include this built-in capability; use an external browser automation MCP server instead. + ## Options | Option | Description | | --- | --- | | Enable ToolHub | Exposes `ccr-toolhub` to agents. If no backend MCP server is available, CCR does not generate a ToolHub MCP config. | -| Resolver model | Choose from configured provider models. Prefer a stable model that understands tool descriptions well. | +| Built-in browser automation | Shown only after ToolHub is enabled. Lets agents use CCR Desktop's built-in browser for web tasks. | +| Resolver model | Choose from configured provider models. Prefer `deepseek-v4-flash`, or another stable lightweight model in a similar flash-price tier with enough tool-description understanding. | | Max tools | Maximum tools returned by one resolve call. Range `1` to `20`, default `10`. | -| Timeout ms | Overall timeout for ToolHub resolving and invocation. Range `8000` to `300000`, default `60000`. | +| Timeout ms | Base timeout for ToolHub resolving and invocation. Range `8000` to `300000`, default `60000`. If a backend MCP server needs a longer request timeout, CCR raises the effective invocation timeout to match the backend. | | MCP servers | Backend tool sources. Each server needs a unique name plus transport, command or URL, environment variables, headers, and timeouts. | | Import JSON | Imports common MCP JSON shapes. Supports a root object, array, `mcpServers`, or `mcp_servers`. | @@ -65,6 +110,7 @@ The desktop app's SQLite config is the effective source, so prefer editing throu { "toolHub": { "enabled": true, + "browserAutomation": true, "llm": { "apiKey": "sk-...", "baseUrl": "https://api.openai.com/v1", @@ -112,8 +158,10 @@ The import dialog also accepts common MCP JSON: ## Troubleshooting -- Agent cannot see ToolHub: make sure ToolHub is enabled, at least one backend MCP server is configured, and reopen Claude Code or Codex from CCR. +- Agent cannot see ToolHub: make sure ToolHub is enabled and at least one backend MCP server is configured or **Built-in browser automation** is turned on, then reopen Claude Code or Codex from CCR. - Missing resolver model or API key: select a configured resolver model and confirm the provider credential works. +- The agent cannot use built-in browser automation: make sure you are using CCR Desktop, turned on **Built-in browser automation** in **Settings → ToolHub**, and reopened Claude Code or Codex from CCR. CLI, server deployments, and pure web environments do not include this built-in capability. +- Chrome login import confirmation keeps waiting for the extension: make sure the unpacked `extension/chrome` extension is loaded in Chrome and has site access for the target domains. If your default browser is not Chrome, copy the confirmation URL into Chrome manually. - No tools are resolved: confirm the MCP server can list tools, improve tool names and descriptions, or increase **Max tools**. - Calls time out: check ToolHub **Timeout ms** and the backend server request/startup timeouts. - Import fails: validate JSON, avoid duplicate server names, make sure `stdio` entries have a command and remote entries have a URL. diff --git a/docs/src/content/docs/zh/configuration/toolhub.md b/docs/src/content/docs/zh/configuration/toolhub.md index 33ec0c34..defadd05 100644 --- a/docs/src/content/docs/zh/configuration/toolhub.md +++ b/docs/src/content/docs/zh/configuration/toolhub.md @@ -2,7 +2,7 @@ title: ToolHub pageTitle: ToolHub eyebrow: 详细配置 -lead: 将多个 MCP server 收束成一个紧凑入口,让 Agent 先按任务检索需要的工具,再通过 ToolHub 调用真实工具。 +lead: 将多个 MCP server 收束成一个紧凑入口,让 Agent 按任务懒加载需要的工具,减少工具列表占用的上下文。 --- ## 适用场景 @@ -12,26 +12,71 @@ lead: 将多个 MCP server 收束成一个紧凑入口,让 Agent 先按任务 - `tool_hub.resolve`:根据用户任务和上下文检索可用 MCP 工具。 - `tool_hub.invoke`:调用已经被本轮任务选中的真实 MCP 工具。 -它适合把内部系统、业务 API、浏览器自动化、订单/账户/门店/优惠券等工具统一交给 Agent 使用。简单本地代码、文件或普通聊天任务通常不需要经过 ToolHub。 +它适合把不常用但偶尔会用到的工具统一懒加载,在任务真正需要时才交给 Agent 使用。核心价值是节省上下文:避免把大量低频工具常驻在 Agent 的工具列表里,减少上下文占用和选错工具的概率。简单本地代码、文件或普通聊天任务通常不需要经过 ToolHub。 ## 工作方式 1. 在 **设置 → ToolHub** 中启用 ToolHub。 -2. 选择一个已配置模型作为 **检索模型**。它负责阅读 MCP 工具目录并挑选本轮任务需要的工具。 +2. 选择一个已配置模型作为 **检索模型**。它负责阅读 MCP 工具目录并挑选本轮任务需要的工具;建议使用 `deepseek-v4-flash`,或同等 Flash 价位、响应稳定的轻量模型。 3. 添加或导入后端 MCP server。ToolHub 支持 `stdio`、`streamable-http` 和 `sse`。 4. 从 CCR 打开 Claude Code 或 Codex。CCR 会在对应 Agent 配置中写入 `ccr-toolhub`。 5. Agent 遇到外部服务、已安装 MCP 能力或业务 API 相关请求时,先调用 `tool_hub.resolve`,再用 `tool_hub.invoke` 执行选中的工具。 ToolHub 会合并 **ToolHub 页面配置的 MCP servers** 和兼容旧配置中的全局 Agent MCP servers,并自动排除 `ccr-toolhub` 自身,避免递归调用。 +## 内置浏览器自动化 + +在 CCR Desktop 中启用 ToolHub,并打开 **内置浏览器自动化** 开关后,Agent 可以使用桌面端内置浏览器完成网页操作。不需要在 ToolHub 页面手动添加浏览器后端,也不需要额外 API Key;CCR 会使用本地网关鉴权连接它。 + +启用步骤: + +1. 打开 **设置 → ToolHub**,先开启 **启用 ToolHub**。 +2. 在同一页打开 **内置浏览器自动化** 开关。该开关只会在 ToolHub 已启用时显示。 +3. 保存设置后,从 CCR 重新打开 Claude Code 或 Codex,让新的 Agent 实例加载最新配置。 + +> 已经运行中的 Agent 实例通常不会立即拿到这个开关变化。要让现有会话生效,请重启该 Agent 实例,或使用 Agent 自身能力重启 ToolHub。 + +内置浏览器自动化适合让 Agent 处理需要真实浏览器状态的任务,例如打开网站、读取页面、填写表单、点击按钮、在页面中滚动,或在没有专用业务能力时完成下单、预约、查询、结账等网页流程。开启后 Agent 可以: + +- 打开或附加内置浏览器标签页、导航 URL 或搜索词。 +- 读取页面内容,并找到按钮、链接、输入框等页面元素。 +- 点击、输入、选择、按键和滚动页面元素。 +- 等待页面加载、跳转、弹窗或人类接管结果,再继续后续步骤。 +- 在登录、验证码、CAPTCHA、人机验证或人工确认时请求用户接管。 + +当网页流程需要登录、验证码、CAPTCHA、人机验证或人工确认时,CCR 会显示内置浏览器窗口,并在顶部工具栏提示用户需要完成的步骤。用户点击 **Done** 或 **Hide** 后,Agent 会收到结果并继续执行。接管等待最长支持 10 分钟。 + +### Chrome 登录态导入扩展 + +内置浏览器自动化还支持把系统 Chrome 中指定域名的登录状态导入 CCR 内置浏览器。这样 Agent 处理网页任务时,可以复用你已经在 Chrome 中登录过的网站状态。该能力需要安装仓库里的 Chrome 解包扩展:`extension/chrome`。 + +安装方式: + +1. 在 Chrome 打开 `chrome://extensions`。 +2. 开启 **Developer mode**。 +3. 点击 **Load unpacked**。 +4. 选择仓库中的 `extension/chrome` 目录。 + +导入流程: + +1. 当任务需要复用 Chrome 登录状态时,Agent 会请求导入;用户也可以在 CCR 内置浏览器工具栏点击钥匙按钮主动发起。 +2. CCR 创建一次性导入任务,并打开确认页。如果默认浏览器不是 Chrome,请把确认页 URL 复制到已安装扩展的 Chrome 中打开。 +3. 用户在确认页检查要导入的域名,点击 **Confirm and Import**。 +4. Chrome 扩展读取这些域名的 cookies 和 localStorage,提交给 CCR;完成后 Agent 可以继续使用内置浏览器执行任务。 + +扩展只读取 CCR 导入任务列出的域名,不会枚举 Chrome 中的全部 cookies。读取 localStorage 时,扩展会临时打开对应 origin 的非激活标签页,读取后自动关闭。若确认页提示扩展没有站点访问权限,请在 Chrome 扩展设置中允许该扩展访问目标域名,然后重新加载解包扩展再重试。 + +> 注意:内置浏览器自动化依赖 CCR Desktop 的内置浏览器,只在桌面端可用。CLI、服务器部署或纯 Web 环境没有这项内置能力,请改用外部浏览器自动化 MCP server。 + ## 配置项 | 配置项 | 说明 | | --- | --- | | 启用 ToolHub | 开启后才会向 Agent 暴露 `ccr-toolhub`。如果没有可用后端 MCP server,CCR 不会生成 ToolHub MCP 配置。 | -| 检索模型 | 从已配置供应商模型中选择。建议选择响应稳定、工具理解能力好的模型。 | +| 内置浏览器自动化 | 仅在启用 ToolHub 后显示。开启后让 Agent 可以使用 CCR Desktop 的内置浏览器完成网页操作。 | +| 检索模型 | 从已配置供应商模型中选择。建议使用 `deepseek-v4-flash`,或同等 Flash 价位、响应稳定、工具理解能力足够的轻量模型。 | | 最大工具数 | 单次解析最多返回的工具数量,范围 `1` 到 `20`,默认 `10`。 | -| 超时毫秒 | ToolHub 解析和调用的总超时时间,范围 `8000` 到 `300000`,默认 `60000`。 | +| 超时毫秒 | ToolHub 解析和调用的基础超时时间,范围 `8000` 到 `300000`,默认 `60000`。如果后端 MCP server 需要更长 request timeout,CCR 会按后端超时自动抬高实际调用超时。 | | MCP servers | 后端工具来源。每个 server 需要唯一名称,并配置 transport、命令或 URL、环境变量、headers 和超时。 | | Import JSON | 导入常见 MCP JSON。支持根对象、数组、`mcpServers` 或 `mcp_servers`。 | @@ -65,6 +110,7 @@ ToolHub 会合并 **ToolHub 页面配置的 MCP servers** 和兼容旧配置中 { "toolHub": { "enabled": true, + "browserAutomation": true, "llm": { "apiKey": "sk-...", "baseUrl": "https://api.openai.com/v1", @@ -112,8 +158,10 @@ ToolHub 会合并 **ToolHub 页面配置的 MCP servers** 和兼容旧配置中 ## 排查 -- Agent 看不到 ToolHub:确认已启用 ToolHub、至少配置了一个后端 MCP server,并从 CCR 重新打开 Claude Code 或 Codex。 +- Agent 看不到 ToolHub:确认已启用 ToolHub,并且至少配置了一个后端 MCP server 或开启了 **内置浏览器自动化**,然后从 CCR 重新打开 Claude Code 或 Codex。 - 提示缺少检索模型或 API Key:在 **检索模型** 中选择已配置模型,并确认供应商凭据可用。 +- Agent 无法使用内置浏览器自动化:确认正在使用 CCR Desktop,并且已在 **设置 → ToolHub** 中开启 **内置浏览器自动化**,然后从 CCR 重新打开 Claude Code 或 Codex。CLI、服务器部署或纯 Web 环境没有这项内置能力。 +- Chrome 登录态导入确认页一直等待扩展:确认已在 Chrome 中加载 `extension/chrome` 解包扩展,并允许扩展访问要导入的目标域名。如果默认浏览器不是 Chrome,请手动把确认页 URL 复制到 Chrome。 - 解析不到工具:检查 MCP server 是否能正常列出工具,工具名称和描述是否足够清楚,必要时提高 **最大工具数**。 - 调用超时:分别检查 ToolHub 的 **超时毫秒** 和单个 MCP server 的 request/startup timeout。 - 导入失败:检查 JSON 是否有效、server 名称是否重复、`stdio` 是否有 command,远程 transport 是否有 URL。 diff --git a/extension/chrome/README.md b/extension/chrome/README.md new file mode 100644 index 00000000..86a5f262 --- /dev/null +++ b/extension/chrome/README.md @@ -0,0 +1,23 @@ +# CCR Login Import Chrome Extension + +This unpacked Chrome extension imports cookies and localStorage for explicitly selected domains into CCR's in-app browser. + +## Development install + +1. Open `chrome://extensions`. +2. Enable **Developer mode**. +3. Click **Load unpacked**. +4. Select this `extension/chrome` directory. + +After changing extension files, click **Reload** for this unpacked extension in `chrome://extensions`. +The confirmation-page flow uses the site access declared in `manifest.json`; it does not request new host permissions from the page click. + +## Flow + +1. An agent calls CCR's Chrome login import browser tool, or the user clicks the key button in CCR's in-app browser. +2. CCR opens a one-time confirmation page in the system browser. +3. Review the requested domains and click **Confirm and Import**. + +The extension reads only the domains listed in the CCR job. It does not enumerate all Chrome cookies. + +For localStorage, the extension temporarily opens non-active tabs for the selected origins, reads `localStorage`, then closes those tabs. diff --git a/extension/chrome/background.js b/extension/chrome/background.js new file mode 100644 index 00000000..e86e03b5 --- /dev/null +++ b/extension/chrome/background.js @@ -0,0 +1,262 @@ +chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => { + if (!message || message.type !== "ccr-login-import-confirm") { + return false; + } + + runImport(message.importUrl) + .then((result) => sendResponse({ ok: true, result })) + .catch((error) => sendResponse({ error: formatError(error), ok: false })); + return true; +}); + +async function runImport(importUrl) { + const normalizedImportUrl = normalizeImportUrl(importUrl); + if (!normalizedImportUrl) { + throw new Error("Invalid CCR import URL."); + } + + const job = await fetchImportJob(normalizedImportUrl); + const domains = Array.isArray(job.domains) ? job.domains.map(normalizeDomain).filter(Boolean) : []; + if (domains.length === 0) { + throw new Error("CCR import job does not include any domains."); + } + + await ensureHostPermissions(domains); + + const cookies = await readCookiesForDomains(domains); + const localStorageEntries = await readLocalStorageForDomains(domains, cookies); + if (cookies.length === 0 && localStorageEntries.length === 0) { + throw new Error("No cookies or localStorage entries were found for the selected domains."); + } + + return await submitLoginState(normalizedImportUrl, cookies, localStorageEntries, domains); +} + +async function fetchImportJob(importUrl) { + const response = await fetch(importUrl, { + headers: { + "x-ccr-login-import": "chrome-extension" + } + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(body?.error?.message || `CCR import job request failed (${response.status}).`); + } + if (!body.job || body.job.status !== "pending") { + throw new Error(`CCR import job is ${body.job?.status || "unavailable"}.`); + } + return body.job; +} + +async function submitLoginState(importUrl, cookies, localStorage, domains) { + const response = await fetch(`${importUrl.replace(/\/+$/, "")}/cookies`, { + body: JSON.stringify({ + cookies, + domains, + localStorage, + source: { + browser: "chrome", + extension: chrome.runtime.getManifest().version + } + }), + headers: { + "content-type": "application/json", + "x-ccr-login-import": "chrome-extension" + }, + method: "POST" + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(body?.error?.message || `CCR login import failed (${response.status}).`); + } + return body.result || {}; +} + +async function readCookiesForDomains(domains) { + const cookies = []; + for (const domain of domains) { + cookies.push(...await chrome.cookies.getAll({ domain })); + } + return dedupeCookies(cookies); +} + +async function readLocalStorageForDomains(domains, cookies) { + const origins = localStorageOriginsForDomains(domains, cookies); + const entries = []; + for (const origin of origins) { + const entry = await readLocalStorageForOrigin(origin).catch(() => undefined); + if (entry && Object.keys(entry.items).length > 0) { + entries.push(entry); + } + } + return entries; +} + +async function readLocalStorageForOrigin(origin) { + const tab = await chrome.tabs.create({ + active: false, + url: `${origin}/` + }); + try { + await waitForTabLoad(tab.id, 15000); + const [frameResult] = await chrome.scripting.executeScript({ + func: () => { + const items = {}; + for (let index = 0; index < window.localStorage.length; index += 1) { + const key = window.localStorage.key(index); + if (key !== null) { + items[key] = window.localStorage.getItem(key) || ""; + } + } + return { + items, + origin: window.location.origin + }; + }, + target: { tabId: tab.id }, + world: "MAIN" + }); + const result = frameResult?.result; + if (!result || !allowedLocalStorageOrigin(result.origin, origin)) { + return undefined; + } + return result; + } finally { + if (tab.id !== undefined) { + await chrome.tabs.remove(tab.id).catch(() => undefined); + } + } +} + +function waitForTabLoad(tabId, timeoutMs) { + return new Promise((resolve) => { + let done = false; + const finish = () => { + if (done) { + return; + } + done = true; + chrome.tabs.onUpdated.removeListener(listener); + clearTimeout(timeout); + resolve(); + }; + const listener = (updatedTabId, changeInfo) => { + if (updatedTabId === tabId && changeInfo.status === "complete") { + finish(); + } + }; + const timeout = setTimeout(finish, timeoutMs); + chrome.tabs.onUpdated.addListener(listener); + }); +} + +function dedupeCookies(cookies) { + const seen = new Set(); + const result = []; + for (const cookie of cookies) { + const partitionKey = cookie.partitionKey ? JSON.stringify(cookie.partitionKey) : ""; + const key = `${cookie.storeId || ""}\n${cookie.domain}\n${cookie.path}\n${cookie.name}\n${partitionKey}`; + if (!seen.has(key)) { + seen.add(key); + result.push(cookie); + } + } + return result; +} + +function localStorageOriginsForDomains(domains, cookies) { + const origins = new Set(); + for (const domain of domains) { + if (domain === "localhost" || isIpAddress(domain)) { + origins.add(`http://${domain}`); + origins.add(`https://${domain}`); + } else { + origins.add(`https://${domain}`); + } + } + + for (const cookie of cookies) { + const host = normalizeDomain(cookie.domain); + if (!host || !cookie.hostOnly || !domains.some((domain) => host === domain || host.endsWith(`.${domain}`))) { + continue; + } + origins.add(`${cookie.secure === false ? "http" : "https"}://${host}`); + } + + return [...origins]; +} + +function originsForDomains(domains) { + return [...new Set(domains.flatMap((domain) => { + if (domain === "localhost" || isIpAddress(domain)) { + return [ + `http://${domain}/*`, + `https://${domain}/*` + ]; + } + return [ + `http://${domain}/*`, + `https://${domain}/*`, + `http://*.${domain}/*`, + `https://*.${domain}/*` + ]; + }))]; +} + +async function ensureHostPermissions(domains) { + const origins = originsForDomains(domains); + const granted = await chrome.permissions.contains({ origins }); + if (granted) { + return; + } + throw new Error( + [ + `CCR Login Import does not have Chrome site access for ${domains.join(", ")}.`, + "Reload the unpacked extension after updating it, then grant the extension site access for the requested domains in Chrome extensions settings." + ].join(" ") + ); +} + +function normalizeImportUrl(value) { + const raw = typeof value === "string" ? value.trim() : ""; + if (!raw) { + return ""; + } + try { + const url = new URL(raw); + if (!["127.0.0.1", "localhost"].includes(url.hostname)) { + return ""; + } + if (!/^\/chrome-import\/jobs\/[^/]+\/?$/.test(url.pathname)) { + return ""; + } + url.pathname = url.pathname.replace(/\/+$/, ""); + return url.toString(); + } catch { + return ""; + } +} + +function normalizeDomain(value) { + return typeof value === "string" + ? value.trim().replace(/^\*\./, "").replace(/^\./, "").toLowerCase() + : ""; +} + +function allowedLocalStorageOrigin(actualOrigin, requestedOrigin) { + try { + const actual = new URL(actualOrigin); + const requested = new URL(requestedOrigin); + return actual.protocol === requested.protocol && actual.hostname === requested.hostname && actual.port === requested.port; + } catch { + return false; + } +} + +function isIpAddress(value) { + return /^\d{1,3}(?:\.\d{1,3}){3}$/.test(value) || value.includes(":"); +} + +function formatError(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/extension/chrome/confirm.js b/extension/chrome/confirm.js new file mode 100644 index 00000000..56a6aa1a --- /dev/null +++ b/extension/chrome/confirm.js @@ -0,0 +1,46 @@ +const root = document.getElementById("ccr-chrome-login-import"); +const button = document.getElementById("ccr-confirm-import"); +const statusElement = document.getElementById("ccr-import-status"); + +if (root && button && statusElement) { + const importUrl = root.getAttribute("data-import-url") || ""; + button.disabled = false; + button.textContent = "Confirm and Import"; + setStatus("CCR Login Import extension is connected. Review the domains, then confirm."); + + button.addEventListener("click", () => { + void confirmImport(importUrl); + }); +} + +async function confirmImport(importUrl) { + button.disabled = true; + setStatus("Importing Chrome cookies and localStorage into CCR..."); + try { + const response = await chrome.runtime.sendMessage({ + importUrl, + type: "ccr-login-import-confirm" + }); + if (!response?.ok) { + throw new Error(response?.error || "Chrome login import failed."); + } + const result = response.result || {}; + setStatus( + `Imported ${result.cookieImported || 0} cookies and ${result.localStorageImported || 0} localStorage items. Skipped ${result.skipped || 0}.`, + "ok" + ); + button.textContent = "Imported"; + } catch (error) { + button.disabled = false; + setStatus(formatError(error), "error"); + } +} + +function setStatus(message, kind = "") { + statusElement.textContent = message; + statusElement.className = `status ${kind}`.trim(); +} + +function formatError(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/extension/chrome/manifest.json b/extension/chrome/manifest.json new file mode 100644 index 00000000..262869a8 --- /dev/null +++ b/extension/chrome/manifest.json @@ -0,0 +1,35 @@ +{ + "manifest_version": 3, + "name": "CCR Login Import", + "description": "Import selected Chrome site login cookies into CCR's in-app browser.", + "version": "0.1.0", + "action": { + "default_popup": "popup.html", + "default_title": "CCR Login Import" + }, + "background": { + "service_worker": "background.js" + }, + "content_scripts": [ + { + "js": [ + "confirm.js" + ], + "matches": [ + "http://127.0.0.1/*", + "http://localhost/*" + ] + } + ], + "permissions": [ + "cookies", + "scripting", + "storage" + ], + "host_permissions": [ + "http://127.0.0.1/*", + "http://localhost/*", + "http://*/*", + "https://*/*" + ] +} diff --git a/extension/chrome/popup.html b/extension/chrome/popup.html new file mode 100644 index 00000000..391063b9 --- /dev/null +++ b/extension/chrome/popup.html @@ -0,0 +1,110 @@ + + + + + + CCR Login Import + + + +
+

CCR Login Import

+ + +
+
+ + + diff --git a/extension/chrome/popup.js b/extension/chrome/popup.js new file mode 100644 index 00000000..4b12be15 --- /dev/null +++ b/extension/chrome/popup.js @@ -0,0 +1,297 @@ +const importUrlInput = document.getElementById("import-url"); +const importButton = document.getElementById("import-button"); +const statusElement = document.getElementById("status"); + +document.addEventListener("DOMContentLoaded", () => { + void restoreLastImportUrl(); +}); + +importButton.addEventListener("click", () => { + void runImport(); +}); + +async function restoreLastImportUrl() { + const stored = await chrome.storage.local.get(["lastImportUrl"]); + if (typeof stored.lastImportUrl === "string") { + importUrlInput.value = stored.lastImportUrl; + } +} + +async function runImport() { + const importUrl = normalizeImportUrl(importUrlInput.value); + if (!importUrl) { + setStatus("Paste the import URL copied from CCR.", "error"); + return; + } + + importButton.disabled = true; + try { + await chrome.storage.local.set({ lastImportUrl: importUrl }); + setStatus("Reading CCR import job..."); + const job = await fetchImportJob(importUrl); + const domains = Array.isArray(job.domains) ? job.domains.map(normalizeDomain).filter(Boolean) : []; + if (domains.length === 0) { + throw new Error("CCR import job does not include any domains."); + } + + await ensureHostPermissions(domains); + + setStatus(`Reading cookies for ${domains.join(", ")}...`); + const cookies = await readCookiesForDomains(domains); + setStatus(`Reading localStorage for ${domains.join(", ")}...`); + const localStorageEntries = await readLocalStorageForDomains(domains, cookies); + + if (cookies.length === 0 && localStorageEntries.length === 0) { + throw new Error("No cookies or localStorage entries were found for the selected domains."); + } + + setStatus(`Sending ${cookies.length} cookies and ${localStorageItemCount(localStorageEntries)} localStorage items to CCR...`); + const result = await submitCookies(importUrl, cookies, localStorageEntries, domains); + setStatus( + `Imported ${result.cookieImported ?? 0} cookies and ${result.localStorageImported ?? 0} localStorage items. Skipped ${result.skipped ?? 0}.`, + "ok" + ); + } catch (error) { + setStatus(formatError(error), "error"); + } finally { + importButton.disabled = false; + } +} + +async function fetchImportJob(importUrl) { + const response = await fetch(importUrl, { + headers: { + "x-ccr-login-import": "chrome-extension" + } + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(body?.error?.message || `CCR import job request failed (${response.status}).`); + } + if (!body.job || body.job.status !== "pending") { + throw new Error(`CCR import job is ${body.job?.status || "unavailable"}.`); + } + return body.job; +} + +async function submitCookies(importUrl, cookies, localStorage, domains) { + const response = await fetch(`${importUrl.replace(/\/+$/, "")}/cookies`, { + body: JSON.stringify({ + cookies, + domains, + localStorage, + source: { + browser: "chrome", + extension: chrome.runtime.getManifest().version + } + }), + headers: { + "content-type": "application/json", + "x-ccr-login-import": "chrome-extension" + }, + method: "POST" + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) { + throw new Error(body?.error?.message || `CCR cookie import failed (${response.status}).`); + } + return body.result || {}; +} + +async function readCookiesForDomains(domains) { + const cookies = []; + for (const domain of domains) { + cookies.push(...await chrome.cookies.getAll({ domain })); + } + return dedupeCookies(cookies); +} + +async function readLocalStorageForDomains(domains, cookies) { + const origins = localStorageOriginsForDomains(domains, cookies); + const entries = []; + for (const origin of origins) { + const entry = await readLocalStorageForOrigin(origin).catch(() => undefined); + if (entry && Object.keys(entry.items).length > 0) { + entries.push(entry); + } + } + return entries; +} + +async function readLocalStorageForOrigin(origin) { + const tab = await chrome.tabs.create({ + active: false, + url: `${origin}/` + }); + try { + await waitForTabLoad(tab.id, 15000); + const [frameResult] = await chrome.scripting.executeScript({ + func: () => { + const items = {}; + for (let index = 0; index < window.localStorage.length; index += 1) { + const key = window.localStorage.key(index); + if (key !== null) { + items[key] = window.localStorage.getItem(key) || ""; + } + } + return { + items, + origin: window.location.origin + }; + }, + target: { tabId: tab.id }, + world: "MAIN" + }); + const result = frameResult?.result; + if (!result || !allowedLocalStorageOrigin(result.origin, origin)) { + return undefined; + } + return result; + } finally { + if (tab.id !== undefined) { + await chrome.tabs.remove(tab.id).catch(() => undefined); + } + } +} + +function waitForTabLoad(tabId, timeoutMs) { + return new Promise((resolve) => { + let done = false; + const finish = () => { + if (done) { + return; + } + done = true; + chrome.tabs.onUpdated.removeListener(listener); + window.clearTimeout(timeout); + resolve(); + }; + const listener = (updatedTabId, changeInfo) => { + if (updatedTabId === tabId && changeInfo.status === "complete") { + finish(); + } + }; + const timeout = window.setTimeout(finish, timeoutMs); + chrome.tabs.onUpdated.addListener(listener); + }); +} + +function dedupeCookies(cookies) { + const seen = new Set(); + const result = []; + for (const cookie of cookies) { + const partitionKey = cookie.partitionKey ? JSON.stringify(cookie.partitionKey) : ""; + const key = `${cookie.storeId || ""}\n${cookie.domain}\n${cookie.path}\n${cookie.name}\n${partitionKey}`; + if (!seen.has(key)) { + seen.add(key); + result.push(cookie); + } + } + return result; +} + +function localStorageOriginsForDomains(domains, cookies) { + const origins = new Set(); + for (const domain of domains) { + if (domain === "localhost" || isIpAddress(domain)) { + origins.add(`http://${domain}`); + origins.add(`https://${domain}`); + } else { + origins.add(`https://${domain}`); + } + } + + for (const cookie of cookies) { + const host = normalizeDomain(cookie.domain); + if (!host || !cookie.hostOnly || !domains.some((domain) => host === domain || host.endsWith(`.${domain}`))) { + continue; + } + origins.add(`${cookie.secure === false ? "http" : "https"}://${host}`); + } + + return [...origins]; +} + +function allowedLocalStorageOrigin(actualOrigin, requestedOrigin) { + try { + const actual = new URL(actualOrigin); + const requested = new URL(requestedOrigin); + return actual.protocol === requested.protocol && actual.hostname === requested.hostname && actual.port === requested.port; + } catch { + return false; + } +} + +function localStorageItemCount(entries) { + return entries.reduce((count, entry) => count + Object.keys(entry.items || {}).length, 0); +} + +function originsForDomains(domains) { + return [...new Set(domains.flatMap((domain) => { + if (domain === "localhost" || isIpAddress(domain)) { + return [ + `http://${domain}/*`, + `https://${domain}/*` + ]; + } + return [ + `http://${domain}/*`, + `https://${domain}/*`, + `http://*.${domain}/*`, + `https://*.${domain}/*` + ]; + }))]; +} + +async function ensureHostPermissions(domains) { + const origins = originsForDomains(domains); + const granted = await chrome.permissions.contains({ origins }); + if (granted) { + return; + } + throw new Error( + [ + `CCR Login Import does not have Chrome site access for ${domains.join(", ")}.`, + "Reload the unpacked extension after updating it, then grant the extension site access for the requested domains in Chrome extensions settings." + ].join(" ") + ); +} + +function normalizeImportUrl(value) { + const raw = value.trim(); + if (!raw) { + return ""; + } + try { + const url = new URL(raw); + if (!["127.0.0.1", "localhost"].includes(url.hostname)) { + return ""; + } + if (!/^\/chrome-import\/jobs\/[^/]+\/?$/.test(url.pathname)) { + return ""; + } + url.pathname = url.pathname.replace(/\/+$/, ""); + return url.toString(); + } catch { + return ""; + } +} + +function normalizeDomain(value) { + return typeof value === "string" + ? value.trim().replace(/^\*\./, "").replace(/^\./, "").toLowerCase() + : ""; +} + +function isIpAddress(value) { + return /^\d{1,3}(?:\.\d{1,3}){3}$/.test(value) || value.includes(":"); +} + +function setStatus(message, kind = "") { + statusElement.textContent = message; + statusElement.className = `status ${kind}`.trim(); +} + +function formatError(error) { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/core/src/config/config.ts b/packages/core/src/config/config.ts index 5e4e4bc7..0cf78ab6 100644 --- a/packages/core/src/config/config.ts +++ b/packages/core/src/config/config.ts @@ -700,6 +700,10 @@ function parseToolHub(value: unknown): Partial | undefined { if (typeof value.enabled === "boolean") { toolHub.enabled = value.enabled; } + const browserAutomation = value.browserAutomation ?? value.browser_automation; + if (typeof browserAutomation === "boolean") { + toolHub.browserAutomation = browserAutomation; + } const maxTools = readNumber(value.maxTools ?? value.max_tools); if (maxTools !== undefined) { toolHub.maxTools = clampNumber(maxTools, 1, 20); diff --git a/packages/core/src/config/default-config.ts b/packages/core/src/config/default-config.ts index 63933cb8..58a38681 100644 --- a/packages/core/src/config/default-config.ts +++ b/packages/core/src/config/default-config.ts @@ -170,6 +170,7 @@ export function createDefaultAppConfig(options: DefaultAppConfigOptions): AppCon trayWidgets: DEFAULT_TRAY_WIDGETS, trayWindowModules: DEFAULT_TRAY_WINDOW_MODULES, toolHub: { + browserAutomation: false, enabled: false, llm: { apiKey: "", diff --git a/packages/core/src/contracts/app.ts b/packages/core/src/contracts/app.ts index 4c00d21e..685a35ee 100644 --- a/packages/core/src/contracts/app.ts +++ b/packages/core/src/contracts/app.ts @@ -642,6 +642,7 @@ export type ToolHubLlmConfig = { }; export type ToolHubConfig = { + browserAutomation: boolean; enabled: boolean; llm: ToolHubLlmConfig; mcpServers: GatewayMcpServerConfig[]; @@ -1468,12 +1469,91 @@ export type BuiltInBrowserTabState = { url: string; }; +export type BuiltInBrowserAutomationHandoffKind = + | "blocked" + | "human_verification" + | "login_required" + | "other" + | "verification_code"; + +export type BuiltInBrowserAutomationHandoff = { + id: string; + kind: BuiltInBrowserAutomationHandoffKind; + message: string; + reason?: string; + requestedAt: number; + sessionId?: string; + status: "pending"; + tabId?: string; +}; + export type BuiltInBrowserState = { activeTabId?: string; apps: InstalledBrowserApp[]; + automationHandoff?: BuiltInBrowserAutomationHandoff; tabs: BuiltInBrowserTabState[]; }; +export type ChromeLoginImportTarget = "browser" | "browser-and-web-search"; + +export type ChromeLoginImportStatus = + | "completed" + | "expired" + | "failed" + | "pending"; + +export type ChromeLoginImportRequest = { + domains: string[]; + openConfirmationPage?: boolean; + target?: ChromeLoginImportTarget; +}; + +export type ChromeLoginImportResult = { + completedAt: number; + cookieImported: number; + cookieSkipped: number; + domains: string[]; + errors?: string[]; + imported: number; + localStorageImported: number; + localStorageSkipped: number; + partitions: string[]; + skipped: number; +}; + +export type ChromeLoginImportJob = { + confirmUrl: string; + createdAt: number; + domains: string[]; + endpointUrl: string; + expiresAt: number; + id: string; + importUrl: string; + result?: ChromeLoginImportResult; + status: ChromeLoginImportStatus; + target: ChromeLoginImportTarget; +}; + +export type ChromeLoginImportCookie = { + domain: string; + expirationDate?: number; + hostOnly?: boolean; + httpOnly?: boolean; + name: string; + partitionKey?: unknown; + path?: string; + sameSite?: "lax" | "no_restriction" | "strict" | "unspecified"; + secure?: boolean; + session?: boolean; + storeId?: string; + value: string; +}; + +export type ChromeLoginImportLocalStorage = { + items: Record; + origin: string; +}; + export type ProxyCertificateInstallResult = { caCertFile: string; manualCommand?: string; diff --git a/packages/core/src/contracts/ipc-channels.ts b/packages/core/src/contracts/ipc-channels.ts index 103b0b0f..f24d040f 100644 --- a/packages/core/src/contracts/ipc-channels.ts +++ b/packages/core/src/contracts/ipc-channels.ts @@ -73,10 +73,13 @@ export const IPC_CHANNELS = { browserBack: "ccr:browser:back", browserCloseTab: "ccr:browser:close-tab", browserForward: "ccr:browser:forward", + browserGetChromeLoginImport: "ccr:browser:get-chrome-login-import", browserGetState: "ccr:browser:get-state", browserNavigate: "ccr:browser:navigate", browserNewTab: "ccr:browser:new-tab", browserReload: "ccr:browser:reload", + browserResolveAutomationHandoff: "ccr:browser:resolve-automation-handoff", browserSelectTab: "ccr:browser:select-tab", + browserStartChromeLoginImport: "ccr:browser:start-chrome-login-import", browserStateChanged: "ccr:browser:state-changed" } as const; diff --git a/packages/core/src/gateway/claude-code-router-plugin.ts b/packages/core/src/gateway/claude-code-router-plugin.ts index b42ea83d..c39b3f98 100644 --- a/packages/core/src/gateway/claude-code-router-plugin.ts +++ b/packages/core/src/gateway/claude-code-router-plugin.ts @@ -365,9 +365,11 @@ function injectClaudeCodeToolHubInstructions(body: Record, conf const invokeName = toolNames.invoke ?? "tool_hub.invoke"; appendSystemInstruction(body, [ ccrToolHubSystemInstructionMarker, - `You MUST call ${toolNames.resolve} before answering any request that asks about external services, installed MCP capabilities, business APIs, orders, coupons, stores, accounts, available tools, or capabilities that are not already obvious from the eager tools.`, - `Do this even if the user did not mention ToolHub or ${toolNames.resolve}. Only skip ${toolNames.resolve} when the request is clearly local code/file/shell work or simple conversation that does not need an external or MCP capability.`, - `If ${toolNames.resolve} returns selected tools, use ${invokeName} to call the selected tool instead of telling the user that no such capability is available.`, + `The ToolHub search/resolution tool is ${toolNames.resolve}; call this actual tool, do not merely mention its name in text.`, + `You MUST call the ToolHub search/resolution tool ${toolNames.resolve} before answering any request that asks about external services, installed MCP capabilities, business APIs, orders, coupons, stores, accounts, available tools, or capabilities that are not already obvious from the eager tools.`, + `Do this even if the user did not mention ToolHub or ${toolNames.resolve}. Only skip the ToolHub search/resolution tool when the request is clearly local code/file/shell work or simple conversation that does not need an external or MCP capability.`, + `If ${toolNames.resolve} returns selected tools, call the ToolHub invocation tool ${invokeName} to run the selected tool instead of telling the user that no such capability is available.`, + "When the ToolHub resolve result includes executionPlanJs or workflowSketch, treat that JavaScript as the invocation dependency plan: await means serial order, and only callTool calls grouped inside the same Promise.all may be issued in parallel.", "Use the user's request as the task and include concise context when resolving. Do not ask the user to name the MCP tool unless the task is genuinely ambiguous after resolution." ].join("\n")); } @@ -380,16 +382,40 @@ function claudeCodeToolHubToolNames(tools: unknown[]): { invoke?: string; resolv } const name = claudeCodeToolName(tool); const normalized = normalizeClaudeCodeToolName(name); - if (!names.resolve && normalized.endsWith("toolhubresolve")) { + if (normalized.endsWith("toolhubresolve") && shouldUseClaudeCodeToolHubName(names.resolve, name)) { names.resolve = name; } - if (!names.invoke && normalized.endsWith("toolhubinvoke")) { + if (normalized.endsWith("toolhubinvoke") && shouldUseClaudeCodeToolHubName(names.invoke, name)) { names.invoke = name; } } return names; } +function shouldUseClaudeCodeToolHubName(current: string | undefined, candidate: string | undefined): boolean { + if (!candidate) { + return false; + } + if (!current) { + return true; + } + return claudeCodeToolHubNameScore(candidate) > claudeCodeToolHubNameScore(current); +} + +function claudeCodeToolHubNameScore(name: string): number { + const normalized = name.toLowerCase(); + if (normalized.startsWith("mcp__ccr-toolhub__") || normalized.startsWith("mcp__ccr_toolhub__")) { + return 3; + } + if (normalized.startsWith("mcp__") && normalized.includes("toolhub")) { + return 2; + } + if (normalized.startsWith("mcp__")) { + return 1; + } + return 0; +} + function appendSystemInstruction(body: Record, instruction: string): void { if (systemContainsInstruction(body.system, ccrToolHubSystemInstructionMarker)) { return; diff --git a/packages/core/src/gateway/service.ts b/packages/core/src/gateway/service.ts index a10ff09d..ef274dfa 100644 --- a/packages/core/src/gateway/service.ts +++ b/packages/core/src/gateway/service.ts @@ -45,7 +45,7 @@ import { loadPersistedApiKeys } from "@ccr/core/config/api-key-store"; import { codexDefaultBaseUrl, readCodexAuth } from "@ccr/core/agents/local-providers/service"; import { fetchWithSystemProxy, getSystemProxyUrlForProtocol } from "@ccr/core/proxy/system-proxy-fetch"; import { handleNetworkCaptureMcpRequest, isNetworkCaptureMcpPath } from "@ccr/core/mcp/network-capture-mcp"; -import { TOOL_HUB_MCP_SERVER_NAME, toolHubMcpRuntimeConfig } from "@ccr/core/mcp/toolhub-config"; +import { BROWSER_AUTOMATION_MCP_PATH, TOOL_HUB_MCP_SERVER_NAME, browserAutomationMcpEnabled, toolHubBuiltInBackendServers, toolHubMcpRuntimeConfig, toolHubRequestTimeoutMs } from "@ccr/core/mcp/toolhub-config"; import { pluginService } from "@ccr/core/plugins/service"; import { proxyService } from "@ccr/core/proxy/service"; import { createSseErrorDetector, recordGatewayRequestLog, updateGatewayRequestLogFromRawTrace, type RequestLogRawTraceUpdateInput } from "@ccr/core/observability/request-log-store"; @@ -130,6 +130,7 @@ export type BrowserWebSearchMcpRegistration = { export type BrowserWebSearchProtocolResult = { content?: string; + diagnostics?: string[]; snippet?: string; title: string; url: string; @@ -151,6 +152,11 @@ export type BrowserWebSearchMcpIntegration = { stopBrowserWebSearchMcpServers: () => Promise; }; +export type BrowserAutomationMcpIntegration = { + handleBrowserAutomationMcpRequest: (request: IncomingMessage, response: ServerResponse) => Promise; + stopBrowserAutomationMcpServer: () => Promise; +}; + type CoreGatewayHealth = { runtimeId?: string; status?: string; @@ -312,6 +318,7 @@ const persistedApiKeyCacheTtlMs = 1000; let persistedApiKeyCache: { loadedAt: number; values: ApiKeyConfig[] } | undefined; class GatewayService { + private browserAutomationMcpIntegration?: BrowserAutomationMcpIntegration; private browserWebSearchMcpIntegration?: BrowserWebSearchMcpIntegration; private child?: ChildProcess; private config?: AppConfig; @@ -332,6 +339,10 @@ class GatewayService { this.browserWebSearchMcpIntegration = integration; } + setBrowserAutomationMcpIntegration(integration: BrowserAutomationMcpIntegration): void { + this.browserAutomationMcpIntegration = integration; + } + async start(config: AppConfig): Promise { const coreHostError = loopbackCoreHostError(config.gateway.coreHost); if (coreHostError) { @@ -439,6 +450,9 @@ class GatewayService { await this.browserWebSearchMcpIntegration?.stopBrowserWebSearchMcpServers().catch((error) => { console.warn(`[gateway] Failed to stop browser web search MCP: ${formatError(error)}`); }); + await this.browserAutomationMcpIntegration?.stopBrowserAutomationMcpServer().catch((error) => { + console.warn(`[gateway] Failed to stop browser automation MCP: ${formatError(error)}`); + }); this.status = { ...this.status, @@ -551,6 +565,31 @@ class GatewayService { return; } + if (path === BROWSER_AUTOMATION_MCP_PATH || path === `${BROWSER_AUTOMATION_MCP_PATH}/`) { + if (!browserAutomationMcpEnabled(this.config)) { + sendJson(response, 404, { + error: { + message: "CCR browser automation MCP is disabled." + } + }); + return; + } + const authorization = await authorize(request, response, this.config); + if (!authorization.ok) { + return; + } + if (!this.browserAutomationMcpIntegration) { + sendJson(response, 503, { + error: { + message: "CCR browser automation MCP is only available in the Electron desktop app." + } + }); + return; + } + await this.browserAutomationMcpIntegration.handleBrowserAutomationMcpRequest(request, response); + return; + } + if (isNetworkCaptureMcpPath(path)) { if (!this.config.proxy.captureNetwork) { sendJson(response, 404, { error: { message: "Network capture MCP is disabled." } }); @@ -819,6 +858,15 @@ class GatewayService { sinceMs: startedAt - 1_000 }); + if (hostedWebSearchProtocolContext && !this.browserWebSearchMcpIntegration) { + const message = browserWebSearchUnavailableMessage(hostedWebSearchProtocolContext.toolName); + const responseHeaders = new Headers({ "content-type": "application/json; charset=utf-8" }); + const responseBody = JSON.stringify({ error: { message } }); + writeRequestLog(503, responseHeaders, responseBody, false, message); + sendJson(response, 503, { error: { message } }); + return; + } + if (hostedWebSearchProtocolContext && this.browserWebSearchMcpIntegration) { const records = await selectHostedWebSearchProtocolRecords( hostedWebSearchProtocolContext, @@ -1546,16 +1594,20 @@ function bundledFusionToolFallbackMcpEntryPath(): string { function toolHubMcpServer(config: AppConfig, backendServers: unknown[]): GatewayMcpServerConfig | undefined { const toolHub = config.toolHub; - const runtimeConfig = toolHubMcpRuntimeConfig(config, backendServers); + const runtimeBackendServers = [ + ...toolHubBuiltInBackendServers(config), + ...backendServers + ]; + const runtimeConfig = toolHubMcpRuntimeConfig(config, runtimeBackendServers); if (!toolHub?.enabled || !runtimeConfig) { return undefined; } return { ...runtimeConfig, - name: uniqueMcpServerName(TOOL_HUB_MCP_SERVER_NAME, backendServers), + name: uniqueMcpServerName(TOOL_HUB_MCP_SERVER_NAME, runtimeBackendServers), protocolVersion: "2024-11-05", - requestTimeoutMs: Math.max(toolHub.requestTimeoutMs ?? 60000, 60000), + requestTimeoutMs: toolHubRequestTimeoutMs(config, runtimeBackendServers), startupTimeoutMs: 600000, stdioMessageMode: "content-length", transport: "stdio" @@ -1565,53 +1617,121 @@ function toolHubMcpServer(config: AppConfig, backendServers: unknown[]): Gateway export function fusionFallbackToolDefinitions( profiles: unknown[], backedToolNames: Set = new Set() -): Array<{ description?: string; inputSchema?: Record; name: string }> { - const byName = new Map; name: string }>(); +): FusionFallbackToolDefinition[] { + const byName = new Map(); for (const profile of profiles) { - if (!isRecord(profile) || profile.enabled === false || !Array.isArray(profile.tools)) { + if (!isRecord(profile) || profile.enabled === false) { continue; } - for (const tool of profile.tools) { - if (!isRecord(tool)) { - continue; - } - const name = stringValue(tool.name); - if (!name) { - continue; - } - if (backedToolNames.has(name)) { - continue; - } - const existing = byName.get(name); - const description = stringValue(tool.description); - const inputSchema = isRecord(tool.inputSchema) - ? tool.inputSchema - : isRecord(tool.input_schema) - ? tool.input_schema - : undefined; - if (existing) { - if (!existing.description && description) { - existing.description = description; + if (Array.isArray(profile.tools)) { + for (const tool of profile.tools) { + if (!isRecord(tool)) { + continue; } - if (!existing.inputSchema && inputSchema) { - existing.inputSchema = inputSchema; + const name = stringValue(tool.name); + if (!name) { + continue; + } + if (backedToolNames.has(name)) { + continue; } - continue; - } - byName.set(name, { - ...(description ? { description } : {}), - ...(inputSchema ? { inputSchema } : {}), - name - }); + const existing = byName.get(name); + const description = stringValue(tool.description); + const inputSchema = isRecord(tool.inputSchema) + ? tool.inputSchema + : isRecord(tool.input_schema) + ? tool.input_schema + : undefined; + const unavailableMessage = fusionFallbackToolUnavailableMessage(profile, name); + if (existing) { + if (!existing.description && description) { + existing.description = description; + } + if (!existing.inputSchema && inputSchema) { + existing.inputSchema = inputSchema; + } + if (!existing.unavailableMessage && unavailableMessage) { + existing.unavailableMessage = unavailableMessage; + } + continue; + } + + byName.set(name, { + ...(description ? { description } : {}), + ...(inputSchema ? { inputSchema } : {}), + ...(unavailableMessage ? { unavailableMessage } : {}), + name + }); + } + } + + const browserFallback = browserWebSearchFallbackToolDefinition(profile, backedToolNames); + if (browserFallback && !byName.has(browserFallback.name)) { + byName.set(browserFallback.name, browserFallback); } } return [...byName.values()]; } +type FusionFallbackToolDefinition = { + description?: string; + inputSchema?: Record; + name: string; + unavailableMessage?: string; +}; + +function fusionFallbackToolUnavailableMessage(profile: unknown, toolName: string): string | undefined { + if (!isRecord(profile)) { + return undefined; + } + const metadata = isRecord(profile.metadata) ? profile.metadata : undefined; + const fusionWebSearch = isRecord(metadata?.fusionWebSearch) ? metadata.fusionWebSearch : undefined; + const webSearchConfig = readFusionWebSearchConfig(fusionWebSearch); + if (webSearchConfig?.provider !== "browser" || webSearchConfig.toolName !== toolName) { + return undefined; + } + return browserWebSearchUnavailableMessage(toolName); +} + +function browserWebSearchUnavailableMessage(toolName: string): string { + return [ + `Fusion MCP tool "${toolName}" is unavailable because In-app Browser web search requires CCR Desktop.`, + "This runtime did not register the Electron browser web search integration, so the hidden browser search tool cannot run here.", + "Run the profile in CCR Desktop or switch the Fusion web search provider to Brave, Bing, Google CSE, Serper, SerpAPI, Tavily, or Exa." + ].join(" "); +} + +function browserWebSearchFallbackToolDefinition( + profile: Record, + backedToolNames: Set +): FusionFallbackToolDefinition | undefined { + const metadata = isRecord(profile.metadata) ? profile.metadata : undefined; + const fusionWebSearch = isRecord(metadata?.fusionWebSearch) ? metadata.fusionWebSearch : undefined; + const webSearchConfig = readFusionWebSearchConfig(fusionWebSearch); + if (webSearchConfig?.provider !== "browser" || !webSearchConfig.toolName || backedToolNames.has(webSearchConfig.toolName)) { + return undefined; + } + return { + description: "Fallback registration for CCR In-app Browser web search when the Electron browser integration is unavailable.", + inputSchema: { + additionalProperties: true, + properties: { + count: { maximum: 20, minimum: 1, type: "number" }, + prompt: { type: "string" }, + query: { type: "string" } + }, + required: ["prompt"], + type: "object" + }, + name: webSearchConfig.toolName, + unavailableMessage: fusionFallbackToolUnavailableMessage(profile, webSearchConfig.toolName) + }; +} + export function fusionToolNamesBackedByMcpServers(servers: unknown[]): Set { const names = new Set(); for (const server of servers) { @@ -3037,7 +3157,8 @@ function hostedWebSearchEvidenceText(records: BrowserWebSearchProtocolRecord[], const content = focusedWebSearchContent(result.content, queryHint); const details = [ result.snippet ? `Search snippet: ${result.snippet}` : "", - content ? `Extracted page content: ${content}` : "" + content ? `Extracted page content: ${content}` : "", + result.diagnostics?.length ? `Diagnostics: ${result.diagnostics.join("; ")}` : "" ].filter(Boolean).join("\n"); return [ `${resultIndex + 1}. ${result.title}`, @@ -5201,7 +5322,8 @@ function anthropicWebSearchResultBlock(result: BrowserWebSearchProtocolResult): function anthropicWebSearchResultSnippet(result: BrowserWebSearchProtocolResult): string | undefined { const parts = [ result.snippet ? `Search snippet: ${sanitizeWebSearchEvidenceText(result.snippet)}` : "", - result.content ? `Extracted page content: ${sanitizeWebSearchEvidenceText(result.content)}` : "" + result.content ? `Extracted page content: ${sanitizeWebSearchEvidenceText(result.content)}` : "", + result.diagnostics?.length ? `Diagnostics: ${result.diagnostics.join("; ")}` : "" ].filter(Boolean); return parts.length > 0 ? parts.join("\n") : undefined; } diff --git a/packages/core/src/mcp/fusion-tool-fallback-mcp.ts b/packages/core/src/mcp/fusion-tool-fallback-mcp.ts index a67635d1..87c9b623 100644 --- a/packages/core/src/mcp/fusion-tool-fallback-mcp.ts +++ b/packages/core/src/mcp/fusion-tool-fallback-mcp.ts @@ -28,6 +28,7 @@ type McpTool = { description: string; inputSchema: JsonValue; name: string; + unavailableMessage?: string; }; type ToolCallResult = { @@ -134,10 +135,11 @@ async function handleJsonRpcRequest(payload: unknown): Promise item.name === toolLabel); const knownSuffix = toolNames.has(toolLabel) ? "" : " The requested tool was not in the fallback catalog."; return { content: [{ - text: + text: tool?.unavailableMessage || `Fusion MCP tool "${toolLabel}" is temporarily unavailable. ` + "CCR registered a fallback definition because the real MCP server did not provide the tool during discovery. " + `Check the Fusion MCP server logs and retry.${knownSuffix}`, @@ -179,7 +181,8 @@ function readFallbackTools(): McpTool[] { readString(item.description) || `Fallback registration for Fusion MCP tool "${name}". The real MCP server should handle successful calls.`, inputSchema: isRecord(item.inputSchema) ? item.inputSchema as JsonValue : objectSchema({}), - name + name, + unavailableMessage: readString(item.unavailableMessage) }); } return result; diff --git a/packages/core/src/mcp/toolhub-config.ts b/packages/core/src/mcp/toolhub-config.ts index be91a80a..5c4cbd18 100644 --- a/packages/core/src/mcp/toolhub-config.ts +++ b/packages/core/src/mcp/toolhub-config.ts @@ -1,9 +1,13 @@ import { join as pathJoin } from "node:path"; import { CONFIGDIR } from "@ccr/core/config/constants"; -import type { AppConfig } from "@ccr/core/contracts/app"; +import type { AppConfig, GatewayMcpServerConfig } from "@ccr/core/contracts/app"; export const TOOL_HUB_MCP_SERVER_NAME = "ccr-toolhub"; export const TOOL_HUB_MCP_RUNTIME_FILE_NAME = "toolhub-mcp.js"; +export const BROWSER_AUTOMATION_MCP_SERVER_NAME = "ccr-browser-automation"; +export const BROWSER_AUTOMATION_MCP_PATH = "/__ccr/browser-automation/mcp"; +export const BROWSER_AUTOMATION_HANDOFF_TIMEOUT_MS = 600000; +export const TOOL_HUB_DEFAULT_REQUEST_TIMEOUT_MS = 60000; export type ToolHubMcpRuntimeConfig = { args: string[]; @@ -26,17 +30,53 @@ export type ToolHubClaudeCodeMcpConfig = { mcpServers: Record; }; -export function toolHubBackendServers(config: AppConfig | undefined, extraServers: unknown[] = []): unknown[] { +export function toolHubBackendServers( + config: AppConfig | undefined, + extraServers: unknown[] = [], + options: { + apiKey?: string; + includeBuiltIns?: boolean; + } = {} +): unknown[] { return [ + ...(options.includeBuiltIns === false ? [] : toolHubBuiltInBackendServers(config, options)), ...(Array.isArray(config?.agent?.mcpServers) ? config.agent.mcpServers : []), ...(Array.isArray(config?.toolHub?.mcpServers) ? config.toolHub.mcpServers : []), ...extraServers ].filter(isToolHubBackendServer); } +export function toolHubBuiltInBackendServers( + config: AppConfig | undefined, + options: { + apiKey?: string; + } = {} +): GatewayMcpServerConfig[] { + if (!config || !browserAutomationMcpEnabled(config) || !hasGatewayEndpoint(config)) { + return []; + } + + return [ + { + apiKey: options.apiKey || firstConfiguredApiKey(config), + headers: {}, + name: BROWSER_AUTOMATION_MCP_SERVER_NAME, + protocolVersion: "2024-11-05", + requestTimeoutMs: BROWSER_AUTOMATION_HANDOFF_TIMEOUT_MS, + startupTimeoutMs: 60000, + transport: "streamable-http", + url: `${gatewayEndpoint(config)}${BROWSER_AUTOMATION_MCP_PATH}` + } + ]; +} + +export function browserAutomationMcpEnabled(config: AppConfig | undefined): boolean { + return Boolean(config?.toolHub?.enabled && config.toolHub.browserAutomation); +} + export function toolHubMcpRuntimeConfig( config: AppConfig | undefined, - backendServers = toolHubBackendServers(config), + backendServers?: unknown[], options: { command?: string; entryPath?: string; @@ -52,10 +92,14 @@ export function toolHubMcpRuntimeConfig( return undefined; } - const normalizedBackendServers = backendServers.filter(isToolHubBackendServer); + const resolvedBackendServers = backendServers ?? toolHubBackendServers(config, [], { + apiKey: options.resolver?.apiKey + }); + const normalizedBackendServers = resolvedBackendServers.filter(isToolHubBackendServer); if (normalizedBackendServers.length === 0) { return undefined; } + const requestTimeoutMs = toolHubRequestTimeoutMs(config, normalizedBackendServers); return { args: [options.entryPath ?? bundledToolHubMcpEntryPath()], @@ -68,11 +112,18 @@ export function toolHubMcpRuntimeConfig( TOOLHUB_OPENAI_API_KEY: options.resolver?.apiKey ?? toolHub.llm?.apiKey ?? "", TOOLHUB_OPENAI_BASE_URL: options.resolver?.baseUrl ?? toolHub.llm?.baseUrl ?? "https://api.openai.com/v1", TOOLHUB_OPENAI_MODEL: options.resolver?.model ?? toolHub.llm?.model ?? "", - TOOLHUB_REQUEST_TIMEOUT_MS: String(toolHub.requestTimeoutMs ?? 60000) + TOOLHUB_REQUEST_TIMEOUT_MS: String(requestTimeoutMs) } }; } +export function toolHubRequestTimeoutMs(config: AppConfig | undefined, backendServers?: unknown[]): number { + const configuredTimeout = positiveInteger(config?.toolHub?.requestTimeoutMs, TOOL_HUB_DEFAULT_REQUEST_TIMEOUT_MS); + const backendTimeouts = (backendServers ?? toolHubBackendServers(config)) + .map((server) => isRecord(server) ? positiveInteger(server.requestTimeoutMs, 0) : 0); + return Math.max(configuredTimeout, ...backendTimeouts); +} + export function toolHubClaudeCodeMcpConfig( config: AppConfig | undefined, options: Parameters[2] = {} @@ -82,7 +133,9 @@ export function toolHubClaudeCodeMcpConfig( return undefined; } - const backendServers = toolHubBackendServers(config); + const backendServers = toolHubBackendServers(config, [], { + apiKey: options.resolver?.apiKey + }); if (backendServers.length === 0) { return undefined; } @@ -126,6 +179,39 @@ function stringValue(value: unknown): string | undefined { return typeof value === "string" && value.trim() ? value.trim() : undefined; } +function firstConfiguredApiKey(config: AppConfig): string | undefined { + return (Array.isArray(config.APIKEYS) ? config.APIKEYS : []) + .find((apiKey) => apiKey.key.trim())?.key.trim() || stringValue(config.APIKEY); +} + +function positiveInteger(value: unknown, fallback: number): number { + return typeof value === "number" && Number.isFinite(value) && value > 0 ? Math.trunc(value) : fallback; +} + +function gatewayEndpoint(config: AppConfig): string { + return `http://${formatHost(clientGatewayHost(config.gateway.host))}:${config.gateway.port}`; +} + +function hasGatewayEndpoint(config: AppConfig): boolean { + const gateway = (config as Partial).gateway; + return Boolean(gateway && stringValue(gateway.host) && Number.isFinite(gateway.port)); +} + +function formatHost(host: string): string { + return host.includes(":") && !host.startsWith("[") ? `[${host}]` : host; +} + +function clientGatewayHost(host: string): string { + const value = stringValue(host) ?? "127.0.0.1"; + if (value === "0.0.0.0") { + return "127.0.0.1"; + } + if (value === "::" || value === "[::]") { + return "::1"; + } + return value; +} + function uniqueStrings(values: string[]): string[] { const seen = new Set(); const result: string[] = []; diff --git a/packages/core/src/mcp/toolhub-mcp.ts b/packages/core/src/mcp/toolhub-mcp.ts index 1a5e9c7f..9d440d98 100644 --- a/packages/core/src/mcp/toolhub-mcp.ts +++ b/packages/core/src/mcp/toolhub-mcp.ts @@ -149,6 +149,8 @@ type LlmToolResolution = { type ResolveOutput = { alreadyResolved?: boolean; + executionPlanInstructions?: string; + executionPlanJs?: string; nextAction?: { confirmationRequiredFor: string[]; firstAction: { @@ -186,6 +188,17 @@ const defaultRequestTimeoutMs = 60_000; const defaultMaxTools = 10; const discoveryCacheMaxAgeMs = 10_000; const repeatedResolveWindowMs = 5 * 60_000; +const executionPlanInstructions = [ + "Treat executionPlanJs as the dependency plan for ToolHub invocations.", + "Each callTool(toolName, args) call maps to one tool_hub.invoke call with tool=toolName and args=args.", + "await means the next ToolHub invocation depends on the previous result and must not be started early.", + "Only callTool calls inside the same Promise.all([...]) expression may be issued as parallel tool calls.", + "If a later call needs values from an earlier result, wait for that earlier result and fill the arguments after it returns." +].join(" "); +const reservedJavaScriptWords = new Set( + "await break case catch class const continue debugger default delete do else enum export extends false finally for function if import in instanceof new null return super switch this throw true try typeof var void while with yield" + .split(" ") +); type RuntimeLike = { callTool(params: unknown): Promise; @@ -365,15 +378,25 @@ class ToolHubRuntime { await this.registry.refreshServers(undefined, true); catalog = await this.registry.listTools(); } + if (taskWantsChromeLoginImport(task) && !catalogHasChromeLoginImportTool(catalog)) { + await this.registry.refreshServers(["ccr-browser-automation"], true); + catalog = await this.registry.listTools(); + } if (catalog.length === 0) { throw new Error("No MCP tools are available to resolve."); } const cached = this.findRecentlyResolvedTask(scopeKey, taskHash); if (cached) { - const selectedTools = cached.toolNames - .map((toolName) => this.resolveCatalogEntry(toolName, catalog)) - .filter((entry): entry is CatalogEntry => Boolean(entry)); + const selectedTools = expandToolBundleWithCompanionTools( + uniqueToolEntries([ + ...cached.toolNames + .map((toolName) => this.resolveCatalogEntry(toolName, catalog)) + .filter((entry): entry is CatalogEntry => Boolean(entry)), + ...getDeterministicTaskTools(task, catalog) + ]), + catalog + ); if (selectedTools.length > 0) { this.markToolsLoaded(scopeKey, selectedTools); return toolResult(this.buildRepeatedResolveOutput(selectedTools)); @@ -383,9 +406,15 @@ class ToolHubRuntime { const inFlightResolve = session.inFlightResolves.get(taskHash); if (inFlightResolve) { const output = await inFlightResolve; - const selectedTools = output.selectedTools - .map((tool) => this.resolveCatalogEntry(tool.toolName, catalog) ?? tool) - .filter((entry): entry is CatalogEntry => Boolean(entry)); + const selectedTools = expandToolBundleWithCompanionTools( + uniqueToolEntries([ + ...output.selectedTools + .map((tool) => this.resolveCatalogEntry(tool.toolName, catalog) ?? tool) + .filter((entry): entry is CatalogEntry => Boolean(entry)), + ...getDeterministicTaskTools(task, catalog) + ]), + catalog + ); if (selectedTools.length > 0) { this.markToolsLoaded(scopeKey, selectedTools); return toolResult(this.buildRepeatedResolveOutput(selectedTools)); @@ -461,7 +490,13 @@ class ToolHubRuntime { usedLlm = false; } - let selectedTools = resolution.selectedTools; + let selectedTools = expandToolBundleWithCompanionTools( + uniqueToolEntries([ + ...resolution.selectedTools, + ...getDeterministicTaskTools(input.task, input.catalog) + ]), + input.catalog + ); if (input.withoutSideEffects) { selectedTools = selectedTools.filter((tool) => !tool.invocation.sideEffect); } @@ -471,13 +506,14 @@ class ToolHubRuntime { this.markToolsLoaded(input.scopeKey, selectedTools); this.rememberResolvedTask(input.scopeKey, input.taskHash, selectedTools); + const executionPlanJs = buildExecutionPlanJs(resolution.workflowSketch, selectedTools); return { - ...this.buildResolveOutput(resolution.summary, selectedTools), + ...this.buildResolveOutput(resolution.summary, selectedTools, executionPlanJs), plannedSteps: resolution.plannedSteps, referencedTokens: resolution.referencedTokens, retriever, usedLlm, - workflowSketch: resolution.workflowSketch + workflowSketch: executionPlanJs }; } @@ -587,39 +623,50 @@ class ToolHubRuntime { return catalog.find((entry) => entry.toolName === resolvedName); } - private buildResolveOutput(summary: string, selectedTools: CatalogEntry[]): ResolveOutput { + private buildResolveOutput(summary: string, selectedTools: CatalogEntry[], executionPlanJs = buildSequentialExecutionPlanJs(selectedTools)): ResolveOutput { return { + executionPlanInstructions, + executionPlanJs, nextAction: this.buildResolveNextAction(selectedTools), reasoningSummary: summary, runtimeContext: { - availableContextKeys: ["selectedTools"], - summary: selectedTools.map((tool) => `${tool.toolName}: ${tool.description || tool.title}`) + availableContextKeys: ["selectedTools", "executionPlanJs", "executionPlanInstructions"], + summary: [ + ...selectedTools.map((tool) => `${tool.toolName}: ${tool.description || tool.title}`), + "Follow executionPlanJs for dependency ordering before invoking selected tools." + ] }, selectedToolNames: selectedTools.map((tool) => tool.toolName), selectedTools, - tsDefinitions: buildTsDefinitions(selectedTools) + tsDefinitions: buildTsDefinitions(selectedTools), + workflowSketch: executionPlanJs }; } private buildRepeatedResolveOutput(selectedTools: CatalogEntry[]): ResolveOutput { const nextAction = this.buildResolveNextAction(selectedTools); + const executionPlanJs = buildSequentialExecutionPlanJs(selectedTools); return { alreadyResolved: true, + executionPlanInstructions, + executionPlanJs, nextAction, reasoningSummary: [ "This task has already been resolved in the current ToolHub session.", nextAction.instruction ].join(" "), runtimeContext: { - availableContextKeys: ["selectedTools", "nextAction"], + availableContextKeys: ["selectedTools", "nextAction", "executionPlanJs", "executionPlanInstructions"], summary: [ "The selected tools are already loaded for tool_hub.invoke.", "Do not repeat discovery for this task.", - nextAction.instruction + nextAction.instruction, + "Follow executionPlanJs for dependency ordering before invoking selected tools." ] }, selectedToolNames: selectedTools.map((tool) => tool.toolName), - selectedTools + selectedTools, + workflowSketch: executionPlanJs }; } @@ -646,7 +693,10 @@ class ToolHubRuntime { toolName: firstTool?.toolName, type: "invoke_tool" as const }; - const instructionParts = ["Do not call tool_hub.resolve again for this task."]; + const instructionParts = [ + "Do not call tool_hub.resolve again for this task.", + "Follow executionPlanJs: await is serial dependency; only Promise.all groups may run in parallel." + ]; if (firstTool && firstRequiredArguments.length > 0) { instructionParts.push(`Ask the user for missing required arguments for ${firstTool.toolName}: ${firstRequiredArguments.join(", ")}.`); } else if (firstTool) { @@ -790,7 +840,7 @@ class ToolHubRegistry { private async ensureDiscoveryFresh(maxAgeMs = discoveryCacheMaxAgeMs): Promise { const staleServerNames = [...this.entries.values()] - .filter((entry) => !entry.loadedFromCache && (!entry.lastCheckedAt || Date.now() - entry.lastCheckedAt > maxAgeMs)) + .filter((entry) => !entry.lastCheckedAt || Date.now() - entry.lastCheckedAt > maxAgeMs) .map((entry) => entry.config.name); if (staleServerNames.length === 0) { return; @@ -1752,6 +1802,7 @@ function metaTools(): Array<{ description: string; inputSchema: Record\", {...}) or tools.call(\"\", {...}).", "3. Call the tree-sitter tool on that sketch.", "4. Inspect the tree-sitter result and check whether the bundle is complete end-to-end.", @@ -2000,10 +2051,18 @@ function buildSearchSystemPrompt(catalog: SearchCatalogItem[], topK: number): st "{ \"summary\": string, \"steps\": string[], \"workflowSketch\": string, \"toolNames\": string[] }", "Rules:", "- toolNames must be exact catalog names or aliases.", + "- workflowSketch is the dependency plan that will be returned to the caller as executionPlanJs.", + "- In workflowSketch, use await to show serial dependencies and Promise.all([...]) only for tool calls that are independent and safe to invoke in parallel.", + "- Do not put side-effecting calls in Promise.all unless they are truly independent and safe to run concurrently.", "- In workflowSketch, prefer string-literal tool calls over reconstructed member chains whenever possible.", "- Include prerequisite and downstream tools you will likely need after the first action.", "- Never invent tool names.", "- Generic browser automation tools are a strong match for web tasks such as ordering, buying, booking, delivery, or checkout when no domain-specific MCP tool exists.", + "- For browser navigation/open calls, prefer omitting waitUntil or using waitUntil: \"interactive\" so the agent can inspect and act as soon as the page is usable.", + "- Do not use waitUntil: \"network_idle\" for Gmail, Google sign-in, SPAs, mail, chat, auth, checkout, verification, or pages with long-lived requests. Use network_idle only when the user explicitly asks for network quiescence.", + "- When selecting CCR browser automation tools, include the human-handoff follow-up tools needed if login, CAPTCHA, verification, blocked navigation, or manual confirmation appears.", + "- For CCR browser automation bundles, include browser_handoff_request and browser_handoff_wait when the workflow may need user help. Do not assume all browser tools are preloaded.", + "- For importing existing Chrome login state into CCR's in-app browser, select browser_chrome_login_import and include browser_chrome_login_import_status to check completion.", "- If using member-call syntax, prefer tools.(...) or mcp..(...).", "- For lookup tasks, do not stop at opening or navigating. Include the tools needed to read or extract the answer.", "- If the catalog has no strong match, return an empty toolNames array.", @@ -2312,6 +2371,21 @@ function getLocalFallbackPreferredTools(taskText: string): Map { add("browser_screenshot", 60); add("browser_events_await", 44); } + if (hasAnyToken(tokens, [ + "auth", + "chrome", + "cookie", + "cookies", + "import", + "localstorage", + "login", + "session", + "signin", + "storage" + ]) || /chrome|cookie|cookies|localstorage|local storage|登录态|登录状态|导入登录|浏览器登录|本地存储/.test(normalizedText)) { + add("browser_chrome_login_import", 128); + add("browser_chrome_login_import_status", 92); + } if (hasAnyToken(tokens, ["address", "delivery", "geo", "geolocation", "location", "nearby", "pickup", "store"]) || /地址|定位|位置|附近|配送|外卖|自取|门店/.test(normalizedText)) { add("location_permission_status", 90); @@ -2324,6 +2398,28 @@ function getLocalFallbackPreferredTools(taskText: string): Map { return scores; } +function getDeterministicTaskTools(taskText: string, catalog: CatalogEntry[]): CatalogEntry[] { + if (!taskWantsChromeLoginImport(taskText)) { + return []; + } + return [ + catalog.find((tool) => isBrowserAutomationTool(tool) && tool.remoteToolName === "browser_chrome_login_import"), + catalog.find((tool) => isBrowserAutomationTool(tool) && tool.remoteToolName === "browser_chrome_login_import_status") + ].filter((tool): tool is CatalogEntry => Boolean(tool)); +} + +function taskWantsChromeLoginImport(taskText: string): boolean { + const normalizedText = taskText.toLowerCase(); + return normalizedText.includes("browser_chrome_login_import") || + /(chrome|谷歌浏览器|浏览器).*(login|signin|auth|cookie|localstorage|local storage|session|登录态|登录状态|登录信息|本地存储|cookie)/.test(normalizedText) || + /(import|导入|迁移|同步).*(chrome|谷歌浏览器).*(login|signin|auth|cookie|localstorage|local storage|session|登录态|登录状态|登录信息|本地存储)/.test(normalizedText) || + /(登录态|登录状态|登录信息).*(chrome|谷歌浏览器|浏览器).*(导入|迁移|同步)/.test(normalizedText); +} + +function catalogHasChromeLoginImportTool(catalog: CatalogEntry[]): boolean { + return catalog.some((tool) => isBrowserAutomationTool(tool) && tool.remoteToolName === "browser_chrome_login_import"); +} + function tokenizeLocalSearchText(text: string): string[] { const tokens = text .replace(/([a-z0-9])([A-Z])/g, "$1 $2") @@ -2350,10 +2446,113 @@ function uniqueToolEntries(entries: CatalogEntry[]): CatalogEntry[] { return output; } +function expandToolBundleWithCompanionTools(selectedTools: CatalogEntry[], catalog: CatalogEntry[]): CatalogEntry[] { + const output = uniqueToolEntries(selectedTools); + const selectedNames = new Set(output.map((tool) => tool.toolName)); + const selectedRemoteNames = new Set(output.map((tool) => tool.remoteToolName)); + const hasBrowserAutomationTool = output.some((tool) => isBrowserAutomationTool(tool)); + const hasHandoffRequestTool = output.some((tool) => + isBrowserAutomationTool(tool) && + (tool.remoteToolName === "browser_handoff_request" || tool.remoteToolName === "askHumanHelp") + ); + const hasChromeLoginImportTool = output.some((tool) => + isBrowserAutomationTool(tool) && + tool.remoteToolName === "browser_chrome_login_import" + ); + const companionRemoteNames = new Set(); + + if (hasBrowserAutomationTool) { + companionRemoteNames.add("browser_handoff_request"); + companionRemoteNames.add("browser_handoff_status"); + companionRemoteNames.add("browser_handoff_wait"); + } + if (hasHandoffRequestTool) { + companionRemoteNames.add("browser_handoff_wait"); + } + if (hasChromeLoginImportTool) { + companionRemoteNames.add("browser_chrome_login_import_status"); + } + + for (const remoteToolName of companionRemoteNames) { + if (selectedRemoteNames.has(remoteToolName)) { + continue; + } + const companion = catalog.find((tool) => + isBrowserAutomationTool(tool) && + tool.remoteToolName === remoteToolName && + !selectedNames.has(tool.toolName) + ); + if (companion) { + output.push(companion); + selectedNames.add(companion.toolName); + selectedRemoteNames.add(companion.remoteToolName); + } + } + + return output; +} + +function isBrowserAutomationTool(tool: CatalogEntry): boolean { + return tool.serverName === "ccr-browser-automation" || + tool.serverId === "ccr-browser-automation" || + tool.serverNamespace === "ccr_browser_automation" || + tool.toolName.startsWith("mcp.ccr_browser_automation."); +} + function buildLocalFallbackWorkflowSketch(selectedTools: CatalogEntry[]): string | undefined { - return selectedTools.length > 0 - ? selectedTools.slice(0, 5).map((tool) => `await callTool(${JSON.stringify(tool.toolName)}, {});`).join("\n") - : undefined; + return selectedTools.length > 0 ? buildSequentialExecutionPlanJs(selectedTools) : undefined; +} + +function isBrowserNavigationTool(tool: CatalogEntry): boolean { + return isBrowserAutomationTool(tool) && ( + tool.toolName.endsWith("browser_session_open") || + tool.toolName.endsWith("browser_navigate") || + tool.remoteToolName === "browser_session_open" || + tool.remoteToolName === "browser_navigate" + ); +} + +function buildExecutionPlanJs(workflowSketch: string | undefined, selectedTools: CatalogEntry[]): string { + const trimmed = typeof workflowSketch === "string" ? workflowSketch.trim() : ""; + return trimmed || buildSequentialExecutionPlanJs(selectedTools); +} + +function buildSequentialExecutionPlanJs(selectedTools: CatalogEntry[]): string { + if (selectedTools.length === 0) { + return [ + "async function runWithToolHub() {", + " // Ask the user for missing task details before invoking tools.", + "}" + ].join("\n"); + } + const lines = [ + "async function runWithToolHub() {", + " // Invoke calls in this order unless the plan explicitly uses Promise.all." + ]; + selectedTools.forEach((tool, index) => { + lines.push(` const step${index + 1} = await callTool(${JSON.stringify(tool.toolName)}, ${buildExecutionPlanArgs(tool)});`); + }); + lines.push("}"); + return lines.join("\n"); +} + +function buildExecutionPlanArgs(tool: CatalogEntry): string { + if (isBrowserNavigationTool(tool)) { + return "{ url, waitUntil: \"interactive\" }"; + } + const required = getSchemaRequiredProperties(tool.inputSchema); + if (required.length === 0) { + return "{}"; + } + return `{ ${required.map((key) => `${JSON.stringify(key)}: ${toPlanVariableName(key)}`).join(", ")} }`; +} + +function toPlanVariableName(value: string): string { + const identifier = toIdentifier(value).replace(/^[A-Z]/, (match) => match.toLowerCase()); + if (!identifier || reservedJavaScriptWords.has(identifier)) { + return "value"; + } + return identifier; } function inferInvocation(tool: ToolDefinition): ToolInvocation { diff --git a/packages/electron/src/main/browser-automation-mcp.ts b/packages/electron/src/main/browser-automation-mcp.ts new file mode 100644 index 00000000..884905f3 --- /dev/null +++ b/packages/electron/src/main/browser-automation-mcp.ts @@ -0,0 +1,3517 @@ +import type { IncomingMessage, ServerResponse } from "node:http"; +import { randomUUID } from "node:crypto"; +import type { Event as ElectronEvent, WebContents } from "electron"; +import { loadAppConfig } from "@ccr/core/config/config"; +import type { + BuiltInBrowserAutomationHandoff, + BuiltInBrowserAutomationHandoffKind, + BuiltInBrowserState, + BuiltInBrowserTabState +} from "@ccr/core/contracts/app"; +import type { BrowserAutomationMcpIntegration } from "@ccr/core/gateway/service"; +import { BROWSER_AUTOMATION_MCP_PATH } from "@ccr/core/mcp/toolhub-config"; +import { builtInBrowserService, type BrowserAutomationEvent } from "./built-in-browser"; +import { chromeLoginImportService } from "./chrome-login-import"; + +type JsonPrimitive = boolean | null | number | string; +type JsonValue = JsonPrimitive | JsonValue[] | { [key: string]: JsonValue }; + +type JsonRpcRequest = { + id?: null | number | string; + jsonrpc?: string; + method?: string; + params?: unknown; +}; + +type JsonRpcResponse = + | { + id: null | number | string; + jsonrpc: "2.0"; + result: JsonValue; + } + | { + error: { + code: number; + data?: JsonValue; + message: string; + }; + id: null | number | string; + jsonrpc: "2.0"; + }; + +type McpTool = { + description: string; + inputSchema: JsonValue; + name: string; + tags?: string[]; + title?: string; +}; + +type ToolCallResult = { + content: Array<{ text: string; type: "text" }>; + isError?: boolean; +}; + +type BrowserTarget = { + axNodeId?: string; + backendNodeId?: number; + exact?: boolean; + index?: number; + ref?: string; + role?: string; + selector?: string; + text?: string; +}; + +type BrowserSessionRef = { + frameId?: string; + sessionId: string; + tabId: string; + userId?: string; +}; + +type AttachedSession = { + attachedAt: number; + leaseId: string; + observeOnly: boolean; + ref: BrowserSessionRef; +}; + +type EventSubscription = { + channels: Set; + dropped: boolean; + events: BrowserAutomationEvent[]; + ref?: BrowserSessionRef; + redactTextInput: boolean; + sampleMouseMove: boolean; + startedAt: number; + subscriptionId: string; + unsubscribe: () => void; +}; + +type AxSnapshotResult = { + handoff?: BuiltInBrowserAutomationHandoff; + handoffRequired?: boolean; + humanHelp?: Record; + humanHelpRequired?: boolean; + nextAction?: "human_help"; + nodes: Array>; + scope: string; + session: BrowserSessionRef; + title: string; + url: string; +}; + +type BrowserHandoffDetection = { + kind: BuiltInBrowserAutomationHandoffKind; + message: string; + reason: string; +}; + +type ReadinessResult = { + matched: boolean; + matchedEvent?: string; + navigationError?: { + errorCode?: number; + errorDescription?: string; + url?: string; + }; + readinessUrl?: string; + timedOut: boolean; +}; + +type ReadinessEvent = { + errorCode?: number; + errorDescription?: string; + isMainFrame?: boolean; + type: string; + url?: string; +}; + +type NavigationReadinessContext = { + expectedUrl?: string; + previousUrl?: string; +}; + +const protocolVersion = "2024-11-05"; +const automationEventReplayMs = 15_000; +const defaultAxSnapshotLimit = 60; +const defaultSnapshotMaxElements = 80; +const defaultSnapshotMaxText = 3_000; +const defaultEventAwaitTimeoutMs = 10_000; +const defaultJavascriptTimeoutMs = 8_000; +const maxMcpRequestBytes = 2 * 1024 * 1024; +const maxSubscriptionEvents = 512; +const maxToolResultChars = 60_000; +const maxToolResultArrayItems = 120; +const maxToolResultStringChars = 2_000; +const maxToolResultObjectKeys = 120; +const maxSnapshotResultElements = 80; +const maxAxSnapshotResultNodes = 80; +const maxBrowserResultTextChars = 3_000; +const defaultWaitTimeoutMs = 10_000; + +const sessionSchema = objectSchema({ + frameId: { description: "Optional frame id. CCR currently targets the main frame.", type: "string" }, + sessionId: { description: "Browser automation session id.", type: "string" }, + tabId: { description: "Built-in browser tab id.", type: "string" }, + userId: { description: "Optional logical user id.", type: "string" } +}, ["sessionId", "tabId"]); + +const cursorSchema = objectSchema({ + dropped: { description: "Whether earlier events were dropped before this cursor.", type: "boolean" }, + seq: { description: "Last observed event sequence.", type: "number" }, + subscriptionId: { description: "Event subscription id.", type: "string" }, + ts: { description: "Last observed event timestamp.", type: "number" } +}, ["subscriptionId", "seq"]); + +const targetSchema = objectSchema({ + axNodeId: { description: "Accessibility node id returned by browser_ax_snapshot/query. In CCR this is a stable element ref.", type: "string" }, + backendNodeId: { description: "Reserved for CDP-compatible clients. CSS/ref targeting is preferred in CCR.", type: "number" }, + exact: { description: "Match text/name exactly instead of by substring.", type: "boolean" }, + index: { description: "Zero-based match index when text/role resolves multiple elements.", minimum: 0, type: "number" }, + ref: { description: "Element ref returned by browser_snapshot. Refs are CSS selectors.", type: "string" }, + role: { description: "Accessible or implicit role such as button, link, textbox, combobox, checkbox.", type: "string" }, + selector: { description: "CSS selector. Used directly when provided.", type: "string" }, + text: { description: "Visible text, accessible name, placeholder, label, or value to match.", type: "string" } +}); + +const browserAutomationTools: McpTool[] = [ + ...browserPublicAutomationTools(), + ...browserLegacyAliasTools() +].filter((tool, index, tools) => tools.findIndex((candidate) => candidate.name === tool.name) === index); + +function browserPublicAutomationTools(): McpTool[] { + return [ + { + description: "Open a URL or attach an existing CCR built-in browser tab and create an automation session.", + inputSchema: objectSchema({ + observeOnly: { description: "If true, action tools reject this session.", type: "boolean" }, + tabId: { description: "Existing tab id to attach.", type: "string" }, + timeoutMs: { description: "Optional navigation timeout in milliseconds.", type: "number" }, + url: { description: "Optional URL or search query to open.", type: "string" }, + userId: { description: "Optional logical user id.", type: "string" }, + waitUntil: { description: "Readiness condition. Default/recommended: interactive, which waits until the page is inspectable/actionable and avoids long-lived network requests. network_idle is rarely appropriate for SPAs, Google, mail, chat, auth, or streaming pages.", enum: ["none", "interactive", "domcontentloaded", "load", "network_idle"], type: "string" }, + windowId: { description: "Ignored in CCR; included for agentic-browser compatibility.", type: "string" } + }), + name: "browser_session_open", + tags: ["browser", "automation", "session", "open", "tab"], + title: "Open Browser Automation Session" + }, + { + description: "Detach and release a browser automation session. Does not close the tab.", + inputSchema: objectSchema({ session: sessionSchema }, ["session"]), + name: "browser_session_close", + tags: ["browser", "automation", "session", "close"], + title: "Close Browser Automation Session" + }, + { + description: "Create a new CCR built-in browser tab. Accepts an existing session or the fixed CCR browser window id.", + inputSchema: objectSchema({ + activate: { description: "Whether the new tab should become active. Defaults to true.", type: "boolean" }, + session: sessionSchema, + url: { description: "Optional URL or search query.", type: "string" }, + windowId: { description: "Ignored in CCR; included for compatibility.", type: "string" } + }), + name: "browser_tab_create", + tags: ["browser", "automation", "tab", "create"], + title: "Create Browser Tab" + }, + { + description: "List all CCR built-in browser tabs with active, loading, URL, title, and navigation state.", + inputSchema: objectSchema({ + session: sessionSchema, + windowId: { description: "Ignored in CCR; included for compatibility.", type: "string" } + }), + name: "browser_tab_list", + tags: ["browser", "automation", "tab", "list"], + title: "List Browser Tabs" + }, + { + description: "Activate an existing CCR built-in browser tab.", + inputSchema: objectSchema({ + session: sessionSchema, + tabId: { description: "Tab id to activate. Defaults to session.tabId.", type: "string" } + }), + name: "browser_tab_activate", + tags: ["browser", "automation", "tab", "activate", "focus"], + title: "Activate Browser Tab" + }, + { + description: "Close a CCR built-in browser tab.", + inputSchema: objectSchema({ + session: sessionSchema, + tabId: { description: "Tab id to close. Defaults to session.tabId.", type: "string" } + }), + name: "browser_tab_close", + tags: ["browser", "automation", "tab", "close"], + title: "Close Browser Tab" + }, + { + description: "Navigate the session tab or a specified tab to a URL or search query.", + inputSchema: objectSchema({ + session: sessionSchema, + tabId: { description: "Optional tab id. Defaults to session.tabId or active tab.", type: "string" }, + timeoutMs: { description: "Optional navigation timeout in milliseconds.", maximum: 120000, minimum: 100, type: "number" }, + url: { description: "URL or search query to load.", type: "string" }, + waitUntil: { description: "Readiness condition. Default/recommended: interactive, which waits until the page is inspectable/actionable and avoids long-lived network requests. network_idle is rarely appropriate for SPAs, Google, mail, chat, auth, or streaming pages.", enum: ["none", "interactive", "domcontentloaded", "load", "network_idle"], type: "string" } + }, ["url"]), + name: "browser_navigate", + tags: ["browser", "automation", "navigate", "tab", "url"], + title: "Navigate Browser Tab" + }, + { + description: "Go back in the session tab or a specified tab.", + inputSchema: objectSchema({ session: sessionSchema, tabId: { type: "string" } }), + name: "browser_tab_go_back", + tags: ["browser", "automation", "tab", "back"], + title: "Browser Tab Back" + }, + { + description: "Go forward in the session tab or a specified tab.", + inputSchema: objectSchema({ session: sessionSchema, tabId: { type: "string" } }), + name: "browser_tab_go_forward", + tags: ["browser", "automation", "tab", "forward"], + title: "Browser Tab Forward" + }, + { + description: "Reload the session tab or a specified tab.", + inputSchema: objectSchema({ session: sessionSchema, tabId: { type: "string" } }), + name: "browser_tab_reload", + tags: ["browser", "automation", "tab", "reload"], + title: "Reload Browser Tab" + }, + { + description: "Read a condensed accessibility-oriented page snapshot. Nodes include axNodeId/ref, role, name, value, text, and rect.", + inputSchema: objectSchema({ + includeIgnored: { description: "Included for compatibility; CCR returns visible/high-signal nodes.", type: "boolean" }, + limit: { description: "Maximum nodes to return.", maximum: 300, minimum: 1, type: "number" }, + maxDepth: { description: "Included for compatibility.", type: "number" }, + rootAxNodeId: { description: "Optional root node/ref to scope the snapshot.", type: "string" }, + scope: { description: "full or outline. CCR returns the same compact shape for both.", enum: ["full", "outline"], type: "string" }, + session: sessionSchema + }, ["session"]), + name: "browser_ax_snapshot", + tags: ["browser", "automation", "accessibility", "snapshot", "dom", "page"], + title: "Browser Accessibility Snapshot" + }, + { + description: "Search the page accessibility outline by role, accessible name, visible text, label, placeholder, or value.", + inputSchema: objectSchema({ + includeIgnored: { description: "Included for compatibility; CCR searches visible/high-signal nodes.", type: "boolean" }, + limit: { description: "Maximum matches.", maximum: 300, minimum: 1, type: "number" }, + name: { description: "Accessible name filter.", type: "string" }, + role: { description: "Role filter such as button, link, textbox, combobox.", type: "string" }, + rootAxNodeId: { description: "Optional root node/ref to scope the query.", type: "string" }, + session: sessionSchema, + text: { description: "Visible text/value/description filter.", type: "string" } + }, ["session"]), + name: "browser_ax_query", + tags: ["browser", "automation", "accessibility", "query", "find"], + title: "Query Browser Accessibility Tree" + }, + { + description: "Click an element resolved by axNodeId/ref, CSS selector, role, or text.", + inputSchema: objectSchema({ force: { type: "boolean" }, session: sessionSchema, target: targetSchema }, ["session", "target"]), + name: "browser_element_click", + tags: ["browser", "automation", "element", "click"], + title: "Click Browser Element" + }, + { + description: "Set text into an input, textarea, contenteditable, or textbox-like element.", + inputSchema: objectSchema({ + replace: { description: "Replace existing text. Defaults to true.", type: "boolean" }, + session: sessionSchema, + target: targetSchema, + text: { type: "string" } + }, ["session", "target", "text"]), + name: "browser_element_input", + tags: ["browser", "automation", "element", "input", "type", "form"], + title: "Input Browser Element" + }, + { + description: "Select an option on a native select by value or visible label.", + inputSchema: objectSchema({ + exact: { type: "boolean" }, + session: sessionSchema, + target: targetSchema, + value: { description: "Requested option value or visible label.", type: "string" } + }, ["session", "target", "value"]), + name: "browser_element_select", + tags: ["browser", "automation", "element", "select", "form"], + title: "Select Browser Element Option" + }, + { + description: "Press a keyboard key against a resolved target or the current focused element.", + inputSchema: objectSchema({ + key: { description: "Keyboard key, e.g. Enter, Tab, Escape, ArrowDown.", type: "string" }, + session: sessionSchema, + target: targetSchema + }, ["session", "key"]), + name: "browser_element_press", + tags: ["browser", "automation", "element", "press", "keyboard"], + title: "Press Browser Element Key" + }, + { + description: "Scroll the page or a target element.", + inputSchema: objectSchema({ + amount: { description: "Scroll amount in CSS pixels.", type: "number" }, + direction: { description: "Scroll direction.", enum: ["up", "down"], type: "string" }, + session: sessionSchema, + target: targetSchema + }, ["session"]), + name: "browser_element_scroll", + tags: ["browser", "automation", "element", "scroll"], + title: "Scroll Browser Element" + }, + { + description: "Subscribe to browser automation events such as tab, navigation, DOM-ready, runtime, and loading signals.", + inputSchema: objectSchema({ + channels: { description: "Event channels: tab, navigation, dom, runtime, dialog, download, input, focus, selection, handoff.", items: { type: "string" }, type: "array" }, + redactTextInput: { type: "boolean" }, + sampleMouseMove: { type: "boolean" }, + session: sessionSchema + }, ["session", "channels"]), + name: "browser_events_subscribe", + tags: ["browser", "automation", "events", "subscribe"], + title: "Subscribe Browser Events" + }, + { + description: "Read buffered browser automation events from a subscription.", + inputSchema: objectSchema({ + cursor: cursorSchema, + limit: { maximum: 200, minimum: 1, type: "number" }, + subscriptionId: { type: "string" } + }, ["subscriptionId"]), + name: "browser_events_read", + tags: ["browser", "automation", "events", "read"], + title: "Read Browser Events" + }, + { + description: "Wait for browser automation events matching kind, tabId, URL pattern, title pattern, or summary pattern.", + inputSchema: objectSchema({ + coalesceMs: { description: "Small delay after a first match to collect related events.", type: "number" }, + cursor: cursorSchema, + kinds: { items: { type: "string" }, type: "array" }, + maxEvents: { maximum: 50, minimum: 1, type: "number" }, + summaryPattern: { type: "string" }, + tabId: { type: "string" }, + timeoutMs: { maximum: 120000, minimum: 100, type: "number" }, + titlePattern: { type: "string" }, + urlPattern: { type: "string" }, + subscriptionId: { type: "string" } + }, ["subscriptionId"]), + name: "browser_events_await", + tags: ["browser", "automation", "events", "await", "wait"], + title: "Await Browser Events" + }, + { + description: "Unsubscribe from browser automation events and release the subscription.", + inputSchema: objectSchema({ subscriptionId: { type: "string" } }, ["subscriptionId"]), + name: "browser_events_unsubscribe", + tags: ["browser", "automation", "events", "unsubscribe"], + title: "Unsubscribe Browser Events" + }, + { + description: "Handle a currently open JavaScript alert/confirm/prompt dialog using Chromium CDP.", + inputSchema: objectSchema({ + accept: { type: "boolean" }, + promptText: { type: "string" }, + session: sessionSchema + }, ["session", "accept"]), + name: "browser_dialog_handle", + tags: ["browser", "automation", "dialog", "alert", "confirm", "prompt"], + title: "Handle Browser Dialog" + }, + { + description: "Request human intervention for the current browser task. Shows the hidden browser window and displays the requested action in the top toolbar.", + inputSchema: objectSchema({ + kind: { + description: "Type of help needed.", + enum: ["login_required", "verification_code", "human_verification", "blocked", "other"], + type: "string" + }, + message: { description: "Short instruction shown in the top toolbar.", type: "string" }, + reason: { description: "Why automation is blocked.", type: "string" }, + session: sessionSchema, + tabId: { description: "Optional tab id. Defaults to session.tabId or active tab.", type: "string" } + }, ["reason"]), + name: "browser_handoff_request", + tags: ["browser", "automation", "human", "handoff", "intervention"], + title: "Request Browser Human Handoff" + }, + { + description: "Read the current browser human handoff status.", + inputSchema: objectSchema({}), + name: "browser_handoff_status", + tags: ["browser", "automation", "human", "handoff"], + title: "Browser Handoff Status" + }, + { + description: "Wait until the user clicks Done or Hide on the current browser handoff toolbar. Use after browser_handoff_request or a tool result with humanHelpRequired.", + inputSchema: objectSchema({ + handoffId: { description: "Optional handoff id returned by browser_handoff_request or humanHelp.handoff.", type: "string" }, + session: sessionSchema, + tabId: { description: "Optional tab id. Defaults to session.tabId.", type: "string" }, + timeoutMs: { description: "Maximum wait time in milliseconds.", maximum: 600000, minimum: 100, type: "number" } + }), + name: "browser_handoff_wait", + tags: ["browser", "automation", "human", "handoff", "wait"], + title: "Wait For Browser Human Handoff" + }, + { + description: "Clear the current browser human handoff prompt.", + inputSchema: objectSchema({ + status: { enum: ["completed", "dismissed"], type: "string" } + }), + name: "browser_handoff_clear", + tags: ["browser", "automation", "human", "handoff"], + title: "Clear Browser Handoff" + }, + { + description: "Ask the user to confirm importing Chrome cookies and localStorage for selected domains into CCR's in-app browser. Opens a browser confirmation page; the Chrome extension performs the import after user confirmation.", + inputSchema: objectSchema({ + domain: { description: "Single domain to import, such as github.com. Used with domains if both are provided.", type: "string" }, + domains: { description: "Domains to import. If omitted, CCR derives the current tab hostname.", items: { type: "string" }, type: "array" }, + openConfirmationPage: { description: "Open the system browser confirmation page. Defaults to true.", type: "boolean" }, + session: sessionSchema, + tabId: { description: "Optional tab id used to derive a domain when domain/domains are omitted.", type: "string" }, + target: { description: "Target CCR browser storage partition.", enum: ["browser", "browser-and-web-search"], type: "string" } + }), + name: "browser_chrome_login_import", + tags: ["browser", "automation", "chrome", "login", "import", "cookies", "localStorage"], + title: "Import Chrome Login State" + }, + { + description: "Read the status of a Chrome login import job created by browser_chrome_login_import.", + inputSchema: objectSchema({ + jobId: { description: "Chrome login import job id.", type: "string" } + }, ["jobId"]), + name: "browser_chrome_login_import_status", + tags: ["browser", "automation", "chrome", "login", "import", "status"], + title: "Chrome Login Import Status" + } + ]; +} + +function browserLegacyAliasTools(): McpTool[] { + return [ + { + description: "Open or focus the CCR built-in browser window. Optionally navigate the active tab to a URL.", + inputSchema: objectSchema({ + url: { description: "Optional URL or search query to load in the active tab.", type: "string" } + }), + name: "browser_open", + tags: ["browser", "automation", "open", "focus"], + title: "Open Browser" + }, + { + description: "List CCR built-in browser tabs, including active tab, URL, title, and loading state.", + inputSchema: objectSchema({}), + name: "browser_tabs", + tags: ["browser", "automation", "tabs"], + title: "List Browser Tabs" + }, + { + description: "Open a new CCR built-in browser tab. Optionally load a URL or search query.", + inputSchema: objectSchema({ + url: { description: "Optional URL or search query to load in the new tab.", type: "string" } + }), + name: "browser_tab_new", + tags: ["browser", "automation", "tab"], + title: "New Browser Tab" + }, + { + description: "Focus a CCR built-in browser tab by tabId.", + inputSchema: objectSchema({ + tabId: { description: "Tab id returned by browser_tabs or browser_tab_new.", type: "string" } + }, ["tabId"]), + name: "browser_tab_activate", + tags: ["browser", "automation", "tab", "focus"], + title: "Activate Browser Tab" + }, + { + description: "Close a CCR built-in browser tab by tabId.", + inputSchema: objectSchema({ + tabId: { description: "Tab id returned by browser_tabs or browser_tab_new.", type: "string" } + }, ["tabId"]), + name: "browser_tab_close", + tags: ["browser", "automation", "tab"], + title: "Close Browser Tab" + }, + { + description: "Navigate a CCR built-in browser tab to a URL or search query.", + inputSchema: objectSchema({ + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + url: { description: "URL or search query to load.", type: "string" } + }, ["url"]), + name: "browser_navigate", + tags: ["browser", "automation", "navigate", "url"], + title: "Navigate Browser" + }, + { + description: "Capture page text plus interactable element refs for browser automation. Use returned refs for browser_click, browser_type, browser_select, browser_press_key, or browser_scroll.", + inputSchema: objectSchema({ + maxElements: { description: "Maximum interactable elements to include.", maximum: 300, minimum: 1, type: "number" }, + maxText: { description: "Maximum page text characters to include.", maximum: 20000, minimum: 0, type: "number" }, + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" } + }), + name: "browser_snapshot", + tags: ["browser", "automation", "snapshot", "accessibility", "dom", "page"], + title: "Browser Page Snapshot" + }, + { + description: "Click an element in the CCR built-in browser by ref, CSS selector, role, or visible text.", + inputSchema: objectSchema({ + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + target: targetSchema + }, ["target"]), + name: "browser_click", + tags: ["browser", "automation", "click", "element"], + title: "Click Browser Element" + }, + { + description: "Type text into an input, textarea, select-like textbox, or contenteditable element in the CCR built-in browser.", + inputSchema: objectSchema({ + replaceExisting: { description: "Replace existing text. Defaults to true.", type: "boolean" }, + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + target: targetSchema, + text: { description: "Text to type.", type: "string" } + }, ["target", "text"]), + name: "browser_type", + tags: ["browser", "automation", "type", "input", "form"], + title: "Type In Browser Element" + }, + { + description: "Choose an option in a select element by value or visible label.", + inputSchema: objectSchema({ + exact: { description: "Match label exactly instead of by substring.", type: "boolean" }, + label: { description: "Visible option label to choose.", type: "string" }, + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + target: targetSchema, + value: { description: "Option value to choose.", type: "string" } + }, ["target"]), + name: "browser_select", + tags: ["browser", "automation", "select", "form"], + title: "Select Browser Option" + }, + { + description: "Press a keyboard key in the CCR built-in browser. Optionally focus an element first.", + inputSchema: objectSchema({ + key: { description: "Electron keyCode, for example Enter, Tab, Escape, Backspace, ArrowDown.", type: "string" }, + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + target: targetSchema + }, ["key"]), + name: "browser_press_key", + tags: ["browser", "automation", "keyboard", "press"], + title: "Press Browser Key" + }, + { + description: "Scroll the page or a target element in the CCR built-in browser.", + inputSchema: objectSchema({ + deltaX: { description: "Horizontal scroll delta in pixels.", type: "number" }, + deltaY: { description: "Vertical scroll delta in pixels.", type: "number" }, + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + target: targetSchema + }), + name: "browser_scroll", + tags: ["browser", "automation", "scroll"], + title: "Scroll Browser" + }, + { + description: "Wait until a browser page condition is true: URL includes/matches, selector is visible, or text is visible.", + inputSchema: objectSchema({ + selector: { description: "CSS selector that must be visible.", type: "string" }, + tabId: { description: "Optional tab id. Defaults to the active tab.", type: "string" }, + text: { description: "Text that must be visible in document body.", type: "string" }, + timeoutMs: { description: "Maximum wait time in milliseconds.", maximum: 120000, minimum: 100, type: "number" }, + urlIncludes: { description: "Substring that must appear in the URL.", type: "string" }, + urlMatches: { description: "JavaScript regular expression that must match the URL.", type: "string" } + }), + name: "browser_wait_for", + tags: ["browser", "automation", "wait", "url", "selector", "text"], + title: "Wait For Browser Page" + }, + { + description: "Agentic-browser-compatible alias for browser_handoff_request. Request human help for login, verification, CAPTCHA, or other manual-only blockers.", + inputSchema: objectSchema({ + browserSessionId: { type: "string" }, + kind: { enum: ["login_required", "verification_code", "human_verification", "blocked", "other"], type: "string" }, + message: { type: "string" }, + reason: { type: "string" }, + tabId: { type: "string" } + }, ["reason", "kind"]), + name: "askHumanHelp", + tags: ["browser", "automation", "human", "handoff", "intervention"], + title: "Ask Human Help" + } + ]; +} + +class BrowserAutomationMcpService implements BrowserAutomationMcpIntegration { + private readonly sessions = new Map(); + private readonly subscriptions = new Map(); + + async handleBrowserAutomationMcpRequest(request: IncomingMessage, response: ServerResponse): Promise { + response.setHeader("MCP-Protocol-Version", protocolVersion); + const path = request.url ? new URL(request.url, "http://127.0.0.1").pathname : "/"; + + if (request.method === "GET" && (path === BROWSER_AUTOMATION_MCP_PATH || path === `${BROWSER_AUTOMATION_MCP_PATH}/`)) { + sendJson(response, 200, { + endpoint: BROWSER_AUTOMATION_MCP_PATH, + name: "ccr-browser-automation", + protocol: "mcp", + transport: "streamable-http" + }); + return; + } + + if (request.method !== "POST") { + sendJson(response, 405, { error: { message: "Browser automation MCP endpoint only supports GET and POST." } }); + return; + } + + let payload: unknown; + try { + payload = JSON.parse((await readRequestBody(request, maxMcpRequestBytes)).toString("utf8")) as unknown; + } catch (error) { + sendJson(response, 400, jsonRpcError(null, -32700, `Invalid JSON-RPC request: ${formatError(error)}`)); + return; + } + + const requests = Array.isArray(payload) ? payload : [payload]; + const responses = await Promise.all(requests.map((item) => this.handleJsonRpcRequest(item))); + const filtered = responses.filter((item): item is JsonRpcResponse => Boolean(item)); + if (filtered.length === 0) { + response.writeHead(204); + response.end(); + return; + } + + sendJson(response, 200, Array.isArray(payload) ? filtered : filtered[0]); + } + + async stopBrowserAutomationMcpServer(): Promise { + for (const subscription of this.subscriptions.values()) { + subscription.unsubscribe(); + } + this.subscriptions.clear(); + this.sessions.clear(); + // The gateway owns this MCP route. Browser tabs remain user-visible state and are + // intentionally not closed when the gateway restarts. + } + + private async handleJsonRpcRequest(payload: unknown): Promise { + if (!isRecord(payload)) { + return jsonRpcError(null, -32600, "JSON-RPC request must be an object."); + } + + const request = payload as JsonRpcRequest; + const id = request.id ?? null; + if (request.id === undefined && request.method?.startsWith("notifications/")) { + return undefined; + } + if (request.jsonrpc !== "2.0" || !request.method) { + return jsonRpcError(id, -32600, "Invalid JSON-RPC 2.0 request."); + } + + try { + switch (request.method) { + case "initialize": + return jsonRpcResult(id, { + capabilities: { + tools: {} + }, + protocolVersion, + serverInfo: { + name: "ccr-browser-automation", + title: "CCR Browser Automation", + version: "1.0.0" + } + }); + case "ping": + return jsonRpcResult(id, {}); + case "tools/list": + return jsonRpcResult(id, { tools: browserAutomationTools as unknown as JsonValue }); + case "tools/call": + return jsonRpcResult(id, await this.callTool(request.params) as unknown as JsonValue); + default: + return jsonRpcError(id, -32601, `Unsupported MCP method: ${request.method}`); + } + } catch (error) { + return jsonRpcError(id, -32603, formatError(error)); + } + } + + private async callTool(params: unknown): Promise { + if (!isRecord(params) || typeof params.name !== "string") { + throw new Error("tools/call params must include a tool name."); + } + const args = isRecord(params.arguments) ? params.arguments : {}; + try { + const result = await this.runTool(params.name, args); + return textResult(formatToolResult(params.name, result)); + } catch (error) { + return { + ...textResult(formatError(error)), + isError: true + }; + } + } + + private async runTool(name: string, args: Record): Promise { + switch (name) { + case "browser_session_open": + return await this.openSession(args); + case "browser_session_close": + return this.closeSession(args); + case "browser_tab_create": + return await this.createTab(args); + case "browser_tab_list": + await this.ensureBrowserOpen(); + return browserWindowState(); + case "browser_tab_activate": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + const tabId = readString(args.tabId) || session?.ref.tabId; + if (!tabId) { + throw new Error("browser_tab_activate requires tabId or session."); + } + const state = builtInBrowserService.selectAutomationTab(tabId); + return { + ...browserWindowState(state), + tab: summarizeTab(requiredTabState(state, tabId), state.activeTabId) + }; + } + case "browser_tab_close": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + this.assertCanMutate(session); + const tabId = readString(args.tabId) || session?.ref.tabId; + if (!tabId) { + throw new Error("browser_tab_close requires tabId or session."); + } + const state = builtInBrowserService.closeAutomationTab(tabId); + this.removeSessionsForTab(tabId); + return browserWindowState(state); + } + case "browser_navigate": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + this.assertCanMutate(session); + const tabId = readString(args.tabId) || session?.ref.tabId || builtInBrowserService.getAutomationState().activeTabId; + if (!tabId) { + throw new Error("browser_navigate could not resolve an active tab."); + } + const url = requiredString(args.url, "browser_navigate requires url."); + const waitUntil = normalizeWaitUntil(readString(args.waitUntil), "interactive"); + const readiness = await waitForReadiness( + builtInBrowserService.getAutomationWebContents(tabId), + waitUntil, + clampInteger(readNumber(args.timeoutMs) ?? defaultWaitTimeoutMs, 100, 120000), + () => builtInBrowserService.navigateAutomationTab(url, tabId), + url + ); + const state = builtInBrowserService.getAutomationState(); + const webContents = builtInBrowserService.getAutomationWebContents(tabId); + const handoff = await maybeRequestHandoffAfterNavigation(webContents, readiness, { + requestedUrl: url, + session: session?.ref, + tabId, + waitUntil + }); + return { + ...browserWindowState(state), + ...(handoff ? humanHelpResultFields(handoff) : {}), + session: session?.ref, + tab: summarizeTab(requiredTabState(state, tabId), state.activeTabId), + ...readiness + }; + } + case "browser_tab_go_back": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + this.assertCanMutate(session); + return browserWindowState(builtInBrowserService.goBackAutomationTab(readString(args.tabId) || session?.ref.tabId)); + } + case "browser_tab_go_forward": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + this.assertCanMutate(session); + return browserWindowState(builtInBrowserService.goForwardAutomationTab(readString(args.tabId) || session?.ref.tabId)); + } + case "browser_tab_reload": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + this.assertCanMutate(session); + return browserWindowState(builtInBrowserService.reloadAutomationTab(readString(args.tabId) || session?.ref.tabId)); + } + case "browser_ax_snapshot": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + return await captureAxSnapshot( + builtInBrowserService.getAutomationWebContents(session.ref.tabId), + session.ref, + { + limit: clampInteger(readNumber(args.limit) ?? defaultAxSnapshotLimit, 1, 300), + rootAxNodeId: readString(args.rootAxNodeId), + scope: readString(args.scope) === "outline" ? "outline" : "full" + } + ); + } + case "browser_ax_query": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + return await queryAxSnapshot( + builtInBrowserService.getAutomationWebContents(session.ref.tabId), + session.ref, + { + limit: clampInteger(readNumber(args.limit) ?? 50, 1, 300), + name: readString(args.name), + role: readString(args.role), + rootAxNodeId: readString(args.rootAxNodeId), + text: readString(args.text) + } + ); + } + case "browser_element_click": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + this.assertCanMutate(session); + const webContents = builtInBrowserService.getAutomationWebContents(session.ref.tabId); + const result = await runTargetAction(webContents, "click", readTarget(args.target)); + return await pageActionResult(webContents, session.ref, "click", result); + } + case "browser_element_input": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + this.assertCanMutate(session); + const webContents = builtInBrowserService.getAutomationWebContents(session.ref.tabId); + const result = await runTargetAction(webContents, "type", readTarget(args.target), { + replaceExisting: args.replace !== false, + text: typeof args.text === "string" ? args.text : "" + }); + return await pageActionResult(webContents, session.ref, "input", result); + } + case "browser_element_select": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + this.assertCanMutate(session); + const value = requiredString(args.value, "browser_element_select requires value."); + const webContents = builtInBrowserService.getAutomationWebContents(session.ref.tabId); + const result = await runTargetAction(webContents, "select", readTarget(args.target), { + exact: args.exact === true, + label: value, + value + }); + return await pageActionResult(webContents, session.ref, "select", result); + } + case "browser_element_press": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + this.assertCanMutate(session); + const webContents = builtInBrowserService.getAutomationWebContents(session.ref.tabId); + const result = await pressKey( + webContents, + requiredString(args.key, "browser_element_press requires key."), + isRecord(args.target) ? readTarget(args.target) : undefined + ); + return await pageActionResult(webContents, session.ref, "press", result); + } + case "browser_element_scroll": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + this.assertCanMutate(session); + const amount = Math.abs(readNumber(args.amount) ?? 700); + const direction = readString(args.direction) === "up" ? "up" : "down"; + const webContents = builtInBrowserService.getAutomationWebContents(session.ref.tabId); + const result = await runTargetAction( + webContents, + "scroll", + isRecord(args.target) ? readTarget(args.target) : undefined, + { + deltaX: 0, + deltaY: direction === "up" ? -amount : amount + } + ); + return await pageActionResult(webContents, session.ref, "scroll", result); + } + case "browser_events_subscribe": + await this.ensureBrowserOpen(); + return this.subscribeEvents(args); + case "browser_events_read": + return this.readEvents(args); + case "browser_events_await": + return await this.awaitEvents(args); + case "browser_events_unsubscribe": + return this.unsubscribeEvents(args); + case "browser_dialog_handle": { + await this.ensureBrowserOpen(); + const session = this.requireSession(args); + this.assertCanMutate(session); + const webContents = builtInBrowserService.getAutomationWebContents(session.ref.tabId); + return await handleJavaScriptDialog(webContents, args.accept === true, readString(args.promptText), session.ref); + } + case "browser_handoff_request": + case "askHumanHelp": + await this.ensureBrowserOpen(); + return this.requestHandoff(args); + case "browser_handoff_status": + await this.ensureBrowserOpen(); + return { + handoff: builtInBrowserService.getAutomationState().automationHandoff, + state: browserWindowState() + }; + case "browser_handoff_wait": + await this.ensureBrowserOpen(); + return await this.waitForHandoff(args); + case "browser_handoff_clear": + return { + handoff: undefined, + state: builtInBrowserService.resolveAutomationHandoff(readString(args.status) === "dismissed" ? "dismissed" : "completed") + }; + case "browser_chrome_login_import": { + await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + const domains = resolveChromeLoginImportDomains(args, session?.ref.tabId); + const job = await chromeLoginImportService.createJob({ + domains, + openConfirmationPage: args.openConfirmationPage !== false, + target: readString(args.target) === "browser-and-web-search" ? "browser-and-web-search" : "browser" + }); + return { + humanHelpRequired: true, + job, + nextAction: "user_confirm_chrome_import", + state: browserWindowState(), + summary: `Opened Chrome login import confirmation for ${job.domains.join(", ")}.` + }; + } + case "browser_chrome_login_import_status": { + const jobId = requiredString(args.jobId, "browser_chrome_login_import_status requires jobId."); + const job = chromeLoginImportService.getJob(jobId); + if (!job) { + throw new Error(`Chrome login import job was not found: ${jobId}`); + } + return { job }; + } + case "browser_open": { + const state = await this.ensureBrowserVisible(); + const url = readString(args.url); + return url ? await builtInBrowserService.navigateAutomationTab(url, state.activeTabId) : builtInBrowserService.getAutomationState(); + } + case "browser_tabs": + await this.ensureBrowserOpen(); + return builtInBrowserService.getAutomationState(); + case "browser_tab_new": { + await this.ensureBrowserOpen(); + const tab = builtInBrowserService.createAutomationTab(readString(args.url) || undefined); + return { state: builtInBrowserService.getAutomationState(), tab }; + } + case "browser_snapshot": + await this.ensureBrowserOpen(); + return await captureSnapshotWithHandoff( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + readString(args.tabId), + { + maxElements: clampInteger(readNumber(args.maxElements) ?? defaultSnapshotMaxElements, 1, 300), + maxText: clampInteger(readNumber(args.maxText) ?? defaultSnapshotMaxText, 0, 20000) + } + ); + case "browser_click": + await this.ensureBrowserOpen(); + return await runTargetAction( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + "click", + readTarget(args.target) + ); + case "browser_type": + await this.ensureBrowserOpen(); + return await runTargetAction( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + "type", + readTarget(args.target), + { + replaceExisting: args.replaceExisting !== false, + text: typeof args.text === "string" ? args.text : "" + } + ); + case "browser_select": + await this.ensureBrowserOpen(); + return await runTargetAction( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + "select", + readTarget(args.target), + { + exact: args.exact === true, + label: readString(args.label), + value: readString(args.value) + } + ); + case "browser_press_key": + await this.ensureBrowserOpen(); + return await pressKey( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + requiredString(args.key, "browser_press_key requires key."), + isRecord(args.target) ? readTarget(args.target) : undefined + ); + case "browser_scroll": + await this.ensureBrowserOpen(); + return await runTargetAction( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + "scroll", + isRecord(args.target) ? readTarget(args.target) : undefined, + { + deltaX: readNumber(args.deltaX) ?? 0, + deltaY: readNumber(args.deltaY) ?? 700 + } + ); + case "browser_wait_for": + await this.ensureBrowserOpen(); + return await waitForPageCondition( + builtInBrowserService.getAutomationWebContents(readString(args.tabId)), + { + selector: readString(args.selector), + text: readString(args.text), + timeoutMs: clampInteger(readNumber(args.timeoutMs) ?? defaultWaitTimeoutMs, 100, 120000), + urlIncludes: readString(args.urlIncludes), + urlMatches: readString(args.urlMatches) + } + ); + default: + throw new Error(`Unknown browser automation tool: ${name}`); + } + } + + private async ensureBrowserOpen(): Promise { + await builtInBrowserService.openHidden(await loadAppConfig()); + return builtInBrowserService.getAutomationState(); + } + + private async ensureBrowserVisible(): Promise { + await builtInBrowserService.open(await loadAppConfig()); + return builtInBrowserService.getAutomationState(); + } + + private async openSession(args: Record): Promise { + let state = await this.ensureBrowserOpen(); + const url = readString(args.url); + const requestedTabId = readString(args.tabId); + const previousActiveTabId = state.activeTabId; + let tabId = requestedTabId; + let createdTab = false; + + if (tabId && !state.tabs.some((tab) => tab.id === tabId)) { + throw new Error(`Browser tab was not found: ${tabId}`); + } + + if (!tabId && url) { + const tab = builtInBrowserService.createAutomationTab(); + tabId = tab.id; + createdTab = true; + } else if (!tabId) { + tabId = state.activeTabId || builtInBrowserService.createAutomationTab().id; + createdTab = !state.activeTabId; + } else if (state.activeTabId !== tabId) { + builtInBrowserService.selectAutomationTab(tabId); + } + + if (!tabId) { + throw new Error("Unable to resolve browser tab for automation session."); + } + + const waitUntil = normalizeWaitUntil(readString(args.waitUntil), url ? "interactive" : "none"); + const timeoutMs = clampInteger(readNumber(args.timeoutMs) ?? defaultWaitTimeoutMs, 100, 120000); + const webContents = builtInBrowserService.getAutomationWebContents(tabId); + const readiness = await waitForReadiness( + webContents, + waitUntil, + timeoutMs, + url ? () => builtInBrowserService.navigateAutomationTab(url, tabId) : undefined, + url + ); + state = builtInBrowserService.getAutomationState(); + const tab = requiredTabState(state, tabId); + const ref: BrowserSessionRef = { + sessionId: randomUUID(), + tabId, + ...(readString(args.userId) ? { userId: readString(args.userId) } : {}) + }; + const attached: AttachedSession = { + attachedAt: Date.now(), + leaseId: randomUUID(), + observeOnly: args.observeOnly === true, + ref + }; + this.sessions.set(ref.sessionId, attached); + const handoff = await maybeRequestHandoffAfterNavigation(webContents, readiness, { + requestedUrl: url, + session: ref, + tabId, + waitUntil + }); + + return { + session: ref, + attachedAt: attached.attachedAt, + createdTab, + ...(handoff ? humanHelpResultFields(handoff) : {}), + matched: readiness.matched, + matchedEvent: readiness.matchedEvent, + ...(readiness.navigationError ? { navigationError: readiness.navigationError } : {}), + previousActiveTabId, + readinessUrl: readiness.readinessUrl, + tabId, + timedOut: readiness.timedOut, + title: tab.title, + url: tab.url, + waitUntil, + windowId: builtInBrowserService.getAutomationWindowId() + }; + } + + private closeSession(args: Record): unknown { + const session = this.requireSession(args); + this.sessions.delete(session.ref.sessionId); + for (const [subscriptionId, subscription] of this.subscriptions) { + if (subscription.ref?.sessionId === session.ref.sessionId) { + subscription.unsubscribe(); + this.subscriptions.delete(subscriptionId); + } + } + return { success: true }; + } + + private async createTab(args: Record): Promise { + const state = await this.ensureBrowserOpen(); + const session = this.resolveOptionalSession(args); + this.assertCanMutate(session); + const previousActiveTabId = state.activeTabId; + const tab = builtInBrowserService.createAutomationTab(readString(args.url) || undefined); + if (args.activate === false && previousActiveTabId) { + builtInBrowserService.selectAutomationTab(previousActiveTabId); + } + const nextState = builtInBrowserService.getAutomationState(); + return summarizeTab(requiredTabState(nextState, tab.id), nextState.activeTabId); + } + + private requireSession(args: Record): AttachedSession { + const session = this.resolveOptionalSession(args); + if (!session) { + throw new Error("A valid browser automation session is required."); + } + return session; + } + + private resolveOptionalSession(args: Record): AttachedSession | undefined { + const ref = readSessionRef(args.session); + if (!ref) { + return undefined; + } + const existing = this.sessions.get(ref.sessionId); + if (existing) { + if (existing.ref.tabId !== ref.tabId) { + throw new Error("Browser automation session tabId does not match the attached session."); + } + return existing; + } + const state = builtInBrowserService.getAutomationState(); + if (!state.tabs.some((tab) => tab.id === ref.tabId)) { + throw new Error(`Browser automation session tab was not found: ${ref.tabId}`); + } + const restored: AttachedSession = { + attachedAt: Date.now(), + leaseId: randomUUID(), + observeOnly: false, + ref + }; + this.sessions.set(ref.sessionId, restored); + return restored; + } + + private assertCanMutate(session?: AttachedSession): void { + if (session?.observeOnly) { + throw new Error("This browser automation session is observeOnly and cannot mutate browser state."); + } + } + + private removeSessionsForTab(tabId: string): void { + for (const [sessionId, session] of this.sessions) { + if (session.ref.tabId === tabId) { + this.sessions.delete(sessionId); + } + } + for (const [subscriptionId, subscription] of this.subscriptions) { + if (subscription.ref?.tabId === tabId) { + subscription.unsubscribe(); + this.subscriptions.delete(subscriptionId); + } + } + } + + private subscribeEvents(args: Record): unknown { + const session = this.requireSession(args); + const channels = normalizeEventChannels(readStringArray(args.channels)); + const subscription: EventSubscription = { + channels, + dropped: false, + events: [], + ref: session.ref, + redactTextInput: args.redactTextInput !== false, + sampleMouseMove: args.sampleMouseMove === true, + startedAt: Date.now(), + subscriptionId: randomUUID(), + unsubscribe: () => undefined + }; + const listener = (event: BrowserAutomationEvent) => { + if (!matchesSubscriptionEvent(subscription, event)) { + return; + } + subscription.events.push(event); + if (subscription.events.length > maxSubscriptionEvents) { + subscription.events.splice(0, subscription.events.length - maxSubscriptionEvents); + subscription.dropped = true; + } + }; + subscription.unsubscribe = builtInBrowserService.subscribeAutomationEvents(listener, { + replayRecentMs: automationEventReplayMs + }); + this.subscriptions.set(subscription.subscriptionId, subscription); + + return { + subscription: subscriptionDescriptor(subscription) + }; + } + + private readEvents(args: Record): unknown { + const subscription = this.requireSubscription(args); + const limit = clampInteger(readNumber(args.limit) ?? 100, 1, 200); + const cursorSeq = readCursorSeq(args.cursor); + const events = eventsAfterCursor(subscription, cursorSeq).slice(0, limit); + return { + dropped: subscription.dropped || cursorDropped(subscription, cursorSeq), + events, + nextCursor: makeEventCursor(subscription, events.at(-1)?.seq ?? cursorSeq), + subscriptionId: subscription.subscriptionId + }; + } + + private async awaitEvents(args: Record): Promise { + const subscription = this.requireSubscription(args); + const timeoutMs = clampInteger(readNumber(args.timeoutMs) ?? defaultEventAwaitTimeoutMs, 100, 120000); + const maxEvents = clampInteger(readNumber(args.maxEvents) ?? 10, 1, 50); + const coalesceMs = clampInteger(readNumber(args.coalesceMs) ?? 100, 0, 2000); + const cursorSeq = readCursorSeq(args.cursor); + const deadline = Date.now() + timeoutMs; + let matched: BrowserAutomationEvent[] = []; + + while (Date.now() <= deadline) { + matched = eventsAfterCursor(subscription, cursorSeq) + .filter((event) => matchesAwaitFilter(event, args)) + .slice(0, maxEvents); + if (matched.length > 0) { + if (coalesceMs > 0) { + await sleep(coalesceMs); + matched = eventsAfterCursor(subscription, cursorSeq) + .filter((event) => matchesAwaitFilter(event, args)) + .slice(0, maxEvents); + } + break; + } + await sleep(100); + } + + return { + dropped: subscription.dropped || cursorDropped(subscription, cursorSeq), + event: matched[0], + events: matched, + matched: matched.length > 0, + nextCursor: makeEventCursor(subscription, matched.at(-1)?.seq ?? cursorSeq), + subscriptionId: subscription.subscriptionId, + timedOut: matched.length === 0 + }; + } + + private unsubscribeEvents(args: Record): unknown { + const subscription = this.requireSubscription(args); + subscription.unsubscribe(); + this.subscriptions.delete(subscription.subscriptionId); + return { + ok: true, + subscriptionId: subscription.subscriptionId + }; + } + + private requireSubscription(args: Record): EventSubscription { + const subscriptionId = requiredString(args.subscriptionId, "Browser event subscriptionId is required."); + const subscription = this.subscriptions.get(subscriptionId); + if (!subscription) { + throw new Error(`Browser event subscription was not found: ${subscriptionId}`); + } + return subscription; + } + + private requestHandoff(args: Record): unknown { + const session = this.resolveOptionalSession(args); + const sessionId = session?.ref.sessionId || readString(args.browserSessionId); + const tabId = readString(args.tabId) || session?.ref.tabId; + const handoff = builtInBrowserService.requestAutomationHandoff({ + kind: readHandoffKind(args.kind), + message: readString(args.message), + reason: requiredString(args.reason, "browser_handoff_request requires reason."), + sessionId, + tabId + }); + return { + ...humanHelpResultFields(handoff), + state: browserWindowState() + }; + } + + private async waitForHandoff(args: Record): Promise { + const session = this.resolveOptionalSession(args); + const handoffId = readString(args.handoffId); + const tabId = readString(args.tabId) || session?.ref.tabId; + const timeoutMs = clampInteger(readNumber(args.timeoutMs) ?? 300000, 100, 600000); + + const current = builtInBrowserService.getAutomationState().automationHandoff; + if (!handoffMatches(current, { handoffId, tabId })) { + const recentResolution = findRecentHandoffResolution({ handoffId, tabId }); + if (recentResolution) { + return handoffResolutionResult(recentResolution, undefined, false); + } + return { + handoff: current, + matched: false, + reason: current ? "A browser handoff is pending, but it does not match the requested handoffId/tabId." : "No browser handoff is currently pending.", + state: browserWindowState(), + timedOut: false + }; + } + + return await new Promise((resolve) => { + let settled = false; + const finish = (result: Record) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + unsubscribe(); + resolve(result); + }; + const unsubscribe = builtInBrowserService.subscribeAutomationEvents((event) => { + if (event.kind !== "handoff.completed" && event.kind !== "handoff.dismissed") { + return; + } + if (handoffId && event.handoffId !== handoffId) { + return; + } + if (tabId && event.tabId !== tabId) { + return; + } + finish(handoffResolutionResult(event, current, false)); + }); + const timer = setTimeout(() => { + finish({ + handoff: current, + matched: false, + state: browserWindowState(), + timedOut: true + }); + }, timeoutMs); + }); + } +} + +export const browserAutomationMcpService = new BrowserAutomationMcpService(); + +function browserWindowState(state = builtInBrowserService.getAutomationState()): Record { + return { + activeTabId: state.activeTabId, + tabs: state.tabs.map((tab) => summarizeTab(tab, state.activeTabId)), + windowId: builtInBrowserService.getAutomationWindowId() + }; +} + +function resolveChromeLoginImportDomains(args: Record, fallbackTabId?: string): string[] { + const explicit = uniqueStrings([ + ...readStringArray(args.domains), + ...(readString(args.domain) ? [readString(args.domain) as string] : []) + ].map(normalizeChromeLoginImportDomain).filter((domain): domain is string => Boolean(domain))); + if (explicit.length > 0) { + return explicit; + } + + const state = builtInBrowserService.getAutomationState(); + const tabId = readString(args.tabId) || fallbackTabId || state.activeTabId; + const tab = state.tabs.find((candidate) => candidate.id === tabId) || state.tabs.find((candidate) => candidate.id === state.activeTabId); + const domain = normalizeChromeLoginImportDomain(tab?.url); + if (!domain) { + throw new Error("browser_chrome_login_import requires domains or an active http(s) tab."); + } + return [domain]; +} + +function normalizeChromeLoginImportDomain(value: unknown): string | undefined { + const raw = readString(value)?.toLowerCase(); + if (!raw) { + return undefined; + } + try { + const url = new URL(raw.includes("://") ? raw : `https://${raw}`); + return url.hostname.replace(/^\*\./, "").replace(/^\./, ""); + } catch { + const domain = raw.replace(/^\*\./, "").replace(/^\./, "").split("/")[0]; + return domain && !domain.includes(" ") ? domain : undefined; + } +} + +function summarizeTab(tab: BuiltInBrowserTabState, activeTabId?: string): Record { + return { + canGoBack: tab.canGoBack, + canGoForward: tab.canGoForward, + displayUrl: tab.url, + isActive: tab.id === activeTabId, + isLoading: tab.isLoading, + tabId: tab.id, + title: tab.title, + url: tab.url, + windowId: builtInBrowserService.getAutomationWindowId() + }; +} + +function requiredTabState(state: BuiltInBrowserState, tabId: string): BuiltInBrowserTabState { + const tab = state.tabs.find((candidate) => candidate.id === tabId); + if (!tab) { + throw new Error(`Browser tab was not found: ${tabId}`); + } + return tab; +} + +function readSessionRef(value: unknown): BrowserSessionRef | undefined { + if (!isRecord(value)) { + return undefined; + } + const sessionId = readString(value.sessionId); + const tabId = readString(value.tabId); + if (!sessionId || !tabId) { + return undefined; + } + return { + sessionId, + tabId, + ...(readString(value.frameId) ? { frameId: readString(value.frameId) } : {}), + ...(readString(value.userId) ? { userId: readString(value.userId) } : {}) + }; +} + +async function pageActionResult( + webContents: WebContents, + session: BrowserSessionRef, + action: string, + result: unknown +): Promise> { + const handoff = await maybeRequestPageHandoff(webContents, { + session, + tabId: session.tabId + }); + return { + action, + ...(handoff ? humanHelpResultFields(handoff) : {}), + result, + session, + title: await webContents.getTitle(), + url: webContents.getURL() + }; +} + +async function captureAxSnapshot( + webContents: WebContents, + session: BrowserSessionRef, + options: { limit: number; rootAxNodeId?: string; scope: string } +): Promise { + const snapshot = await captureSnapshot(webContents, { + maxElements: options.limit, + maxText: options.scope === "outline" ? 0 : defaultSnapshotMaxText + }); + const record = isRecord(snapshot) ? snapshot : {}; + const title = readString(record.title) || await webContents.getTitle(); + const url = readString(record.url) || webContents.getURL(); + const elements = Array.isArray(record.elements) ? record.elements.filter(isRecord) : []; + const elementNodes = elements.map((element, index) => axNodeFromSnapshotElement(element, index)); + const rootNode = { + axNodeId: "document", + childAxNodeIds: elementNodes.map((node) => String(node.axNodeId)), + ignored: false, + name: title || url, + role: "document", + url + }; + let nodes: Array> = [rootNode, ...elementNodes]; + if (options.rootAxNodeId && options.rootAxNodeId !== "document") { + nodes = nodes.filter((node) => node.axNodeId === options.rootAxNodeId || node.ref === options.rootAxNodeId); + } + const result: AxSnapshotResult = { + nodes: nodes.slice(0, options.limit), + scope: options.scope, + session, + title, + url + }; + const handoff = await maybeRequestPageHandoff(webContents, { + session, + snapshot, + tabId: session.tabId + }); + return handoff ? { ...result, ...humanHelpResultFields(handoff) } : result; +} + +async function queryAxSnapshot( + webContents: WebContents, + session: BrowserSessionRef, + options: { limit: number; name?: string; role?: string; rootAxNodeId?: string; text?: string } +): Promise> { + const snapshot = await captureAxSnapshot(webContents, session, { + limit: 300, + rootAxNodeId: options.rootAxNodeId, + scope: "outline" + }); + const role = options.role?.toLowerCase(); + const name = options.name?.toLowerCase(); + const text = options.text?.toLowerCase(); + const matches = snapshot.nodes + .filter((node) => node.role !== "document") + .filter((node) => { + if (role && String(node.role || "").toLowerCase() !== role) { + return false; + } + if (name && !String(node.name || "").toLowerCase().includes(name)) { + return false; + } + if (text) { + const haystack = [ + node.name, + node.value, + node.description, + node.text, + node.placeholder + ].join(" ").toLowerCase(); + if (!haystack.includes(text)) { + return false; + } + } + return true; + }) + .slice(0, options.limit); + return { + ...(snapshot.handoff ? humanHelpResultFields(snapshot.handoff) : {}), + matches, + session, + title: snapshot.title, + url: snapshot.url + }; +} + +function axNodeFromSnapshotElement(element: Record, index: number): Record { + const ref = readString(element.ref) || `element-${index}`; + const text = readString(element.text); + const name = readString(element.name) || text || readString(element.placeholder) || ref; + const description = text && text !== name ? text : undefined; + return { + axNodeId: ref, + backendNodeId: readNumber(element.backendNodeId), + checked: element.checked === true ? true : undefined, + description, + disabled: element.disabled === true ? true : undefined, + href: readString(element.href), + ignored: false, + index, + name, + placeholder: readString(element.placeholder), + rect: isRecord(element.rect) ? element.rect : undefined, + ref, + role: readString(element.role) || readString(element.tag) || "generic", + tag: readString(element.tag), + text: description, + value: readString(element.value) + }; +} + +async function waitForReadiness( + webContents: WebContents, + waitUntil: string, + timeoutMs: number, + requestNavigation?: () => Promise, + expectedUrl?: string +): Promise { + if (waitUntil === "none") { + if (requestNavigation) { + await requestNavigation(); + } + return { matched: true, matchedEvent: "none", readinessUrl: safeWebContentsUrl(webContents), timedOut: false }; + } + if (webContents.isDestroyed()) { + return { matched: false, matchedEvent: "destroyed", timedOut: false }; + } + if (requestNavigation) { + const readiness = waitForNextReadinessEvent(webContents, waitUntil, timeoutMs, { + expectedUrl: normalizeNavigationUrlForMatch(expectedUrl), + previousUrl: safeWebContentsUrl(webContents) + }); + try { + await requestNavigation(); + } catch (error) { + return navigationFailureResult("navigation_request_failed", { + errorDescription: formatError(error), + url: normalizeNavigationUrlForMatch(expectedUrl) || safeWebContentsUrl(webContents) + }); + } + return await readiness; + } + + const deadline = Date.now() + timeoutMs; + while (Date.now() <= deadline) { + if (webContents.isDestroyed()) { + return { matched: false, matchedEvent: "destroyed", timedOut: false }; + } + + if (!webContents.isLoading()) { + const readinessUrl = safeWebContentsUrl(webContents); + if (isChromiumErrorPage(readinessUrl)) { + return navigationFailureResult("chrome-error", { + errorDescription: "Chromium displayed an internal error page.", + url: readinessUrl + }); + } + if (waitUntil === "interactive") { + const interactive = await interactiveReadinessResult(webContents, readinessUrl); + if (interactive) { + return interactive; + } + } + if (waitUntil === "domcontentloaded") { + return { matched: true, matchedEvent: "domcontentloaded", readinessUrl, timedOut: false }; + } + if (waitUntil === "load") { + return { matched: true, matchedEvent: "load", readinessUrl, timedOut: false }; + } + if (waitUntil === "network_idle") { + const idle = await waitForNoLoadingStart(webContents, Math.min(500, Math.max(0, deadline - Date.now()))); + if (idle && !webContents.isDestroyed() && !webContents.isLoading()) { + return { matched: true, matchedEvent: "network_idle", readinessUrl: safeWebContentsUrl(webContents), timedOut: false }; + } + } + } + + const event = await waitForReadinessEvent(webContents, Math.max(0, deadline - Date.now())); + if (event.type === "timeout") { + break; + } + if (event.type === "destroyed") { + return { matched: false, matchedEvent: "destroyed", timedOut: false }; + } + if (event.type === "did-fail-load" && event.errorCode !== -3) { + return navigationFailureResult("did-fail-load", event); + } + const eventUrl = event.url || safeWebContentsUrl(webContents); + if (isChromiumErrorPage(eventUrl)) { + return navigationFailureResult("chrome-error", { + errorDescription: "Chromium displayed an internal error page.", + url: eventUrl + }); + } + if (waitUntil === "interactive" && isInteractiveReadinessEvent(event)) { + const interactive = await interactiveReadinessResult(webContents, eventUrl); + if (interactive) { + return interactive; + } + } + if (waitUntil === "domcontentloaded" && event.type === "dom-ready") { + return { matched: true, matchedEvent: "domcontentloaded", readinessUrl: eventUrl, timedOut: false }; + } + if (waitUntil === "load" && (event.type === "did-finish-load" || event.type === "did-stop-loading") && !webContents.isDestroyed() && !webContents.isLoading()) { + return { matched: true, matchedEvent: "load", readinessUrl: eventUrl, timedOut: false }; + } + if (waitUntil === "network_idle" && (event.type === "did-finish-load" || event.type === "did-stop-loading") && !webContents.isDestroyed() && !webContents.isLoading()) { + const idle = await waitForNoLoadingStart(webContents, Math.min(500, Math.max(0, deadline - Date.now()))); + if (idle && !webContents.isDestroyed() && !webContents.isLoading()) { + return { matched: true, matchedEvent: "network_idle", readinessUrl: safeWebContentsUrl(webContents), timedOut: false }; + } + } + } + if (waitUntil === "interactive" || waitUntil === "network_idle") { + const fallback = await interactiveReadinessResult( + webContents, + undefined, + waitUntil === "network_idle" ? "interactive_after_network_idle_timeout" : "interactive" + ); + if (fallback) { + return fallback; + } + } + return { + matched: false, + matchedEvent: webContents.isDestroyed() ? "destroyed" : undefined, + timedOut: !webContents.isDestroyed() + }; +} + +async function waitForNextReadinessEvent( + webContents: WebContents, + waitUntil: string, + timeoutMs: number, + context: NavigationReadinessContext +): Promise { + const deadline = Date.now() + timeoutMs; + let navigationStarted = false; + let lastNavigationUrl: string | undefined; + + while (Date.now() <= deadline) { + if (webContents.isDestroyed()) { + return { matched: false, matchedEvent: "destroyed", timedOut: false }; + } + + const event = await waitForReadinessEvent(webContents, Math.max(0, deadline - Date.now())); + if (event.type === "timeout") { + break; + } + if (event.type === "destroyed") { + return { matched: false, matchedEvent: "destroyed", timedOut: false }; + } + + const eventUrl = event.url || safeWebContentsUrl(webContents); + if (isNavigationStartEvent(event)) { + if (event.isMainFrame !== false && isRelevantNavigationUrl(eventUrl, context)) { + navigationStarted = true; + lastNavigationUrl = eventUrl; + } + continue; + } + + if (event.type === "did-fail-load") { + if (event.errorCode === -3) { + continue; + } + if (event.isMainFrame !== false && (navigationStarted || isRelevantNavigationUrl(eventUrl, context))) { + return navigationFailureResult("did-fail-load", event); + } + continue; + } + + if (!navigationStarted) { + continue; + } + + if (isChromiumErrorPage(eventUrl)) { + return navigationFailureResult("chrome-error", { + errorDescription: "Chromium displayed an internal error page.", + url: eventUrl + }); + } + if (waitUntil === "interactive" && isInteractiveReadinessEvent(event)) { + const interactive = await interactiveReadinessResult(webContents, eventUrl || lastNavigationUrl); + if (interactive) { + return interactive; + } + } + if (waitUntil === "domcontentloaded" && event.type === "dom-ready") { + return { matched: true, matchedEvent: "domcontentloaded", readinessUrl: eventUrl || lastNavigationUrl, timedOut: false }; + } + if (waitUntil === "load" && (event.type === "did-finish-load" || event.type === "did-stop-loading") && !webContents.isDestroyed() && !webContents.isLoading()) { + return { matched: true, matchedEvent: "load", readinessUrl: eventUrl || lastNavigationUrl, timedOut: false }; + } + if (waitUntil === "network_idle" && (event.type === "did-finish-load" || event.type === "did-stop-loading") && !webContents.isDestroyed() && !webContents.isLoading()) { + const idle = await waitForNoLoadingStart(webContents, Math.min(500, Math.max(0, deadline - Date.now()))); + if (idle && !webContents.isDestroyed() && !webContents.isLoading()) { + return { matched: true, matchedEvent: "network_idle", readinessUrl: safeWebContentsUrl(webContents) || lastNavigationUrl, timedOut: false }; + } + } + } + if (waitUntil === "interactive" || waitUntil === "network_idle") { + const fallback = await interactiveReadinessResult( + webContents, + lastNavigationUrl, + waitUntil === "network_idle" ? "interactive_after_network_idle_timeout" : "interactive" + ); + if (fallback && (navigationStarted || isRelevantNavigationUrl(fallback.readinessUrl, context))) { + return fallback; + } + } + return { + matched: false, + matchedEvent: webContents.isDestroyed() ? "destroyed" : undefined, + timedOut: !webContents.isDestroyed() + }; +} + +function waitForReadinessEvent(webContents: WebContents, timeoutMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const finish = (event: ReadinessEvent) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + webContents.off("did-navigate", onDidNavigate); + webContents.off("did-redirect-navigation", onDidRedirectNavigation); + webContents.off("did-fail-load", onDidFailLoad); + webContents.off("did-finish-load", onDidFinishLoad); + webContents.off("did-start-navigation", onDidStartNavigation); + webContents.off("did-start-loading", onDidStartLoading); + webContents.off("did-stop-loading", onDidStopLoading); + webContents.off("dom-ready", onDomReady); + webContents.off("destroyed", onDestroyed); + resolve(event); + }; + const onDidFailLoad = ( + _event: ElectronEvent, + errorCode: number, + errorDescription: string, + validatedUrl: string, + isMainFrame?: boolean + ) => finish({ + errorCode, + errorDescription, + isMainFrame, + type: "did-fail-load", + url: validatedUrl || safeWebContentsUrl(webContents) + }); + const onDidFinishLoad = () => finish({ type: "did-finish-load", url: safeWebContentsUrl(webContents) }); + const onDidNavigate = (_event: ElectronEvent, url: string) => finish({ isMainFrame: true, type: "did-navigate", url }); + const onDidRedirectNavigation = (_event: ElectronEvent, url: string, isInPlace?: boolean, isMainFrame?: boolean) => { + finish({ isMainFrame, type: isInPlace ? "did-navigate-in-page" : "did-redirect-navigation", url }); + }; + const onDidStartNavigation = (_event: ElectronEvent, url: string, isInPlace?: boolean, isMainFrame?: boolean) => { + finish({ isMainFrame, type: isInPlace ? "did-navigate-in-page" : "did-start-navigation", url }); + }; + const onDidStartLoading = () => finish({ type: "did-start-loading", url: safeWebContentsUrl(webContents) }); + const onDidStopLoading = () => finish({ type: "did-stop-loading", url: safeWebContentsUrl(webContents) }); + const onDomReady = () => finish({ type: "dom-ready", url: safeWebContentsUrl(webContents) }); + const onDestroyed = () => finish({ type: "destroyed" }); + const timer = setTimeout(() => finish({ type: "timeout" }), Math.max(0, timeoutMs)); + + webContents.once("did-navigate", onDidNavigate); + webContents.once("did-redirect-navigation", onDidRedirectNavigation); + webContents.once("did-fail-load", onDidFailLoad); + webContents.once("did-finish-load", onDidFinishLoad); + webContents.once("did-start-navigation", onDidStartNavigation); + webContents.once("did-start-loading", onDidStartLoading); + webContents.once("did-stop-loading", onDidStopLoading); + webContents.once("dom-ready", onDomReady); + webContents.once("destroyed", onDestroyed); + }); +} + +function waitForNoLoadingStart(webContents: WebContents, timeoutMs: number): Promise { + return new Promise((resolve) => { + let settled = false; + const finish = (idle: boolean) => { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + webContents.off("did-start-loading", onDidStartLoading); + webContents.off("destroyed", onDestroyed); + resolve(idle); + }; + const onDidStartLoading = () => finish(false); + const onDestroyed = () => finish(false); + const timer = setTimeout(() => finish(true), Math.max(0, timeoutMs)); + webContents.once("did-start-loading", onDidStartLoading); + webContents.once("destroyed", onDestroyed); + }); +} + +async function interactiveReadinessResult( + webContents: WebContents, + fallbackUrl?: string, + matchedEvent = "interactive" +): Promise { + if (webContents.isDestroyed()) { + return undefined; + } + const readinessUrl = safeWebContentsUrl(webContents) || fallbackUrl; + if (!isUsableInteractiveUrl(readinessUrl)) { + return undefined; + } + const readyState = await documentReadyState(webContents); + if (readyState === "interactive" || readyState === "complete" || !webContents.isLoading()) { + return { matched: true, matchedEvent, readinessUrl, timedOut: false }; + } + return undefined; +} + +async function documentReadyState(webContents: WebContents): Promise { + try { + const value = await executeJavaScriptWithTimeout( + webContents, + "document.readyState", + 1000, + "Document readiness check" + ); + return typeof value === "string" ? value : undefined; + } catch { + return undefined; + } +} + +function isInteractiveReadinessEvent(event: ReadinessEvent): boolean { + return event.type === "dom-ready" || + event.type === "did-finish-load" || + event.type === "did-stop-loading"; +} + +function isUsableInteractiveUrl(value: string | undefined): value is string { + return Boolean(value) && !isBlankPageUrl(value) && !isChromiumErrorPage(value); +} + +function navigationFailureResult(matchedEvent: string, event: Pick): ReadinessResult { + return { + matched: false, + matchedEvent, + navigationError: { + ...(event.errorCode !== undefined ? { errorCode: event.errorCode } : {}), + ...(event.errorDescription ? { errorDescription: event.errorDescription } : {}), + ...(event.url ? { url: event.url } : {}) + }, + readinessUrl: event.url, + timedOut: false + }; +} + +function isNavigationStartEvent(event: ReadinessEvent): boolean { + return event.type === "did-start-navigation" || + event.type === "did-redirect-navigation" || + event.type === "did-navigate"; +} + +function isRelevantNavigationUrl(url: string | undefined, context: NavigationReadinessContext): boolean { + if (!url) { + return false; + } + const normalizedUrl = normalizeUrlForComparison(url); + if (!normalizedUrl) { + return false; + } + if (context.expectedUrl && urlsMatchForNavigation(normalizedUrl, context.expectedUrl)) { + return true; + } + if (isChromiumErrorPage(normalizedUrl)) { + return true; + } + if (isBlankPageUrl(normalizedUrl)) { + return isBlankPageUrl(context.expectedUrl); + } + if (!context.previousUrl) { + return true; + } + return !urlsMatchForNavigation(normalizedUrl, context.previousUrl); +} + +function normalizeNavigationUrlForMatch(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + if (!trimmed) { + return undefined; + } + if (isBlankPageUrl(trimmed) || isChromiumErrorPage(trimmed)) { + return trimmed; + } + if (/^[a-z][a-z0-9+.-]*:\/\//i.test(trimmed)) { + return normalizeUrlForComparison(trimmed); + } + if (/^(localhost|127\.0\.0\.1|\[::1\])(?::\d+)?(?:\/|$)/i.test(trimmed) || trimmed.includes(".")) { + return normalizeUrlForComparison(`https://${trimmed}`); + } + return normalizeUrlForComparison(`https://www.google.com/search?q=${encodeURIComponent(trimmed)}`); +} + +function normalizeUrlForComparison(value: string | undefined): string | undefined { + const trimmed = value?.trim(); + if (!trimmed) { + return undefined; + } + try { + const url = new URL(trimmed); + url.hash = ""; + return url.toString(); + } catch { + return trimmed; + } +} + +function urlsMatchForNavigation(left: string | undefined, right: string | undefined): boolean { + if (!left || !right) { + return false; + } + const normalizedLeft = normalizeUrlForComparison(left); + const normalizedRight = normalizeUrlForComparison(right); + if (!normalizedLeft || !normalizedRight) { + return false; + } + if (normalizedLeft === normalizedRight) { + return true; + } + try { + const leftUrl = new URL(normalizedLeft); + const rightUrl = new URL(normalizedRight); + return leftUrl.protocol === rightUrl.protocol && + leftUrl.hostname === rightUrl.hostname && + leftUrl.pathname === rightUrl.pathname && + leftUrl.search === rightUrl.search; + } catch { + return normalizedLeft === normalizedRight; + } +} + +function isBlankPageUrl(value: string | undefined): boolean { + return value === "about:blank"; +} + +function isChromiumErrorPage(value: string | undefined): boolean { + return value === "chrome-error://chromewebdata/"; +} + +function safeWebContentsUrl(webContents: WebContents): string | undefined { + if (webContents.isDestroyed()) { + return undefined; + } + try { + return webContents.getURL(); + } catch { + return undefined; + } +} + +function normalizeWaitUntil(value: string | undefined, fallback: string): string { + const normalized = (value || fallback).toLowerCase().replace(/[-\s]/g, "_"); + if (normalized === "auto" || normalized === "interactive" || normalized === "ready") { + return "interactive"; + } + if (normalized === "dom_ready" || normalized === "domcontentloaded") { + return "domcontentloaded"; + } + if (normalized === "networkidle" || normalized === "network_idle") { + return "network_idle"; + } + if (normalized === "none" || normalized === "load") { + return normalized; + } + return fallback; +} + +function readHandoffKind(value: unknown): BuiltInBrowserAutomationHandoffKind { + const kind = readString(value); + if ( + kind === "blocked" || + kind === "human_verification" || + kind === "login_required" || + kind === "other" || + kind === "verification_code" + ) { + return kind; + } + return "other"; +} + +function normalizeEventChannels(values: string[]): Set { + const normalized = values + .map((value) => value.toLowerCase().trim()) + .filter(Boolean); + if (normalized.length === 0) { + return new Set(["tab", "navigation", "dom", "runtime", "dialog", "download", "input", "focus", "selection", "handoff"]); + } + return new Set(normalized); +} + +function matchesSubscriptionEvent(subscription: EventSubscription, event: BrowserAutomationEvent): boolean { + const channel = eventChannel(event.kind); + if (!subscription.channels.has(channel) && !subscription.channels.has(event.kind) && !subscription.channels.has("*")) { + return false; + } + if (!subscription.ref) { + return true; + } + if (!event.tabId || event.tabId === subscription.ref.tabId) { + return true; + } + return channel === "tab"; +} + +function eventChannel(kind: string): string { + if (kind.startsWith("tab.")) return "tab"; + if (kind.startsWith("page.navigation") || kind.startsWith("page.loading")) return "navigation"; + if (kind === "page.dom_ready") return "dom"; + if (kind.startsWith("runtime.")) return "runtime"; + if (kind.startsWith("dialog.")) return "dialog"; + if (kind.startsWith("download.")) return "download"; + if (kind.startsWith("handoff.")) return "handoff"; + if (kind.startsWith("input.")) return "input"; + if (kind.startsWith("focus.")) return "focus"; + if (kind.startsWith("selection.")) return "selection"; + return kind.split(".")[0] || kind; +} + +function subscriptionDescriptor(subscription: EventSubscription): Record { + return { + channels: [...subscription.channels], + cursor: makeEventCursor(subscription), + redactTextInput: subscription.redactTextInput, + sampleMouseMove: subscription.sampleMouseMove, + session: subscription.ref, + startedAt: subscription.startedAt, + subscriptionId: subscription.subscriptionId + }; +} + +function findRecentHandoffResolution(filter: { handoffId?: string; tabId?: string }): BrowserAutomationEvent | undefined { + const events = builtInBrowserService.getAutomationEvents({ replayRecentMs: automationEventReplayMs }); + for (let index = events.length - 1; index >= 0; index -= 1) { + const event = events[index]; + if (event.kind !== "handoff.completed" && event.kind !== "handoff.dismissed") { + continue; + } + if (filter.handoffId && event.handoffId !== filter.handoffId) { + continue; + } + if (filter.tabId && event.tabId !== filter.tabId) { + continue; + } + return event; + } + return undefined; +} + +function handoffResolutionResult( + event: BrowserAutomationEvent, + handoff: BuiltInBrowserAutomationHandoff | undefined, + timedOut: boolean +): Record { + return { + event, + handoff, + matched: !timedOut, + state: browserWindowState(), + status: event.handoffStatus || (event.kind === "handoff.completed" ? "completed" : "dismissed"), + timedOut, + title: event.title, + url: event.url + }; +} + +function eventsAfterCursor(subscription: EventSubscription, cursorSeq?: number): BrowserAutomationEvent[] { + const seq = cursorSeq ?? 0; + return subscription.events.filter((event) => event.seq > seq); +} + +function cursorDropped(subscription: EventSubscription, cursorSeq?: number): boolean { + if (cursorSeq === undefined || subscription.events.length === 0) { + return subscription.dropped; + } + return subscription.dropped && cursorSeq < subscription.events[0].seq - 1; +} + +function makeEventCursor(subscription: EventSubscription, seq?: number): Record { + const lastEvent = subscription.events.findLast((event) => seq === undefined || event.seq <= seq) || subscription.events.at(-1); + return { + dropped: subscription.dropped, + seq: seq ?? lastEvent?.seq ?? 0, + subscriptionId: subscription.subscriptionId, + ts: lastEvent?.ts ?? subscription.startedAt + }; +} + +function readCursorSeq(value: unknown): number | undefined { + if (!isRecord(value)) { + return undefined; + } + return readNumber(value.seq); +} + +function matchesAwaitFilter(event: BrowserAutomationEvent, args: Record): boolean { + const match = isRecord(args.match) ? args.match : args; + const kinds = readStringArray(match.kinds); + const kind = readString(match.kind); + if (kind && event.kind !== kind) { + return false; + } + if (kinds.length > 0 && !kinds.includes(event.kind)) { + return false; + } + const tabId = readString(match.tabId); + if (tabId && event.tabId !== tabId) { + return false; + } + const windowId = readString(match.windowId); + if (windowId && event.windowId !== windowId) { + return false; + } + return stringMatchesPattern(event.url, readString(match.urlPattern)) + && stringMatchesPattern(event.title, readString(match.titlePattern)) + && stringMatchesPattern(event.summary, readString(match.summaryPattern)); +} + +function stringMatchesPattern(value: string | undefined, pattern: string | undefined): boolean { + if (!pattern) { + return true; + } + const haystack = value || ""; + try { + return new RegExp(pattern).test(haystack); + } catch { + return haystack.toLowerCase().includes(pattern.toLowerCase()); + } +} + +async function handleJavaScriptDialog( + webContents: WebContents, + accept: boolean, + promptText: string | undefined, + session: BrowserSessionRef +): Promise> { + let attachedHere = false; + try { + if (!webContents.debugger.isAttached()) { + webContents.debugger.attach("1.3"); + attachedHere = true; + } + await withTimeout(webContents.debugger.sendCommand("Page.enable"), defaultJavascriptTimeoutMs, "Timed out enabling browser dialog handling."); + await withTimeout( + webContents.debugger.sendCommand("Page.handleJavaScriptDialog", { + accept, + ...(promptText !== undefined ? { promptText } : {}) + }), + defaultJavascriptTimeoutMs, + "Timed out handling browser dialog." + ); + return { + accepted: accept, + ok: true, + promptText, + session, + title: await webContents.getTitle(), + url: webContents.getURL() + }; + } catch (error) { + throw new Error(`Failed to handle browser dialog: ${formatError(error)}`); + } finally { + if (attachedHere && webContents.debugger.isAttached()) { + try { + webContents.debugger.detach(); + } catch { + // Ignore detach errors after the dialog is resolved. + } + } + } +} + +async function executeJavaScriptWithTimeout( + webContents: WebContents, + script: string, + timeoutMs: number, + label: string +): Promise { + if (webContents.isDestroyed()) { + throw new Error(`${label} failed because the browser tab is destroyed.`); + } + return await withTimeout( + webContents.executeJavaScript(script, true) as Promise, + timeoutMs, + `${label} timed out after ${timeoutMs}ms.` + ); +} + +async function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error(message)), Math.max(0, timeoutMs)); + }) + ]); + } finally { + if (timer) { + clearTimeout(timer); + } + } +} + +async function captureSnapshot(webContents: WebContents, options: { maxElements: number; maxText: number }): Promise { + return await executeJavaScriptWithTimeout( + webContents, + `(${snapshotScript})(${JSON.stringify(options)})`, + defaultJavascriptTimeoutMs, + "Browser snapshot" + ) as unknown; +} + +async function captureSnapshotWithHandoff( + webContents: WebContents, + tabId: string | undefined, + options: { maxElements: number; maxText: number } +): Promise { + const snapshot = await captureSnapshot(webContents, options); + const handoff = await maybeRequestPageHandoff(webContents, { + snapshot, + tabId + }); + if (!handoff || !isRecord(snapshot)) { + return snapshot; + } + return { + ...snapshot, + ...humanHelpResultFields(handoff) + }; +} + +async function maybeRequestHandoffAfterNavigation( + webContents: WebContents, + readiness: ReadinessResult, + options: { + requestedUrl?: string; + session?: BrowserSessionRef; + tabId?: string; + waitUntil?: string; + } +): Promise { + if (webContents.isDestroyed()) { + return undefined; + } + + if (!readiness.navigationError) { + const pageHandoff = await maybeRequestPageHandoff(webContents, { + session: options.session, + tabId: options.tabId + }); + if (pageHandoff) { + return pageHandoff; + } + } + + return maybeRequestNavigationHandoff(webContents, readiness, options); +} + +function maybeRequestNavigationHandoff( + webContents: WebContents, + readiness: ReadinessResult, + options: { + requestedUrl?: string; + session?: BrowserSessionRef; + tabId?: string; + waitUntil?: string; + } +): BuiltInBrowserAutomationHandoff | undefined { + if (readiness.matched || webContents.isDestroyed()) { + return undefined; + } + + const tabId = options.tabId || options.session?.tabId; + const existing = existingHandoffForTab(tabId); + if (existing) { + return existing; + } + + const currentUrl = safeWebContentsUrl(webContents); + const target = options.requestedUrl || readiness.readinessUrl || currentUrl || "the requested page"; + const error = readiness.navigationError; + const reason = error + ? `Browser navigation was blocked or failed while opening ${target}${error.errorCode !== undefined ? ` (error ${error.errorCode})` : ""}${error.errorDescription ? `: ${error.errorDescription}` : "."}` + : `Browser automation did not reach ${options.waitUntil || "the requested readiness state"} while opening ${target}.`; + + return builtInBrowserService.requestAutomationHandoff({ + kind: "blocked", + message: "Please inspect this browser page and complete or retry the blocked step, then click Done.", + reason, + sessionId: options.session?.sessionId, + tabId + }); +} + +async function maybeRequestPageHandoff( + webContents: WebContents, + options: { + session?: BrowserSessionRef; + snapshot?: unknown; + tabId?: string; + } = {} +): Promise { + if (webContents.isDestroyed()) { + return undefined; + } + + let snapshot = options.snapshot; + if (snapshot === undefined) { + try { + snapshot = await captureSnapshot(webContents, { + maxElements: 20, + maxText: 1200 + }); + } catch { + snapshot = undefined; + } + } + + const detection = detectPageHandoff(snapshot, { + title: await webContents.getTitle(), + url: webContents.getURL() + }); + if (!detection) { + return undefined; + } + + const tabId = options.tabId || options.session?.tabId; + const existing = existingHandoffForTab(tabId); + if (existing) { + return existing; + } + + return builtInBrowserService.requestAutomationHandoff({ + kind: detection.kind, + message: detection.message, + reason: detection.reason, + sessionId: options.session?.sessionId, + tabId + }); +} + +function existingHandoffForTab(tabId: string | undefined): BuiltInBrowserAutomationHandoff | undefined { + const existing = builtInBrowserService.getAutomationState().automationHandoff; + if (existing && (!tabId || existing.tabId === tabId)) { + return existing; + } + return undefined; +} + +function handoffMatches( + handoff: BuiltInBrowserAutomationHandoff | undefined, + filter: { handoffId?: string; tabId?: string } +): boolean { + if (!handoff) { + return false; + } + if (filter.handoffId && handoff.id !== filter.handoffId) { + return false; + } + if (filter.tabId && handoff.tabId !== filter.tabId) { + return false; + } + return true; +} + +function detectPageHandoff(snapshot: unknown, fallback: { title?: string; url?: string }): BrowserHandoffDetection | undefined { + const record = isRecord(snapshot) ? snapshot : {}; + const title = readString(record.title) || fallback.title || ""; + const url = readString(record.url) || fallback.url || ""; + const signal = pageHandoffSignal(record, title, url); + if (isChromiumErrorPage(url)) { + return { + kind: "blocked", + message: "Please inspect this browser error page and retry or complete the blocked step, then click Done.", + reason: "Chromium displayed an internal error page for this navigation." + }; + } + if (!signal) { + return undefined; + } + + const verificationCode = /verification code|security code|one[-\s]?time code|2[-\s]?step|two[-\s]?step|authenticator|check your (?:phone|email)|enter the code/.test(signal); + if (verificationCode) { + return { + kind: "verification_code", + message: "Please enter the verification code in this browser window, then click Done.", + reason: `The page appears to require a verification code: ${title || url || "current page"}.` + }; + } + + const cloudflare = signal.includes("cloudflare") || /\bray id\s*:/.test(signal); + const botVerification = /performing security verification|security verification|verif(?:y|ies|ying)[^.\n]{0,80}(?:not a bot|human)|not a bot|not a robot|human verification|are you human|checking your browser|security check/.test(signal); + const challenge = /captcha|hcaptcha|recaptcha|turnstile|cf-challenge|cf-turnstile/.test(signal); + const justAMoment = title.trim().toLowerCase() === "just a moment..." || signal.includes("just a moment..."); + if (challenge || (cloudflare && (botVerification || justAMoment))) { + return { + kind: "human_verification", + message: "Please complete the security verification in this browser window, then click Done.", + reason: `The page appears to be a human verification or bot-protection challenge: ${title || url || "current page"}.` + }; + } + + const loginUrl = /(?:^|[/.?=&_-])(?:login|signin|sign-in|accounts)(?:[/.?=&_-]|$)/.test(url.toLowerCase()) || + /accounts\.google\.com|mail\.google\.com/.test(url.toLowerCase()); + const loginTitle = /\bsign in\b|\blog in\b|\blogin\b|google accounts|account login/.test(title.toLowerCase()); + const credentialFields = /email or phone|email address|username|password|forgot (?:email|password)|create account|\bnext\b/.test(signal); + if ((loginUrl || loginTitle) && credentialFields) { + return { + kind: "login_required", + message: "Please sign in in this browser window, then click Done.", + reason: `The page appears to require a human login: ${title || url || "current page"}.` + }; + } + + return undefined; +} + +function humanHelpResultFields(handoff: BuiltInBrowserAutomationHandoff): Record { + return { + handoff, + handoffRequired: true, + humanHelp: { + aliasTool: "askHumanHelp", + instruction: "Browser automation needs human help. The browser window has been shown with a top toolbar instruction. Call browser_handoff_wait to receive the user's Done/Hide result before continuing automation.", + handoffId: handoff.id, + message: handoff.message, + reason: handoff.reason, + required: true, + status: "requested", + tool: "browser_handoff_wait" + }, + humanHelpRequired: true, + nextAction: "human_help" + }; +} + +function pageHandoffSignal(record: Record, title: string, url: string): string { + const parts = [title, url, rawString(record.text)]; + const activeElement = isRecord(record.activeElement) ? record.activeElement : undefined; + if (activeElement) { + parts.push(rawString(activeElement.name), rawString(activeElement.text), rawString(activeElement.role)); + } + const elements = Array.isArray(record.elements) ? record.elements : []; + for (const element of elements.slice(0, 40)) { + if (!isRecord(element)) { + continue; + } + parts.push( + rawString(element.name), + rawString(element.text), + rawString(element.role), + rawString(element.tag), + rawString(element.href) + ); + } + return parts + .filter((part): part is string => Boolean(part)) + .join("\n") + .replace(/\s+/g, " ") + .toLowerCase(); +} + +async function runTargetAction( + webContents: WebContents, + action: "click" | "focus" | "scroll" | "select" | "type", + target?: BrowserTarget, + value: Record = {} +): Promise { + const result = await executeJavaScriptWithTimeout( + webContents, + `(${targetActionScript})(${JSON.stringify({ action, target, value })})`, + defaultJavascriptTimeoutMs, + `Browser target action ${action}` + ) as { error?: string; ok?: boolean }; + if (!result?.ok) { + throw new Error(result?.error || `Browser target action failed: ${action}`); + } + return result; +} + +async function pressKey(webContents: WebContents, key: string, target?: BrowserTarget): Promise { + if (target) { + await runTargetAction(webContents, "focus", target); + } + webContents.sendInputEvent({ keyCode: key, type: "keyDown" }); + webContents.sendInputEvent({ keyCode: key, type: "keyUp" }); + return { + key, + ok: true, + title: await webContents.getTitle(), + url: webContents.getURL() + }; +} + +async function waitForPageCondition( + webContents: WebContents, + condition: { + selector?: string; + text?: string; + timeoutMs: number; + urlIncludes?: string; + urlMatches?: string; + } +): Promise { + const deadline = Date.now() + condition.timeoutMs; + let last: unknown; + while (Date.now() <= deadline) { + try { + last = await executeJavaScriptWithTimeout( + webContents, + `(${waitForConditionScript})(${JSON.stringify(condition)})`, + Math.max(100, Math.min(1000, deadline - Date.now())), + "Browser wait condition check" + ) as unknown; + if (isRecord(last) && last.matched === true) { + return last; + } + } catch (error) { + last = { + error: formatError(error), + matched: false + }; + } + await sleep(150); + } + return { + last, + matched: false, + timedOut: true, + title: await webContents.getTitle(), + url: webContents.getURL() + }; +} + +const snapshotScript = function(options: { maxElements: number; maxText: number }) { + const maxElements = Math.max(1, Math.min(300, Math.floor(options.maxElements || 80))); + const maxText = Math.max(0, Math.min(20000, Math.floor(options.maxText || 3000))); + const refAttribute = "data-ccr-browser-ref"; + const elementSelector = [ + "a[href]", + "button", + "input", + "textarea", + "select", + "summary", + "[role]", + "[contenteditable='true']", + "[contenteditable='plaintext-only']", + "[tabindex]:not([tabindex='-1'])" + ].join(","); + + function visible(el: Element) { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== "none" && + style.visibility !== "hidden" && + Number(style.opacity || "1") > 0 && + rect.width > 0 && + rect.height > 0; + } + + function cssEscape(value: string) { + return window.CSS && typeof window.CSS.escape === "function" + ? window.CSS.escape(value) + : value.replace(/[^a-zA-Z0-9_-]/g, "\\$&"); + } + + function uniqueIdSelector(id: string) { + if (!id) return ""; + const selector = `#${cssEscape(id)}`; + try { + return document.querySelectorAll(selector).length === 1 ? selector : ""; + } catch { + return ""; + } + } + + function refFor(el: Element) { + const win = window as Window & { __ccrBrowserRefSeq?: number }; + const existing = el.getAttribute(refAttribute); + if (existing) { + return `[${refAttribute}="${cssEscape(existing)}"]`; + } + for (let attempt = 0; attempt < 1000; attempt += 1) { + win.__ccrBrowserRefSeq = (win.__ccrBrowserRefSeq || 0) + 1; + const nextRef = `ccr-${win.__ccrBrowserRefSeq}`; + const selector = `[${refAttribute}="${cssEscape(nextRef)}"]`; + if (!document.querySelector(selector)) { + el.setAttribute(refAttribute, nextRef); + return selector; + } + } + + const idSelector = uniqueIdSelector((el as HTMLElement).id || ""); + if (idSelector) return idSelector; + const parts: string[] = []; + let current: Element | null = el; + while (current && current.nodeType === 1 && current !== document.documentElement && parts.length < 8) { + const tag = current.tagName.toLowerCase(); + const parent: HTMLElement | null = current.parentElement; + if (!parent) { + parts.unshift(tag); + break; + } + const parentIdSelector = uniqueIdSelector(parent.id || ""); + const sameTag = Array.from(parent.children).filter((child): child is Element => child instanceof Element && child.tagName === current!.tagName); + const index = sameTag.indexOf(current) + 1; + parts.unshift(`${tag}:nth-of-type(${Math.max(index, 1)})`); + if (parentIdSelector) { + parts.unshift(parentIdSelector); + break; + } + current = parent; + } + return parts.join(" > "); + } + + function text(el: Element) { + return ((el as HTMLElement).innerText || el.textContent || "").replace(/\s+/g, " ").trim(); + } + + function labelText(el: Element) { + const labelledBy = el.getAttribute("aria-labelledby"); + if (labelledBy) { + const value = labelledBy + .split(/\s+/) + .map((id) => document.getElementById(id)?.innerText || "") + .join(" ") + .replace(/\s+/g, " ") + .trim(); + if (value) return value; + } + const id = (el as HTMLInputElement).id; + if (id) { + const label = document.querySelector(`label[for="${cssEscape(id)}"]`); + if (label) return text(label); + } + const wrappingLabel = el.closest("label"); + return wrappingLabel ? text(wrappingLabel) : ""; + } + + function unsafeField(el: Element) { + const input = el as HTMLInputElement; + const haystack = [ + input.type, + input.name, + input.id, + input.autocomplete, + el.getAttribute("aria-label"), + labelText(el) + ].join(" ").toLowerCase(); + return /\b(password|secret|token|api[-_ ]?key|bearer|credential)\b/.test(haystack); + } + + function valueOf(el: Element) { + if (!(el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement || el instanceof HTMLSelectElement)) { + return ""; + } + if (unsafeField(el)) { + return ""; + } + return el.value || ""; + } + + function roleOf(el: Element) { + const explicit = el.getAttribute("role"); + if (explicit) return explicit; + const tag = el.tagName.toLowerCase(); + const input = el as HTMLInputElement; + if (tag === "a") return "link"; + if (tag === "button") return "button"; + if (tag === "textarea") return "textbox"; + if (tag === "select") return "combobox"; + if (tag === "summary") return "button"; + if (tag === "input") { + if (["button", "submit", "reset"].includes(input.type)) return "button"; + if (input.type === "checkbox") return "checkbox"; + if (input.type === "radio") return "radio"; + if (input.type === "range") return "slider"; + return "textbox"; + } + if (el.getAttribute("contenteditable")) return "textbox"; + return ""; + } + + function nameOf(el: Element) { + const input = el as HTMLInputElement; + return ( + el.getAttribute("aria-label") || + labelText(el) || + el.getAttribute("alt") || + el.getAttribute("title") || + input.placeholder || + (input.type && ["button", "submit", "reset"].includes(input.type) ? input.value : "") || + text(el) + ).replace(/\s+/g, " ").trim(); + } + + const elements = Array.from(document.querySelectorAll(elementSelector)) + .filter(visible) + .slice(0, maxElements) + .map((el, index) => { + const rect = el.getBoundingClientRect(); + return { + checked: el instanceof HTMLInputElement && ["checkbox", "radio"].includes(el.type) ? el.checked : undefined, + disabled: (el as HTMLButtonElement | HTMLInputElement | HTMLSelectElement | HTMLTextAreaElement).disabled || undefined, + href: el instanceof HTMLAnchorElement ? el.href : undefined, + index, + name: nameOf(el).slice(0, 240), + placeholder: (el as HTMLInputElement).placeholder || undefined, + rect: { + height: Math.round(rect.height), + width: Math.round(rect.width), + x: Math.round(rect.x), + y: Math.round(rect.y) + }, + ref: refFor(el), + role: roleOf(el), + tag: el.tagName.toLowerCase(), + text: text(el).slice(0, 180), + value: valueOf(el) || undefined + }; + }); + + return { + activeElement: document.activeElement ? { + name: nameOf(document.activeElement).slice(0, 240), + ref: refFor(document.activeElement), + role: roleOf(document.activeElement), + tag: document.activeElement.tagName.toLowerCase() + } : undefined, + elements, + text: (document.body?.innerText || "").replace(/\s+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim().slice(0, maxText), + title: document.title, + url: location.href + }; +}; + +const targetActionScript = function(request: { + action: "click" | "focus" | "scroll" | "select" | "type"; + target?: BrowserTarget; + value?: Record; +}) { + function visible(el: Element) { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + return style.display !== "none" && style.visibility !== "hidden" && rect.width > 0 && rect.height > 0; + } + function text(el: Element) { + return ((el as HTMLElement).innerText || el.textContent || "").replace(/\s+/g, " ").trim(); + } + function cssEscape(value: string) { + return window.CSS && typeof window.CSS.escape === "function" + ? window.CSS.escape(value) + : value.replace(/[^a-zA-Z0-9_-]/g, "\\$&"); + } + function labelText(el: Element) { + const id = (el as HTMLInputElement).id; + if (id) { + const label = document.querySelector(`label[for="${cssEscape(id)}"]`); + if (label) return text(label); + } + const wrappingLabel = el.closest("label"); + return wrappingLabel ? text(wrappingLabel) : ""; + } + function roleOf(el: Element) { + const explicit = el.getAttribute("role"); + if (explicit) return explicit.toLowerCase(); + const tag = el.tagName.toLowerCase(); + const input = el as HTMLInputElement; + if (tag === "a") return "link"; + if (tag === "button") return "button"; + if (tag === "textarea") return "textbox"; + if (tag === "select") return "combobox"; + if (tag === "summary") return "button"; + if (tag === "input") { + if (["button", "submit", "reset"].includes(input.type)) return "button"; + if (input.type === "checkbox") return "checkbox"; + if (input.type === "radio") return "radio"; + return "textbox"; + } + if (el.getAttribute("contenteditable")) return "textbox"; + return ""; + } + function nameOf(el: Element) { + const input = el as HTMLInputElement; + return ( + el.getAttribute("aria-label") || + labelText(el) || + el.getAttribute("alt") || + el.getAttribute("title") || + input.placeholder || + input.value || + text(el) + ).replace(/\s+/g, " ").trim(); + } + function summarize(el: Element) { + const rect = el.getBoundingClientRect(); + return { + name: nameOf(el).slice(0, 240), + rect: { + height: Math.round(rect.height), + width: Math.round(rect.width), + x: Math.round(rect.x), + y: Math.round(rect.y) + }, + role: roleOf(el), + tag: el.tagName.toLowerCase(), + text: text(el).slice(0, 180) + }; + } + function selectorElement(selector: string) { + try { + return document.querySelector(selector); + } catch { + return null; + } + } + function matchesText(value: string, needle: string, exact?: boolean) { + const left = value.trim().toLowerCase(); + const right = needle.trim().toLowerCase(); + return exact ? left === right : left.includes(right); + } + function findTarget(target?: BrowserTarget) { + if (!target && request.action !== "scroll") return null; + const directSelector = target?.selector || target?.ref || target?.axNodeId; + if (directSelector) { + return selectorElement(directSelector); + } + const selector = [ + "a[href]", + "button", + "input", + "textarea", + "select", + "summary", + "[role]", + "[contenteditable='true']", + "[contenteditable='plaintext-only']", + "[tabindex]:not([tabindex='-1'])" + ].join(","); + const candidates = Array.from(document.querySelectorAll(selector)).filter(visible); + const role = target?.role?.trim().toLowerCase(); + const needle = target?.text?.trim(); + const matches = candidates.filter((el) => { + if (role && roleOf(el) !== role) return false; + if (!needle) return true; + const haystack = [ + nameOf(el), + text(el), + (el as HTMLInputElement).placeholder || "", + (el as HTMLInputElement).value || "" + ].join(" "); + return matchesText(haystack, needle, target?.exact); + }); + return matches[Math.max(0, Math.floor(target?.index || 0))] || null; + } + function dispatchValueEvents(el: Element) { + el.dispatchEvent(new Event("input", { bubbles: true })); + el.dispatchEvent(new Event("change", { bubbles: true })); + } + + const el = findTarget(request.target); + if (!el && request.action !== "scroll") { + return { error: "No matching browser element found.", ok: false }; + } + if (el) { + el.scrollIntoView({ block: "center", inline: "center" }); + if (el instanceof HTMLElement) { + el.focus({ preventScroll: true }); + } + } + + if (request.action === "focus") { + return { element: el ? summarize(el) : undefined, ok: true }; + } + + if (request.action === "click") { + if (!(el instanceof HTMLElement)) { + return { error: "Matched element is not clickable.", ok: false }; + } + el.click(); + return { element: summarize(el), ok: true }; + } + + if (request.action === "type") { + const nextText = typeof request.value?.text === "string" ? request.value.text : ""; + const replaceExisting = request.value?.replaceExisting !== false; + if (el instanceof HTMLInputElement || el instanceof HTMLTextAreaElement) { + el.value = replaceExisting ? nextText : `${el.value}${nextText}`; + dispatchValueEvents(el); + return { element: summarize(el), ok: true, value: el.type === "password" ? "" : el.value }; + } + if (el instanceof HTMLElement && el.isContentEditable) { + if (replaceExisting) { + el.textContent = nextText; + } else { + el.textContent = `${el.textContent || ""}${nextText}`; + } + dispatchValueEvents(el); + return { element: summarize(el), ok: true }; + } + return { error: "Matched element cannot receive typed text.", ok: false }; + } + + if (request.action === "select") { + if (!(el instanceof HTMLSelectElement)) { + return { error: "Matched element is not a select.", ok: false }; + } + const requestedValue = typeof request.value?.value === "string" ? request.value.value : ""; + const requestedLabel = typeof request.value?.label === "string" ? request.value.label : ""; + const exact = request.value?.exact === true; + const option = Array.from(el.options).find((candidate) => { + if (requestedValue && candidate.value === requestedValue) return true; + if (!requestedLabel) return false; + return matchesText(candidate.textContent || "", requestedLabel, exact); + }); + if (!option) { + return { error: "No matching select option found.", ok: false }; + } + el.value = option.value; + dispatchValueEvents(el); + return { element: summarize(el), label: option.textContent || "", ok: true, value: option.value }; + } + + if (request.action === "scroll") { + const deltaX = typeof request.value?.deltaX === "number" ? request.value.deltaX : 0; + const deltaY = typeof request.value?.deltaY === "number" ? request.value.deltaY : 700; + if (el instanceof HTMLElement) { + el.scrollBy({ behavior: "auto", left: deltaX, top: deltaY }); + return { element: summarize(el), ok: true }; + } + window.scrollBy({ behavior: "auto", left: deltaX, top: deltaY }); + return { ok: true, scrollX: window.scrollX, scrollY: window.scrollY }; + } + + return { error: "Unsupported browser target action.", ok: false }; +}; + +const waitForConditionScript = function(condition: { + selector?: string; + text?: string; + urlIncludes?: string; + urlMatches?: string; +}) { + const checks: Array<{ matched: boolean; type: string }> = []; + if (condition.urlIncludes) { + checks.push({ matched: location.href.includes(condition.urlIncludes), type: "urlIncludes" }); + } + if (condition.urlMatches) { + let matched = false; + try { + matched = new RegExp(condition.urlMatches).test(location.href); + } catch { + matched = false; + } + checks.push({ matched, type: "urlMatches" }); + } + if (condition.selector) { + let matched = false; + try { + const el = document.querySelector(condition.selector); + if (el) { + const style = window.getComputedStyle(el); + const rect = el.getBoundingClientRect(); + matched = style.display !== "none" && style.visibility !== "hidden" && rect.width > 0 && rect.height > 0; + } + } catch { + matched = false; + } + checks.push({ matched, type: "selector" }); + } + if (condition.text) { + const pageText = (document.body?.innerText || "").toLowerCase(); + checks.push({ matched: pageText.includes(condition.text.toLowerCase()), type: "text" }); + } + return { + checks, + matched: checks.length > 0 && checks.every((check) => check.matched), + title: document.title, + url: location.href + }; +}; + +function readTarget(value: unknown): BrowserTarget { + if (!isRecord(value)) { + throw new Error("A browser target object is required."); + } + return { + axNodeId: readString(value.axNodeId), + backendNodeId: readNumber(value.backendNodeId), + exact: value.exact === true, + index: clampInteger(readNumber(value.index) ?? 0, 0, 10000), + ref: readString(value.ref) || readString(value.axNodeId), + role: readString(value.role), + selector: readString(value.selector), + text: readString(value.text) + }; +} + +function requiredString(value: unknown, message: string): string { + const text = readString(value); + if (!text) { + throw new Error(message); + } + return text; +} + +function readString(value: unknown): string | undefined { + return typeof value === "string" && value.trim() ? value.trim() : undefined; +} + +function readStringArray(value: unknown): string[] { + return Array.isArray(value) ? value.filter((item): item is string => typeof item === "string" && item.trim().length > 0) : []; +} + +function uniqueStrings(values: string[]): string[] { + return [...new Set(values)]; +} + +function readNumber(value: unknown): number | undefined { + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function clampInteger(value: number, min: number, max: number): number { + return Math.min(max, Math.max(min, Math.trunc(value))); +} + +function objectSchema(properties: Record, required: string[] = []): JsonValue { + return { + additionalProperties: false, + properties, + required, + type: "object" + }; +} + +type CompactResultOptions = { + maxArrayItems: number; + maxDepth: number; + maxObjectKeys: number; + maxStringChars: number; +}; + +function formatToolResult(toolName: string, result: unknown): string { + const compacted = compactToolResult(toolName, result); + const serialized = stringifyToolResult(compacted); + if (serialized.length <= maxToolResultChars) { + return serialized; + } + + const tighter = compactJsonValue(compacted, { + maxArrayItems: 50, + maxDepth: 6, + maxObjectKeys: 80, + maxStringChars: 600 + }); + const tightSerialized = stringifyToolResult({ + guidance: largeResultGuidance(toolName), + maxChars: maxToolResultChars, + originalChars: serialized.length, + result: tighter, + tool: toolName, + truncated: true + }); + if (tightSerialized.length <= maxToolResultChars) { + return tightSerialized; + } + + return stringifyToolResult({ + guidance: largeResultGuidance(toolName), + maxChars: maxToolResultChars, + originalChars: serialized.length, + preview: truncateString(tightSerialized, 12_000), + tool: toolName, + truncated: true + }); +} + +function compactToolResult(toolName: string, result: unknown): unknown { + if (toolName === "browser_snapshot") { + return compactSnapshotResult(result); + } + if (toolName === "browser_ax_snapshot") { + return compactAxSnapshotResult(result); + } + if (toolName === "browser_ax_query") { + return compactArrayResult(result, "matches", 80, 600); + } + if (toolName === "browser_events_read" || toolName === "browser_events_await") { + return compactArrayResult(result, "events", 80, 800); + } + return compactJsonValue(result, defaultCompactResultOptions()); +} + +function compactSnapshotResult(result: unknown): unknown { + if (!isRecord(result)) { + return compactJsonValue(result, defaultCompactResultOptions()); + } + + const elements = Array.isArray(result.elements) ? result.elements : []; + const text = rawString(result.text); + const compacted: Record = {}; + for (const [key, value] of Object.entries(result)) { + if (key === "elements" || key === "text") { + continue; + } + compacted[key] = compactJsonValue(value, { + maxArrayItems: 40, + maxDepth: 5, + maxObjectKeys: 60, + maxStringChars: 600 + }); + } + + if (text !== undefined) { + compacted.text = truncateString(text, maxBrowserResultTextChars); + } + compacted.elements = elements + .slice(0, maxSnapshotResultElements) + .map((element) => compactJsonValue(element, { + maxArrayItems: 20, + maxDepth: 4, + maxObjectKeys: 40, + maxStringChars: 500 + })); + compacted.elementCount = elements.length; + + const truncated: Record = {}; + if (text !== undefined && text.length > maxBrowserResultTextChars) { + truncated.textChars = text.length - maxBrowserResultTextChars; + } + if (elements.length > maxSnapshotResultElements) { + truncated.elements = elements.length - maxSnapshotResultElements; + } + if (Object.keys(truncated).length > 0) { + compacted.truncated = truncated; + } + return compacted; +} + +function compactAxSnapshotResult(result: unknown): unknown { + if (!isRecord(result)) { + return compactJsonValue(result, defaultCompactResultOptions()); + } + + const nodes = Array.isArray(result.nodes) ? result.nodes : []; + const compacted: Record = {}; + for (const [key, value] of Object.entries(result)) { + if (key === "nodes") { + continue; + } + compacted[key] = compactJsonValue(value, { + maxArrayItems: 40, + maxDepth: 5, + maxObjectKeys: 60, + maxStringChars: 400 + }); + } + compacted.nodes = nodes + .slice(0, maxAxSnapshotResultNodes) + .map((node) => compactJsonValue(node, { + maxArrayItems: 80, + maxDepth: 5, + maxObjectKeys: 40, + maxStringChars: 260 + })); + compacted.nodeCount = nodes.length; + if (nodes.length > maxAxSnapshotResultNodes) { + compacted.truncated = { + nodes: nodes.length - maxAxSnapshotResultNodes + }; + } + return compacted; +} + +function compactArrayResult(result: unknown, key: string, maxItems: number, maxStringChars: number): unknown { + if (!isRecord(result)) { + return compactJsonValue(result, defaultCompactResultOptions()); + } + + const items = Array.isArray(result[key]) ? result[key] : []; + const compacted: Record = {}; + for (const [entryKey, value] of Object.entries(result)) { + if (entryKey === key) { + continue; + } + compacted[entryKey] = compactJsonValue(value, { + maxArrayItems: 40, + maxDepth: 5, + maxObjectKeys: 60, + maxStringChars + }); + } + compacted[key] = items + .slice(0, maxItems) + .map((item) => compactJsonValue(item, { + maxArrayItems: 40, + maxDepth: 5, + maxObjectKeys: 60, + maxStringChars + })); + compacted[`${key}Count`] = items.length; + if (items.length > maxItems) { + compacted.truncated = { + [key]: items.length - maxItems + }; + } + return compacted; +} + +function compactJsonValue( + value: unknown, + options: CompactResultOptions, + depth = 0, + seen: WeakSet = new WeakSet() +): unknown { + if (value === null || typeof value === "boolean" || typeof value === "number") { + return value; + } + if (typeof value === "string") { + return truncateString(value, options.maxStringChars); + } + if (typeof value === "bigint") { + return value.toString(); + } + if (typeof value === "undefined" || typeof value === "function" || typeof value === "symbol") { + return undefined; + } + if (typeof value !== "object") { + return String(value); + } + if (seen.has(value)) { + return "[Circular]"; + } + if (depth >= options.maxDepth) { + return { + reason: "Maximum result depth reached.", + truncated: true + }; + } + + seen.add(value); + if (Array.isArray(value)) { + const limit = Math.max(0, options.maxArrayItems); + const items = value + .slice(0, limit) + .map((item) => compactJsonValue(item, options, depth + 1, seen)); + if (value.length > limit) { + items.push({ + omittedItems: value.length - limit, + truncated: true + }); + } + seen.delete(value); + return items; + } + + const entries = Object.entries(value); + const compacted: Record = {}; + for (const [key, entryValue] of entries.slice(0, options.maxObjectKeys)) { + compacted[key] = compactJsonValue(entryValue, options, depth + 1, seen); + } + if (entries.length > options.maxObjectKeys) { + compacted.truncatedKeys = entries.length - options.maxObjectKeys; + } + seen.delete(value); + return compacted; +} + +function defaultCompactResultOptions(): CompactResultOptions { + return { + maxArrayItems: maxToolResultArrayItems, + maxDepth: 8, + maxObjectKeys: maxToolResultObjectKeys, + maxStringChars: maxToolResultStringChars + }; +} + +function stringifyToolResult(value: unknown): string { + return JSON.stringify(value, null, 2) || "null"; +} + +function rawString(value: unknown): string | undefined { + return typeof value === "string" ? value : undefined; +} + +function truncateString(value: string, maxChars: number): string { + if (value.length <= maxChars) { + return value; + } + let suffix = "\n[truncated]"; + for (let index = 0; index < 2; index += 1) { + const keep = Math.max(0, maxChars - suffix.length); + suffix = `\n[truncated ${value.length - keep} chars]`; + } + return `${value.slice(0, Math.max(0, maxChars - suffix.length))}${suffix}`; +} + +function largeResultGuidance(toolName: string): string { + if (toolName === "browser_snapshot") { + return "Result was compacted. Re-run browser_snapshot with lower maxElements/maxText, or use browser_ax_query to fetch targeted elements."; + } + if (toolName === "browser_ax_snapshot") { + return "Result was compacted. Re-run browser_ax_snapshot with lower limit or scope=outline, or use browser_ax_query with role/name/text filters."; + } + if (toolName === "browser_ax_query") { + return "Result was compacted. Re-run browser_ax_query with a lower limit or narrower role/name/text filters."; + } + if (toolName === "browser_events_read" || toolName === "browser_events_await") { + return "Result was compacted. Read events with a lower limit/maxEvents or a more specific cursor/filter."; + } + return "Result was compacted before returning to the MCP client. Use narrower arguments or lower limits for more detail."; +} + +function textResult(text: string): ToolCallResult { + return { + content: [{ text, type: "text" }] + }; +} + +function jsonRpcResult(id: null | number | string, result: JsonValue): JsonRpcResponse { + return { + id, + jsonrpc: "2.0", + result + }; +} + +function jsonRpcError(id: null | number | string, code: number, message: string): JsonRpcResponse { + return { + error: { + code, + message + }, + id, + jsonrpc: "2.0" + }; +} + +function sendJson(response: ServerResponse, status: number, payload: unknown): void { + const body = `${JSON.stringify(payload)}\n`; + response.writeHead(status, { + "content-length": Buffer.byteLength(body), + "content-type": "application/json; charset=utf-8" + }); + response.end(body); +} + +function readRequestBody(request: IncomingMessage, maxBytes: number): Promise { + return new Promise((resolve, reject) => { + const chunks: Buffer[] = []; + let total = 0; + request.on("data", (chunk: Buffer | string) => { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + total += buffer.byteLength; + if (total > maxBytes) { + reject(new Error("MCP request body is too large.")); + request.destroy(); + return; + } + chunks.push(buffer); + }); + request.on("end", () => resolve(Buffer.concat(chunks))); + request.on("error", reject); + }); +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/electron/src/main/browser-preload.ts b/packages/electron/src/main/browser-preload.ts index 2e1930f4..9d571fff 100644 --- a/packages/electron/src/main/browser-preload.ts +++ b/packages/electron/src/main/browser-preload.ts @@ -1,16 +1,19 @@ import { contextBridge, ipcRenderer, type IpcRendererEvent } from "electron"; import { IPC_CHANNELS } from "@ccr/core/contracts/ipc-channels"; -import type { BuiltInBrowserState } from "@ccr/core/contracts/app"; +import type { BuiltInBrowserState, ChromeLoginImportJob, ChromeLoginImportRequest } from "@ccr/core/contracts/app"; contextBridge.exposeInMainWorld("ccrBrowser", { back: (tabId?: string) => ipcRenderer.invoke(IPC_CHANNELS.browserBack, tabId) as Promise, closeTab: (tabId: string) => ipcRenderer.invoke(IPC_CHANNELS.browserCloseTab, tabId) as Promise, forward: (tabId?: string) => ipcRenderer.invoke(IPC_CHANNELS.browserForward, tabId) as Promise, + getChromeLoginImport: (id: string) => ipcRenderer.invoke(IPC_CHANNELS.browserGetChromeLoginImport, id) as Promise, getState: () => ipcRenderer.invoke(IPC_CHANNELS.browserGetState) as Promise, navigate: (url: string, tabId?: string) => ipcRenderer.invoke(IPC_CHANNELS.browserNavigate, url, tabId) as Promise, newTab: (url?: string) => ipcRenderer.invoke(IPC_CHANNELS.browserNewTab, url) as Promise, reload: (tabId?: string) => ipcRenderer.invoke(IPC_CHANNELS.browserReload, tabId) as Promise, + resolveAutomationHandoff: (status: "completed" | "dismissed") => ipcRenderer.invoke(IPC_CHANNELS.browserResolveAutomationHandoff, status) as Promise, selectTab: (tabId: string) => ipcRenderer.invoke(IPC_CHANNELS.browserSelectTab, tabId) as Promise, + startChromeLoginImport: (request: ChromeLoginImportRequest) => ipcRenderer.invoke(IPC_CHANNELS.browserStartChromeLoginImport, request) as Promise, onStateChanged: (callback: (state: BuiltInBrowserState) => void) => { const handler = (_event: IpcRendererEvent, state: BuiltInBrowserState) => callback(state); ipcRenderer.on(IPC_CHANNELS.browserStateChanged, handler); diff --git a/packages/electron/src/main/built-in-browser.ts b/packages/electron/src/main/built-in-browser.ts index a8ae1b08..c9410160 100644 --- a/packages/electron/src/main/built-in-browser.ts +++ b/packages/electron/src/main/built-in-browser.ts @@ -1,4 +1,5 @@ import { randomUUID } from "node:crypto"; +import { EventEmitter } from "node:events"; import { BrowserWindow, clipboard, @@ -10,28 +11,64 @@ import { WebContentsView, type ContextMenuParams, type IpcMainInvokeEvent, - type MenuItemConstructorOptions + type MenuItemConstructorOptions, + type WebContents } from "electron"; import path from "node:path"; import { pathToFileURL } from "node:url"; -import type { AppConfig, BuiltInBrowserState, BuiltInBrowserTabState, GatewayPluginAppConfig, InstalledBrowserApp } from "@ccr/core/contracts/app"; +import type { + AppConfig, + BuiltInBrowserAutomationHandoff, + BuiltInBrowserAutomationHandoffKind, + BuiltInBrowserState, + BuiltInBrowserTabState, + ChromeLoginImportRequest, + GatewayPluginAppConfig, + InstalledBrowserApp +} from "@ccr/core/contracts/app"; import { IPC_CHANNELS } from "@ccr/core/contracts/ipc-channels"; import { APP_NAME } from "@ccr/core/config/constants"; import { pluginService } from "@ccr/core/plugins/service"; -import { proxyService } from "@ccr/core/proxy/service"; +import { chromeLoginImportService } from "./chrome-login-import"; type BrowserTab = BuiltInBrowserTabState & { view: WebContentsView; }; -const browserChromeHeight = 82; +export type BrowserAutomationEvent = { + errorCode?: number; + errorDescription?: string; + handoffId?: string; + handoffStatus?: "completed" | "dismissed"; + kind: string; + seq: number; + summary?: string; + tabId?: string; + title?: string; + ts: number; + url?: string; + windowId?: string; +}; + +export type BrowserAutomationEventListener = (event: BrowserAutomationEvent) => void; + +const browserChromeBaseHeight = 82; +const browserHandoffToolbarHeight = 44; const browserHomeUrl = "about:blank"; const browserPartition = "persist:ccr-built-in-browser"; +const browserAutomationWindowId = "ccr-built-in-browser"; +const maxAutomationEventHistory = 512; +const maxAutomationEventAgeMs = 60_000; const titleBarHeight = 46; class BuiltInBrowserService { private activeTabId?: string; private apps: InstalledBrowserApp[] = []; + private automationHandoff?: BuiltInBrowserAutomationHandoff; + private automationEventSeq = 0; + private readonly automationEvents = new EventEmitter(); + private readonly automationEventHistory: BrowserAutomationEvent[] = []; + private hideWindowAfterAutomationHandoff = false; private proxyConfigKey = ""; private tabOrder: string[] = []; private tabs = new Map(); @@ -44,10 +81,7 @@ class BuiltInBrowserService { async open(config: AppConfig): Promise { await this.syncProxy(config); - const window = this.window && !this.window.isDestroyed() ? this.window : this.createWindow(); - if (this.tabs.size === 0) { - this.createTab(browserHomeUrl); - } + const window = this.ensureWindow(); if (window.isMinimized()) { window.restore(); } @@ -57,28 +91,18 @@ class BuiltInBrowserService { this.sendState(); } + async openHidden(config: AppConfig): Promise { + await this.syncProxy(config); + this.ensureWindow(); + this.layoutActiveView(); + this.sendState(); + } + async syncProxy(config: AppConfig): Promise { this.syncApps(config); const browserSession = session.fromPartition(browserPartition); - if (config.proxy.enabled) { - await proxyService.refreshUpstreamProxyFromCurrentSystem(); - } - const proxyStatus = proxyService.getStatus(); - const shouldUseProxy = Boolean( - config.proxy.enabled && - proxyStatus.state === "running" && - proxyStatus.endpoint - ); - const proxyConfig = shouldUseProxy - ? { - mode: "fixed_servers" as const, - proxyBypassRules: "<-loopback>", - proxyRules: electronProxyRules(proxyStatus.endpoint) - } - : { - mode: "direct" as const - }; + const proxyConfig = { mode: "system" as const }; const nextKey = JSON.stringify(proxyConfig); if (nextKey === this.proxyConfigKey) { return; @@ -100,9 +124,211 @@ class BuiltInBrowserService { async clearProxy(): Promise { const browserSession = session.fromPartition(browserPartition); - await browserSession.setProxy({ mode: "direct" }); + const proxyConfig = { mode: "system" as const }; + await browserSession.setProxy(proxyConfig); await browserSession.forceReloadProxyConfig(); - this.proxyConfigKey = JSON.stringify({ mode: "direct" }); + this.proxyConfigKey = JSON.stringify(proxyConfig); + } + + getAutomationState(): BuiltInBrowserState { + return this.getState(); + } + + getAutomationWindowId(): string { + return browserAutomationWindowId; + } + + getAutomationEvents(options: { replayRecentMs?: number } = {}): BrowserAutomationEvent[] { + const replayRecentMs = Math.max(0, Math.floor(options.replayRecentMs ?? 0)); + const cutoff = replayRecentMs > 0 ? Date.now() - replayRecentMs : 0; + this.pruneAutomationEventHistory(); + return this.automationEventHistory + .filter((event) => event.ts >= cutoff) + .map((event) => ({ ...event })); + } + + requestAutomationHandoff(request: { + kind?: BuiltInBrowserAutomationHandoffKind; + message?: string; + reason?: string; + sessionId?: string; + tabId?: string; + }): BuiltInBrowserAutomationHandoff { + const existingWindow = this.window && !this.window.isDestroyed() ? this.window : undefined; + if (!this.automationHandoff) { + this.hideWindowAfterAutomationHandoff = !existingWindow || !existingWindow.isVisible() || existingWindow.isMinimized(); + } + const tab = this.getTab(request.tabId); + if (request.tabId && tab?.id) { + this.selectTab(request.tabId); + } + const targetTab = tab || this.getTab(); + const handoff: BuiltInBrowserAutomationHandoff = { + id: randomUUID(), + kind: request.kind || "other", + message: request.message?.trim() || defaultAutomationHandoffMessage(request.kind), + ...(request.reason?.trim() ? { reason: request.reason.trim() } : {}), + requestedAt: Date.now(), + ...(request.sessionId?.trim() ? { sessionId: request.sessionId.trim() } : {}), + status: "pending", + tabId: targetTab?.id || request.tabId || this.activeTabId + }; + this.automationHandoff = handoff; + this.layoutActiveView(); + this.showAutomationWindow(); + this.sendState(); + this.emitAutomationEvent({ + handoffId: handoff.id, + kind: "handoff.requested", + summary: handoff.message, + tabId: handoff.tabId, + title: targetTab?.title, + url: targetTab?.url + }); + return handoff; + } + + resolveAutomationHandoff(status: "completed" | "dismissed" = "completed"): BuiltInBrowserState { + const handoff = this.automationHandoff; + const shouldHideWindow = this.hideWindowAfterAutomationHandoff; + this.automationHandoff = undefined; + this.hideWindowAfterAutomationHandoff = false; + this.layoutActiveView(); + this.sendState(); + if (handoff) { + const tab = this.getTab(handoff.tabId); + if (shouldHideWindow) { + this.hideAutomationWindow(); + } + this.emitAutomationEvent({ + handoffId: handoff.id, + handoffStatus: status, + kind: status === "completed" ? "handoff.completed" : "handoff.dismissed", + summary: status === "completed" ? "User completed the requested browser handoff." : "User dismissed the requested browser handoff.", + tabId: handoff.tabId, + title: tab?.title, + url: tab?.url + }); + } + return this.getState(); + } + + subscribeAutomationEvents( + listener: BrowserAutomationEventListener, + options: { replayRecentMs?: number } = {} + ): () => void { + this.automationEvents.on("event", listener); + const replayRecentMs = Math.max(0, Math.floor(options.replayRecentMs ?? 0)); + if (replayRecentMs > 0) { + const cutoff = Date.now() - replayRecentMs; + this.pruneAutomationEventHistory(); + for (const event of this.automationEventHistory) { + if (event.ts >= cutoff) { + listener(event); + } + } + } + return () => this.automationEvents.off("event", listener); + } + + createAutomationTab(url = browserHomeUrl): BuiltInBrowserTabState { + const tab = this.createTab(url); + const { view: _view, ...state } = tab; + return state; + } + + selectAutomationTab(tabId: string): BuiltInBrowserState { + this.selectTab(tabId); + return this.getState(); + } + + closeAutomationTab(tabId: string): BuiltInBrowserState { + this.closeTab(tabId); + return this.getState(); + } + + async navigateAutomationTab(url: string, tabId?: string): Promise { + const tab = this.getTab(tabId); + if (!tab) { + throw new Error("Browser tab was not found."); + } + + const nextUrl = normalizeBrowserUrl(url); + tab.url = nextUrl; + if (tab.id === this.activeTabId) { + this.layoutActiveView(); + } + this.sendState(); + void tab.view.webContents.loadURL(nextUrl).catch((error) => { + this.emitAutomationEvent({ + kind: "page.load_failed", + summary: `Navigation request failed: ${formatError(error)}`, + tabId: tab.id, + title: tab.title, + url: nextUrl + }); + }); + this.emitAutomationEvent({ + kind: "page.navigation_requested", + summary: `Navigation requested: ${nextUrl}`, + tabId: tab.id, + title: tab.title, + url: nextUrl + }); + return this.getState(); + } + + goBackAutomationTab(tabId?: string): BuiltInBrowserState { + const tab = this.getTab(tabId); + tab?.view.webContents.navigationHistory.goBack(); + if (tab) { + this.emitAutomationEvent({ + kind: "tab.go_back", + summary: `Go back in tab ${tab.id}.`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); + } + return this.getState(); + } + + goForwardAutomationTab(tabId?: string): BuiltInBrowserState { + const tab = this.getTab(tabId); + tab?.view.webContents.navigationHistory.goForward(); + if (tab) { + this.emitAutomationEvent({ + kind: "tab.go_forward", + summary: `Go forward in tab ${tab.id}.`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); + } + return this.getState(); + } + + reloadAutomationTab(tabId?: string): BuiltInBrowserState { + const tab = this.getTab(tabId); + tab?.view.webContents.reload(); + if (tab) { + this.emitAutomationEvent({ + kind: "tab.reload", + summary: `Reload tab ${tab.id}.`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); + } + return this.getState(); + } + + getAutomationWebContents(tabId?: string): WebContents { + const tab = this.getTab(tabId); + if (!tab || tab.view.webContents.isDestroyed()) { + throw new Error("Browser tab is unavailable."); + } + return tab.view.webContents; } private registerIpcHandlers(): void { @@ -110,6 +336,14 @@ class BuiltInBrowserService { this.assertBrowserSender(event); return this.getState(); }); + ipcMain.handle(IPC_CHANNELS.browserStartChromeLoginImport, async (event, request: ChromeLoginImportRequest) => { + this.assertBrowserSender(event); + return await chromeLoginImportService.createJob(request); + }); + ipcMain.handle(IPC_CHANNELS.browserGetChromeLoginImport, (event, id: string) => { + this.assertBrowserSender(event); + return chromeLoginImportService.getJob(typeof id === "string" ? id : ""); + }); ipcMain.handle(IPC_CHANNELS.browserNewTab, (event, url?: string) => { this.assertBrowserSender(event); this.createTab(url); @@ -145,6 +379,34 @@ class BuiltInBrowserService { this.getTab(tabId)?.view.webContents.reload(); return this.getState(); }); + ipcMain.handle(IPC_CHANNELS.browserResolveAutomationHandoff, (event, status?: string) => { + this.assertBrowserSender(event); + return this.resolveAutomationHandoff(status === "dismissed" ? "dismissed" : "completed"); + }); + } + + private ensureWindow(): BrowserWindow { + const window = this.window && !this.window.isDestroyed() ? this.window : this.createWindow(); + if (this.tabs.size === 0) { + this.createTab(browserHomeUrl); + } + return window; + } + + private showAutomationWindow(): void { + const window = this.ensureWindow(); + if (window.isMinimized()) { + window.restore(); + } + window.show(); + window.focus(); + } + + private hideAutomationWindow(): void { + const window = this.window; + if (window && !window.isDestroyed()) { + window.hide(); + } } private createWindow(): BrowserWindow { @@ -187,15 +449,12 @@ class BuiltInBrowserService { this.window = undefined; } }); - window.once("ready-to-show", () => { - if (!window.isDestroyed()) { - window.show(); - } - }); window.webContents.setWindowOpenHandler(() => ({ action: "deny" })); window.webContents.on("did-finish-load", () => this.sendState()); - void window.loadURL(this.resolveRendererUrl("pages/browser/index.html")); + void window.loadURL(this.resolveRendererUrl("pages/browser/index.html")).catch((error) => { + console.warn(`[browser] Failed to load browser chrome: ${formatError(error)}`); + }); return window; } @@ -225,8 +484,23 @@ class BuiltInBrowserService { this.window?.contentView.addChildView(view); view.setVisible(false); this.selectTab(tab.id); - void view.webContents.loadURL(tab.url); + void view.webContents.loadURL(tab.url).catch((error) => { + this.emitAutomationEvent({ + kind: "page.load_failed", + summary: `Initial tab load failed: ${formatError(error)}`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); + }); this.sendState(); + this.emitAutomationEvent({ + kind: "tab.created", + summary: `Created tab ${tab.id}.`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); return tab; } @@ -234,7 +508,14 @@ class BuiltInBrowserService { const { webContents } = tab.view; webContents.setWindowOpenHandler(({ url }) => { if (isHttpUrl(url)) { - this.createTab(url); + const opened = this.createTab(url); + this.emitAutomationEvent({ + kind: "tab.opened", + summary: `Opened new tab from window.open: ${url}`, + tabId: opened.id, + title: opened.title, + url: opened.url + }); } return { action: "deny" }; }); @@ -244,14 +525,35 @@ class BuiltInBrowserService { webContents.on("page-title-updated", (_event, title) => { tab.title = title || titleFromUrl(tab.url); this.sendState(); + this.emitAutomationEvent({ + kind: "page.title", + summary: `Title updated: ${tab.title}`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); }); webContents.on("did-start-loading", () => { tab.isLoading = true; this.updateTabNavigationState(tab); + this.emitAutomationEvent({ + kind: "page.loading_started", + summary: `Loading started: ${tab.url}`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); }); webContents.on("did-stop-loading", () => { tab.isLoading = false; this.updateTabNavigationState(tab); + this.emitAutomationEvent({ + kind: "page.loading_stopped", + summary: `Loading stopped: ${tab.url}`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); }); webContents.on("did-navigate", (_event, url) => { tab.url = url; @@ -260,6 +562,13 @@ class BuiltInBrowserService { this.layoutActiveView(); } this.updateTabNavigationState(tab); + this.emitAutomationEvent({ + kind: "page.navigation", + summary: `Navigated to ${url}`, + tabId: tab.id, + title: tab.title, + url + }); }); webContents.on("did-navigate-in-page", (_event, url) => { tab.url = url; @@ -267,8 +576,15 @@ class BuiltInBrowserService { this.layoutActiveView(); } this.updateTabNavigationState(tab); + this.emitAutomationEvent({ + kind: "page.navigation_in_page", + summary: `In-page navigation to ${url}`, + tabId: tab.id, + title: tab.title, + url + }); }); - webContents.on("did-fail-load", (_event, errorCode, _errorDescription, validatedUrl) => { + webContents.on("did-fail-load", (_event, errorCode, errorDescription, validatedUrl) => { if (errorCode !== -3) { tab.isLoading = false; tab.url = validatedUrl || tab.url; @@ -276,6 +592,35 @@ class BuiltInBrowserService { this.layoutActiveView(); } this.updateTabNavigationState(tab); + this.emitAutomationEvent({ + errorCode, + errorDescription, + kind: "page.load_failed", + summary: `Load failed (${errorCode}): ${errorDescription}`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); + } + }); + webContents.on("dom-ready", () => { + this.emitAutomationEvent({ + kind: "page.dom_ready", + summary: `DOM ready: ${tab.url}`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); + }); + webContents.on("console-message", (_event, level, message) => { + if (level >= 2) { + this.emitAutomationEvent({ + kind: "runtime.console", + summary: message, + tabId: tab.id, + title: tab.title, + url: tab.url + }); } }); webContents.on("destroyed", () => { @@ -286,6 +631,13 @@ class BuiltInBrowserService { this.activeTabId = this.tabOrder[0]; } this.sendState(); + this.emitAutomationEvent({ + kind: "tab.destroyed", + summary: `Destroyed tab ${tab.id}.`, + tabId: tab.id, + title: tab.title, + url: tab.url + }); } }); } @@ -403,6 +755,13 @@ class BuiltInBrowserService { selected.view.webContents.focus(); } this.sendState(); + this.emitAutomationEvent({ + kind: "tab.activated", + summary: `Activated tab ${tabId}.`, + tabId, + title: selected.title, + url: selected.url + }); } private closeTab(tabId: string): void { @@ -415,6 +774,13 @@ class BuiltInBrowserService { this.tabs.delete(tabId); this.tabOrder = this.tabOrder.filter((id) => id !== tabId); tab.view.webContents.close({ waitForBeforeUnload: false }); + this.emitAutomationEvent({ + kind: "tab.closed", + summary: `Closed tab ${tabId}.`, + tabId, + title: tab.title, + url: tab.url + }); if (this.tabOrder.length === 0) { this.createTab(browserHomeUrl); @@ -440,8 +806,23 @@ class BuiltInBrowserService { if (tab.id === this.activeTabId) { this.layoutActiveView(); } - void tab.view.webContents.loadURL(nextUrl); + void tab.view.webContents.loadURL(nextUrl).catch((error) => { + this.emitAutomationEvent({ + kind: "page.load_failed", + summary: `Navigation request failed: ${formatError(error)}`, + tabId: tab.id, + title: tab.title, + url: nextUrl + }); + }); this.sendState(); + this.emitAutomationEvent({ + kind: "page.navigation_requested", + summary: `Navigation requested: ${nextUrl}`, + tabId: tab.id, + title: tab.title, + url: nextUrl + }); } private getTab(tabId?: string): BrowserTab | undefined { @@ -469,17 +850,22 @@ class BuiltInBrowserService { const { height, width } = window.getContentBounds(); activeTab.view.setVisible(true); activeTab.view.setBounds({ - height: Math.max(0, height - browserChromeHeight), + height: Math.max(0, height - this.browserChromeHeight()), width, x: 0, - y: browserChromeHeight + y: this.browserChromeHeight() }); } + private browserChromeHeight(): number { + return browserChromeBaseHeight + (this.automationHandoff ? browserHandoffToolbarHeight : 0); + } + private getState(): BuiltInBrowserState { return { activeTabId: this.activeTabId, apps: this.apps.map((app) => ({ ...app })), + ...(this.automationHandoff ? { automationHandoff: { ...this.automationHandoff } } : {}), tabs: this.tabOrder .map((id) => this.tabs.get(id)) .filter((tab): tab is BrowserTab => Boolean(tab)) @@ -512,6 +898,27 @@ class BuiltInBrowserService { this.activeTabId = undefined; } + private emitAutomationEvent(event: Omit): void { + const nextEvent: BrowserAutomationEvent = { + ...event, + seq: ++this.automationEventSeq, + ts: Date.now(), + windowId: browserAutomationWindowId + }; + this.automationEventHistory.push(nextEvent); + this.pruneAutomationEventHistory(nextEvent.ts); + this.automationEvents.emit("event", nextEvent); + } + + private pruneAutomationEventHistory(now = Date.now()): void { + while (this.automationEventHistory.length > 0 && now - this.automationEventHistory[0].ts > maxAutomationEventAgeMs) { + this.automationEventHistory.shift(); + } + if (this.automationEventHistory.length > maxAutomationEventHistory) { + this.automationEventHistory.splice(0, this.automationEventHistory.length - maxAutomationEventHistory); + } + } + private resolveRendererUrl(relativeHtmlPath: string): string { return pathToFileURL(path.join(__dirname, "../renderer", relativeHtmlPath)).toString(); } @@ -619,9 +1026,22 @@ function titleFromUrl(value: string): string { } } -function electronProxyRules(endpoint: string): string { - const parsed = new URL(endpoint); - const host = parsed.hostname.includes(":") ? `[${parsed.hostname}]` : parsed.hostname; - const port = parsed.port || (parsed.protocol === "https:" ? "443" : "80"); - return `http=${host}:${port};https=${host}:${port}`; +function defaultAutomationHandoffMessage(kind?: BuiltInBrowserAutomationHandoffKind): string { + if (kind === "login_required") { + return "Please sign in on this page, then click Done."; + } + if (kind === "verification_code") { + return "Please enter the verification code, then click Done."; + } + if (kind === "human_verification") { + return "Please complete the human verification, then click Done."; + } + if (kind === "blocked") { + return "Please resolve the blocker on this page, then click Done."; + } + return "Please complete the requested browser step, then click Done."; +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); } diff --git a/packages/electron/src/main/chrome-login-import.ts b/packages/electron/src/main/chrome-login-import.ts new file mode 100644 index 00000000..850dd171 --- /dev/null +++ b/packages/electron/src/main/chrome-login-import.ts @@ -0,0 +1,593 @@ +import { randomUUID } from "node:crypto"; +import http, { type IncomingMessage, type Server, type ServerResponse } from "node:http"; +import { BrowserWindow, session, shell } from "electron"; +import type { + ChromeLoginImportCookie, + ChromeLoginImportJob, + ChromeLoginImportLocalStorage, + ChromeLoginImportRequest, + ChromeLoginImportResult, + ChromeLoginImportTarget +} from "@ccr/core/contracts/app"; + +type ServerInfo = { + server: Server; + url: string; +}; + +type StoredChromeLoginImportJob = ChromeLoginImportJob; + +type CookieSetDetails = Parameters[0]; + +const browserPartition = "persist:ccr-built-in-browser"; +const webSearchPartition = "persist:ccr-browser-web-search-mcp"; +const importJobTtlMs = 5 * 60_000; +const maxImportRequestBytes = 16 * 1024 * 1024; +const maxStoredErrors = 20; +const localStorageWriteTimeoutMs = 15_000; + +class ChromeLoginImportService { + private readonly jobs = new Map(); + private serverInfo?: ServerInfo; + private serverStartPromise?: Promise; + + async createJob(request: ChromeLoginImportRequest): Promise { + const domains = uniqueStrings(request.domains.map(normalizeImportDomain).filter((item): item is string => Boolean(item))); + if (domains.length === 0) { + throw new Error("At least one Chrome import domain is required."); + } + + const target = normalizeImportTarget(request.target); + const serverInfo = await this.ensureServer(); + const id = randomUUID(); + const now = Date.now(); + const job: StoredChromeLoginImportJob = { + confirmUrl: `${serverInfo.url}/chrome-import/jobs/${id}/confirm`, + createdAt: now, + domains, + endpointUrl: serverInfo.url, + expiresAt: now + importJobTtlMs, + id, + importUrl: `${serverInfo.url}/chrome-import/jobs/${id}`, + status: "pending", + target + }; + this.jobs.set(id, job); + this.pruneJobs(); + if (request.openConfirmationPage) { + await this.openConfirmationPage(job.id); + } + return cloneJob(job); + } + + getJob(id: string): ChromeLoginImportJob | undefined { + const job = this.jobs.get(id); + if (!job) { + return undefined; + } + this.refreshExpiredJob(job); + return cloneJob(job); + } + + async openConfirmationPage(id: string): Promise { + const job = this.jobs.get(id); + if (!job) { + throw new Error("Chrome login import job was not found."); + } + this.refreshExpiredJob(job); + if (job.status !== "pending") { + throw new Error(`Chrome login import job is ${job.status}.`); + } + await shell.openExternal(job.confirmUrl); + return cloneJob(job); + } + + private async ensureServer(): Promise { + if (this.serverInfo) { + return this.serverInfo; + } + if (this.serverStartPromise) { + return this.serverStartPromise; + } + + this.serverStartPromise = new Promise((resolve, reject) => { + const server = http.createServer((request, response) => { + void this.handleRequest(request, response).catch((error) => { + sendJson(response, 500, { error: { message: formatError(error) } }); + }); + }); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + server.off("error", reject); + const address = server.address(); + if (!address || typeof address === "string") { + server.close(); + reject(new Error("Chrome login import server failed to start.")); + return; + } + const info = { + server, + url: `http://127.0.0.1:${address.port}` + }; + this.serverInfo = info; + resolve(info); + }); + }).finally(() => { + this.serverStartPromise = undefined; + }); + return this.serverStartPromise; + } + + private async handleRequest(request: IncomingMessage, response: ServerResponse): Promise { + setCorsHeaders(response); + if (request.method === "OPTIONS") { + response.writeHead(204); + response.end(); + return; + } + + const path = request.url ? new URL(request.url, "http://127.0.0.1").pathname : "/"; + const match = path.match(/^\/chrome-import\/jobs\/([^/]+)(?:\/(confirm|cookies))?$/); + if (!match) { + sendJson(response, 404, { error: { message: "Chrome login import endpoint not found." } }); + return; + } + + const id = decodeURIComponent(match[1]); + const job = this.jobs.get(id); + if (!job) { + sendJson(response, 404, { error: { message: "Chrome login import job was not found." } }); + return; + } + this.refreshExpiredJob(job); + if (job.status === "expired") { + sendJson(response, 410, { error: { message: "Chrome login import job expired." }, job: cloneJob(job) }); + return; + } + + const action = match[2]; + if (request.method === "GET" && action === "confirm") { + sendHtml(response, 200, confirmationPageHtml(job)); + return; + } + + if (request.method === "GET" && !action) { + sendJson(response, 200, { job: cloneJob(job) }); + return; + } + + if (request.method === "POST" && action === "cookies") { + if (job.status !== "pending") { + sendJson(response, 409, { + error: { + message: `Chrome login import job is ${job.status}.` + }, + job: cloneJob(job) + }); + return; + } + const payload = await readJsonRequest(request); + const result = await this.importLoginState(job, payload); + sendJson(response, 200, { job: cloneJob(job), result }); + return; + } + + sendJson(response, 405, { error: { message: "Unsupported Chrome login import method." } }); + } + + private async importLoginState(job: StoredChromeLoginImportJob, payload: unknown): Promise { + const importPayload = readImportPayload(payload); + const partitions = targetPartitions(job.target); + const result: ChromeLoginImportResult = { + completedAt: Date.now(), + cookieImported: 0, + cookieSkipped: 0, + domains: [...job.domains], + imported: 0, + localStorageImported: 0, + localStorageSkipped: 0, + partitions, + skipped: 0 + }; + const errors: string[] = []; + + for (const cookie of importPayload.cookies) { + const normalized = normalizeChromeCookie(cookie, job.domains); + if (!normalized) { + result.cookieSkipped += 1; + continue; + } + + try { + await Promise.all(partitions.map((partition) => session.fromPartition(partition).cookies.set(normalized))); + result.cookieImported += 1; + } catch (error) { + result.cookieSkipped += 1; + if (errors.length < maxStoredErrors) { + errors.push(`${normalized.domain || new URL(normalized.url).hostname}:${normalized.name}: ${formatError(error)}`); + } + } + } + + await Promise.all(partitions.map((partition) => session.fromPartition(partition).cookies.flushStore())); + + for (const localStorageEntry of importPayload.localStorage) { + const normalized = normalizeLocalStorageEntry(localStorageEntry, job.domains); + if (!normalized) { + result.localStorageSkipped += 1; + continue; + } + + const itemCount = Object.keys(normalized.items).length; + if (itemCount === 0) { + result.localStorageSkipped += 1; + continue; + } + + try { + await Promise.all(partitions.map((partition) => writeLocalStorage(partition, normalized.origin, normalized.items))); + result.localStorageImported += itemCount; + } catch (error) { + result.localStorageSkipped += itemCount; + if (errors.length < maxStoredErrors) { + errors.push(`${normalized.origin}: localStorage: ${formatError(error)}`); + } + } + } + + result.imported = result.cookieImported + result.localStorageImported; + result.skipped = result.cookieSkipped + result.localStorageSkipped; + if (errors.length > 0) { + result.errors = errors; + } + job.result = result; + job.status = result.imported > 0 || result.skipped > 0 ? "completed" : "failed"; + return result; + } + + private refreshExpiredJob(job: StoredChromeLoginImportJob): void { + if (job.status === "pending" && Date.now() > job.expiresAt) { + job.status = "expired"; + } + } + + private pruneJobs(): void { + const cutoff = Date.now() - importJobTtlMs; + for (const [id, job] of this.jobs) { + this.refreshExpiredJob(job); + if (job.expiresAt < cutoff) { + this.jobs.delete(id); + } + } + } +} + +export const chromeLoginImportService = new ChromeLoginImportService(); + +function targetPartitions(target: ChromeLoginImportTarget): string[] { + return target === "browser-and-web-search" + ? [browserPartition, webSearchPartition] + : [browserPartition]; +} + +function normalizeChromeCookie(cookie: ChromeLoginImportCookie, allowedDomains: string[]): CookieSetDetails | undefined { + if (!isRecord(cookie) || typeof cookie.name !== "string" || typeof cookie.value !== "string") { + return undefined; + } + if (cookie.partitionKey !== undefined) { + return undefined; + } + + const rawDomain = readString(cookie.domain).toLowerCase(); + const cookieHost = normalizeCookieHost(rawDomain); + if (!cookieHost || !allowedDomains.some((domain) => cookieDomainMatches(cookieHost, domain))) { + return undefined; + } + + const path = normalizeCookiePath(cookie.path); + const expirationDate = typeof cookie.expirationDate === "number" && Number.isFinite(cookie.expirationDate) + ? cookie.expirationDate + : undefined; + if (expirationDate !== undefined && expirationDate <= Date.now() / 1000) { + return undefined; + } + + const secure = cookie.secure === true; + const details: CookieSetDetails = { + httpOnly: cookie.httpOnly === true, + name: cookie.name, + path, + secure, + url: cookieUrl(cookieHost, path, secure), + value: cookie.value + }; + if (cookie.hostOnly !== true) { + details.domain = rawDomain || cookieHost; + } + if (expirationDate !== undefined && cookie.session !== true) { + details.expirationDate = expirationDate; + } + const sameSite = normalizeSameSite(cookie.sameSite); + if (sameSite) { + details.sameSite = sameSite; + } + return details; +} + +function normalizeLocalStorageEntry( + entry: ChromeLoginImportLocalStorage, + allowedDomains: string[] +): ChromeLoginImportLocalStorage | undefined { + if (!isRecord(entry) || !isRecord(entry.items)) { + return undefined; + } + + let origin: URL; + try { + origin = new URL(readString(entry.origin)); + } catch { + return undefined; + } + + if (!["http:", "https:"].includes(origin.protocol)) { + return undefined; + } + if (!allowedDomains.some((domain) => cookieDomainMatches(origin.hostname.toLowerCase(), domain))) { + return undefined; + } + + const items: Record = {}; + for (const [key, value] of Object.entries(entry.items)) { + if (typeof key === "string" && typeof value === "string") { + items[key] = value; + } + } + return { + items, + origin: origin.origin + }; +} + +async function writeLocalStorage(partition: string, origin: string, items: Record): Promise { + const window = new BrowserWindow({ + height: 480, + paintWhenInitiallyHidden: true, + show: false, + skipTaskbar: true, + title: "CCR Chrome Login Import", + webPreferences: { + contextIsolation: true, + nodeIntegration: false, + partition, + sandbox: true, + webSecurity: true + }, + width: 640 + }); + try { + await withTimeout(window.webContents.loadURL(`${origin}/`), localStorageWriteTimeoutMs, "Timed out loading localStorage origin."); + const loadedOrigin = new URL(window.webContents.getURL()).origin; + if (loadedOrigin !== origin) { + throw new Error(`Origin redirected to ${loadedOrigin}.`); + } + const entries = Object.entries(items); + await window.webContents.executeJavaScript( + `(() => { + const entries = ${JSON.stringify(entries)}; + for (const [key, value] of entries) { + window.localStorage.setItem(key, value); + } + return entries.length; + })()`, + true + ); + } finally { + if (!window.isDestroyed()) { + window.destroy(); + } + } +} + +function normalizeImportTarget(value: unknown): ChromeLoginImportTarget { + return value === "browser-and-web-search" ? "browser-and-web-search" : "browser"; +} + +function normalizeImportDomain(value: unknown): string | undefined { + const raw = readString(value).toLowerCase(); + if (!raw) { + return undefined; + } + try { + const parsed = new URL(raw.includes("://") ? raw : `https://${raw}`); + return normalizeCookieHost(parsed.hostname); + } catch { + return normalizeCookieHost(raw.replace(/^\*\./, "").split("/")[0] || ""); + } +} + +function normalizeCookieHost(value: string): string | undefined { + const host = value.trim().replace(/^\./, "").toLowerCase(); + if (!host || host.includes("*") || host.includes("/") || host.includes(" ")) { + return undefined; + } + return host; +} + +function normalizeCookiePath(value: unknown): string { + const path = readString(value); + return path.startsWith("/") ? path : "/"; +} + +function cookieUrl(host: string, path: string, secure: boolean): string { + const normalizedPath = path.startsWith("/") ? path : "/"; + return `${secure ? "https" : "http"}://${host}${normalizedPath}`; +} + +function cookieDomainMatches(cookieHost: string, allowedDomain: string): boolean { + return cookieHost === allowedDomain || cookieHost.endsWith(`.${allowedDomain}`); +} + +function normalizeSameSite(value: unknown): CookieSetDetails["sameSite"] | undefined { + return value === "lax" || value === "strict" || value === "no_restriction" || value === "unspecified" + ? value + : undefined; +} + +function readImportPayload(payload: unknown): { + cookies: ChromeLoginImportCookie[]; + localStorage: ChromeLoginImportLocalStorage[]; +} { + if (!isRecord(payload)) { + throw new Error("Chrome login import payload must be an object."); + } + const cookies = Array.isArray(payload.cookies) + ? payload.cookies.filter(isRecord) as ChromeLoginImportCookie[] + : []; + const localStorage = Array.isArray(payload.localStorage) + ? payload.localStorage.filter(isRecord) as ChromeLoginImportLocalStorage[] + : []; + if (cookies.length === 0 && localStorage.length === 0) { + throw new Error("Chrome login import payload must include cookies or localStorage."); + } + return { cookies, localStorage }; +} + +function cloneJob(job: StoredChromeLoginImportJob): ChromeLoginImportJob { + return { + ...job, + domains: [...job.domains], + ...(job.result + ? { + result: { + ...job.result, + domains: [...job.result.domains], + ...(job.result.errors ? { errors: [...job.result.errors] } : {}), + partitions: [...job.result.partitions] + } + } + : {}) + }; +} + +async function readJsonRequest(request: IncomingMessage): Promise { + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of request) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk); + total += buffer.byteLength; + if (total > maxImportRequestBytes) { + throw new Error("Chrome login import payload is too large."); + } + chunks.push(buffer); + } + return JSON.parse(Buffer.concat(chunks).toString("utf8")) as unknown; +} + +function sendJson(response: ServerResponse, statusCode: number, body: unknown): void { + if (!response.headersSent) { + response.writeHead(statusCode, { "content-type": "application/json" }); + } + response.end(`${JSON.stringify(body)}\n`); +} + +function sendHtml(response: ServerResponse, statusCode: number, body: string): void { + if (!response.headersSent) { + response.writeHead(statusCode, { + "content-security-policy": "default-src 'none'; style-src 'unsafe-inline'; script-src 'none'; connect-src http://127.0.0.1:* http://localhost:*", + "content-type": "text/html; charset=utf-8" + }); + } + response.end(body); +} + +function confirmationPageHtml(job: StoredChromeLoginImportJob): string { + const domains = job.domains.map((domain) => `
  • ${escapeHtml(domain)}
  • `).join(""); + return ` + + + + + CCR Chrome Login Import + + + +
    +

    Import Chrome Login State into CCR

    +

    CCR is requesting permission to import cookies and localStorage for these domains into the in-app browser.

    +
      ${domains}
    + +
    Install or enable the CCR Login Import extension in Chrome to continue.
    +
    + +`; +} + +function setCorsHeaders(response: ServerResponse): void { + response.setHeader("access-control-allow-headers", "content-type, x-ccr-login-import"); + response.setHeader("access-control-allow-methods", "GET, POST, OPTIONS"); + response.setHeader("access-control-allow-origin", "*"); +} + +function uniqueStrings(values: string[]): string[] { + return [...new Set(values)]; +} + +function readString(value: unknown): string { + return typeof value === "string" ? value.trim() : ""; +} + +function isRecord(value: unknown): value is Record { + return Boolean(value) && typeof value === "object" && !Array.isArray(value); +} + +function escapeHtml(value: string): string { + return value.replace(/[&<>"']/g, (char) => { + switch (char) { + case "&": + return "&"; + case "<": + return "<"; + case ">": + return ">"; + case "\"": + return """; + default: + return "'"; + } + }); +} + +function withTimeout(promise: Promise, timeoutMs: number, message: string): Promise { + let timeout: NodeJS.Timeout | undefined; + const timeoutPromise = new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error(message)), timeoutMs); + }); + return Promise.race([promise, timeoutPromise]).finally(() => { + if (timeout) { + clearTimeout(timeout); + } + }); +} + +function formatError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} diff --git a/packages/electron/src/main/electron-web-search-mcp.ts b/packages/electron/src/main/electron-web-search-mcp.ts index 0badf3b0..e59ec3a8 100644 --- a/packages/electron/src/main/electron-web-search-mcp.ts +++ b/packages/electron/src/main/electron-web-search-mcp.ts @@ -70,6 +70,7 @@ type BrowserSearchRequest = BrowserSearchInput & { type BrowserSearchResult = { content?: string; + diagnostics?: string[]; snippet?: string; title: string; url: string; @@ -597,11 +598,15 @@ class HiddenBrowserSearchPool { return { ...result, ...(content ? { content } : {}), + ...(!content ? { diagnostics: appendSearchResultDiagnostic(result.diagnostics, "No extractable page content found.") } : {}), ...(snippet ? { snippet } : {}), ...(page.title && page.title.length > result.title.length ? { title: page.title.slice(0, 240) } : {}) }; - } catch { - return result; + } catch (error) { + return { + ...result, + diagnostics: appendSearchResultDiagnostic(result.diagnostics, `Page extraction failed: ${formatError(error)}`) + }; } finally { if (worker) { this.release(worker); @@ -635,7 +640,16 @@ async function pollSearchResults(webContents: WebContents, request: BrowserSearc const deadline = Date.now() + request.timeoutMs; let last: BrowserSearchPageResult = { results: [] }; while (Date.now() < deadline) { - last = await webContents.executeJavaScript(searchResultExtractionScript(request.engine, request.count), true) as BrowserSearchPageResult; + try { + last = await withTimeout( + webContents.executeJavaScript(searchResultExtractionScript(request.engine, request.count), true) as Promise, + Math.max(100, Math.min(1000, deadline - Date.now())), + "Browser search result extraction timed out." + ); + } catch { + await sleep(150); + continue; + } if (last.blocked || (last.results?.length ?? 0) >= Math.min(request.count, 3)) { return last; } @@ -823,18 +837,46 @@ function searchResultExtractionScript(engine: BrowserSearchEngine, count: number const host = url.hostname.toLowerCase(); const path = url.pathname.toLowerCase(); if (host === location.hostname.toLowerCase() && (path === "/" || path === "/search" || path.startsWith("/search/"))) return true; - if (host.includes("google.") && /\\/(preferences|advanced_search|intl|policies|support|sorry)/.test(path)) return true; - if (host.endsWith("bing.com") && /\\/(search|account|profile|images|videos|maps|news)/.test(path)) return true; - if (host.endsWith("duckduckgo.com") && path === "/") return true; + if (host.includes("google.") && /\\/(preferences|advanced_search|intl|policies|support|sorry|search|aclk|imgres)/.test(path)) return true; + if (host.endsWith("bing.com") && /\\/(search|account|profile|images|videos|maps|news|aclick)/.test(path)) return true; + if (host.endsWith("bing.com") && path.startsWith("/ck/")) return true; + if (host.endsWith("duckduckgo.com") && (path === "/" || path.startsWith("/settings") || path.startsWith("/y.js"))) return true; return false; } function resultContainer(anchor, title) { + const root = anchor && anchor.closest("li.b_algo, li.b_ad, div.g, div[data-sokoban-container], div[data-testid='result'], article, .result, .result__body, .web-result"); + if (root) return root; let node = anchor; for (let depth = 0; node && depth < 8; depth += 1, node = node.parentElement) { + if (node.matches && node.matches("nav, header, footer, form, aside")) return undefined; + if (node.matches && node.matches("main, #search, #b_results, #rso")) break; const text = compact(node.textContent || ""); if (text.length > title.length + 40) return node; } - return anchor.closest("li.b_algo, article, div.g, div[data-sokoban-container], div[data-testid='result'], .result, li, div"); + return undefined; + } + function isLikelyOrganicResult(anchor, container) { + if (!anchor || !container) return false; + if (anchor.closest("nav, header, footer, form, aside")) return false; + if (anchor.closest("li.b_algo, div.g, div[data-sokoban-container], div[data-testid='result'], article, .result, .result__body, .web-result")) return true; + const heading = anchor.closest("h1, h2, h3, [role='heading']"); + const searchRegion = anchor.closest("main, #search, #b_results, #rso"); + return Boolean(heading && searchRegion); + } + function isAdOrUtilityResult(anchor, container) { + let node = anchor; + for (let depth = 0; node && depth < 6; depth += 1, node = node.parentElement) { + const attributes = [ + node.getAttribute && node.getAttribute("aria-label"), + node.getAttribute && node.getAttribute("data-text-ad"), + node.getAttribute && node.getAttribute("data-testid"), + node.className + ].filter(Boolean).join(" "); + if (/\\b(?:ad|ads|sponsored|promo|shopping)\\b|广告|赞助|推廣/i.test(attributes)) return true; + } + const firstText = compact(container && container.textContent).slice(0, 180); + if (/^(?:ad|ads|sponsored|promo|shopping)\\b|^(?:广告|赞助|推廣)/i.test(firstText)) return true; + return false; } function cleanSnippetCandidate(value, title, url) { const host = (() => { try { return new URL(url).hostname.replace(/^www\\./, ""); } catch (_) { return ""; } })(); @@ -881,6 +923,7 @@ function searchResultExtractionScript(engine: BrowserSearchEngine, count: number const title = titleFor(anchor, titleOverride); if (!title || title.length < 2) return; const container = resultContainer(anchor, title); + if (!isLikelyOrganicResult(anchor, container) || isAdOrUtilityResult(anchor, container)) return; seen.add(url); results.push({ snippet: snippetFor(anchor, container, title, url), @@ -908,19 +951,22 @@ function searchResultExtractionScript(engine: BrowserSearchEngine, count: number collect(".result__a"); collect("article h2 a[href]"); } + function collectGenericHeadings() { + collect("main h1 a[href], main h2 a[href], main h3 a[href], #search h3, #b_results h2 a[href], #rso h3, article h2 a[href], article h3 a[href]"); + } if (engine === "google") collectGoogle(); if (engine === "bing") collectBing(); if (engine === "duckduckgo") collectDuckDuckGo(); - collect("main a[href], #search a[href], #b_results a[href], article a[href], a[href]"); + collectGenericHeadings(); return { results, title: document.title || "" }; function detectBlocked() { const text = compact(document.body && document.body.innerText).toLowerCase(); if (!text) return ""; - if (text.includes("unusual traffic") || text.includes("detected unusual") || text.includes("not a robot") || text.includes("captcha")) { + if (text.includes("unusual traffic") || text.includes("detected unusual") || text.includes("not a robot") || text.includes("captcha") || text.includes("验证码") || text.includes("人机验证")) { return "Search engine returned an anti-bot or CAPTCHA page."; } - if (text.includes("before you continue to google") || text.includes("consent.google")) { + if (text.includes("before you continue to google") || text.includes("consent.google") || text.includes("cookie consent")) { return "Google returned a consent page instead of search results."; } return ""; @@ -996,11 +1042,16 @@ function formatSearchResponse( `${index + 1}. ${result.title}`, `URL: ${result.url}`, result.snippet ? `Snippet: ${result.snippet}` : "", - result.content ? `Extracted content: ${result.content}` : "" + result.content ? `Extracted content: ${result.content}` : "", + result.diagnostics?.length ? `Diagnostics: ${result.diagnostics.join("; ")}` : "" ].filter(Boolean).join("\n")) ].filter(Boolean).join("\n\n"); } +function appendSearchResultDiagnostic(existing: string[] | undefined, message: string): string[] { + return uniqueStrings([...(existing ?? []), message]); +} + function uniqueSearchResults(results: BrowserSearchResult[]): BrowserSearchResult[] { const seen = new Set(); const unique: BrowserSearchResult[] = []; diff --git a/packages/electron/src/main/ipc.ts b/packages/electron/src/main/ipc.ts index 53afe4cb..a639fbc3 100644 --- a/packages/electron/src/main/ipc.ts +++ b/packages/electron/src/main/ipc.ts @@ -1,7 +1,6 @@ import { app, BrowserWindow, dialog, ipcMain, nativeImage, shell, type OpenDialogOptions, type Rectangle, type SaveDialogOptions } from "electron"; import { randomUUID } from "node:crypto"; import { existsSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; -import net from "node:net"; import path from "node:path"; import { deflateSync, inflateSync } from "node:zlib"; import { loadPersistedAppSetting, replacePersistedAppSetting } from "@ccr/core/config/app-config-store"; @@ -133,7 +132,6 @@ ipcMain.handle(IPC_CHANNELS.appListMcpServerTools, async (_event, serverName: st }); ipcMain.handle(IPC_CHANNELS.appOpenBuiltInBrowser, async () => { const config = await loadAppConfig(); - await ensureBuiltInBrowserProxyReady(config); await builtInBrowserService.open(config); }); ipcMain.handle(IPC_CHANNELS.appCloseTray, () => { @@ -361,157 +359,6 @@ function logProfileApplyResult(result: ProfileApplyResult): void { } } -type ProxyConnectProbeResult = { - detail?: string; - ok: boolean; -}; - -const browserProxyConnectProbeTarget = "claude.ai:443"; -const proxyConnectProbeTimeoutMs = 3000; - -async function ensureBuiltInBrowserProxyReady(config: AppConfig): Promise { - if (!config.proxy.enabled) { - return; - } - - let proxyStatus = proxyService.getStatus(); - if (proxyStatus.state === "running" && proxyStatus.endpoint) { - const probe = await probeProxyConnect(proxyStatus.endpoint); - if (probe.ok) { - return; - } - console.warn(`[browser] Proxy CONNECT probe failed at ${proxyStatus.endpoint}; restarting proxy: ${probe.detail || "unknown error"}`); - } - - const status = await gatewayService.start(config); - if (status.state === "error") { - throw new Error(status.lastError || "Failed to start proxy mode."); - } - - proxyStatus = proxyService.getStatus(); - if (proxyStatus.state !== "running" || !proxyStatus.endpoint) { - throw new Error(proxyStatus.lastError || "Proxy mode is not running."); - } - - const probe = await probeProxyConnect(proxyStatus.endpoint); - if (probe.ok) { - return; - } - - console.warn( - `[browser] Shared proxy endpoint ${proxyStatus.endpoint} still does not accept CONNECT after restart; starting dedicated proxy endpoint: ${probe.detail || "unknown error"}` - ); - const dedicatedProxyConfig = await createBuiltInBrowserProxyConfig(config); - const dedicatedProxyStatus = await proxyService.start(dedicatedProxyConfig); - if (dedicatedProxyStatus.state !== "running" || !dedicatedProxyStatus.endpoint) { - throw new Error(dedicatedProxyStatus.lastError || "Failed to start the dedicated proxy endpoint for the built-in browser."); - } - - const dedicatedProbe = await probeProxyConnect(dedicatedProxyStatus.endpoint); - if (!dedicatedProbe.ok) { - const detail = dedicatedProbe.detail ? `: ${dedicatedProbe.detail}` : ""; - throw new Error(`Proxy mode is running at ${dedicatedProxyStatus.endpoint}, but HTTPS CONNECT is not available${detail}.`); - } -} - -async function createBuiltInBrowserProxyConfig(config: AppConfig): Promise { - return { - ...config, - proxy: { - ...config.proxy, - host: "127.0.0.1", - port: await findAvailableLoopbackPort(), - systemProxy: false - } - }; -} - -function findAvailableLoopbackPort(): Promise { - return new Promise((resolve, reject) => { - const server = net.createServer(); - server.once("error", reject); - server.listen(0, "127.0.0.1", () => { - const address = server.address(); - server.close((error) => { - if (error) { - reject(error); - return; - } - if (!address || typeof address === "string") { - reject(new Error("Failed to allocate a local proxy port.")); - return; - } - resolve(address.port); - }); - }); - }); -} - -function probeProxyConnect(endpoint: string): Promise { - return new Promise((resolve) => { - let parsed: URL; - try { - parsed = new URL(endpoint); - } catch { - resolve({ detail: `Invalid proxy endpoint: ${endpoint}`, ok: false }); - return; - } - - const port = Number(parsed.port || (parsed.protocol === "https:" ? 443 : 80)); - if (!Number.isInteger(port) || port <= 0 || port > 65535) { - resolve({ detail: `Invalid proxy port in endpoint: ${endpoint}`, ok: false }); - return; - } - - const host = parsed.hostname.replace(/^\[|\]$/g, ""); - const socket = net.connect({ host, port }); - let response = ""; - let settled = false; - - const finish = (result: ProxyConnectProbeResult) => { - if (settled) { - return; - } - settled = true; - socket.destroy(); - resolve(result); - }; - const parseResponse = (): ProxyConnectProbeResult | undefined => { - const firstLine = response.split(/\r?\n/, 1)[0]?.trim(); - if (!firstLine) { - return undefined; - } - if (/^HTTP\/1\.[01]\s+200\b/i.test(firstLine)) { - return { ok: true }; - } - if (/^HTTP\/1\.[01]\s+\d{3}\b/i.test(firstLine)) { - return { detail: firstLine, ok: false }; - } - return { detail: `Unexpected response: ${firstLine}`, ok: false }; - }; - - socket.setTimeout(proxyConnectProbeTimeoutMs, () => { - finish({ detail: `Timed out after ${proxyConnectProbeTimeoutMs}ms`, ok: false }); - }); - socket.once("error", (error) => { - finish({ detail: formatError(error), ok: false }); - }); - socket.once("connect", () => { - socket.write(`CONNECT ${browserProxyConnectProbeTarget} HTTP/1.1\r\nHost: ${browserProxyConnectProbeTarget}\r\n\r\n`); - }); - socket.on("data", (chunk) => { - response += chunk.toString("latin1"); - const result = parseResponse(); - if (result) { - finish(result); - } - }); - socket.once("close", () => { - finish(parseResponse() ?? { detail: "Connection closed without a CONNECT response", ok: false }); - }); - }); -} - async function exportAppData(window: BrowserWindow | null): Promise { const exportedAt = new Date().toISOString(); const result = window diff --git a/packages/electron/src/main/main-app.ts b/packages/electron/src/main/main-app.ts index f71892a7..9f3ab9a9 100644 --- a/packages/electron/src/main/main-app.ts +++ b/packages/electron/src/main/main-app.ts @@ -11,6 +11,7 @@ import { syncLaunchAtLogin } from "./launch-at-login"; import { proxyService } from "@ccr/core/proxy/service"; import trayController from "./tray-controller"; import { appUpdateService } from "./update-service"; +import { browserAutomationMcpService } from "./browser-automation-mcp"; import { browserWebSearchMcpService } from "./electron-web-search-mcp"; import windowsManager from "./windows"; @@ -29,6 +30,7 @@ if (!gotTheLock) { } function startPrimaryInstance(): void { + gatewayService.setBrowserAutomationMcpIntegration(browserAutomationMcpService); gatewayService.setBrowserWebSearchMcpIntegration(browserWebSearchMcpService); deepLinkService.register(); deepLinkService.handleArgv(process.argv); diff --git a/packages/ui/src/pages/browser/main.tsx b/packages/ui/src/pages/browser/main.tsx index 578481e7..11b5e901 100644 --- a/packages/ui/src/pages/browser/main.tsx +++ b/packages/ui/src/pages/browser/main.tsx @@ -1,7 +1,7 @@ import { useEffect, useMemo, useState, type FormEvent } from "react"; import { createRoot } from "react-dom/client"; -import { ArrowLeft, ArrowRight, Globe2, LoaderCircle, Plus, RotateCw, Search, X } from "lucide-react"; -import type { BuiltInBrowserState } from "@ccr/core/contracts/app"; +import { ArrowLeft, ArrowRight, Check, KeyRound, LoaderCircle, Plus, RotateCw, Search, UserRound, X } from "lucide-react"; +import type { BuiltInBrowserState, ChromeLoginImportJob, ChromeLoginImportRequest } from "@ccr/core/contracts/app"; declare global { interface Window { @@ -9,11 +9,14 @@ declare global { back: (tabId?: string) => Promise; closeTab: (tabId: string) => Promise; forward: (tabId?: string) => Promise; + getChromeLoginImport: (id: string) => Promise; getState: () => Promise; navigate: (url: string, tabId?: string) => Promise; newTab: (url?: string) => Promise; reload: (tabId?: string) => Promise; + resolveAutomationHandoff: (status: "completed" | "dismissed") => Promise; selectTab: (tabId: string) => Promise; + startChromeLoginImport: (request: ChromeLoginImportRequest) => Promise; onStateChanged: (callback: (state: BuiltInBrowserState) => void) => () => void; }; } @@ -29,10 +32,12 @@ function BrowserChrome() { const [state, setState] = useState(emptyState); const [addressDraft, setAddressDraft] = useState(""); const [homeDraft, setHomeDraft] = useState(""); + const [chromeImportJob, setChromeImportJob] = useState(); const activeTab = useMemo( () => state.tabs.find((tab) => tab.id === state.activeTabId), [state.activeTabId, state.tabs] ); + const handoff = state.automationHandoff; const homeVisible = activeTab?.url === browserHomeUrl; useEffect(() => { @@ -57,6 +62,25 @@ function BrowserChrome() { setHomeDraft(""); }, [activeTab?.id]); + useEffect(() => { + if (!chromeImportJob || chromeImportJob.status !== "pending") { + return; + } + const interval = window.setInterval(() => { + void window.ccrBrowser?.getChromeLoginImport(chromeImportJob.id).then((job) => { + if (!job) { + setChromeImportJob(undefined); + return; + } + setChromeImportJob(job); + if (job.status === "completed" && activeTab?.id && activeTabMatchesImport(activeTab.url, job.domains)) { + void run(window.ccrBrowser?.reload(activeTab.id)); + } + }); + }, 2000); + return () => window.clearInterval(interval); + }, [activeTab?.id, activeTab?.url, chromeImportJob]); + async function run(action: Promise | undefined): Promise { if (!action) { return; @@ -79,8 +103,53 @@ function BrowserChrome() { void run(window.ccrBrowser?.navigate(url, activeTab?.id)); } + async function startChromeLoginImport() { + const defaultDomain = activeTabDomain(activeTab?.url); + const rawDomains = window.prompt("Chrome domain(s) to import. Separate multiple domains with commas.", defaultDomain); + if (!rawDomains) { + return; + } + const domains = parseImportDomains(rawDomains); + if (domains.length === 0) { + return; + } + const job = await window.ccrBrowser?.startChromeLoginImport({ domains, openConfirmationPage: true }); + if (!job) { + return; + } + setChromeImportJob(job); + } + + function chromeImportTitle(): string { + if (!chromeImportJob) { + return "Import login from Chrome"; + } + if (chromeImportJob.status === "completed") { + return `Chrome import complete: ${chromeImportJob.result?.imported ?? 0} imported, ${chromeImportJob.result?.skipped ?? 0} skipped`; + } + if (chromeImportJob.status === "expired") { + return "Chrome import expired"; + } + if (chromeImportJob.status === "failed") { + return "Chrome import failed"; + } + return "Chrome import pending. Confirm it in the browser window."; + } + + async function handleChromeImportButton() { + if (chromeImportJob?.status === "pending") { + try { + await navigator.clipboard.writeText(chromeImportJob.confirmUrl); + } catch { + window.prompt("Open this Chrome import confirmation URL.", chromeImportJob.confirmUrl); + } + return; + } + await startChromeLoginImport(); + } + return ( -
    +
    @@ -149,8 +218,45 @@ function BrowserChrome() { spellCheck={false} value={addressDraft} /> + + {handoff ? ( +
    +
    + + {handoff.message} + {handoff.reason ? {handoff.reason} : null} +
    +
    + + +
    +
    + ) : null} + {homeVisible ? (
    @@ -189,6 +295,42 @@ function BrowserChrome() { const root = createRoot(document.getElementById("root") as HTMLElement); root.render(); +function activeTabDomain(url: string | undefined): string { + if (!url || url === browserHomeUrl) { + return ""; + } + try { + return new URL(url).hostname; + } catch { + return ""; + } +} + +function activeTabMatchesImport(url: string | undefined, domains: string[]): boolean { + const host = activeTabDomain(url).toLowerCase(); + return Boolean(host && domains.some((domain) => host === domain || host.endsWith(`.${domain}`))); +} + +function parseImportDomains(value: string): string[] { + return [...new Set(value + .split(/[,\n]/) + .map((item) => normalizeImportDomain(item)) + .filter((item): item is string => Boolean(item)))]; +} + +function normalizeImportDomain(value: string): string | undefined { + const raw = value.trim().toLowerCase(); + if (!raw) { + return undefined; + } + try { + return new URL(raw.includes("://") ? raw : `https://${raw}`).hostname; + } catch { + const domain = raw.replace(/^\*\./, "").split("/")[0]; + return domain && !domain.includes(" ") ? domain : undefined; + } +} + const style = document.createElement("style"); style.textContent = ` :root { @@ -231,6 +373,10 @@ style.textContent = ` width: 100%; } + .browser-shell.has-handoff { + grid-template-rows: 38px 44px 44px minmax(0, 1fr); + } + .tabs-row { -webkit-app-region: drag; align-items: end; @@ -325,7 +471,7 @@ style.textContent = ` border-bottom: 1px solid color-mix(in srgb, CanvasText 12%, transparent); display: grid; gap: 4px; - grid-template-columns: 32px 32px 32px minmax(0, 1fr); + grid-template-columns: 32px 32px 32px minmax(0, 1fr) 32px; padding: 6px 10px; } @@ -339,6 +485,11 @@ style.textContent = ` opacity: 0.4; } + .icon-button.active-import { + background: color-mix(in srgb, #0f766e 16%, transparent); + color: color-mix(in srgb, #0f766e 82%, CanvasText); + } + input { background: color-mix(in srgb, CanvasText 4%, Canvas); border: 1px solid color-mix(in srgb, CanvasText 12%, transparent); @@ -355,6 +506,79 @@ style.textContent = ` border-color: color-mix(in srgb, #2563eb 70%, CanvasText 30%); } + .automation-handoff { + -webkit-app-region: drag; + align-items: center; + background: color-mix(in srgb, #f59e0b 16%, Canvas); + border-bottom: 1px solid color-mix(in srgb, #92400e 24%, transparent); + display: grid; + gap: 10px; + grid-template-columns: minmax(0, 1fr) auto; + min-width: 0; + padding: 6px 10px; + } + + .handoff-copy { + align-items: center; + color: color-mix(in srgb, #78350f 76%, CanvasText); + display: flex; + gap: 8px; + min-width: 0; + } + + .handoff-message, + .handoff-reason { + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + } + + .handoff-message { + font-size: 13px; + font-weight: 700; + } + + .handoff-reason { + color: color-mix(in srgb, CanvasText 58%, transparent); + font-size: 12px; + } + + .handoff-actions { + -webkit-app-region: no-drag; + align-items: center; + display: flex; + flex: 0 0 auto; + gap: 6px; + } + + .handoff-button { + align-items: center; + background: color-mix(in srgb, CanvasText 5%, Canvas); + border: 1px solid color-mix(in srgb, CanvasText 12%, transparent); + border-radius: 7px; + display: inline-flex; + gap: 5px; + height: 30px; + justify-content: center; + min-width: 0; + padding: 0 9px; + white-space: nowrap; + } + + .handoff-button:hover { + background: color-mix(in srgb, CanvasText 9%, Canvas); + } + + .handoff-button.primary { + background: #166534; + border-color: #166534; + color: white; + } + + .handoff-button.primary:hover { + background: #14532d; + } + .home-page { align-items: flex-start; background: @@ -487,6 +711,24 @@ style.textContent = ` .installed-apps { grid-template-columns: 1fr; } + + .automation-handoff { + gap: 6px; + grid-template-columns: minmax(0, 1fr) auto; + } + + .handoff-reason { + display: none; + } + + .handoff-button span { + display: none; + } + + .handoff-button { + padding: 0 8px; + width: 32px; + } } @keyframes spin { diff --git a/packages/ui/src/pages/home/components/settings.tsx b/packages/ui/src/pages/home/components/settings.tsx index 0b12d403..2b6657fe 100644 --- a/packages/ui/src/pages/home/components/settings.tsx +++ b/packages/ui/src/pages/home/components/settings.tsx @@ -4,6 +4,7 @@ import { botGatewaySavedConfigFromDraft, botGatewaySavedConfigLabel, BotGatewayQrLoginStartResult, BotGatewayQrLoginWaitResult, BotGatewayQrWindowOpenResult, BotGatewaySavedConfig, Button, CircleAlert, closestCenter, cn, CSS, Database, Dialog, DialogBody, DialogContent, DialogFooter, DialogHeader, DialogTitle, Field, formatAppError, formatProviderAccountMeterValue, formatSystemOption, Gauge, + Globe, createBotGatewayConfigDraft, createMcpServerDraft, createMcpServerDraftFromConfig, createMcpServerDraftFromUnknown, createRouteModelOptions, DndContext, DragEndEvent, GatewayMcpServerConfig, GatewayProviderConfig, Input, isBotGatewayConfigDraftSubmittable, KeyboardSensor, KeyRound, KeyValueRowsControl, languageDisplayName, Layers3, LoaderCircle, mcpServerConfigFromDraft, mcpServerEndpointSummary, mcpServerTransportOptions, mcpStdioMessageModeOptions, McpServerDraft, normalizeBotGatewayAuthType, normalizeBotGatewayPlatform, normalizeToolHubConfig, Palette, Pencil, Plus, ProfileConfig, profileAgentLabel, PanelLeftOpen, Power, ProviderAccountMeter, ProviderAccountSnapshot, ReactNode, ResolvedLanguage, ResolvedTheme, Select, SelectControl, @@ -512,6 +513,13 @@ function ToolHubSettingsPage({
    {toolHub.enabled ? ( <> + onChange({ browserAutomation })} + />
    {t("MCP servers")}
    -
    {String(toolHub.mcpServers.length)}