feat(chrome-ai): draft 1

This commit is contained in:
Simon
2026-07-20 15:47:58 +08:00
parent da1db95955
commit 2c0161e0b6
7 changed files with 1175 additions and 5 deletions
@@ -0,0 +1,438 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Page Agent · Chrome on-device lab</title>
<style>
:root {
font-family:
Inter,
ui-sans-serif,
-apple-system,
BlinkMacSystemFont,
'Segoe UI',
sans-serif;
color: #1b1d22;
background: #f1f0ec;
font-synthesis: none;
}
* {
box-sizing: border-box;
}
body {
min-height: 100vh;
margin: 0;
background:
radial-gradient(circle at 12% 2%, rgba(95, 80, 246, 0.12), transparent 31rem), #f1f0ec;
}
button,
input,
select,
textarea {
font: inherit;
}
button {
min-height: 2.7rem;
border: 1px solid #1b1d22;
border-radius: 999px;
padding: 0 1.1rem;
color: #fff;
background: #1b1d22;
cursor: pointer;
transition:
transform 120ms ease,
box-shadow 120ms ease;
}
button:hover:not(:disabled) {
transform: translateY(-1px);
box-shadow: 0 6px 18px rgba(27, 29, 34, 0.14);
}
button:disabled {
cursor: not-allowed;
opacity: 0.45;
}
button.secondary {
color: #1b1d22;
background: transparent;
}
.lab {
width: min(1180px, calc(100% - 2rem));
margin: 0 auto;
padding: 3.5rem 0 2.5rem;
}
.lab[hidden] {
display: none;
}
.hero {
display: grid;
grid-template-columns: minmax(0, 1fr) minmax(17rem, 0.62fr);
gap: 2rem;
align-items: end;
margin-bottom: 2rem;
}
.eyebrow {
margin: 0 0 0.65rem;
color: #5a4bf4;
font-size: 0.76rem;
font-weight: 750;
letter-spacing: 0.12em;
text-transform: uppercase;
}
h1,
h2,
p {
margin-top: 0;
}
h1 {
max-width: 14ch;
margin-bottom: 0;
font-size: clamp(2.7rem, 6vw, 5.4rem);
line-height: 0.94;
letter-spacing: -0.065em;
}
.lede {
margin-bottom: 0;
color: #5d6069;
font-size: 1.04rem;
line-height: 1.6;
}
.grid {
display: grid;
grid-template-columns: repeat(12, minmax(0, 1fr));
gap: 1rem;
}
.card {
grid-column: span 6;
border: 1px solid rgba(27, 29, 34, 0.13);
border-radius: 1.15rem;
padding: 1.25rem;
background: rgba(255, 255, 255, 0.82);
box-shadow: 0 18px 50px rgba(34, 36, 42, 0.06);
backdrop-filter: blur(12px);
}
.card.full {
grid-column: 1 / -1;
}
.card h2 {
margin-bottom: 1rem;
font-size: 1rem;
letter-spacing: -0.015em;
}
.status-grid {
display: grid;
grid-template-columns: repeat(3, minmax(0, 1fr));
gap: 0.75rem;
}
.stat {
min-width: 0;
border-radius: 0.85rem;
padding: 0.85rem;
background: #f4f3f0;
}
.stat span {
display: block;
margin-bottom: 0.35rem;
color: #777a82;
font-size: 0.73rem;
text-transform: uppercase;
letter-spacing: 0.08em;
}
.stat output {
display: block;
overflow: hidden;
font-weight: 700;
text-overflow: ellipsis;
white-space: nowrap;
}
progress {
width: 100%;
height: 0.55rem;
margin: 1rem 0 0.55rem;
accent-color: #5a4bf4;
}
.progress-label,
.hint {
margin-bottom: 0;
color: #777a82;
font-size: 0.82rem;
line-height: 1.5;
}
.actions {
display: flex;
flex-wrap: wrap;
gap: 0.65rem;
margin-top: 1rem;
}
label {
display: block;
margin-bottom: 0.45rem;
font-size: 0.82rem;
font-weight: 700;
}
textarea,
input,
select {
width: 100%;
border: 1px solid #c9c8c3;
border-radius: 0.75rem;
padding: 0.78rem 0.85rem;
color: #1b1d22;
background: #fff;
outline: none;
}
textarea:focus,
input:focus,
select:focus {
border-color: #5a4bf4;
box-shadow: 0 0 0 3px rgba(90, 75, 244, 0.13);
}
textarea {
min-height: 7.6rem;
resize: vertical;
line-height: 1.55;
}
pre {
max-height: 25rem;
overflow: auto;
margin: 0;
border-radius: 0.85rem;
padding: 1rem;
color: #dcdde4;
background: #202228;
font:
0.76rem/1.55 ui-monospace,
SFMono-Regular,
Menlo,
Consolas,
monospace;
white-space: pre-wrap;
word-break: break-word;
}
details + details {
margin-top: 0.7rem;
}
summary {
margin-bottom: 0.65rem;
cursor: pointer;
font-size: 0.84rem;
font-weight: 700;
}
.fixture-shell {
width: min(720px, calc(100% - 2rem));
margin: 2rem auto 4rem;
}
.fixture {
border: 1px solid rgba(27, 29, 34, 0.15);
border-radius: 1.25rem;
padding: clamp(1.35rem, 4vw, 2.2rem);
background: #fff;
box-shadow: 0 24px 70px rgba(34, 36, 42, 0.09);
}
.fixture h2 {
margin-bottom: 0.35rem;
font-size: 1.7rem;
letter-spacing: -0.035em;
}
.fixture-description {
margin-bottom: 1.4rem;
color: #696c74;
}
.fixture-form {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 1rem;
}
.fixture-form button {
grid-column: 1 / -1;
justify-self: start;
margin-top: 0.25rem;
}
.fixture-result {
display: block;
margin-top: 1.2rem;
border-radius: 0.8rem;
padding: 0.9rem;
color: #555963;
background: #f1f0ec;
font-weight: 700;
}
.fixture-result[data-saved='true'] {
color: #215f42;
background: #e5f5eb;
}
.fixture-running {
display: none;
margin: 0 0 1rem;
color: #5a4bf4;
font-size: 0.8rem;
font-weight: 750;
}
body[data-agent-running='true'] .fixture-running {
display: block;
}
@media (max-width: 760px) {
.hero,
.fixture-form {
grid-template-columns: 1fr;
}
.card {
grid-column: 1 / -1;
}
.status-grid {
grid-template-columns: 1fr;
}
}
</style>
</head>
<body>
<main id="lab" class="lab" data-page-agent-ignore="true">
<header class="hero">
<div>
<p class="eyebrow">Chrome on-device experiment</p>
<h1>Structured action lab.</h1>
</div>
<p class="lede">
Runs Page Agent's real macro-tool schema through Chrome's Prompt API. No request leaves
this browser.
</p>
</header>
<section class="grid" aria-label="Experiment controls">
<article class="card">
<h2>Model readiness</h2>
<div class="status-grid">
<div class="stat">
<span>Prompt API</span>
<output id="api-status">Checking…</output>
</div>
<div class="stat">
<span>Availability</span>
<output id="availability-status">Unknown</output>
</div>
<div class="stat">
<span>Base session</span>
<output id="session-status">Cold</output>
</div>
</div>
<progress id="download-progress" value="0" max="1"></progress>
<p id="progress-label" class="progress-label">No download started.</p>
<div class="actions">
<button id="check-button" class="secondary" type="button">Check availability</button>
<button id="prepare-button" type="button">Prepare model</button>
</div>
</article>
<article class="card">
<h2>Controlled run</h2>
<label for="task-input">Task</label>
<textarea id="task-input">
Set the profile name to "Ramen", choose "Researcher" as the role, click "Save profile", then finish with the exact confirmation shown on the page.</textarea>
<p class="hint">
The lab UI is excluded from Page Agent's DOM view. Press Escape to stop.
</p>
<div class="actions">
<button id="run-button" type="button" disabled>Run Page Agent</button>
<button id="reset-button" class="secondary" type="button">Reset fixture</button>
</div>
</article>
<article class="card full">
<h2>Measurements</h2>
<pre id="metrics-output">No requests yet.</pre>
</article>
<article class="card full">
<h2>Trace</h2>
<details open>
<summary>Agent result</summary>
<pre id="result-output">No run yet.</pre>
</details>
<details>
<summary>Latest AgentOutput JSON Schema</summary>
<pre id="schema-output">Captured on first request.</pre>
</details>
<details>
<summary>Errors</summary>
<pre id="errors-output">No errors.</pre>
</details>
</article>
</section>
</main>
<section class="fixture-shell" aria-label="Page Agent test fixture">
<article class="fixture">
<p class="fixture-running">Page Agent is running</p>
<p class="eyebrow">Controlled fixture</p>
<h2>Profile editor</h2>
<p class="fixture-description">Create a small team profile and save it.</p>
<form id="profile-form" class="fixture-form">
<div>
<label for="profile-name">Profile name</label>
<input id="profile-name" name="name" type="text" autocomplete="off" />
</div>
<div>
<label for="profile-role">Role</label>
<select id="profile-role" name="role">
<option value="">Choose a role</option>
<option value="Designer">Designer</option>
<option value="Engineer">Engineer</option>
<option value="Researcher">Researcher</option>
</select>
</div>
<button type="submit">Save profile</button>
</form>
<output id="fixture-result" class="fixture-result" data-saved="false"
>No profile saved.</output
>
</article>
</section>
<script type="module" src="./main.ts"></script>
</body>
</html>
@@ -0,0 +1,613 @@
import { type AgentStatus, type ExecutionResult, PageAgent } from '../../src/PageAgent'
type Availability = 'unavailable' | 'downloadable' | 'downloading' | 'available'
interface TextMessage {
role: 'system' | 'user' | 'assistant'
content: string
}
interface PromptOptions {
responseConstraint: Record<string, unknown>
}
interface LanguageModelSession {
readonly contextUsage?: number
readonly contextWindow?: number
readonly inputUsage?: number
readonly inputQuota?: number
clone(options?: { signal?: AbortSignal }): Promise<LanguageModelSession>
destroy(): void
measureContextUsage?(input: TextMessage[], options: PromptOptions): Promise<number>
measureInputUsage?(input: TextMessage[], options: PromptOptions): Promise<number>
prompt(input: TextMessage[], options: PromptOptions & { signal?: AbortSignal }): Promise<string>
}
interface LanguageModelCreateOptions {
expectedInputs: { type: 'text'; languages: string[] }[]
expectedOutputs: { type: 'text'; languages: string[] }[]
initialPrompts?: TextMessage[]
monitor?: (monitor: EventTarget) => void
signal?: AbortSignal
}
interface LanguageModelFactory {
availability(options: LanguageModelCreateOptions): Promise<Availability>
create(options: LanguageModelCreateOptions): Promise<LanguageModelSession>
}
interface ToolDefinition {
type: 'function'
function: {
name: string
parameters: Record<string, unknown>
}
}
interface RequestMeasurement {
id: number
actionVariants?: number
cloneMs?: number
contextUsageAfter?: number
contextUsageBefore?: number
contextWindow?: number
error?: string
jsonOutput?: unknown
measureContextUsageMs?: number
measuredContextUsage?: number
measurementError?: string
messageBytes?: number
promptMs?: number
promptBytes?: number
schemaBytes?: number
systemBaseCreateMs?: number
systemBytes?: number
totalMs?: number
}
interface LabState {
agentStatus: AgentStatus | 'not-created'
askUserRequests: { question: string; answer: string }[]
availability: Availability | 'unknown'
downloadProgress: number
downloadCompletedAt?: number
errors: { at: string; scope: string; message: string }[]
latestSchema?: Record<string, unknown>
postDownloadLoadMs?: number
prepareMs?: number
preparing: boolean
promptApiSupported: boolean
requests: RequestMeasurement[]
result?: ExecutionResult
running: boolean
userAgent: string
warmSession: boolean
warmSessionContextUsage?: number
warmSessionContextWindow?: number
}
interface ChromeOnDeviceLabApi {
checkAvailability(): Promise<Availability | 'unknown'>
getState(): LabState
prepare(): Promise<void>
resetFixture(): void
run(task?: string): Promise<ExecutionResult>
stop(): Promise<void>
}
declare global {
interface Window {
chromeOnDeviceLab: ChromeOnDeviceLabApi
}
}
const textEncoder = new TextEncoder()
const modelOptions: LanguageModelCreateOptions = {
expectedInputs: [{ type: 'text', languages: ['en'] }],
expectedOutputs: [{ type: 'text', languages: ['en'] }],
}
const labElement = getElement<HTMLElement>('lab')
const apiStatusOutput = getElement<HTMLOutputElement>('api-status')
const availabilityOutput = getElement<HTMLOutputElement>('availability-status')
const sessionOutput = getElement<HTMLOutputElement>('session-status')
const progressElement = getElement<HTMLProgressElement>('download-progress')
const progressLabel = getElement<HTMLElement>('progress-label')
const checkButton = getElement<HTMLButtonElement>('check-button')
const prepareButton = getElement<HTMLButtonElement>('prepare-button')
const runButton = getElement<HTMLButtonElement>('run-button')
const resetButton = getElement<HTMLButtonElement>('reset-button')
const taskInput = getElement<HTMLTextAreaElement>('task-input')
const metricsOutput = getElement<HTMLElement>('metrics-output')
const resultOutput = getElement<HTMLElement>('result-output')
const schemaOutput = getElement<HTMLElement>('schema-output')
const errorsOutput = getElement<HTMLElement>('errors-output')
const profileForm = getElement<HTMLFormElement>('profile-form')
const profileNameInput = getElement<HTMLInputElement>('profile-name')
const profileRoleSelect = getElement<HTMLSelectElement>('profile-role')
const fixtureResult = getElement<HTMLOutputElement>('fixture-result')
const state: LabState = {
agentStatus: 'not-created',
askUserRequests: [],
availability: 'unknown',
downloadProgress: 0,
errors: [],
preparing: false,
promptApiSupported: getPromptApi() !== undefined,
requests: [],
running: false,
userAgent: navigator.userAgent,
warmSession: false,
}
let baseSession: LanguageModelSession | undefined
let baseSystemPrompt: string | undefined
let agent: PageAgent | undefined
function getElement<T extends HTMLElement>(id: string): T {
const element = document.getElementById(id)
if (!element) throw new Error(`Missing #${id}`)
return element as T
}
function getPromptApi(): LanguageModelFactory | undefined {
return (globalThis as typeof globalThis & { LanguageModel?: LanguageModelFactory }).LanguageModel
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === 'object' && value !== null && !Array.isArray(value)
}
function formatError(error: unknown): string {
if (error instanceof DOMException || error instanceof Error) {
return `${error.name}: ${error.message}`
}
return String(error)
}
function isAbortError(error: unknown): boolean {
return isRecord(error) && error.name === 'AbortError'
}
function round(value: number): number {
return Math.round(value * 10) / 10
}
function byteLength(value: unknown): number {
return textEncoder.encode(typeof value === 'string' ? value : JSON.stringify(value)).byteLength
}
function readContextUsage(session: LanguageModelSession): number | undefined {
return session.contextUsage ?? session.inputUsage
}
function readContextWindow(session: LanguageModelSession): number | undefined {
return session.contextWindow ?? session.inputQuota
}
function snapshot(): LabState {
return JSON.parse(JSON.stringify(state)) as LabState
}
function recordError(scope: string, error: unknown): void {
state.errors.push({ at: new Date().toISOString(), scope, message: formatError(error) })
render()
}
function render(): void {
apiStatusOutput.value = state.promptApiSupported ? 'Present' : 'Missing'
availabilityOutput.value = state.availability
sessionOutput.value = state.preparing ? 'Preparing…' : state.warmSession ? 'Warm' : 'Cold'
progressElement.value = state.downloadProgress
if (state.preparing && state.downloadProgress >= 1) {
progressLabel.textContent = 'Download complete. Chrome is loading the model.'
} else if (state.preparing && state.downloadProgress > 0) {
progressLabel.textContent = `Downloading model: ${Math.round(state.downloadProgress * 100)}%`
} else if (state.warmSession) {
progressLabel.textContent = `Ready in ${state.prepareMs ?? 0} ms. Context window: ${state.warmSessionContextWindow ?? 'unknown'}.`
} else if (!state.promptApiSupported) {
progressLabel.textContent = 'LanguageModel is not exposed in this page.'
} else {
progressLabel.textContent = 'No download started.'
}
const busy = state.preparing || state.running
checkButton.disabled = busy || !state.promptApiSupported
prepareButton.disabled =
busy || !state.promptApiSupported || state.availability === 'unavailable' || state.warmSession
runButton.disabled = busy || !state.warmSession
resetButton.disabled = busy
taskInput.disabled = busy
metricsOutput.textContent = JSON.stringify(
{
environment: { userAgent: state.userAgent },
warmup: {
availability: state.availability,
prepareMs: state.prepareMs,
postDownloadLoadMs: state.postDownloadLoadMs,
contextUsage: state.warmSessionContextUsage,
contextWindow: state.warmSessionContextWindow,
},
requests: state.requests,
askUserRequests: state.askUserRequests,
},
null,
2
)
resultOutput.textContent = state.result ? JSON.stringify(state.result, null, 2) : 'No run yet.'
schemaOutput.textContent = state.latestSchema
? JSON.stringify(state.latestSchema, null, 2)
: 'Captured on first request.'
errorsOutput.textContent = state.errors.length
? state.errors.map((entry) => `[${entry.at}] ${entry.scope}\n${entry.message}`).join('\n\n')
: 'No errors.'
}
async function checkAvailability(): Promise<Availability | 'unknown'> {
const promptApi = getPromptApi()
state.promptApiSupported = promptApi !== undefined
if (!promptApi) {
state.availability = 'unknown'
render()
return state.availability
}
try {
state.availability = await promptApi.availability(modelOptions)
return state.availability
} catch (error) {
recordError('availability', error)
state.availability = 'unknown'
return state.availability
} finally {
render()
}
}
async function prepareModel(): Promise<void> {
if (baseSession) return
const promptApi = getPromptApi()
if (!promptApi) throw new Error('LanguageModel is not available in this page.')
state.preparing = true
state.downloadProgress = 0
state.downloadCompletedAt = undefined
state.postDownloadLoadMs = undefined
render()
const startedAt = performance.now()
const sessionPromise = promptApi.create({
...modelOptions,
monitor(monitor) {
monitor.addEventListener('downloadprogress', (event) => {
const loaded = (event as Event & { loaded?: number }).loaded ?? 0
state.downloadProgress = Math.min(1, Math.max(0, loaded))
if (state.downloadProgress >= 1 && state.downloadCompletedAt === undefined) {
state.downloadCompletedAt = performance.now()
}
render()
})
},
})
try {
baseSession = await sessionPromise
const preparedAt = performance.now()
state.prepareMs = round(preparedAt - startedAt)
state.postDownloadLoadMs = state.downloadCompletedAt
? round(preparedAt - state.downloadCompletedAt)
: undefined
state.warmSession = true
state.warmSessionContextUsage = readContextUsage(baseSession)
state.warmSessionContextWindow = readContextWindow(baseSession)
state.downloadProgress = 1
void checkAvailability()
} catch (error) {
recordError('prepare', error)
throw error
} finally {
state.preparing = false
render()
}
}
function parseOpenAIRequest(init?: RequestInit): {
messages: TextMessage[]
promptMessages: TextMessage[]
schema: Record<string, unknown>
systemPrompt: string
tool: ToolDefinition
} {
if (typeof init?.body !== 'string') throw new TypeError('Expected a JSON string request body.')
const body = JSON.parse(init.body) as unknown
if (!isRecord(body)) throw new TypeError('Expected an OpenAI request object.')
if (!Array.isArray(body.tools) || body.tools.length !== 1) {
throw new TypeError('Expected exactly one OpenAI function tool.')
}
const toolValue = body.tools[0]
if (!isRecord(toolValue) || toolValue.type !== 'function' || !isRecord(toolValue.function)) {
throw new TypeError('Expected an OpenAI function tool definition.')
}
if (toolValue.function.name !== 'AgentOutput' || !isRecord(toolValue.function.parameters)) {
throw new TypeError('Expected the AgentOutput tool and its JSON Schema.')
}
const tool = toolValue as unknown as ToolDefinition
if (!Array.isArray(body.messages) || body.messages.length === 0) {
throw new TypeError('Expected at least one prompt message.')
}
const messages = body.messages.map((value): TextMessage => {
if (!isRecord(value) || typeof value.content !== 'string') {
throw new TypeError('Only text prompt messages are supported in this experiment.')
}
if (value.role !== 'system' && value.role !== 'user' && value.role !== 'assistant') {
throw new TypeError(`Unsupported prompt role: ${String(value.role)}`)
}
return { role: value.role, content: value.content }
})
const [systemMessage, ...promptMessages] = messages
if (
systemMessage.role !== 'system' ||
promptMessages.some((message) => message.role === 'system')
) {
throw new TypeError('Expected one leading system message.')
}
if (promptMessages.length === 0) throw new TypeError('Expected at least one user message.')
return {
messages,
promptMessages,
schema: tool.function.parameters,
systemPrompt: systemMessage.content,
tool,
}
}
async function ensureSystemBase(
systemPrompt: string,
signal: AbortSignal | undefined,
measurement: RequestMeasurement
): Promise<LanguageModelSession> {
if (baseSession && baseSystemPrompt === systemPrompt) return baseSession
const promptApi = getPromptApi()
if (!promptApi) throw new Error('LanguageModel is not available in this page.')
const startedAt = performance.now()
const replacement = await promptApi.create({
...modelOptions,
initialPrompts: [{ role: 'system', content: systemPrompt }],
signal,
})
measurement.systemBaseCreateMs = round(performance.now() - startedAt)
const previous = baseSession
baseSession = replacement
baseSystemPrompt = systemPrompt
state.warmSessionContextUsage = readContextUsage(replacement)
state.warmSessionContextWindow = readContextWindow(replacement)
previous?.destroy()
return replacement
}
function countActionVariants(schema: Record<string, unknown>): number | undefined {
const properties = schema.properties
if (!isRecord(properties) || !isRecord(properties.action)) return undefined
const variants = properties.action.anyOf
return Array.isArray(variants) ? variants.length : undefined
}
function errorResponse(error: unknown): Response {
const errorName = error instanceof DOMException || error instanceof Error ? error.name : 'Error'
return Response.json(
{
error: {
code: errorName,
message: formatError(error),
type: 'chrome_on_device_error',
},
},
{ status: 422, statusText: 'Chrome on-device inference failed' }
)
}
const chromeOnDeviceFetch: typeof globalThis.fetch = async (_input, init) => {
const startedAt = performance.now()
const measurement: RequestMeasurement = { id: state.requests.length + 1 }
state.requests.push(measurement)
let requestSession: LanguageModelSession | undefined
try {
init?.signal?.throwIfAborted()
if (!baseSession) throw new Error('Prepare the Chrome model before running Page Agent.')
const { messages, promptMessages, schema, systemPrompt, tool } = parseOpenAIRequest(init)
state.latestSchema = schema
measurement.schemaBytes = byteLength(schema)
measurement.messageBytes = byteLength(messages)
measurement.promptBytes = byteLength(promptMessages)
measurement.systemBytes = byteLength(systemPrompt)
measurement.actionVariants = countActionVariants(schema)
const systemBase = await ensureSystemBase(systemPrompt, init?.signal ?? undefined, measurement)
const cloneStartedAt = performance.now()
requestSession = await systemBase.clone({ signal: init?.signal ?? undefined })
measurement.cloneMs = round(performance.now() - cloneStartedAt)
measurement.contextUsageBefore = readContextUsage(requestSession)
measurement.contextWindow = readContextWindow(requestSession)
const measure =
requestSession.measureContextUsage?.bind(requestSession) ??
requestSession.measureInputUsage?.bind(requestSession)
if (measure) {
const measureStartedAt = performance.now()
try {
measurement.measuredContextUsage = await measure(promptMessages, {
responseConstraint: schema,
})
} catch (error) {
measurement.measurementError = formatError(error)
recordError(`request ${measurement.id} measureContextUsage`, error)
} finally {
measurement.measureContextUsageMs = round(performance.now() - measureStartedAt)
}
}
const promptStartedAt = performance.now()
const rawOutput = await requestSession.prompt(promptMessages, {
responseConstraint: schema,
signal: init?.signal ?? undefined,
})
measurement.promptMs = round(performance.now() - promptStartedAt)
measurement.contextUsageAfter = readContextUsage(requestSession)
measurement.jsonOutput = JSON.parse(rawOutput) as unknown
measurement.totalMs = round(performance.now() - startedAt)
render()
return Response.json({
choices: [
{
finish_reason: 'tool_calls',
index: 0,
message: {
content: null,
role: 'assistant',
tool_calls: [
{
function: { arguments: rawOutput, name: tool.function.name },
id: `chrome-on-device-${measurement.id}`,
type: 'function',
},
],
},
},
],
chrome_on_device: measurement,
usage: { completion_tokens: 0, prompt_tokens: 0, total_tokens: 0 },
})
} catch (error) {
measurement.error = formatError(error)
measurement.totalMs = round(performance.now() - startedAt)
recordError(`request ${measurement.id}`, error)
if (isAbortError(error)) throw error
return errorResponse(error)
} finally {
requestSession?.destroy()
render()
}
}
function createAgent(): PageAgent {
const pageAgent = new PageAgent({
baseURL: 'https://chrome-on-device.invalid/v1',
customFetch: chromeOnDeviceFetch,
enableMask: false,
language: 'en-US',
maxRetries: 0,
maxSteps: 8,
model: 'chrome-on-device',
promptForNextTask: false,
stepDelay: 0,
})
pageAgent.onAskUser = async (question, options) => {
options?.signal.throwIfAborted()
const answer = 'The task is fully specified. Continue using the visible page.'
state.askUserRequests.push({ question, answer })
render()
return answer
}
pageAgent.addEventListener('statuschange', () => {
state.agentStatus = pageAgent.status
render()
})
state.agentStatus = pageAgent.status
return pageAgent
}
function resetFixture(): void {
profileForm.reset()
fixtureResult.value = 'No profile saved.'
fixtureResult.dataset.saved = 'false'
}
async function runAgent(task = taskInput.value.trim()): Promise<ExecutionResult> {
if (!baseSession) throw new Error('Prepare the Chrome model before running Page Agent.')
if (!task) throw new Error('Enter a task before running Page Agent.')
if (state.running) throw new Error('Page Agent is already running.')
resetFixture()
state.result = undefined
state.running = true
labElement.hidden = true
document.body.dataset.agentRunning = 'true'
render()
try {
if (!agent || agent.disposed) agent = createAgent()
state.result = await agent.execute(task)
return state.result
} catch (error) {
recordError('agent', error)
throw error
} finally {
state.running = false
labElement.hidden = false
delete document.body.dataset.agentRunning
render()
}
}
async function stopAgent(): Promise<void> {
await agent?.stop()
}
profileForm.addEventListener('submit', (event) => {
event.preventDefault()
const name = profileNameInput.value.trim()
const role = profileRoleSelect.value
if (!name || !role) {
fixtureResult.value = 'Complete both profile fields before saving.'
fixtureResult.dataset.saved = 'false'
return
}
fixtureResult.value = `Profile saved: ${name}${role}.`
fixtureResult.dataset.saved = 'true'
})
checkButton.addEventListener('click', () => {
void checkAvailability()
})
prepareButton.addEventListener('click', () => {
void prepareModel().catch(() => undefined)
})
runButton.addEventListener('click', () => {
void runAgent().catch(() => undefined)
})
resetButton.addEventListener('click', resetFixture)
document.addEventListener('keydown', (event) => {
if (event.key === 'Escape' && state.running) void stopAgent()
})
window.addEventListener('pagehide', () => {
agent?.dispose()
baseSession?.destroy()
})
window.chromeOnDeviceLab = {
checkAvailability,
getState: snapshot,
prepare: prepareModel,
resetFixture,
run: runAgent,
stop: stopAgent,
}
render()
void checkAvailability()
+1
View File
@@ -49,6 +49,7 @@
"scripts": {
"build": "vite build && npm run build:demo",
"build:demo": "vite build --config vite.iife.config.js",
"dev:chrome-on-device": "vite --host 127.0.0.1 --port 5175",
"dev:demo": "concurrently \"vite build --config vite.iife.config.js --watch\" \"npx serve dist/iife -p 5174\"",
"prepublishOnly": "node ../../scripts/pre-publish.js",
"postpublish": "node ../../scripts/post-publish.js"
+1 -1
View File
@@ -3,6 +3,6 @@
"compilerOptions": {
"tsBuildInfoFile": "./node_modules/.tmp/tsconfig.tsbuildinfo"
},
"include": ["src/**/*.ts"],
"include": ["src/**/*.ts", "experiments/**/*.ts"],
"exclude": ["dist", "node_modules"]
}
+25 -1
View File
@@ -18,7 +18,7 @@ import * as dom from './dom'
import type { FlatDomTree, InteractiveElementDomNode } from './dom/dom_tree/type'
import { getPageInfo } from './dom/getPageInfo'
import { patchReact } from './patches/react'
import { isAnchorElement } from './utils'
import { isAnchorElement, isInputElement, isSelectElement } from './utils'
/**
* Configuration for PageController
@@ -246,6 +246,12 @@ export class PageController extends EventTarget {
this.assertIndexed()
const element = getElementByIndex(this.selectorMap, index)
const elemText = this.elementTextMap.get(index)
if (isSelectElement(element)) {
return {
success: false,
message: `⚠️ Element ${index} is a <select>. Use select_dropdown_option({ index: ${index}, text: "<visible option text>" }).`,
}
}
await clickElement(element)
// Handle links that open in new tabs
@@ -276,6 +282,24 @@ export class PageController extends EventTarget {
this.assertIndexed()
const element = getElementByIndex(this.selectorMap, index)
const elemText = this.elementTextMap.get(index)
if (isSelectElement(element)) {
return {
success: false,
message: `⚠️ Element ${index} is a <select>. Use select_dropdown_option({ index: ${index}, text: "<visible option text>" }).`,
}
}
if (isInputElement(element) && ['checkbox', 'radio'].includes(element.type)) {
return {
success: false,
message: `❌ Element ${index} is input[type="${element.type}"]. Use click_element_by_index({ index: ${index} }) to toggle it.`,
}
}
if (isInputElement(element) && element.type === 'file') {
return {
success: false,
message: `❌ Element ${index} is input[type="file"], which input_text cannot operate. Ask the user to choose a file.`,
}
}
await inputTextElement(element, text)
return {
@@ -0,0 +1,63 @@
import { describe, expect, it, vi } from 'vitest'
import { clickElement, inputTextElement, selectOptionElement } from './actions'
describe('element actions', () => {
it('rejects clicking a native select with actionable feedback', async () => {
const select = document.createElement('select')
await expect(clickElement(select)).rejects.toThrow('Use select_dropdown_option')
})
it('rejects typing into a native select with actionable feedback', async () => {
const select = document.createElement('select')
await expect(inputTextElement(select, 'Researcher')).rejects.toThrow(
'Use select_dropdown_option'
)
})
it('reports when an input value does not persist', async () => {
const input = document.createElement('input')
document.body.append(input)
input.addEventListener('input', () => {
input.value = ''
})
await expect(inputTextElement(input, 'Ramen')).rejects.toThrow('Input did not persist')
})
it('selects an option and dispatches input and change events', async () => {
const select = document.createElement('select')
select.innerHTML = `
<option value="">Choose a role</option>
<option value="researcher">Researcher</option>
`
const onInput = vi.fn()
const onChange = vi.fn()
select.addEventListener('input', onInput)
select.addEventListener('change', onChange)
await selectOptionElement(select, 'Researcher')
expect(select.value).toBe('researcher')
expect(select.selectedOptions.item(0)?.textContent).toBe('Researcher')
expect(onInput).toHaveBeenCalledOnce()
expect(onChange).toHaveBeenCalledOnce()
})
it('reports when a selected value does not persist', async () => {
const select = document.createElement('select')
select.innerHTML = `
<option value="">Choose a role</option>
<option value="researcher">Researcher</option>
`
select.addEventListener('change', () => {
select.value = ''
})
await expect(selectOptionElement(select, 'Researcher')).rejects.toThrow(
'Selection did not persist'
)
})
})
+34 -3
View File
@@ -62,6 +62,12 @@ function blurLastClickedElement() {
* @private Internal method, subject to change at any time.
*/
export async function clickElement(element: HTMLElement) {
if (isSelectElement(element)) {
throw new Error(
'Native <select> cannot be completed with click_element_by_index. Use select_dropdown_option with this element index and the visible option text.'
)
}
blurLastClickedElement()
lastClickedElement = element
@@ -131,7 +137,12 @@ export async function clickElement(element: HTMLElement) {
export async function inputTextElement(element: HTMLElement, text: string) {
const isContentEditable = element.isContentEditable
if (!isInputElement(element) && !isTextAreaElement(element) && !isContentEditable) {
throw new Error('Element is not an input, textarea, or contenteditable')
if (isSelectElement(element)) {
throw new Error(
'Native <select> does not accept text input. Use select_dropdown_option with this element index and the visible option text.'
)
}
throw new Error('Element does not accept text input')
}
await clickElement(element)
@@ -226,9 +237,18 @@ export async function inputTextElement(element: HTMLElement, text: string) {
element.dispatchEvent(new Event('input', { bubbles: true }))
}
blurLastClickedElement()
await waitFor(0.1)
blurLastClickedElement()
const actualText = isContentEditable
? element.innerText.trim()
: (element as HTMLInputElement | HTMLTextAreaElement).value
const expectedText = isContentEditable ? text.trim() : text
if (actualText !== expectedText) {
throw new Error(
`Input did not persist. Expected value ${JSON.stringify(expectedText)}, but the element contains ${JSON.stringify(actualText)}. Inspect the current page state before retrying.`
)
}
}
/**
@@ -247,10 +267,21 @@ export async function selectOptionElement(selectElement: HTMLSelectElement, opti
throw new Error(`Option with text "${optionText}" not found in select element`)
}
selectElement.value = option.value
const expectedText = option.textContent?.trim() ?? ''
const expectedValue = option.value
selectElement.selectedIndex = option.index
selectElement.dispatchEvent(new Event('input', { bubbles: true }))
selectElement.dispatchEvent(new Event('change', { bubbles: true }))
await waitFor(0.1) // Wait to ensure change event processing completes
const selectedOption = selectElement.selectedOptions.item(0)
const actualText = selectedOption?.textContent?.trim() ?? ''
if (!selectedOption || selectElement.value !== expectedValue || actualText !== expectedText) {
throw new Error(
`Selection did not persist. Expected option ${JSON.stringify(expectedText)} with value ${JSON.stringify(expectedValue)}, but the current option is ${JSON.stringify(actualText)} with value ${JSON.stringify(selectElement.value)}. Inspect the current page state before retrying.`
)
}
}
interface ScrollableElement extends Element {