opensource/proxycast
3
0
Fork 0
mirror of https://github.com/aiclientproxy/proxycast.git synced 2026-09-24 23:10:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

chore: release v1.37.0

Browse Source
This commit is contained in:
coso
2026-05-13 07:14:35 +08:00
parent c896c40897
commit b3111949ff
149 changed files with 13369 additions and 6791 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/agentui-ttft-live-sample.mjs
+547
View File
@@ -0,0 +1,547 @@
#!/usr/bin/env node
import fs from "node:fs";
import path from "node:path";
import process from "node:process";
const DEFAULT_HEALTH_URL = "http://127.0.0.1:3030/health";
const DEFAULT_INVOKE_URL = "http://127.0.0.1:3030/invoke";
const DEFAULT_TIMEOUT_MS = 120_000;
const DEFAULT_INTERVAL_MS = 1_000;
const DEFAULT_MESSAGE = "Reply with exactly: OK";
const DEFAULT_MODE = "responsive-auto";
const MAX_SAMPLES_PER_RUN = 6;
const TERMINAL_RUN_STATUSES = new Set([
"success",
"error",
"failed",
"timeout",
"cancelled",
"canceled",
]);
function printHelp() {
console.log(`
Lime AgentUI TTFT Live Sample
用途:
通过 DevBridge 提交真实 Agent Runtime 回合，采集脱敏 routing / TTFT 证据。
默认走 responsive_chat 自动路由，不输出 prompt、assistant 正文、error_message、密钥或 run id。
用法:
node scripts/agentui-ttft-live-sample.mjs
node scripts/agentui-ttft-live-sample.mjs --samples 3 --output /tmp/agentui-ttft-live-sample.json
选项:
--mode responsive-auto|request-override
默认 responsive-auto；request-override 仅用于对照样本
--provider-preference ID
request-override 模式下的 provider
--model-preference ID
request-override 模式下的 model
--samples N 本次采样条数，默认 1，单次最多 ${MAX_SAMPLES_PER_RUN}
--message TEXT 提交给模型的最短测试消息；不会写入脚本输出，默认 ${DEFAULT_MESSAGE}
--output PATH 写入脱敏 JSON；默认 stdout
--health-url URL DevBridge health 地址，默认 ${DEFAULT_HEALTH_URL}
--invoke-url URL DevBridge invoke 地址，默认 ${DEFAULT_INVOKE_URL}
--timeout-ms MS 单条样本等待超时，默认 ${DEFAULT_TIMEOUT_MS}
--interval-ms MS 轮询间隔，默认 ${DEFAULT_INTERVAL_MS}
-h, --help 显示帮助
`);
}
function parseArgs(argv) {
const options = {
healthUrl: DEFAULT_HEALTH_URL,
invokeUrl: DEFAULT_INVOKE_URL,
intervalMs: DEFAULT_INTERVAL_MS,
message: DEFAULT_MESSAGE,
mode: DEFAULT_MODE,
modelPreference: "",
outputPath: "",
providerPreference: "",
samples: 1,
timeoutMs: DEFAULT_TIMEOUT_MS,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--help" || arg === "-h") {
printHelp();
process.exit(0);
}
if (arg === "--health-url" && argv[index + 1]) {
options.healthUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--invoke-url" && argv[index + 1]) {
options.invokeUrl = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--timeout-ms" && argv[index + 1]) {
options.timeoutMs = Number(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--interval-ms" && argv[index + 1]) {
options.intervalMs = Number(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--samples" && argv[index + 1]) {
options.samples = Number.parseInt(String(argv[index + 1]).trim(), 10);
index += 1;
continue;
}
if (arg === "--mode" && argv[index + 1]) {
options.mode = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--message" && argv[index + 1]) {
options.message = String(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
options.outputPath = path.resolve(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--provider-preference" && argv[index + 1]) {
options.providerPreference = String(argv[index + 1]).trim();
index += 1;
continue;
}
if (arg === "--model-preference" && argv[index + 1]) {
options.modelPreference = String(argv[index + 1]).trim();
index += 1;
}
}
if (!["responsive-auto", "request-override"].includes(options.mode)) {
throw new Error("--mode 只能是 responsive-auto 或 request-override");
}
if (!Number.isFinite(options.timeoutMs) || options.timeoutMs < 10_000) {
throw new Error("--timeout-ms 必须是 >= 10000 的数字");
}
if (!Number.isFinite(options.intervalMs) || options.intervalMs < 100) {
throw new Error("--interval-ms 必须是 >= 100 的数字");
}
if (!Number.isFinite(options.samples) || options.samples < 1) {
throw new Error("--samples 必须是 >= 1 的数字");
}
options.samples = Math.min(Math.floor(options.samples), MAX_SAMPLES_PER_RUN);
if (options.mode === "request-override") {
if (!options.providerPreference || !options.modelPreference) {
throw new Error(
"request-override 模式必须同时提供 --provider-preference 和 --model-preference",
);
}
}
return options;
}
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
function asRecord(value) {
return value && typeof value === "object" && !Array.isArray(value)
? value
: null;
}
function stringField(record, keys) {
const source = asRecord(record);
if (!source) {
return null;
}
for (const key of keys) {
const value = source[key];
if (typeof value === "string" && value.trim()) {
return value.trim();
}
}
return null;
}
function numberField(record, keys) {
const source = asRecord(record);
if (!source) {
return null;
}
for (const key of keys) {
const value = source[key];
if (typeof value === "number" && Number.isFinite(value)) {
return value;
}
if (typeof value === "string" && value.trim()) {
const parsed = Number(value);
if (Number.isFinite(parsed)) {
return parsed;
}
}
}
return null;
}
function stringArrayField(record, keys) {
const source = asRecord(record);
if (!source) {
return [];
}
for (const key of keys) {
const value = source[key];
if (Array.isArray(value)) {
return value.filter((item) => typeof item === "string" && item.trim());
}
}
return [];
}
async function waitForHealth(options) {
const startedAt = Date.now();
let lastError = null;
while (Date.now() - startedAt < options.timeoutMs) {
try {
const response = await fetch(options.healthUrl, {
method: "GET",
signal: AbortSignal.timeout(Math.min(options.intervalMs, 5_000)),
});
const text = await response.text();
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
}
return text ? JSON.parse(text) : {};
} catch (error) {
lastError = error;
await sleep(options.intervalMs);
}
}
throw new Error(
`DevBridge health timeout: ${
lastError instanceof Error ? lastError.message : String(lastError)
}`,
);
}
async function fetchWithHardTimeout(url, init, timeoutMs, label) {
const controller = new AbortController();
let rejectTimeout;
const abortTimeout = setTimeout(() => controller.abort(), timeoutMs);
const timeoutPromise = new Promise((_, reject) => {
rejectTimeout = setTimeout(() => {
reject(new Error(`${label} timeout after ${timeoutMs}ms`));
}, timeoutMs + 100);
});
try {
return await Promise.race([
fetch(url, { ...init, signal: controller.signal }),
timeoutPromise,
]);
} finally {
clearTimeout(abortTimeout);
clearTimeout(rejectTimeout);
}
}
async function invoke(options, cmd, args, timeoutMs = options.timeoutMs) {
const response = await fetchWithHardTimeout(
options.invokeUrl,
{
method: "POST",
headers: {
"content-type": "application/json",
},
body: JSON.stringify({ cmd, args }),
},
timeoutMs,
cmd,
);
const text = await response.text();
if (!response.ok) {
throw new Error(`${cmd} HTTP ${response.status}`);
}
const payload = text ? JSON.parse(text) : null;
if (payload?.error) {
throw new Error(`${cmd} error (redacted)`);
}
return payload?.result;
}
function buildFastResponseMetadata() {
return {
harness: {
fast_response_routing: {
mode: "auto",
label: "快速响应",
reason: "first-turn-short-prompt",
service_model_slot: "responsive_chat",
routing_slot: "responsive_chat_model",
routing_changed: false,
resolver: "backend_service_model",
runtime_status_presentation: "transient",
},
},
};
}
function buildTurnConfig(options) {
const turnConfig = {
metadata: buildFastResponseMetadata(),
search_mode: "disabled",
web_search: false,
thinking_enabled: false,
};
if (options.mode === "request-override") {
turnConfig.provider_preference = options.providerPreference;
turnConfig.model_preference = options.modelPreference;
}
return turnConfig;
}
function latestTurnStatus(threadRead) {
const diagnostics = asRecord(threadRead?.diagnostics);
const runtimeSummary = asRecord(
threadRead?.runtime_summary ?? threadRead?.runtimeSummary,
);
return (
stringField(diagnostics, ["latest_turn_status", "latestTurnStatus"]) ||
stringField(runtimeSummary, ["latestTurnStatus", "latest_turn_status"]) ||
stringField(threadRead, ["status"])
);
}
function summarizeThreadRead(threadRead) {
const modelRouting = asRecord(
threadRead?.model_routing ?? threadRead?.modelRouting,
);
const latestTiming = asRecord(
modelRouting?.latestModelDeltaTiming ??
modelRouting?.latest_model_delta_timing,
);
const latestRouting = asRecord(latestTiming?.routing);
const runtimeSummary = asRecord(
threadRead?.runtime_summary ?? threadRead?.runtimeSummary,
);
const routing = {
decisionSource:
stringField(modelRouting, ["decisionSource", "decision_source"]) ||
stringField(latestRouting, ["decisionSource", "decision_source"]) ||
stringField(runtimeSummary, ["decisionSource", "decision_source"]),
settingsSource:
stringField(modelRouting, ["settingsSource", "settings_source"]) ||
stringField(latestRouting, ["settingsSource", "settings_source"]),
serviceModelSlot:
stringField(modelRouting, ["serviceModelSlot", "service_model_slot"]) ||
stringField(latestRouting, ["serviceModelSlot", "service_model_slot"]),
selectedProvider:
stringField(modelRouting, ["selectedProvider", "selected_provider"]) ||
stringField(latestRouting, ["selectedProvider", "selected_provider"]),
selectedModel:
stringField(modelRouting, ["selectedModel", "selected_model"]) ||
stringField(latestRouting, ["selectedModel", "selected_model"]),
decisionReason:
stringField(modelRouting, ["decisionReason", "decision_reason"]) ||
stringField(latestRouting, ["decisionReason", "decision_reason"]),
fallbackChain: [
...new Set([
...stringArrayField(modelRouting, ["fallbackChain", "fallback_chain"]),
...stringArrayField(latestRouting, ["fallbackChain", "fallback_chain"]),
]),
],
};
const timing = {
source: stringField(latestTiming, ["source"]),
runStatus: stringField(latestTiming, ["runStatus", "run_status"]),
durationMs: numberField(latestTiming, ["durationMs", "duration_ms"]),
firstVisibleDeltaMs: numberField(latestTiming, [
"firstVisibleDeltaMs",
"first_visible_delta_ms",
]),
firstThinkingDeltaMs: numberField(latestTiming, [
"firstThinkingDeltaMs",
"first_thinking_delta_ms",
]),
firstTextDeltaMs: numberField(latestTiming, [
"firstTextDeltaMs",
"first_text_delta_ms",
]),
};
return {
status: {
threadStatus: stringField(threadRead, ["status"]),
latestTurnStatus: latestTurnStatus(threadRead),
latestRunStatus: timing.runStatus,
},
routing,
timing,
assertions: {
hasFirstTextDelta: timing.firstTextDeltaMs !== null,
hasRoutingEvidence: Boolean(
routing.decisionSource ||
routing.selectedProvider ||
routing.selectedModel ||
routing.decisionReason,
),
isResponsiveChatAuto: routing.decisionSource === "responsive_chat_auto",
},
};
}
function isTerminalSample(summary) {
if (summary.assertions.hasFirstTextDelta) {
return true;
}
if (TERMINAL_RUN_STATUSES.has(String(summary.status.latestRunStatus || ""))) {
return true;
}
if (TERMINAL_RUN_STATUSES.has(String(summary.status.latestTurnStatus || ""))) {
return true;
}
return false;
}
async function waitForSampleSummary(options, sessionId) {
const startedAt = Date.now();
let lastSummary = null;
while (Date.now() - startedAt < options.timeoutMs) {
const threadRead = await invoke(options, "agent_runtime_get_thread_read", {
sessionId,
});
lastSummary = summarizeThreadRead(threadRead);
if (isTerminalSample(lastSummary)) {
return lastSummary;
}
await sleep(options.intervalMs);
}
return {
...(lastSummary || summarizeThreadRead(null)),
timedOut: true,
};
}
async function runSample(options, workspaceId, sampleIndex) {
const stamp = `${Date.now()}-${process.pid}-${sampleIndex}`;
const sessionId = await invoke(options, "agent_runtime_create_session", {
workspaceId,
name: `AgentUI TTFT live sample ${options.mode} ${stamp}`,
runStartHooks: false,
});
const turnId = `agentui-ttft-${options.mode}-${stamp}`;
const eventName = `aster_stream_${sessionId}_${turnId}`;
await invoke(options, "agent_runtime_submit_turn", {
request: {
message: options.message,
session_id: sessionId,
workspace_id: workspaceId,
event_name: eventName,
turn_id: turnId,
turn_config: buildTurnConfig(options),
skip_pre_submit_resume: true,
},
});
const summary = await waitForSampleSummary(options, sessionId);
return {
sampleIndex,
sessionId,
turnId,
...summary,
};
}
function buildSummary(samples) {
return {
samples: samples.length,
firstTextSamples: samples.filter(
(sample) => sample.assertions?.hasFirstTextDelta,
).length,
responsiveAutoSamples: samples.filter(
(sample) => sample.assertions?.isResponsiveChatAuto,
).length,
routingEvidenceSamples: samples.filter(
(sample) => sample.assertions?.hasRoutingEvidence,
).length,
selectedGroups: samples.map((sample) => ({
decisionSource: sample.routing?.decisionSource || null,
provider: sample.routing?.selectedProvider || null,
model: sample.routing?.selectedModel || null,
firstTextDeltaMs: sample.timing?.firstTextDeltaMs ?? null,
runStatus: sample.status?.latestRunStatus || null,
})),
};
}
function writeOutput(options, payload) {
const text = `${JSON.stringify(payload, null, 2)}\n`;
if (!options.outputPath) {
process.stdout.write(text);
return;
}
fs.mkdirSync(path.dirname(options.outputPath), { recursive: true });
fs.writeFileSync(options.outputPath, text);
}
async function main() {
const options = parseArgs(process.argv.slice(2));
const health = await waitForHealth(options);
const workspace = await invoke(options, "get_or_create_default_project", {});
const workspaceId = workspace?.id;
if (!workspaceId) {
throw new Error("get_or_create_default_project 缺少 workspace id");
}
const samples = [];
for (let index = 0; index < options.samples; index += 1) {
samples.push(await runSample(options, workspaceId, index + 1));
}
writeOutput(options, {
schemaVersion: "v1",
generatedAt: new Date().toISOString(),
mode: options.mode,
health: {
status: health?.status || null,
service: health?.service || null,
version: health?.version || null,
},
workspaceId,
summary: buildSummary(samples),
samples,
privacy: {
exportedFields: [
"session/turn id",
"routing decision",
"first visible/thinking/text TTFT",
"status summary",
],
omittedFields: [
"prompt",
"assistant response",
"error_message",
"secrets",
"run id",
],
},
});
}
main().catch((error) => {
console.error(error instanceof Error ? error.message : String(error));
process.exit(1);
});
scripts/agentui-ttft-sample-matrix.mjs
+440 -4
View File
@@ -8,15 +8,48 @@ import process from "node:process";
const DEFAULT_LIMIT_RUNS = 5000;
const DEFAULT_LIMIT_GROUPS = 24;
const DEFAULT_MIN_FIRST_TEXT_PER_RESPONSIVE_GROUP = 3;
const MATRIX_PRESETS = {
"agentui-responsive-chat-ttft": {
minFirstTextPerResponsiveGroup: 3,
requiredResponsiveModels: ["deepseek-v4-flash", "MiniMax-M2.7"],
requiredResponsiveProviders: ["siliconflow-cn", "openrouter", "lime-hub"],
},
};
function applyPreset(result, presetName) {
const preset = MATRIX_PRESETS[presetName];
if (!preset) {
throw new Error(
`Unknown preset: ${presetName}. Available presets: ${Object.keys(MATRIX_PRESETS).join(", ")}`,
);
}
result.checkResponsiveMatrix = true;
result.matrixPreset = presetName;
result.minFirstTextPerResponsiveGroup = preset.minFirstTextPerResponsiveGroup;
result.requiredResponsiveModels = [...new Set([
...result.requiredResponsiveModels,
...preset.requiredResponsiveModels,
])];
result.requiredResponsiveProviders = [...new Set([
...result.requiredResponsiveProviders,
...preset.requiredResponsiveProviders,
])];
}
function parseArgs(argv) {
const result = {
dbPath: defaultLimeDbPath(),
format: "markdown",
help: false,
checkResponsiveMatrix: false,
limitGroups: DEFAULT_LIMIT_GROUPS,
limitRuns: DEFAULT_LIMIT_RUNS,
minFirstTextPerResponsiveGroup: DEFAULT_MIN_FIRST_TEXT_PER_RESPONSIVE_GROUP,
matrixPreset: "",
outputPath: "",
requiredResponsiveModels: [],
requiredResponsiveProviders: [],
};
for (let index = 0; index < argv.length; index += 1) {
@@ -41,6 +74,30 @@ function parseArgs(argv) {
index += 1;
continue;
}
if (arg === "--min-first-text-per-responsive-group" && argv[index + 1]) {
result.minFirstTextPerResponsiveGroup = Number.parseInt(String(argv[index + 1]), 10);
index += 1;
continue;
}
if (arg === "--check-responsive-matrix") {
result.checkResponsiveMatrix = true;
continue;
}
if (arg === "--preset" && argv[index + 1]) {
applyPreset(result, String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--require-responsive-provider" && argv[index + 1]) {
result.requiredResponsiveProviders.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--require-responsive-model" && argv[index + 1]) {
result.requiredResponsiveModels.push(String(argv[index + 1]).trim());
index += 1;
continue;
}
if (arg === "--output" && argv[index + 1]) {
result.outputPath = String(argv[index + 1]).trim();
index += 1;
@@ -68,6 +125,15 @@ Lime AgentUI TTFT Sample Matrix
--format FMT markdown | json，默认 markdown
--limit-runs N 最近 agent_runs 扫描上限，默认 ${DEFAULT_LIMIT_RUNS}
--limit-groups N Markdown 表格输出的 group 上限，默认 ${DEFAULT_LIMIT_GROUPS}
--check-responsive-matrix
检查 responsive_chat latency group 是否满足 first-text TTFT 样本下限
--preset NAME 套用检查预设：${Object.keys(MATRIX_PRESETS).join(" | ")}
--min-first-text-per-responsive-group N
每个 responsive_chat latency group 的 first-text TTFT 样本下限，默认 ${DEFAULT_MIN_FIRST_TEXT_PER_RESPONSIVE_GROUP}
--require-responsive-provider PROVIDER
必须出现在 responsive_chat latency group 中的 provider；可重复
--require-responsive-model MODEL
必须出现在 responsive_chat latency group 中的 model；可重复
--output PATH 写入文件；默认 stdout
-h, --help 显示帮助
@@ -95,6 +161,10 @@ function normalizePositiveInt(value, fallback) {
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback;
}
function normalizeNonNegativeInt(value, fallback) {
return Number.isFinite(value) && value >= 0 ? Math.floor(value) : fallback;
}
function abbreviateHomePath(filePath) {
const home = os.homedir();
return filePath.startsWith(home) ? `~${filePath.slice(home.length)}` : filePath;
@@ -156,10 +226,22 @@ function numberValue(value) {
return null;
}
function saneFirstTextValue(value) {
const number = numberValue(value);
if (number === null || number < 0 || number > 600_000) {
return null;
}
return number;
}
function stringValue(value) {
return typeof value === "string" && value.trim() ? value.trim() : null;
}
function normalizeRoutingToken(value) {
return stringValue(value)?.toLowerCase().replaceAll("-", "_") || "";
}
function redactProvider(provider) {
const normalized = stringValue(provider);
if (!normalized) {
@@ -242,6 +324,9 @@ function parseRouting(metadata) {
: Array.isArray(routingDecision?.fallback_chain)
? routingDecision.fallback_chain
: [],
settingsSource: stringValue(
firstDefined(routingDecision?.settingsSource, routingDecision?.settings_source),
),
selectedModel,
selectedProvider,
serviceModelSlot: stringValue(
@@ -250,20 +335,38 @@ function parseRouting(metadata) {
};
}
function isResponsiveChatLatencyRun(routing) {
return (
normalizeRoutingToken(routing.decisionSource) === "responsive_chat_auto" ||
normalizeRoutingToken(routing.serviceModelSlot) === "responsive_chat" ||
normalizeRoutingToken(routing.settingsSource) === "service_models.responsive_chat"
);
}
function parseRun(row) {
const metadata = parseJsonObject(row.metadata) || {};
const routing = parseRouting(metadata);
const provider = redactProvider(routing.selectedProvider) || "(unknown)";
const model = routing.selectedModel || "(unknown)";
const decisionSource = routing.decisionSource || "(unknown)";
const metadataFirstTextMs = saneFirstTextValue(
firstDefined(metadata.model_first_text_delta_ms, metadata.modelFirstTextDeltaMs),
);
const timelineFirstTextMs =
row.status === "success" ? saneFirstTextValue(row.timeline_first_text_delta_ms) : null;
const firstTextMs = metadataFirstTextMs ?? timelineFirstTextMs;
return {
createdAt: row.created_at || null,
decisionSource,
durationMs: numberValue(row.duration_ms),
firstTextMs: numberValue(
firstDefined(metadata.model_first_text_delta_ms, metadata.modelFirstTextDeltaMs),
),
firstTextMs,
firstTextSource:
metadataFirstTextMs !== null
? "agent_runs.metadata"
: timelineFirstTextMs !== null
? "thread_items.timeline"
: null,
firstThinkingMs: numberValue(
firstDefined(metadata.model_first_thinking_delta_ms, metadata.modelFirstThinkingDeltaMs),
),
@@ -274,6 +377,7 @@ function parseRun(row) {
model,
provider,
routing,
responsiveLatencyEligible: isResponsiveChatLatencyRun(routing),
status: row.status || "(unknown)",
};
}
@@ -321,6 +425,10 @@ function buildGroupKey(run) {
return `${run.decisionSource}\u0000${run.provider}\u0000${run.model}`;
}
function buildResponsiveGroupKey(run) {
return `${run.provider}\u0000${run.model}`;
}
function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns }) {
if (!fs.existsSync(dbPath)) {
throw new Error(`DB not found: ${dbPath}`);
@@ -329,7 +437,31 @@ function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns
const safeLimit = normalizePositiveInt(limitRuns, DEFAULT_LIMIT_RUNS);
const rows = sqliteJson(
dbPath,
`SELECT id, status, duration_ms, metadata, created_at
`SELECT id,
status,
duration_ms,
metadata,
created_at,
session_id,
(
SELECT CAST((julianday(agent_item.started_at) - julianday(user_item.started_at)) * 86400000 AS INTEGER)
FROM agent_thread_items AS user_item
JOIN agent_thread_items AS agent_item
ON agent_item.session_id = user_item.session_id
AND agent_item.item_type = 'agent_message'
AND (
json_extract(agent_runs.metadata, '$.turn_state.turn_id') IS NULL
OR agent_item.turn_id = json_extract(agent_runs.metadata, '$.turn_state.turn_id')
)
WHERE user_item.session_id = agent_runs.session_id
AND user_item.item_type = 'user_message'
AND (
json_extract(agent_runs.metadata, '$.turn_state.turn_id') IS NULL
OR user_item.turn_id = json_extract(agent_runs.metadata, '$.turn_state.turn_id')
)
ORDER BY user_item.sequence ASC, agent_item.sequence ASC
LIMIT 1
) AS timeline_first_text_delta_ms
FROM agent_runs
WHERE metadata IS NOT NULL
ORDER BY created_at DESC
@@ -337,6 +469,7 @@ function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns
);
const runs = rows.map(parseRun);
const groups = new Map();
const responsiveLatencyGroups = new Map();
for (const run of runs) {
const key = buildGroupKey(run);
@@ -346,6 +479,7 @@ function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns
durationStats: null,
firstTextStats: null,
firstTextCount: 0,
firstTextSourceCounts: {},
model: run.model,
provider: run.provider,
runs: [],
@@ -358,10 +492,44 @@ function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns
group.statusCounts[run.status] = (group.statusCounts[run.status] || 0) + 1;
if (run.firstTextMs !== null) {
group.firstTextCount += 1;
group.firstTextSourceCounts[run.firstTextSource] =
(group.firstTextSourceCounts[run.firstTextSource] || 0) + 1;
}
if (run.routing.selectedProvider || run.routing.selectedModel || run.routing.decisionReason) {
group.withRoutingEvidence += 1;
}
if (run.responsiveLatencyEligible) {
const responsiveKey = buildResponsiveGroupKey(run);
if (!responsiveLatencyGroups.has(responsiveKey)) {
responsiveLatencyGroups.set(responsiveKey, {
decisionSourceCounts: {},
durationStats: null,
firstTextStats: null,
firstTextCount: 0,
firstTextSourceCounts: {},
model: run.model,
provider: run.provider,
runs: [],
statusCounts: {},
withRoutingEvidence: 0,
});
}
const responsiveGroup = responsiveLatencyGroups.get(responsiveKey);
responsiveGroup.runs.push(run);
responsiveGroup.statusCounts[run.status] =
(responsiveGroup.statusCounts[run.status] || 0) + 1;
responsiveGroup.decisionSourceCounts[run.decisionSource] =
(responsiveGroup.decisionSourceCounts[run.decisionSource] || 0) + 1;
if (run.firstTextMs !== null) {
responsiveGroup.firstTextCount += 1;
responsiveGroup.firstTextSourceCounts[run.firstTextSource] =
(responsiveGroup.firstTextSourceCounts[run.firstTextSource] || 0) + 1;
}
if (run.routing.selectedProvider || run.routing.selectedModel || run.routing.decisionReason) {
responsiveGroup.withRoutingEvidence += 1;
}
}
}
const groupSummaries = [...groups.values()]
@@ -371,6 +539,32 @@ function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns
decisionSource: group.decisionSource,
durationStats: summarizeNumbers(group.runs.map((run) => run.durationMs)),
firstTextCount: group.firstTextCount,
firstTextSourceCounts: group.firstTextSourceCounts,
firstTextStats: summarizeNumbers(group.runs.map((run) => run.firstTextMs)),
model: group.model,
provider: group.provider,
runCount,
statusCounts: group.statusCounts,
withRoutingEvidence: group.withRoutingEvidence,
};
})
.sort((left, right) => {
const firstTextDelta = right.firstTextCount - left.firstTextCount;
if (firstTextDelta !== 0) {
return firstTextDelta;
}
return right.runCount - left.runCount;
});
const responsiveLatencyGroupSummaries = [...responsiveLatencyGroups.values()]
.map((group) => {
const runCount = group.runs.length;
return {
decisionSourceCounts: group.decisionSourceCounts,
decisionSources: Object.keys(group.decisionSourceCounts).sort(),
durationStats: summarizeNumbers(group.runs.map((run) => run.durationMs)),
firstTextCount: group.firstTextCount,
firstTextSourceCounts: group.firstTextSourceCounts,
firstTextStats: summarizeNumbers(group.runs.map((run) => run.firstTextMs)),
model: group.model,
provider: group.provider,
@@ -398,13 +592,17 @@ function buildReport({ dbPath, generatedAt = new Date().toISOString(), limitRuns
"status counts",
"duration stats",
"first text TTFT stats",
"first text source counts",
"routing evidence count",
],
omittedFields: ["prompt", "assistant response", "error_message", "secrets", "run ids"],
},
responsiveLatencyGroups: responsiveLatencyGroupSummaries,
summary: {
firstTextRuns: runs.filter((run) => run.firstTextMs !== null).length,
groups: groupSummaries.length,
responsiveLatencyGroups: responsiveLatencyGroupSummaries.length,
responsiveLatencyRuns: runs.filter((run) => run.responsiveLatencyEligible).length,
runs: runs.length,
routingEvidenceRuns: runs.filter(
(run) => run.routing.selectedProvider || run.routing.selectedModel || run.routing.decisionReason,
@@ -420,6 +618,196 @@ function statusSummary(statusCounts) {
.join(" / ");
}
function statusCount(statusCounts, status) {
return Number(statusCounts?.[status] || 0);
}
function errorLikeSampleCount(statusCounts) {
return Object.entries(statusCounts || {}).reduce((total, [status, count]) => {
if (status === "success" || status === "running") {
return total;
}
return total + Number(count || 0);
}, 0);
}
function isFallbackOnlyResponsiveGroup(group, minimum) {
return (
group.firstTextCount < minimum &&
statusCount(group.statusCounts, "success") === 0 &&
errorLikeSampleCount(group.statusCounts) > 0
);
}
function buildResponsiveMatrixCheck(
report,
{
minFirstTextPerGroup,
preset = "",
requiredModels = [],
requiredProviders = [],
},
) {
const minimum = normalizeNonNegativeInt(
minFirstTextPerGroup,
DEFAULT_MIN_FIRST_TEXT_PER_RESPONSIVE_GROUP,
);
const responsiveGroups = report.responsiveLatencyGroups || [];
const fallbackOnlyGroups = responsiveGroups
.filter((group) => isFallbackOnlyResponsiveGroup(group, minimum))
.map((group) => ({
decisionSources: group.decisionSources || [],
firstTextCount: group.firstTextCount,
minimum,
model: group.model,
provider: group.provider,
runCount: group.runCount,
status: statusSummary(group.statusCounts) || "n/a",
}));
const missingGroups = responsiveGroups
.filter((group) => group.firstTextCount < minimum)
.filter((group) => !isFallbackOnlyResponsiveGroup(group, minimum))
.map((group) => ({
decisionSources: group.decisionSources || [],
firstTextCount: group.firstTextCount,
minimum,
model: group.model,
neededFirstTextSamples: minimum - group.firstTextCount,
provider: group.provider,
runCount: group.runCount,
status: statusSummary(group.statusCounts) || "n/a",
}));
const missingRequiredProviders = requiredProviders
.filter((provider) => provider)
.filter(
(provider) => !responsiveGroups.some((group) => group.provider === provider),
);
const missingRequiredModels = requiredModels
.filter((model) => model)
.filter((model) => !responsiveGroups.some((group) => group.model === model));
return {
fallbackOnlyGroups,
firstTextTargetPerGroup: minimum,
missingGroups,
missingRequiredModels,
missingRequiredProviders,
passingGroups: responsiveGroups.length - missingGroups.length - fallbackOnlyGroups.length,
preset,
requiredModels: requiredModels.filter(Boolean),
requiredProviders: requiredProviders.filter(Boolean),
responsiveGroupKind: "responsive_chat_latency",
responsiveGroups: responsiveGroups.length,
status:
responsiveGroups.length > 0 &&
missingGroups.length === 0 &&
missingRequiredProviders.length === 0 &&
missingRequiredModels.length === 0
? "pass"
: "fail",
totalNeededFirstTextSamples: missingGroups.reduce(
(total, group) => total + group.neededFirstTextSamples,
0,
),
};
}
function renderMatrixCheckMarkdown(matrixCheck) {
const lines = [
"",
"## Responsive matrix check",
"",
`- status: ${matrixCheck.status}`,
`- responsive_chat latency groups: ${matrixCheck.responsiveGroups}`,
`- passing groups: ${matrixCheck.passingGroups}`,
`- fallback-only groups: ${matrixCheck.fallbackOnlyGroups.length}`,
`- first-text target per group: ${matrixCheck.firstTextTargetPerGroup}`,
`- additional first-text samples needed: ${matrixCheck.totalNeededFirstTextSamples}`,
];
if (matrixCheck.preset) {
lines.push(`- preset: ${matrixCheck.preset}`);
}
if (matrixCheck.requiredProviders.length > 0) {
lines.push(`- required providers: ${matrixCheck.requiredProviders.join(", ")}`);
}
if (matrixCheck.requiredModels.length > 0) {
lines.push(`- required models: ${matrixCheck.requiredModels.join(", ")}`);
}
if (matrixCheck.missingRequiredProviders.length > 0) {
lines.push(`- missing required providers: ${matrixCheck.missingRequiredProviders.join(", ")}`);
}
if (matrixCheck.missingRequiredModels.length > 0) {
lines.push(`- missing required models: ${matrixCheck.missingRequiredModels.join(", ")}`);
}
if (matrixCheck.missingGroups.length === 0) {
lines.push("- missing groups: none");
} else {
lines.push("- missing groups:");
for (const group of matrixCheck.missingGroups) {
lines.push(
[
` - ${markdownInlineCode(`${group.provider}/${group.model}`)}`,
`${group.firstTextCount}/${group.minimum} first-text samples,`,
`need ${group.neededFirstTextSamples} more,`,
`${group.runCount} runs, status ${group.status}`,
group.decisionSources?.length ? `sources ${group.decisionSources.join("/")}` : "",
].join(" "),
);
}
}
if (matrixCheck.fallbackOnlyGroups.length === 0) {
lines.push("- fallback-only groups: none");
} else {
lines.push("- fallback-only groups:");
for (const group of matrixCheck.fallbackOnlyGroups) {
lines.push(
[
` - ${markdownInlineCode(`${group.provider}/${group.model}`)}`,
`${group.runCount} runs, status ${group.status},`,
"kept as routing fallback evidence rather than first-text baseline",
group.decisionSources?.length ? `sources ${group.decisionSources.join("/")}` : "",
].join(" "),
);
}
}
return lines;
}
function reportMatrixCheckFailure(matrixCheck) {
const missing = matrixCheck.missingGroups
.map((group) => `${group.provider}/${group.model} (${group.firstTextCount}/${group.minimum})`)
.join(", ");
const missingProviders =
matrixCheck.missingRequiredProviders.length > 0
? `Missing required providers: ${matrixCheck.missingRequiredProviders.join(", ")}.`
: "";
const missingModels =
matrixCheck.missingRequiredModels.length > 0
? `Missing required models: ${matrixCheck.missingRequiredModels.join(", ")}.`
: "";
const fallbackOnly =
matrixCheck.fallbackOnlyGroups.length > 0
? `Fallback-only groups: ${matrixCheck.fallbackOnlyGroups
.map((group) => `${group.provider}/${group.model} (${group.status})`)
.join(", ")}.`
: "";
console.error(
[
"Responsive matrix check failed:",
`${matrixCheck.passingGroups}/${matrixCheck.responsiveGroups} groups passed`,
`with target ${matrixCheck.firstTextTargetPerGroup} first-text samples per responsive_chat latency group.`,
`Need ${matrixCheck.totalNeededFirstTextSamples} additional first-text samples.`,
missing ? `Missing: ${missing}` : "",
fallbackOnly,
missingProviders,
missingModels,
]
.filter(Boolean)
.join(" "),
);
}
function renderMarkdown(report, { limitGroups }) {
const safeLimitGroups = normalizePositiveInt(limitGroups, DEFAULT_LIMIT_GROUPS);
const groups = report.groups.slice(0, safeLimitGroups);
@@ -435,6 +823,8 @@ function renderMarkdown(report, { limitGroups }) {
`- groups: ${report.summary.groups}`,
`- runs with routing evidence: ${report.summary.routingEvidenceRuns}`,
`- runs with first text TTFT: ${report.summary.firstTextRuns}`,
`- responsive_chat latency runs: ${report.summary.responsiveLatencyRuns}`,
`- responsive_chat latency groups: ${report.summary.responsiveLatencyGroups}`,
"- privacy: prompt / response / error_message are not exported",
"",
"## Group summary",
@@ -458,6 +848,32 @@ function renderMarkdown(report, { limitGroups }) {
);
}
const responsiveLatencyGroups = (report.responsiveLatencyGroups || []).slice(
0,
safeLimitGroups,
);
lines.push(
"",
"## Responsive chat latency groups",
"",
"| provider/model | runs | decision sources | status | routing evidence | first text samples | first text min/p50/avg/max | duration min/p50/avg/max |",
"| --- | ---: | --- | --- | ---: | ---: | --- | --- |",
);
for (const group of responsiveLatencyGroups) {
lines.push(
[
markdownInlineCode(`${group.provider}/${group.model}`),
String(group.runCount),
markdownCell(group.decisionSources.join(" / ") || "n/a"),
markdownCell(statusSummary(group.statusCounts) || "n/a"),
String(group.withRoutingEvidence),
String(group.firstTextCount),
formatStats(group.firstTextStats),
formatStats(group.durationStats),
].join(" | ").replace(/^/, "| ").replace(/$/, " |"),
);
}
lines.push("", "## Missing first-text TTFT candidates", "");
if (missingFirstText.length === 0) {
lines.push("- none with routing evidence");
@@ -473,6 +889,10 @@ function renderMarkdown(report, { limitGroups }) {
}
}
if (report.matrixCheck) {
lines.push(...renderMatrixCheckMarkdown(report.matrixCheck));
}
return `${lines.join("\n")}\n`;
}
@@ -496,14 +916,30 @@ function main() {
dbPath: path.resolve(args.dbPath),
limitRuns: args.limitRuns,
});
if (args.checkResponsiveMatrix) {
report.matrixCheck = buildResponsiveMatrixCheck(report, {
minFirstTextPerGroup: args.minFirstTextPerResponsiveGroup,
preset: args.matrixPreset,
requiredModels: args.requiredResponsiveModels,
requiredProviders: args.requiredResponsiveProviders,
});
}
if (args.format === "json") {
writeOutput(args.outputPath, `${JSON.stringify(report, null, 2)}\n`);
if (report.matrixCheck && report.matrixCheck.status !== "pass") {
reportMatrixCheckFailure(report.matrixCheck);
process.exitCode = 1;
}
return;
}
if (args.format !== "markdown") {
throw new Error(`Unsupported format: ${args.format}`);
}
writeOutput(args.outputPath, renderMarkdown(report, { limitGroups: args.limitGroups }));
if (report.matrixCheck && report.matrixCheck.status !== "pass") {
reportMatrixCheckFailure(report.matrixCheck);
process.exitCode = 1;
}
}
main();
scripts/at-command-registry-e2e.mjs
+99 -1
View File
@@ -226,6 +226,99 @@ function parseInvokeRequest(request, invokeUrl) {
}
}
async function invokeBridgeCommand(options, cmd, args, timeoutMs = 30_000) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(options.invokeUrl, {
method: "POST",
headers: { "content-type": "application/json" },
body: JSON.stringify({ cmd, args }),
signal: controller.signal,
});
const text = await response.text();
const payload = text ? JSON.parse(text) : null;
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${text}`);
}
return payload;
} finally {
clearTimeout(timeout);
}
}
async function interruptSubmittedTurn(options, request) {
const sessionId = String(
request.session_id || request.sessionId || "",
).trim();
const turnId = String(request.turn_id || request.turnId || "").trim();
if (!sessionId) {
return { attempted: false, reason: "missing-session-id" };
}
const interruptRequest = {
session_id: sessionId,
...(turnId ? { turn_id: turnId } : {}),
};
try {
await invokeBridgeCommand(
options,
"agent_runtime_interrupt_turn",
{ request: interruptRequest },
20_000,
);
} catch (error) {
return {
attempted: true,
status: "failed",
sessionId,
turnId: turnId || null,
error: error instanceof Error ? error.message : String(error),
};
}
const drained = await waitForCondition(
"等待 @配图 smoke 提交任务清理",
async () => {
const threadRead = await invokeBridgeCommand(
options,
"agent_runtime_get_thread_read",
{ sessionId },
20_000,
).catch(() => null);
if (!threadRead) {
return null;
}
const status = String(threadRead.status || "").toLowerCase();
const queuedTurns = Array.isArray(threadRead.queued_turns)
? threadRead.queued_turns.length
: 0;
const activeTurnId =
threadRead.active_turn_id || threadRead.activeTurnId || null;
return !activeTurnId &&
queuedTurns === 0 &&
(status === "idle" || status === "completed" || status === "cancelled")
? { status: threadRead.status || null, queuedTurns }
: null;
},
30_000,
options.intervalMs,
).catch((error) => ({
status: "unknown",
error: error instanceof Error ? error.message : String(error),
}));
return {
attempted: true,
status: "sent",
sessionId,
turnId: turnId || null,
drained,
};
}
async function main() {
if (typeof fetch !== "function") {
throw new Error("当前 Node 运行时不支持 fetch，请使用 Node 18+");
@@ -372,7 +465,7 @@ async function main() {
.waitFor({ state: "visible", timeout: options.timeoutMs });
const imageCommandItem = commandItemByText(page, "@配图").first();
await imageCommandItem.waitFor({
state: "visible",
state: "attached",
timeout: options.timeoutMs,
});
@@ -403,6 +496,7 @@ async function main() {
});
logStage(options, "select-image-command");
await imageCommandItem.scrollIntoViewIfNeeded();
await imageCommandItem.click();
await textarea.fill(IMAGE_PROMPT);
@@ -459,6 +553,8 @@ async function main() {
summary.assertions.chatModelPreferenceSuppressed = true;
summary.submitRequest = {
message: request.message,
sessionId: request.session_id || null,
turnId: request.turn_id || null,
providerPreference: turnConfig.provider_preference ?? null,
modelPreference: turnConfig.model_preference ?? null,
contractKey,
@@ -473,6 +569,8 @@ async function main() {
fullPage: true,
});
summary.cleanup = await interruptSubmittedTurn(options, request);
const summaryPath = path.join(
options.evidenceDir,
`${options.prefix}-summary.json`,
scripts/claw-chat-ready-streaming-smoke.mjs
+213 -50
View File
@@ -34,15 +34,16 @@ const DEFAULTS = {
const POST_HEALTH_SETTLE_MS = 1_500;
const ONBOARDING_VERSION = "1.1.0";
const LONG_TURN_LINE_COUNT = 480;
const LONG_PROMPT = [
"E2E 中断测试：请输出 160 行。",
`E2E 中断测试：请输出 ${LONG_TURN_LINE_COUNT} 行。`,
"每一行都必须使用格式：中断测试第 N 行。",
"从 1 开始递增，不要合并行，不要提前总结。",
"如果收到停止请求应立即停止，不要补完剩余行。",
].join("\n");
const RECOVERY_EXPECTED_TEXT = "复原完成";
const RECOVERY_PROMPT =
"停止后恢复测试：这是一个新的独立回合，请忽略上一条输出 160 行的要求。只输出“复原完成”这四个字，不要输出行号、解释或其他内容。";
`停止后恢复测试：这是一个新的独立回合，请忽略上一条输出 ${LONG_TURN_LINE_COUNT} 行的要求。只输出“复原完成”这四个字，不要输出行号、解释或其他内容。`;
const MODEL_AVAILABILITY_PROMPT = "请只回复 QC_OK。";
const MAX_MODEL_AVAILABILITY_CANDIDATES = 12;
const FAST_RESPONSE_MODE_STORAGE_KEY = "lime:agent-fast-response-mode";
@@ -660,6 +661,50 @@ function queuedTurnCount(session) {
return Array.isArray(session?.queued_turns) ? session.queued_turns.length : 0;
}
function activeTurnIdFromThreadRead(threadRead) {
return threadRead?.active_turn_id || threadRead?.activeTurnId || null;
}
function latestTurnStatusFromThreadRead(threadRead) {
return (
threadRead?.diagnostics?.latest_turn_status ||
threadRead?.runtime_summary?.latestTurnStatus ||
threadRead?.status ||
null
);
}
function syntheticInterruptedTurn(turnId, threadRead) {
return {
id: turnId,
status: "aborted",
synthetic: true,
source: "thread-read-idle-after-interrupt",
threadStatus: latestTurnStatusFromThreadRead(threadRead),
};
}
function interruptDrainedWithoutRecordedTurn({
session,
threadRead,
snapshot,
interruptScoped,
}) {
const threadStatus = String(
latestTurnStatusFromThreadRead(threadRead) || "",
).toLowerCase();
return (
interruptScoped &&
queuedTurnCount(session) === 0 &&
queuedTurnCount(threadRead) === 0 &&
!activeTurnIdFromThreadRead(threadRead) &&
(threadStatus === "idle" || threadStatus === "aborted") &&
snapshot &&
!snapshot.stopVisible &&
!snapshot.finalLineSeen
);
}
function normalizeConsoleLine(item) {
const location =
item.location && item.location.url
@@ -668,7 +713,7 @@ function normalizeConsoleLine(item) {
return `[${item.type}] ${item.text}${location}`;
}
function buildPageSnapshotScript(recoveryExpectedText) {
function buildPageSnapshotScript(recoveryExpectedText, longTurnLineCount) {
return `(() => {
const textareas = Array.from(
document.querySelectorAll('textarea[name="agent-chat-message"]'),
@@ -684,8 +729,13 @@ function buildPageSnapshotScript(recoveryExpectedText) {
) ?? null;
const bodyText = document.body?.innerText || "";
const streamLinePattern = /中断测试第\\s*\\d+\\s*行/;
const finalLinePattern = /中断测试第\\s*160\\s*行/;
const finalLinePattern = new RegExp("中断测试第\\\\s*" + ${JSON.stringify(
String(longTurnLineCount),
)} + "\\\\s*行");
const recoveryExpected = ${JSON.stringify(recoveryExpectedText)};
const recoveryTextCount = recoveryExpected
? bodyText.split(recoveryExpected).length - 1
: 0;
const streamText = bodyText
.split("\\n")
.filter((line) => streamLinePattern.test(line))
@@ -700,6 +750,7 @@ function buildPageSnapshotScript(recoveryExpectedText) {
longPromptVisible: bodyText.includes("E2E 中断测试"),
stoppedMarker: bodyText.includes("用户已停止当前执行"),
recoveryVisible: bodyText.includes(recoveryExpected),
recoveryTextCount,
streamLineCount: streamText ? streamText.split("\\n").filter(Boolean).length : 0,
streamTextLength: streamText.length,
finalLineSeen: finalLinePattern.test(streamText),
@@ -715,7 +766,9 @@ function buildPageSnapshotScript(recoveryExpectedText) {
async function readPageSnapshot(page) {
return page
.evaluate(buildPageSnapshotScript(RECOVERY_EXPECTED_TEXT))
.evaluate(
buildPageSnapshotScript(RECOVERY_EXPECTED_TEXT, LONG_TURN_LINE_COUNT),
)
.catch(() => null);
}
@@ -1009,6 +1062,8 @@ async function main() {
};
let submittedSessionId = "";
let submittedLongTurnId = "";
let submittedFollowSessionId = "";
let submittedFollowTurnId = "";
try {
const scopedProviderKey = `agent_pref_provider_${workspaceId}`;
@@ -1203,12 +1258,26 @@ async function main() {
}`,
);
}
return snapshot?.stopVisible && snapshot.streamTextLength > 0
? snapshot
const runtimeStreamEventSeen = consoleMessages.some(
(item) =>
String(item?.text || "").includes("AgentStream.firstEvent") ||
String(item?.text || "").includes(
"AgentStream.firstRuntimeStatus",
) ||
String(item?.text || "").includes("AgentStream.runtimeKeepalive"),
);
return snapshot?.stopVisible &&
(snapshot.streamTextLength > 0 || runtimeStreamEventSeen)
? {
...snapshot,
runtimeStreamEventSeen,
runtimeTurn: turn || null,
runtimeTurnStatus: turn?.status || null,
}
: null;
},
90_000,
500,
250,
);
summary.firstDelta = firstDelta;
summary.steps.push("running-stop-visible");
@@ -1217,22 +1286,7 @@ async function main() {
fullPage: true,
});
const runningSession = await waitForCondition(
"等待 turn 进入 running",
async () => {
const session = await invoke(
options,
"agent_runtime_get_session",
{ sessionId },
20_000,
).catch(() => null);
const turn = findTurn(session, longTurnId);
return turn?.status === "running" ? { turn, session } : null;
},
60_000,
1_000,
);
summary.runningTurnObserved = runningSession.turn;
summary.runningTurnObserved = firstDelta.runtimeTurn || null;
logStage("interrupt-long-turn");
const interruptStart = invokes.length;
@@ -1264,16 +1318,42 @@ async function main() {
const interrupted = await waitForCondition(
"等待 turn 中断完成",
async () => {
const session = await invoke(
options,
"agent_runtime_get_session",
{ sessionId },
20_000,
).catch(() => null);
const [session, threadRead, snapshot] = await Promise.all([
invoke(
options,
"agent_runtime_get_session",
{ sessionId },
20_000,
).catch(() => null),
invoke(
options,
"agent_runtime_get_thread_read",
{ sessionId },
20_000,
).catch(() => null),
readPageSnapshot(page),
]);
const turn = findTurn(session, longTurnId);
return turn && ["aborted", "failed", "completed"].includes(turn.status)
? { turn, session }
: null;
if (turn && ["aborted", "failed", "completed"].includes(turn.status)) {
return { turn, session, threadRead, snapshot };
}
if (
interruptDrainedWithoutRecordedTurn({
session,
threadRead,
snapshot,
interruptScoped: summary.interruptHasTurnScope,
})
) {
return {
turn: syntheticInterruptedTurn(longTurnId, threadRead),
session,
threadRead,
snapshot,
inferredInterruptDrained: true,
};
}
return null;
},
120_000,
1_000,
@@ -1281,6 +1361,11 @@ async function main() {
let latestSession = interrupted.session;
summary.interruptedTurn = interrupted.turn;
summary.interruptedTurnStatus = interrupted.turn?.status || null;
summary.interruptedThreadRead = interrupted.threadRead || null;
summary.interruptedSnapshot = interrupted.snapshot || null;
summary.interruptDrainedInferred = Boolean(
interrupted.inferredInterruptDrained,
);
summary.queueCountAfterInterrupt = queuedTurnCount(latestSession);
await page
@@ -1297,6 +1382,34 @@ async function main() {
)
.catch(() => undefined);
logStage("restore-interrupted-session-before-recovery");
summary.beforeRecoveryOpenDispatch = await dispatchTaskCenterOpenTask(
page,
sessionId,
workspaceId,
);
summary.beforeRecoverySnapshot = await waitForCondition(
"等待中断会话重新挂载",
async () => {
const snapshot = await readPageSnapshot(page);
return snapshot &&
!isLikelyDetachedBlankTaskSnapshot(snapshot) &&
(snapshot.longPromptVisible ||
snapshot.stoppedMarker ||
snapshot.streamTextLength > 0)
? snapshot
: null;
},
15_000,
250,
).catch((error) => {
summary.beforeRecoverySessionRestore = {
passed: false,
error: error instanceof Error ? error.message : String(error),
};
return null;
});
logStage("submit-recovery-turn");
const followSubmitStart = invokes.length;
await textarea.fill(RECOVERY_PROMPT);
@@ -1317,9 +1430,15 @@ async function main() {
250,
);
const followRequest = followSubmit.args?.request || {};
const followSessionId = String(followRequest.session_id || sessionId);
const followTurnId = String(followRequest.turn_id || "");
assert(followSessionId, "恢复 turn 缺少 session_id");
assert(followTurnId, "恢复 turn 缺少 turn_id");
submittedFollowSessionId = followSessionId;
submittedFollowTurnId = followTurnId;
summary.followSessionId = followSessionId;
summary.followTurnId = followTurnId;
summary.followUsesOriginalSession = followSessionId === sessionId;
summary.followSubmitTurnConfig = followRequest.turn_config || null;
const followCompleted = await waitForCondition(
@@ -1328,7 +1447,7 @@ async function main() {
const session = await invoke(
options,
"agent_runtime_get_session",
{ sessionId },
{ sessionId: followSessionId },
20_000,
).catch(() => null);
const turn = findTurn(session, followTurnId);
@@ -1342,13 +1461,25 @@ async function main() {
latestSession = followCompleted.session || latestSession;
summary.followTurn = followCompleted.turn;
summary.followTurnStatus = followCompleted.turn?.status || null;
const followAssistantText = [
allAssistantText(latestSession),
allAgentItemText(latestSession),
]
.filter(Boolean)
.join("\n");
summary.followPersistedRecoveryBeforeGuiWait =
followAssistantText.includes(RECOVERY_EXPECTED_TEXT);
let recoverySnapshot = null;
try {
recoverySnapshot = await waitForCondition(
"等待 GUI 出现恢复结果",
async () => {
const snapshot = await readPageSnapshot(page);
return snapshot?.recoveryVisible ? snapshot : null;
return snapshot?.recoveryVisible &&
(Number(snapshot.recoveryTextCount || 0) >= 2 ||
summary.followPersistedRecoveryBeforeGuiWait)
? snapshot
: null;
},
60_000,
500,
@@ -1367,7 +1498,7 @@ async function main() {
(await invoke(
options,
"agent_runtime_get_session",
{ sessionId },
{ sessionId: followSessionId },
20_000,
).catch(() => null)) || latestSession;
const persistedAssistantText = [
@@ -1388,12 +1519,16 @@ async function main() {
if (isLikelyDetachedBlankTaskSnapshot(detachedSnapshot)) {
logStage("restore-session-after-recovery-persisted");
summary.recoveryVisibleRestoreDispatch =
await dispatchTaskCenterOpenTask(page, sessionId, workspaceId);
await dispatchTaskCenterOpenTask(page, followSessionId, workspaceId);
recoverySnapshot = await waitForCondition(
"等待重新打开目标会话后 GUI 出现恢复结果",
async () => {
const snapshot = await readPageSnapshot(page);
return snapshot?.recoveryVisible ? snapshot : null;
return snapshot?.recoveryVisible &&
(Number(snapshot.recoveryTextCount || 0) >= 2 ||
summary.recoveryPersistedBeforeRefresh)
? snapshot
: null;
},
45_000,
500,
@@ -1429,7 +1564,7 @@ async function main() {
const threadRead = await invoke(
options,
"agent_runtime_get_thread_read",
{ sessionId },
{ sessionId: followSessionId },
20_000,
).catch((error) => ({
__error: error instanceof Error ? error.message : String(error),
@@ -1446,8 +1581,7 @@ async function main() {
mockFallbackLines,
networkErrorTop,
} = consoleNetworkSummary(consoleMessages, failedRequests);
const activeTurnId =
threadRead?.active_turn_id || threadRead?.activeTurnId || null;
const activeTurnId = activeTurnIdFromThreadRead(threadRead);
const runtimeMockLines = mockFallbackLines.filter((line) =>
/agent_runtime_(submit_turn|interrupt_turn|get_session|get_thread_read)/.test(
line,
@@ -1460,12 +1594,11 @@ async function main() {
summary.threadRead = threadRead?.__error
? { error: threadRead.__error }
: threadRead;
summary.threadReadStatus =
summary.threadRead?.diagnostics?.latest_turn_status ||
summary.threadRead?.runtime_summary?.latestTurnStatus ||
null;
summary.threadReadStatus = latestTurnStatusFromThreadRead(summary.threadRead);
const latestRuntimeRouting =
latestSession?.execution_runtime?.routing_decision || {};
latestSession?.execution_runtime?.routing_decision ||
threadRead?.model_routing ||
{};
const followProviderPreferenceHonored =
followRequest.turn_config?.provider_preference === preferredProvider ||
latestRuntimeRouting?.selectedProvider === preferredProvider ||
@@ -1495,7 +1628,7 @@ async function main() {
assistantText.match(/中断测试第\s*\d+\s*行/g) || []
).length;
summary.interruptedAssistantContainsFinalLine =
assistantText.includes("中断测试第 160 行");
assistantText.includes(`中断测试第 ${LONG_TURN_LINE_COUNT} 行`);
summary.consoleErrorCount = consoleErrors.length;
summary.consoleWarningCount = consoleWarnings.length;
summary.networkErrorCount = failedRequests.length;
@@ -1505,7 +1638,9 @@ async function main() {
summary.assertions = {
workspaceReady: Boolean(readySnapshot?.ready),
devBridgeHealthy: health?.status === "ok" || Boolean(health),
streamFirstDeltaSeen: Boolean(firstDelta?.streamTextLength > 0),
streamFirstDeltaSeen: Boolean(
firstDelta?.streamTextLength > 0 || firstDelta?.runtimeStreamEventSeen,
),
streamGrowthObserved:
Number(recoverySnapshot?.streamTextLength || 0) >=
Number(firstDelta?.streamTextLength || 0) ||
@@ -1657,10 +1792,38 @@ async function main() {
? interruptError.message
: String(interruptError),
}));
if (
submittedFollowSessionId &&
(submittedFollowSessionId !== submittedSessionId ||
submittedFollowTurnId !== submittedLongTurnId)
) {
summary.failureCleanupFollowInterrupt = await invoke(
options,
"agent_runtime_interrupt_turn",
{
request: {
session_id: submittedFollowSessionId,
...(submittedFollowTurnId
? { turn_id: submittedFollowTurnId }
: {}),
},
},
20_000,
)
.then(() => ({ attempted: true, status: "sent" }))
.catch((interruptError) => ({
attempted: true,
status: "failed",
error:
interruptError instanceof Error
? interruptError.message
: String(interruptError),
}));
}
const failureSession = await invoke(
options,
"agent_runtime_get_session",
{ sessionId: submittedSessionId },
{ sessionId: submittedFollowSessionId || submittedSessionId },
20_000,
).catch((sessionError) => ({
__error:
@@ -1671,7 +1834,7 @@ async function main() {
const failureThreadRead = await invoke(
options,
"agent_runtime_get_thread_read",
{ sessionId: submittedSessionId },
{ sessionId: submittedFollowSessionId || submittedSessionId },
20_000,
).catch((threadError) => ({
__error: