From 2a7aa9286b6f9e4a48be8410c96925396e60caa3 Mon Sep 17 00:00:00 2001 From: musistudio Date: Tue, 11 Aug 2026 16:15:53 +0800 Subject: [PATCH] Update router configuration and provider handling --- .../docs/en/configuration/observability.md | 2 +- .../docs/zh/configuration/observability.md | 2 +- packages/core/src/config/constants.ts | 1 + packages/core/src/contracts/app.ts | 24 + packages/core/src/contracts/ipc-channels.ts | 1 + .../core/src/observability/raw-trace-sync.ts | 4 +- .../src/observability/request-log-limits.ts | 8 +- .../src/observability/request-log-runtime.ts | 13 +- .../src/observability/request-log-store.ts | 528 ++++++++++++++++-- .../src/observability/request-log-worker.ts | 40 +- packages/core/src/web/management-server.ts | 4 +- .../request-log-runtime.test.mjs | 77 ++- .../observability/request-log-store.test.mjs | 98 ++++ .../observability/raw-trace-sync.test.mjs | 9 +- packages/electron/src/main/ipc.ts | 3 +- packages/electron/src/main/preload.ts | 3 + .../pages/home/components/network-logs.tsx | 361 ++++++++++-- .../ui/src/pages/home/shared/external.tsx | 3 +- packages/ui/src/pages/home/shared/i18n.tsx | 3 +- .../home/shared/log-body-worker-protocol.ts | 28 +- .../src/pages/home/shared/log-body.worker.ts | 17 +- packages/ui/src/pages/home/shared/logs.ts | 2 + packages/ui/src/types/electron.d.ts | 3 + packages/ui/src/web-client-bridge.ts | 1 + packages/ui/test/unit/network-format.test.ts | 22 + 25 files changed, 1119 insertions(+), 138 deletions(-) diff --git a/docs/src/content/docs/en/configuration/observability.md b/docs/src/content/docs/en/configuration/observability.md index 63d071a6..6651b632 100644 --- a/docs/src/content/docs/en/configuration/observability.md +++ b/docs/src/content/docs/en/configuration/observability.md @@ -33,5 +33,5 @@ Request body / response body logging is governed by three low-level config optio | Option | Default | Description | | --- | --- | --- | | `requestLogBodyCapture` | `all` | Whether to record request and response bodies: `all` records all, `errors` records bodies only for failed requests, `none` records no bodies. | -| `requestLogMaxBodyBytes` | `52428800` (50 MiB) | Maximum bytes for a single request or response body; anything larger is truncated. The hard cap is also 50 MiB. | +| `requestLogMaxBodyBytes` | `52428800` (50 MiB) | In-memory body capture and preview budget for a single request or response body. File-backed raw trace bodies can exceed this limit; the full body is stored in sidecar storage and loaded on demand from the Logs page. | | `requestLogSuccessSampleRate` | `1` | Sampling rate for successful requests, between `0` and `1`. `1` records all, `0.1` records roughly one in ten. | diff --git a/docs/src/content/docs/zh/configuration/observability.md b/docs/src/content/docs/zh/configuration/observability.md index 2ad49756..84724eec 100644 --- a/docs/src/content/docs/zh/configuration/observability.md +++ b/docs/src/content/docs/zh/configuration/observability.md @@ -33,5 +33,5 @@ lead: 本页是日志与观测相关开关和面板能力的配置参考：在 | 配置项 | 默认值 | 说明 | | --- | --- | --- | | `requestLogBodyCapture` | `all` | 控制是否记录请求体和响应体：`all` 记录全部，`errors` 只记录失败请求的 body，`none` 不记录任何 body。 | -| `requestLogMaxBodyBytes` | `52428800`（50 MiB） | 单个请求体或响应体的最大字节数，超出部分会被截断；硬上限同样是 50 MiB。 | +| `requestLogMaxBodyBytes` | `52428800`（50 MiB） | 单个请求体或响应体的内存捕获与预览预算。基于文件的 raw trace body 可以超过该限制；完整 body 会保存到 sidecar 存储，并在日志页按需加载。 | | `requestLogSuccessSampleRate` | `1` | 成功请求的采样率，取 `0` 到 `1` 之间，`1` 表示全部记录，`0.1` 表示记录约十分之一。 | diff --git a/packages/core/src/config/constants.ts b/packages/core/src/config/constants.ts index 3c216a37..5f59b38f 100644 --- a/packages/core/src/config/constants.ts +++ b/packages/core/src/config/constants.ts @@ -27,6 +27,7 @@ export const PROXY_CA_CERT_FILE = path.join(CERTDIR, "ca.pem"); export const PROXY_CA_CERT_DER_FILE = path.join(CERTDIR, "ca.cer"); export const PROXY_CA_KEY_FILE = path.join(CERTDIR, "key.pem"); export const REQUEST_LOGS_DB_FILE = path.join(DATADIR, "request-logs.sqlite"); +export const REQUEST_LOG_BODIES_DIR = path.join(DATADIR, "request-log-bodies"); export const CONTEXT_ARCHIVE_DB_FILE = path.join(DATADIR, "context-archive.sqlite"); export const RAW_TRACE_SPOOL_DIR = path.join(DATADIR, "raw-trace-spool"); export const USAGE_DB_FILE = path.join(DATADIR, "usage.sqlite"); diff --git a/packages/core/src/contracts/app.ts b/packages/core/src/contracts/app.ts index 2103e0cb..4d316a3b 100644 --- a/packages/core/src/contracts/app.ts +++ b/packages/core/src/contracts/app.ts @@ -1935,10 +1935,12 @@ export type ProxyCertificateInstallResult = { export type ProxyNetworkCaptureState = "complete" | "error" | "pending"; export type ProxyNetworkBody = { + bodyRef?: string; contentType?: string; decodedFrom?: string; encoding: "base64" | "utf8"; error?: string; + preview?: boolean; sizeBytes: number; text: string; truncated: boolean; @@ -1993,6 +1995,28 @@ export type RequestLogDetailRequest = { export type RequestLogBody = ProxyNetworkBody; +export type RequestLogBodySide = "request" | "response"; + +export type RequestLogBodyChunkRequest = { + id: number; + length?: number; + offset?: number; + side: RequestLogBodySide; +}; + +export type RequestLogBodyChunk = { + bodyRef?: string; + contentType?: string; + encoding: "base64" | "utf8"; + eof: boolean; + length: number; + nextOffset?: number; + offset: number; + sizeBytes: number; + text: string; + truncated: boolean; +}; + export type RequestLogRetryAttempt = { attempt: number; delayMs: number; diff --git a/packages/core/src/contracts/ipc-channels.ts b/packages/core/src/contracts/ipc-channels.ts index 8b736481..5cb5cdce 100644 --- a/packages/core/src/contracts/ipc-channels.ts +++ b/packages/core/src/contracts/ipc-channels.ts @@ -21,6 +21,7 @@ export const IPC_CHANNELS = { appGetProxyNetworkCaptures: "ccr:app:get-proxy-network-captures", appGetProxyStatus: "ccr:app:get-proxy-status", appGetRequestLogDetail: "ccr:app:get-request-log-detail", + appGetRequestLogBodyChunk: "ccr:app:get-request-log-body-chunk", appGetRequestLogs: "ccr:app:get-request-logs", appGetUpdateStatus: "ccr:app:get-update-status", appGetUsageStats: "ccr:app:get-usage-stats", diff --git a/packages/core/src/observability/raw-trace-sync.ts b/packages/core/src/observability/raw-trace-sync.ts index 02b5f0ed..109c74d3 100644 --- a/packages/core/src/observability/raw-trace-sync.ts +++ b/packages/core/src/observability/raw-trace-sync.ts @@ -16,7 +16,7 @@ import { suppressRequestLogRawTraceBodies, type RequestLogEnqueueResult } from "@ccr/core/observability/request-log-runtime"; -import { resolveRawTraceBodyLimit } from "@ccr/core/observability/request-log-limits"; +import { rawTraceMaxPartBytes, resolveRawTraceBodyLimit } from "@ccr/core/observability/request-log-limits"; import { isRecord, numberValue, stringValue } from "@ccr/core/gateway/internal/value"; import { formatError, parseJsonObject, readHeader, readRequestBody, sendJson } from "@ccr/core/gateway/http/io"; import { endpoint } from "@ccr/core/gateway/core-runtime/supervisor"; @@ -1415,7 +1415,7 @@ export function buildRawTraceConfig(config: AppConfig, rawTraceSyncToken: string return { deleteLocalAfterUpload: false, enabled, - maxPartBytes: maxBodyBytes, + maxPartBytes: rawTraceMaxPartBytes, mode: "wire_raw", spoolDir: RAW_TRACE_SPOOL_DIR, sync: { diff --git a/packages/core/src/observability/request-log-limits.ts b/packages/core/src/observability/request-log-limits.ts index 169d6567..4153eb2e 100644 --- a/packages/core/src/observability/request-log-limits.ts +++ b/packages/core/src/observability/request-log-limits.ts @@ -1,9 +1,11 @@ -// Raw traces are produced outside the request-log runtime, so keep an -// independent hard ceiling as defense in depth if configuration validation is -// bypassed or its public range grows in the future. +// Keep the public body-capture setting bounded for in-memory previews and +// admission accounting. Raw trace sources can write larger parts because the +// request-log store moves them into sidecar files instead of keeping them +// inline in SQLite. export const rawTraceHardMaxBodyBytes = 50 * 1024 * 1024; export const maxRequestLogBodyBytes = rawTraceHardMaxBodyBytes; export const defaultRequestLogBodyBytes = rawTraceHardMaxBodyBytes; +export const rawTraceMaxPartBytes = Number.MAX_SAFE_INTEGER; export function resolveRawTraceBodyLimit(value: number | undefined): number { const configured = value ?? defaultRequestLogBodyBytes; diff --git a/packages/core/src/observability/request-log-runtime.ts b/packages/core/src/observability/request-log-runtime.ts index afc07319..81cb7520 100644 --- a/packages/core/src/observability/request-log-runtime.ts +++ b/packages/core/src/observability/request-log-runtime.ts @@ -10,6 +10,8 @@ import type { AgentAnalysisTracePayloadFullResult, AgentAnalysisTracePayloadRequest, RequestLogDetailRequest, + RequestLogBodyChunk, + RequestLogBodyChunkRequest, RequestLogEntry, RequestLogListFilter, RequestLogPage @@ -309,6 +311,10 @@ export class RequestLogRuntime { return await this.query("getDetail", [request]); } + async getBodyChunk(request: RequestLogBodyChunkRequest): Promise { + return await this.query("getBodyChunk", [request]); + } + async analyze(filter?: AgentAnalysisFilter): Promise { return await this.query("analyze", [filter]); } @@ -1027,12 +1033,17 @@ function constrainRawTraceFiles( export function suppressRequestLogRawTraceBodies( input: RequestLogRawTraceUpdateInput ): RequestLogRawTraceUpdateInput { + const { + requestBodyRef: _requestBodyRef, + responseBodyRef: _responseBodyRef, + ...metadata + } = input; const requestSize = input.requestBodySizeBytes ?? (input.requestBodyText === undefined ? undefined : Buffer.byteLength(input.requestBodyText)); const responseSize = input.responseBodySizeBytes ?? (input.responseBodyText === undefined ? undefined : Buffer.byteLength(input.responseBodyText)); return { - ...input, + ...metadata, ...(requestSize === undefined ? {} : { requestBodySizeBytes: requestSize, requestBodyText: "", diff --git a/packages/core/src/observability/request-log-store.ts b/packages/core/src/observability/request-log-store.ts index b84574c9..9682dc61 100644 --- a/packages/core/src/observability/request-log-store.ts +++ b/packages/core/src/observability/request-log-store.ts @@ -1,7 +1,8 @@ -import { mkdirSync } from "node:fs"; -import { dirname } from "node:path"; +import { randomUUID } from "node:crypto"; +import { closeSync, copyFileSync, existsSync, fstatSync, mkdirSync, openSync, readSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; import { StringDecoder } from "node:string_decoder"; -import { REQUEST_LOGS_DB_FILE } from "@ccr/core/config/constants"; +import { REQUEST_LOG_BODIES_DIR, REQUEST_LOGS_DB_FILE } from "@ccr/core/config/constants"; import { decodeClaudeAppGatewayRouteId } from "@ccr/core/agents/claude-app/gateway-routes"; import { estimateUsageCostUsd, @@ -45,6 +46,9 @@ import type { GatewayProviderProtocol, ProviderModelPricing, RequestLogBody, + RequestLogBodyChunk, + RequestLogBodyChunkRequest, + RequestLogBodySide, RequestLogDetailRequest, RequestLogEntry, RequestLogFilterOptions, @@ -147,6 +151,7 @@ export type RequestLogRawTraceUpdateInput = { path?: string; provider?: string; requestBodyContentType?: string; + requestBodyRef?: string; requestBodySizeBytes?: number; requestBodyText?: string; requestBodyTruncated?: boolean; @@ -154,6 +159,7 @@ export type RequestLogRawTraceUpdateInput = { requestId: string; isStream?: boolean; responseBodyContentType?: string; + responseBodyRef?: string; responseBodySizeBytes?: number; responseBodyText?: string; responseBodyTruncated?: boolean; @@ -299,6 +305,8 @@ type ToolCallStreamState = { }; const maxBodyBytes = maxRequestLogBodyBytes; +const requestLogInlineBodyBytes = 160 * 1024; +const requestLogBodyChunkMaxBytes = 1024 * 1024; const maxAgentAnalysisRows = 5000; const maxAgentSessionDetailRequests = 250; const maxTracePayloadPreviewChars = 1600; @@ -326,7 +334,9 @@ const terminalSseResponseStatuses = new Set([ ]); const requestLogBodyMetadataSelect = ` '' AS request_body_text, - '' AS response_body_text + '' AS response_body_text, + request_body_ref, + response_body_ref `; const emptyAgentAnalysisTotals: AgentAnalysisTotals = { avgDurationMs: 0, @@ -365,7 +375,12 @@ export class RequestLogStore { private revision = 0; private analysisCache?: AgentAnalysisCacheEntry; - constructor(private readonly dbFile: string) {} + constructor( + private readonly dbFile: string, + private readonly bodyDir = dbFile === REQUEST_LOGS_DB_FILE + ? REQUEST_LOG_BODIES_DIR + : join(dirname(dbFile), "request-log-bodies") + ) {} async initialize(): Promise { await this.getDatabase(); @@ -404,6 +419,9 @@ export class RequestLogStore { if (requestId) { const pending = this.takePendingRawTraceUpdate(database, requestId); if (pending) { + if (command.input.captureBody === false) { + deleteRequestLogBodyRefs(this.bodyDir, bodyRefsFromRawTraceInput(pending)); + } const pendingInput = command.input.captureBody === false ? suppressRequestLogRawTraceBodies(pending) : pending; @@ -419,11 +437,12 @@ export class RequestLogStore { if (bundleId && hasProcessedRawTraceBundle(database, bundleId)) { continue; } - const applied = await this.updateFromRawTrace(command.input); + const rawTraceInput = this.prepareRawTraceInput(command.input, command.rawTraceFiles); + const applied = await this.updateFromRawTrace(rawTraceInput); if (bundleId && applied) { - rememberProcessedRawTraceBundle(database, bundleId, command.input.requestId); + rememberProcessedRawTraceBundle(database, bundleId, rawTraceInput.requestId); } else if (!applied) { - this.storePendingRawTraceUpdate(database, command.input); + this.storePendingRawTraceUpdate(database, rawTraceInput); } } database.exec("COMMIT"); @@ -573,7 +592,8 @@ export class RequestLogStore { : await estimateUsageCostUsd(costInput); const capturedRequestBody = bodyFromBuffer( input.requestBody, - headerValue(requestHeaders, "content-type") + headerValue(requestHeaders, "content-type"), + { bodyDir: this.bodyDir, side: "request" } ); const requestBody: RequestLogBody = { ...capturedRequestBody, @@ -584,7 +604,9 @@ export class RequestLogStore { responseBodyText, headerValue(responseHeaders, "content-type"), Boolean(input.responseBodyTruncated), - input.responseBodySizeBytes + input.responseBodySizeBytes, + undefined, + { bodyDir: this.bodyDir, side: "response" } ); const isStream = inferRequestLogIsStream({ path: input.path, @@ -636,13 +658,15 @@ export class RequestLogStore { request_body_content_type, request_body_size_bytes, request_body_truncated, + request_body_ref, response_body_text, response_body_encoding, response_body_content_type, response_body_size_bytes, response_body_truncated, + response_body_ref, error - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) `); let inserted = false; @@ -687,11 +711,13 @@ export class RequestLogStore { requestBody.contentType ?? "", requestBody.sizeBytes, requestBody.truncated ? 1 : 0, + requestBody.bodyRef ?? "", responseBody.text, responseBody.encoding, responseBody.contentType ?? "", responseBody.sizeBytes, responseBody.truncated ? 1 : 0, + responseBody.bodyRef ?? "", responseError ?? "" ); if (result.changes === 0) return; @@ -774,6 +800,9 @@ export class RequestLogStore { rawInput, finalSuccessful ); + if (captureResolution.bodiesSuppressed) { + deleteRequestLogBodyRefs(this.bodyDir, bodyRefsFromRawTraceInput(rawInput)); + } const input = captureResolution.input; const existingUsageContext = readRequestLogUsageContext(database, requestId); @@ -908,27 +937,33 @@ export class RequestLogStore { url }) ? 1 : 0); } - if (input.requestBodyText !== undefined && ( - captureResolution.bodiesSuppressed || input.requestBodyText.length > 0 || !existingOutcome.hasRequestBody + const shouldApplyRequestBody = input.requestBodyText !== undefined || + Boolean(input.requestBodyRef && (!input.requestBodyTruncated || !existingOutcome.hasRequestBody)); + if (shouldApplyRequestBody && ( + captureResolution.bodiesSuppressed || Boolean(input.requestBodyRef) || (input.requestBodyText?.length ?? 0) > 0 || !existingOutcome.hasRequestBody )) { const requestBody = bodyFromText( - input.requestBodyText, + input.requestBodyText ?? "", input.requestBodyContentType ?? headerValue(mergedRequestHeaders ?? {}, "content-type"), Boolean(input.requestBodyTruncated), input.requestBodySizeBytes, - rawTraceHardMaxBodyBytes + rawTraceHardMaxBodyBytes, + { bodyDir: this.bodyDir, bodyRef: input.requestBodyRef, side: "request" } ); pushBodyValues(sets, params, "request", requestBody); } - if (input.responseBodyText !== undefined && ( - captureResolution.bodiesSuppressed || input.responseBodyText.length > 0 || !existingOutcome.hasResponseBody + const shouldApplyResponseBody = input.responseBodyText !== undefined || + Boolean(input.responseBodyRef && (!input.responseBodyTruncated || !existingOutcome.hasResponseBody)); + if (shouldApplyResponseBody && ( + captureResolution.bodiesSuppressed || Boolean(input.responseBodyRef) || (input.responseBodyText?.length ?? 0) > 0 || !existingOutcome.hasResponseBody )) { const responseBody = bodyFromText( - input.responseBodyText, + input.responseBodyText ?? "", responseBodyContentType, Boolean(input.responseBodyTruncated), input.responseBodySizeBytes, - rawTraceHardMaxBodyBytes + rawTraceHardMaxBodyBytes, + { bodyDir: this.bodyDir, bodyRef: input.responseBodyRef, side: "response" } ); pushBodyValues(sets, params, "response", responseBody); } @@ -1036,6 +1071,75 @@ export class RequestLogStore { return entry; } + async getBodyChunk(request: RequestLogBodyChunkRequest): Promise { + const database = await this.getDatabase(); + const requestLogId = normalizeCount(request.id); + const side = request.side === "response" ? "response" : "request"; + if (requestLogId <= 0) { + return undefined; + } + + const row = queryRows( + database, + ` + SELECT + ${side}_body_text AS body_text, + ${side}_body_encoding AS body_encoding, + ${side}_body_content_type AS body_content_type, + ${side}_body_size_bytes AS body_size_bytes, + ${side}_body_truncated AS body_truncated, + ${side}_body_ref AS body_ref + FROM request_logs + WHERE rowid = ? + LIMIT 1 + `, + [requestLogId] + )[0]; + if (!row) { + return undefined; + } + + const offset = clampInteger(request.offset, 0, Number.MAX_SAFE_INTEGER, 0); + const length = clampInteger(request.length, 1, requestLogBodyChunkMaxBytes, requestLogBodyChunkMaxBytes); + const encoding = String(row.body_encoding ?? "utf8") === "base64" ? "base64" : "utf8"; + const contentType = normalizeFilterValue(String(row.body_content_type ?? "")); + const sizeBytes = normalizeCount(row.body_size_bytes); + const truncated = normalizeCount(row.body_truncated) === 1; + const bodyRef = normalizeFilterValue(String(row.body_ref ?? "")); + + if (bodyRef) { + const filePath = requestLogBodyPath(this.bodyDir, bodyRef); + if (filePath && existsSync(filePath)) { + return readRequestLogBodyChunkFromFile({ + bodyRef, + contentType, + encoding, + filePath, + length, + offset, + sizeBytes, + truncated + }); + } + } + + const text = String(row.body_text ?? ""); + const visible = text.slice(offset, offset + length); + const nextOffset = offset + visible.length; + return { + ...(bodyRef ? { bodyRef } : {}), + contentType, + encoding, + eof: nextOffset >= text.length, + length: visible.length, + ...(nextOffset < text.length ? { nextOffset } : {}), + offset, + sizeBytes: Math.max(sizeBytes, text.length), + text: visible, + truncated + }; + } + async analyze(filter: AgentAnalysisFilter = {}): Promise { const database = await this.getDatabase(); this.pruneOldRequestLogs(database); @@ -1091,11 +1195,13 @@ export class RequestLogStore { request_body_content_type, request_body_size_bytes, request_body_truncated, + request_body_ref, response_body_text, response_body_encoding, response_body_content_type, response_body_size_bytes, response_body_truncated, + response_body_ref, error FROM request_logs WHERE source_usage_id IS NULL @@ -1254,11 +1360,13 @@ export class RequestLogStore { request_body_content_type TEXT NOT NULL DEFAULT '', request_body_size_bytes INTEGER NOT NULL DEFAULT 0, request_body_truncated INTEGER NOT NULL DEFAULT 0, + request_body_ref TEXT NOT NULL DEFAULT '', response_body_text TEXT NOT NULL DEFAULT '', response_body_encoding TEXT NOT NULL DEFAULT 'utf8', response_body_content_type TEXT NOT NULL DEFAULT '', response_body_size_bytes INTEGER NOT NULL DEFAULT 0, response_body_truncated INTEGER NOT NULL DEFAULT 0, + response_body_ref TEXT NOT NULL DEFAULT '', error TEXT NOT NULL DEFAULT '' ); @@ -1349,9 +1457,23 @@ export class RequestLogStore { return; } + const refs = queryRows( + database, + ` + SELECT request_body_ref, response_body_ref + FROM request_logs + WHERE source_usage_id IS NULL AND created_at < ? + `, + [cutoff] + ).flatMap((row) => [ + normalizeFilterValue(String(row.request_body_ref ?? "")), + normalizeFilterValue(String(row.response_body_ref ?? "")) + ]).filter((value): value is string => Boolean(value)); + database.prepare( "DELETE FROM request_logs WHERE source_usage_id IS NULL AND created_at < ?", ).run(cutoff); + deleteRequestLogBodyRefs(this.bodyDir, refs); this.lastRetentionCleanupDay = dayKey; } @@ -1370,7 +1492,7 @@ export class RequestLogStore { update_json = excluded.update_json `).run(requestId, now, serialized.bytes, serialized.json); } - prunePendingRawTraceUpdates(database, now); + prunePendingRawTraceUpdates(database, now, this.bodyDir); } private takePendingRawTraceUpdate(database: SqlDatabase, requestId: string): RequestLogRawTraceUpdateInput | undefined { @@ -1384,6 +1506,49 @@ export class RequestLogStore { const parsed = parseJson(String(row.update_json ?? "")); return isRecord(parsed) ? parsed as RequestLogRawTraceUpdateInput : undefined; } + + private prepareRawTraceInput( + input: RequestLogRawTraceUpdateInput, + rawTraceFiles?: RequestLogRawTraceFiles + ): RequestLogRawTraceUpdateInput { + const next: RequestLogRawTraceUpdateInput = { ...input }; + const requestBody = storeRawTraceBodyFile(this.bodyDir, rawTraceFiles?.requestBody); + if (requestBody) { + next.requestBodyRef = requestBody.bodyRef; + if (!requestBody.truncated) next.requestBodyText ??= requestBody.previewText; + next.requestBodyContentType ??= requestBody.contentType; + next.requestBodySizeBytes = Math.max(normalizeCount(next.requestBodySizeBytes), requestBody.sizeBytes); + next.requestBodyTruncated = Boolean(next.requestBodyTruncated) || requestBody.truncated; + } + const responseBody = storeRawTraceBodyFile(this.bodyDir, rawTraceFiles?.responseBody); + if (responseBody) { + next.responseBodyRef = responseBody.bodyRef; + if (!responseBody.truncated) next.responseBodyText ??= responseBody.previewText; + next.responseBodyContentType ??= responseBody.contentType; + next.responseBodySizeBytes = Math.max(normalizeCount(next.responseBodySizeBytes), responseBody.sizeBytes); + next.responseBodyTruncated = Boolean(next.responseBodyTruncated) || responseBody.truncated; + } + return this.prepareRawTraceTextBodies(next); + } + + private prepareRawTraceTextBodies(input: RequestLogRawTraceUpdateInput): RequestLogRawTraceUpdateInput { + const next: RequestLogRawTraceUpdateInput = { ...input }; + if (!next.requestBodyRef && next.requestBodyText !== undefined) { + const stored = storeBodyBuffer(this.bodyDir, Buffer.from(next.requestBodyText), next.requestBodyRef); + if (stored) { + next.requestBodyRef = stored.bodyRef; + next.requestBodySizeBytes = Math.max(Buffer.byteLength(next.requestBodyText), normalizeCount(next.requestBodySizeBytes)); + } + } + if (!next.responseBodyRef && next.responseBodyText !== undefined) { + const stored = storeBodyBuffer(this.bodyDir, Buffer.from(next.responseBodyText), next.responseBodyRef); + if (stored) { + next.responseBodyRef = stored.bodyRef; + next.responseBodySizeBytes = Math.max(Buffer.byteLength(next.responseBodyText), normalizeCount(next.responseBodySizeBytes)); + } + } + return next; + } } export const requestLogStore = new RequestLogStore(REQUEST_LOGS_DB_FILE); @@ -1430,6 +1595,15 @@ export async function getRequestLogDetail(request: RequestLogDetailRequest): Pro } } +export async function getRequestLogBodyChunk(request: RequestLogBodyChunkRequest): Promise { + try { + return await requestLogRuntime.getBodyChunk(request); + } catch (error) { + console.warn(`[request-log] Failed to read request log body chunk: ${formatError(error)}`); + throw error; + } +} + export async function getAgentAnalysis(filter?: AgentAnalysisFilter): Promise { try { return await requestLogRuntime.analyze(filter); @@ -3452,11 +3626,13 @@ function ensureRequestLogSchema(database: SqlDatabase): void { addColumn("request_body_content_type", "TEXT NOT NULL DEFAULT ''"); addColumn("request_body_size_bytes", "INTEGER NOT NULL DEFAULT 0"); addColumn("request_body_truncated", "INTEGER NOT NULL DEFAULT 0"); + addColumn("request_body_ref", "TEXT NOT NULL DEFAULT ''"); addColumn("response_body_text", "TEXT NOT NULL DEFAULT ''"); addColumn("response_body_encoding", "TEXT NOT NULL DEFAULT 'utf8'"); addColumn("response_body_content_type", "TEXT NOT NULL DEFAULT ''"); addColumn("response_body_size_bytes", "INTEGER NOT NULL DEFAULT 0"); addColumn("response_body_truncated", "INTEGER NOT NULL DEFAULT 0"); + addColumn("response_body_ref", "TEXT NOT NULL DEFAULT ''"); addColumn("error", "TEXT NOT NULL DEFAULT ''"); if (needsModelSummaryMigration) { @@ -4048,11 +4224,13 @@ function readRequestLogById(database: SqlDatabase, id: number): StoredRequestLog request_body_content_type, request_body_size_bytes, request_body_truncated, + request_body_ref, response_body_text, response_body_encoding, response_body_content_type, response_body_size_bytes, response_body_truncated, + response_body_ref, error FROM request_logs WHERE rowid = ? @@ -4126,7 +4304,9 @@ function bodyFromRow(row: Record, prefix: "request" | "respons const encoding = String(row[`${prefix}_body_encoding`] ?? "utf8") === "base64" ? "base64" : "utf8"; const contentType = normalizeFilterValue(String(row[`${prefix}_body_content_type`] ?? "")); + const bodyRef = normalizeFilterValue(String(row[`${prefix}_body_ref`] ?? "")); return { + ...(bodyRef ? { bodyRef, preview: true } : {}), contentType, encoding, sizeBytes, @@ -4183,17 +4363,35 @@ function readDistinctValues(database: SqlDatabase, column: "credential_id" | "mo .filter(Boolean); } -function bodyFromBuffer(buffer: Buffer, contentType?: string): RequestLogBody { +type RequestLogBodyStorageOptions = { + bodyDir: string; + bodyRef?: string; + side: RequestLogBodySide; +}; + +function bodyFromBuffer( + buffer: Buffer, + contentType?: string, + storage?: RequestLogBodyStorageOptions +): RequestLogBody { const compacted = compactBase64ImagePayloads(buffer); const exceedsCaptureLimit = compacted.buffer.byteLength > maxBodyBytes; const data = exceedsCaptureLimit ? compacted.buffer.subarray(0, maxBodyBytes) : compacted.buffer; const textLike = isTextLikeContentType(contentType); + const stored = textLike && storage + ? storeBodyBuffer(storage.bodyDir, buffer, storage.bodyRef, compacted.buffer) + : undefined; + const text = textLike + ? stored?.previewText ?? data.toString("utf8") + : data.toString("base64"); + const truncated = !stored && (compacted.compacted || exceedsCaptureLimit); return { + ...(stored ? { bodyRef: stored.bodyRef, preview: true } : {}), contentType, encoding: textLike ? "utf8" : "base64", sizeBytes: buffer.byteLength, - text: textLike ? data.toString("utf8") : data.toString("base64"), - truncated: compacted.compacted || exceedsCaptureLimit + text, + truncated }; } @@ -4202,20 +4400,29 @@ function bodyFromText( contentType?: string, alreadyTruncated = false, originalSizeBytes?: number, - captureLimitBytes = maxBodyBytes + captureLimitBytes = maxBodyBytes, + storage?: RequestLogBodyStorageOptions ): RequestLogBody { const buffer = Buffer.from(text); const sizeBytes = Math.max(buffer.byteLength, normalizeCount(originalSizeBytes)); const compacted = compactBase64ImagePayloads(buffer); - const truncated = alreadyTruncated || compacted.compacted || buffer.byteLength < sizeBytes || - compacted.buffer.byteLength > captureLimitBytes; const exceedsCaptureLimit = compacted.buffer.byteLength > captureLimitBytes; const data = exceedsCaptureLimit ? compacted.buffer.subarray(0, captureLimitBytes) : compacted.buffer; + const stored = storage + ? storeBodyBuffer(storage.bodyDir, buffer, storage.bodyRef, compacted.buffer) + : undefined; + const truncated = alreadyTruncated || (!stored && ( + compacted.compacted || + buffer.byteLength < sizeBytes || + exceedsCaptureLimit + )); + const bodyText = stored?.previewText ?? (exceedsCaptureLimit ? new StringDecoder("utf8").write(data) : data.toString("utf8")); return { + ...(stored ? { bodyRef: stored.bodyRef, preview: true } : {}), contentType, encoding: "utf8", sizeBytes, - text: exceedsCaptureLimit ? new StringDecoder("utf8").write(data) : data.toString("utf8"), + text: bodyText, truncated }; } @@ -4236,6 +4443,231 @@ function pushBodyValues( params.push(body.sizeBytes); sets.push(`${prefix}_body_truncated = ?`); params.push(body.truncated ? 1 : 0); + sets.push(`${prefix}_body_ref = ?`); + params.push(body.bodyRef ?? ""); +} + +type StoredRequestLogBodyFile = { + bodyRef: string; + contentType?: string; + previewText: string; + sizeBytes: number; + truncated: boolean; +}; + +function storeRawTraceBodyFile( + bodyDir: string, + file: RequestLogRawTraceFile | undefined +): StoredRequestLogBodyFile | undefined { + if (!file || file.sizeBytes <= 0 || !existsSync(file.filePath)) { + return undefined; + } + const bodyRef = createRequestLogBodyRef(); + const target = requestLogBodyPath(bodyDir, bodyRef, true); + if (!target) { + return undefined; + } + copyFileSync(file.filePath, target); + const storedBytes = statSync(target).size; + const sizeBytes = Math.max(file.sizeBytes, storedBytes); + return { + bodyRef, + contentType: file.contentType, + previewText: readRequestLogBodyPreview(target), + sizeBytes, + truncated: Boolean(file.truncated) || storedBytes < sizeBytes + }; +} + +function storeBodyBuffer( + bodyDir: string, + buffer: Buffer, + existingBodyRef?: string, + previewBuffer = buffer +): { bodyRef: string; previewText: string } | undefined { + const shouldStore = Boolean(existingBodyRef) || buffer.byteLength > requestLogInlineBodyBytes; + if (!shouldStore) { + return undefined; + } + const bodyRef = normalizeBodyRef(existingBodyRef) ?? createRequestLogBodyRef(); + const target = requestLogBodyPath(bodyDir, bodyRef, true); + if (!target) { + return undefined; + } + if (existingBodyRef && existsSync(target)) { + return { + bodyRef, + previewText: previewBuffer.byteLength > 0 + ? createRequestLogBodyPreviewText(previewBuffer) + : readRequestLogBodyPreview(target) + }; + } + if (!existingBodyRef || !existsSync(target)) { + writeFileSync(target, buffer); + } + return { + bodyRef, + previewText: createRequestLogBodyPreviewText(buffer) + }; +} + +function readRequestLogBodyChunkFromFile({ + bodyRef, + contentType, + encoding, + filePath, + length, + offset, + sizeBytes, + truncated +}: { + bodyRef: string; + contentType?: string; + encoding: "base64" | "utf8"; + filePath: string; + length: number; + offset: number; + sizeBytes: number; + truncated: boolean; +}): RequestLogBodyChunk { + const descriptor = openSync(filePath, "r"); + try { + const storedBytes = fstatSync(descriptor).size; + const boundedOffset = Math.max(0, Math.min(offset, storedBytes)); + const readLength = Math.max(0, Math.min( + encoding === "utf8" ? Math.max(length, 4) : length, + storedBytes - boundedOffset + )); + const buffer = Buffer.allocUnsafe(readLength); + let bytesRead = 0; + while (bytesRead < readLength) { + const count = readSync(descriptor, buffer, bytesRead, readLength - bytesRead, boundedOffset + bytesRead); + if (count === 0) break; + bytesRead += count; + } + const data = bytesRead === buffer.byteLength ? buffer : buffer.subarray(0, bytesRead); + const safeLength = encoding === "utf8" && boundedOffset + data.byteLength < storedBytes + ? validUtf8PrefixLength(data) + : data.byteLength; + const safeData = safeLength > 0 ? data.subarray(0, safeLength) : data; + const nextOffset = boundedOffset + safeData.byteLength; + return { + bodyRef, + contentType, + encoding, + eof: nextOffset >= storedBytes, + length: safeData.byteLength, + ...(nextOffset < storedBytes ? { nextOffset } : {}), + offset: boundedOffset, + sizeBytes: Math.max(sizeBytes, storedBytes), + text: encoding === "base64" ? safeData.toString("base64") : new StringDecoder("utf8").write(safeData), + truncated + }; + } finally { + closeSync(descriptor); + } +} + +function validUtf8PrefixLength(buffer: Buffer): number { + if (buffer.byteLength === 0) return 0; + let leadIndex = buffer.byteLength - 1; + while (leadIndex >= 0 && (buffer[leadIndex] & 0xc0) === 0x80) { + leadIndex -= 1; + } + if (leadIndex < 0) return 0; + const lead = buffer[leadIndex]; + if ((lead & 0x80) === 0) return buffer.byteLength; + const continuationBytes = buffer.byteLength - leadIndex - 1; + const expectedContinuationBytes = (lead & 0xe0) === 0xc0 + ? 1 + : (lead & 0xf0) === 0xe0 + ? 2 + : (lead & 0xf8) === 0xf0 + ? 3 + : 0; + if (expectedContinuationBytes === 0) return leadIndex; + return continuationBytes >= expectedContinuationBytes ? buffer.byteLength : leadIndex; +} + +function readRequestLogBodyPreview(filePath: string): string { + const descriptor = openSync(filePath, "r"); + try { + const size = fstatSync(descriptor).size; + if (size <= requestLogInlineBodyBytes) { + const buffer = Buffer.allocUnsafe(size); + readSync(descriptor, buffer, 0, size, 0); + return new StringDecoder("utf8").write(buffer); + } + const headBytes = Math.floor(requestLogInlineBodyBytes * 0.65); + const tailBytes = requestLogInlineBodyBytes - headBytes; + const head = Buffer.allocUnsafe(headBytes); + const tail = Buffer.allocUnsafe(tailBytes); + const headRead = readSync(descriptor, head, 0, headBytes, 0); + const tailRead = readSync(descriptor, tail, 0, tailBytes, Math.max(0, size - tailBytes)); + return createRequestLogPreviewFromParts( + head.subarray(0, headRead), + tail.subarray(0, tailRead), + Math.max(0, size - headRead - tailRead) + ); + } finally { + closeSync(descriptor); + } +} + +function createRequestLogBodyPreviewText(buffer: Buffer): string { + if (buffer.byteLength <= requestLogInlineBodyBytes) { + return new StringDecoder("utf8").write(buffer); + } + const headBytes = Math.floor(requestLogInlineBodyBytes * 0.65); + const tailBytes = requestLogInlineBodyBytes - headBytes; + return createRequestLogPreviewFromParts( + buffer.subarray(0, headBytes), + buffer.subarray(Math.max(0, buffer.byteLength - tailBytes)), + Math.max(0, buffer.byteLength - headBytes - tailBytes) + ); +} + +function createRequestLogPreviewFromParts(head: Buffer, tail: Buffer, omittedBytes: number): string { + return [ + new StringDecoder("utf8").write(head), + "", + `... ${omittedBytes} bytes omitted from preview ...`, + "", + new StringDecoder("utf8").write(tail) + ].join("\n"); +} + +function createRequestLogBodyRef(): string { + return randomUUID(); +} + +function requestLogBodyPath(bodyDir: string, bodyRef: string, createDirectory = false): string | undefined { + const normalized = normalizeBodyRef(bodyRef); + if (!normalized) { + return undefined; + } + const shard = normalized.slice(0, 2); + const directory = join(bodyDir, shard); + if (createDirectory) { + mkdirSync(directory, { recursive: true }); + } + return join(directory, normalized); +} + +function normalizeBodyRef(value: string | undefined): string | undefined { + const normalized = value?.trim(); + if (!normalized || !/^[A-Za-z0-9._-]+$/.test(normalized)) { + return undefined; + } + return normalized; +} + +function deleteRequestLogBodyRefs(bodyDir: string, refs: string[]): void { + for (const ref of refs) { + const filePath = requestLogBodyPath(bodyDir, ref); + if (!filePath) continue; + rmSync(filePath, { force: true }); + } } function isTextLikeContentType(contentType: string | undefined): boolean { @@ -4292,6 +4724,8 @@ function readRequestLogStoredOutcome(database: SqlDatabase, requestId: string): gateway_status_code, length(request_body_text) AS request_body_chars, length(response_body_text) AS response_body_chars, + request_body_ref, + response_body_ref, ok, status_code FROM request_logs @@ -4306,8 +4740,8 @@ function readRequestLogStoredOutcome(database: SqlDatabase, requestId: string): gatewayError: String(row?.gateway_error ?? ""), gatewayOk: normalizeCount(row?.gateway_ok) === 1, gatewayStatusCode: normalizeCount(row?.gateway_status_code), - hasRequestBody: normalizeCount(row?.request_body_chars) > 0, - hasResponseBody: normalizeCount(row?.response_body_chars) > 0, + hasRequestBody: normalizeCount(row?.request_body_chars) > 0 || Boolean(normalizeFilterValue(String(row?.request_body_ref ?? ""))), + hasResponseBody: normalizeCount(row?.response_body_chars) > 0 || Boolean(normalizeFilterValue(String(row?.response_body_ref ?? ""))), ok: normalizeCount(row?.ok) === 1, statusCode: normalizeCount(row?.status_code) }; @@ -4359,12 +4793,14 @@ function withBoundedRawTraceBodyTexts( ...(requestBodyText === undefined ? {} : { requestBodySizeBytes: Math.max(requestBytes, normalizeCount(input.requestBodySizeBytes)), requestBodyText, - requestBodyTruncated: Boolean(input.requestBodyTruncated) || Buffer.byteLength(requestBodyText) < requestBytes + requestBodyTruncated: Boolean(input.requestBodyTruncated) || + (!input.requestBodyRef && Buffer.byteLength(requestBodyText) < requestBytes) }), ...(responseBodyText === undefined ? {} : { responseBodySizeBytes: Math.max(responseBytes, normalizeCount(input.responseBodySizeBytes)), responseBodyText, - responseBodyTruncated: Boolean(input.responseBodyTruncated) || Buffer.byteLength(responseBodyText) < responseBytes + responseBodyTruncated: Boolean(input.responseBodyTruncated) || + (!input.responseBodyRef && Buffer.byteLength(responseBodyText) < responseBytes) }) }; } @@ -4389,14 +4825,14 @@ function withoutRawTraceBodyTexts(input: RequestLogRawTraceUpdateInput): Request Buffer.byteLength(requestBodyText), normalizeCount(input.requestBodySizeBytes) ), - requestBodyTruncated: true + requestBodyTruncated: Boolean(input.requestBodyTruncated) || !input.requestBodyRef }), ...(responseBodyText === undefined ? {} : { responseBodySizeBytes: Math.max( Buffer.byteLength(responseBodyText), normalizeCount(input.responseBodySizeBytes) ), - responseBodyTruncated: true + responseBodyTruncated: Boolean(input.responseBodyTruncated) || !input.responseBodyRef }) }; } @@ -4405,13 +4841,19 @@ function rawTraceHasBodyText(input: RequestLogRawTraceUpdateInput): boolean { return input.requestBodyText !== undefined || input.responseBodyText !== undefined; } -function prunePendingRawTraceUpdates(database: SqlDatabase, now: number): void { +function prunePendingRawTraceUpdates(database: SqlDatabase, now: number, bodyDir?: string): void { + const expiredRows = queryRows( + database, + "SELECT update_json FROM request_log_pending_updates WHERE received_at < ?", + [now - pendingRawTraceTtlMs] + ); + if (bodyDir) deleteRequestLogBodyRefs(bodyDir, bodyRefsFromPendingRawTraceRows(expiredRows)); database.prepare("DELETE FROM request_log_pending_updates WHERE received_at < ?") .run(now - pendingRawTraceTtlMs); const rows = queryRows( database, ` - SELECT request_id, update_bytes + SELECT request_id, update_bytes, update_json FROM request_log_pending_updates ORDER BY received_at DESC, request_id DESC ` @@ -4423,6 +4865,7 @@ function prunePendingRawTraceUpdates(database: SqlDatabase, now: number): void { const bytes = normalizeCount(row.update_bytes); if (retainedEntries >= maxPendingRawTraceEntries || retainedBytes + bytes > maxPendingRawTraceTotalBytes) { + if (bodyDir) deleteRequestLogBodyRefs(bodyDir, bodyRefsFromPendingRawTraceRows([row])); remove.run(String(row.request_id ?? "")); continue; } @@ -4431,6 +4874,21 @@ function prunePendingRawTraceUpdates(database: SqlDatabase, now: number): void { } } +function bodyRefsFromPendingRawTraceRows(rows: Record[]): string[] { + return rows.flatMap((row) => { + const parsed = parseJson(String(row.update_json ?? "")); + if (!isRecord(parsed)) return []; + return bodyRefsFromRawTraceInput(parsed as RequestLogRawTraceUpdateInput); + }); +} + +function bodyRefsFromRawTraceInput(input: RequestLogRawTraceUpdateInput): string[] { + return [ + normalizeFilterValue(String(input.requestBodyRef ?? "")), + normalizeFilterValue(String(input.responseBodyRef ?? "")) + ].filter((value): value is string => Boolean(value)); +} + function readRequestHeadersForRequestId(database: SqlDatabase, requestId: string): Record { const row = queryRows(database, "SELECT request_headers FROM request_logs WHERE request_id = ? LIMIT 1", [requestId])[0]; return row ? parseHeaderJson(row.request_headers) : {}; diff --git a/packages/core/src/observability/request-log-worker.ts b/packages/core/src/observability/request-log-worker.ts index f0385df8..17f5c87d 100644 --- a/packages/core/src/observability/request-log-worker.ts +++ b/packages/core/src/observability/request-log-worker.ts @@ -6,6 +6,7 @@ import { RAW_TRACE_SPOOL_DIR } from "@ccr/core/config/constants"; import { RequestLogStore, type RequestLogRawTraceFile, + type RequestLogRawTraceFiles, type RequestLogRecordInput, type RequestLogStoreWriteCommand } from "@ccr/core/observability/request-log-store"; @@ -13,6 +14,11 @@ import { resolveRawTraceBodyLimit } from "@ccr/core/observability/request-log-li import { compactBase64ImagePayloads } from "@ccr/core/observability/request-log-body"; import { preloadUsagePriceCatalog } from "@ccr/core/models/pricing-service"; +type RevivedRawTraceBody = RequestLogRawTraceFile & { + text: string; + textTruncated: boolean; +}; + type WorkerConfiguration = { dbFile: string; mode: "query" | "writer"; @@ -80,6 +86,9 @@ async function handleMessage(message: WorkerMessage): Promise { case "getDetail": result = await store.getDetail(args[0] as Parameters[0]); break; + case "getBodyChunk": + result = await store.getBodyChunk(args[0] as Parameters[0]); + break; case "getTracePayload": result = await store.getTracePayload(args[0] as Parameters[0]); break; @@ -140,9 +149,16 @@ function reviveCommand(command: RequestLogStoreWriteCommand): RequestLogStoreWri if (command.kind === "raw-trace-update") { const input = { ...command.input }; const maxBodyBytes = resolveRawTraceBodyLimit(command.rawTraceFiles?.maxBodyBytes); + const rawTraceFiles: RequestLogRawTraceFiles | undefined = command.rawTraceFiles + ? { + cleanupDirectory: command.rawTraceFiles.cleanupDirectory, + maxBodyBytes: command.rawTraceFiles.maxBodyBytes + } + : undefined; if (command.rawTraceFiles?.requestBody) { const body = readRawTraceBody(command.rawTraceFiles.requestBody, maxBodyBytes); if (body) { + if (rawTraceFiles) rawTraceFiles.requestBody = fileMetadataFromRawTraceBody(body); input.requestBodyContentType = body.contentType ?? input.requestBodyContentType; input.requestBodySizeBytes = body.sizeBytes; input.requestBodyTruncated = body.truncated; @@ -152,6 +168,7 @@ function reviveCommand(command: RequestLogStoreWriteCommand): RequestLogStoreWri if (command.rawTraceFiles?.responseBody) { const body = readRawTraceBody(command.rawTraceFiles.responseBody, maxBodyBytes); if (body) { + if (rawTraceFiles) rawTraceFiles.responseBody = fileMetadataFromRawTraceBody(body); input.responseBodyContentType = body.contentType ?? input.responseBodyContentType; input.responseBodySizeBytes = body.sizeBytes; input.responseBodyTruncated = body.truncated; @@ -161,6 +178,7 @@ function reviveCommand(command: RequestLogStoreWriteCommand): RequestLogStoreWri return { input, kind: "raw-trace-update", + ...(rawTraceFiles ? { rawTraceFiles } : {}), sequence: command.sequence }; } @@ -174,10 +192,19 @@ function reviveCommand(command: RequestLogStoreWriteCommand): RequestLogStoreWri }; } +function fileMetadataFromRawTraceBody(body: RevivedRawTraceBody): RequestLogRawTraceFile { + return { + contentType: body.contentType, + filePath: body.filePath, + sizeBytes: body.sizeBytes, + truncated: body.truncated + }; +} + function readRawTraceBody( file: RequestLogRawTraceFile, maxBodyBytes: number -): (RequestLogRawTraceFile & { text: string }) | undefined { +): RevivedRawTraceBody | undefined { let descriptor: number | undefined; try { const filePath = verifiedRawTracePath(file.filePath); @@ -193,13 +220,14 @@ function readRawTraceBody( } const captured = offset === buffer.byteLength ? buffer : buffer.subarray(0, offset); const compacted = compactBase64ImagePayloads(captured); + const sourceSizeBytes = Math.max(file.sizeBytes, storedBytes); return { ...file, filePath, - sizeBytes: Math.max(file.sizeBytes, storedBytes), + sizeBytes: sourceSizeBytes, text: new StringDecoder("utf8").write(compacted.buffer), - truncated: Boolean(file.truncated) || compacted.compacted || - offset < Math.max(file.sizeBytes, storedBytes) + textTruncated: offset < storedBytes, + truncated: Boolean(file.truncated) || storedBytes < sourceSizeBytes }; } catch (error) { if (errorCode(error) === "ENOENT") return undefined; @@ -209,8 +237,8 @@ function readRawTraceBody( } } -function shouldApplyRawTraceBodyText(body: RequestLogRawTraceFile & { text: string }): boolean { - if (!body.truncated) return true; +function shouldApplyRawTraceBodyText(body: RevivedRawTraceBody): boolean { + if (!body.truncated && !body.textTruncated) return true; const contentType = body.contentType?.split(";", 1)[0].trim().toLowerCase() ?? ""; const firstContentIndex = skipJsonWhitespace(body.text, 0); const firstContent = body.text.charCodeAt(firstContentIndex); diff --git a/packages/core/src/web/management-server.ts b/packages/core/src/web/management-server.ts index b5a1094f..6e232297 100644 --- a/packages/core/src/web/management-server.ts +++ b/packages/core/src/web/management-server.ts @@ -40,7 +40,7 @@ import { getPluginMarketplace } from "@ccr/core/plugins/marketplace"; import { ensureProxyCertificateAuthority } from "@ccr/core/proxy/certificates"; import { proxyService } from "@ccr/core/proxy/service"; import { listMcpServerTools } from "@ccr/core/mcp/tool-discovery"; -import { closeRequestLogRuntime, getAgentAnalysis, getAgentTracePayload, getRequestLogDetail, getRequestLogs } from "@ccr/core/observability/request-log-store"; +import { closeRequestLogRuntime, getAgentAnalysis, getAgentTracePayload, getRequestLogBodyChunk, getRequestLogDetail, getRequestLogs } from "@ccr/core/observability/request-log-store"; import { shouldRecordRequestLogs } from "@ccr/core/observability/raw-trace-sync"; import { getUsageStats } from "@ccr/core/usage/store"; import { gatewayService } from "@ccr/core/gateway/service"; @@ -78,6 +78,7 @@ import type { ProviderCatalogModelsRequest, ProviderIconDetectionRequest, ProviderManifestFetchRequest, + RequestLogBodyChunkRequest, RequestLogDetailRequest, RequestLogListFilter, RouteScriptTestRequest, @@ -334,6 +335,7 @@ const rpcHandlers: Record = { getProxyNetworkCaptures: () => proxyService.getNetworkCaptures(), getProxyStatus: () => proxyService.getStatus(), getRequestLogDetail: (request) => getRequestLogDetail(request as RequestLogDetailRequest), + getRequestLogBodyChunk: (request) => getRequestLogBodyChunk(request as RequestLogBodyChunkRequest), getRequestLogs: (filter) => getRequestLogs(filter as RequestLogListFilter | undefined), getUpdateStatus: () => unsupportedUpdateStatus, getUsageStats: (range, filter) => getUsageStats(range as UsageStatsRange | undefined, filter as UsageStatsFilter | undefined), diff --git a/packages/core/test/integration/observability/request-log-runtime.test.mjs b/packages/core/test/integration/observability/request-log-runtime.test.mjs index 95e317bf..073f78bf 100644 --- a/packages/core/test/integration/observability/request-log-runtime.test.mjs +++ b/packages/core/test/integration/observability/request-log-runtime.test.mjs @@ -323,13 +323,14 @@ test("RequestLogRuntime persists unmatched raw trace updates across worker resta } }); -test("RequestLogStore retains a bounded prefix for oversized unmatched raw trace bodies", async () => { +test("RequestLogStore stores oversized unmatched raw trace bodies in sidecar storage", async () => { const dir = mkdtempSync(path.join(tmpdir(), "ccr-request-log-pending-entry-budget-test-")); const store = new RequestLogStore(path.join(dir, "request-logs.sqlite")); try { + const body = "s".repeat(3 * 1024 * 1024); await store.writeBatch([{ input: { - requestBodyText: "s".repeat(3 * 1024 * 1024), + requestBodyText: body, requestId: "oversized-pending-raw" }, kind: "raw-trace-update", @@ -344,25 +345,29 @@ test("RequestLogStore retains a bounded prefix for oversized unmatched raw trace const page = await store.list({ pageSize: 25 }); const detail = await store.getDetail({ id: page.items[0].id }); - assert.match(detail.requestBody.text, /^s+$/); - assert.equal(Buffer.byteLength(detail.requestBody.text), 512 * 1024); - assert.equal(detail.requestBody.sizeBytes, 3 * 1024 * 1024); - assert.equal(detail.requestBody.truncated, true); + assert.ok(detail.requestBody.bodyRef); + assert.equal(detail.requestBody.preview, true); + assert.match(detail.requestBody.text, /bytes omitted from preview/); + assert.equal(detail.requestBody.sizeBytes, Buffer.byteLength(body)); + assert.equal(detail.requestBody.truncated, false); + assert.equal(await readStoredBodyText(store, page.items[0].id, "request"), body); } finally { await store.close(); rmSync(dir, { force: true, recursive: true }); } }); -test("RequestLogStore bounds unmatched request and response bodies independently", async () => { +test("RequestLogStore stores unmatched request and response bodies independently", async () => { const dir = mkdtempSync(path.join(tmpdir(), "ccr-request-log-pending-body-budget-test-")); const store = new RequestLogStore(path.join(dir, "request-logs.sqlite")); try { + const requestBody = "q".repeat(3 * 1024 * 1024); + const responseBody = "p".repeat(3 * 1024 * 1024); await store.writeBatch([{ input: { - requestBodyText: "q".repeat(3 * 1024 * 1024), + requestBodyText: requestBody, requestId: "oversized-pending-pair", - responseBodyText: "p".repeat(3 * 1024 * 1024) + responseBodyText: responseBody }, kind: "raw-trace-update", sequence: 1 @@ -376,12 +381,14 @@ test("RequestLogStore bounds unmatched request and response bodies independently const page = await store.list({ pageSize: 25 }); const detail = await store.getDetail({ id: page.items[0].id }); - assert.equal(Buffer.byteLength(detail.requestBody.text), 512 * 1024); - assert.equal(Buffer.byteLength(detail.responseBody.text), 512 * 1024); - assert.equal(detail.requestBody.sizeBytes, 3 * 1024 * 1024); - assert.equal(detail.responseBody.sizeBytes, 3 * 1024 * 1024); - assert.equal(detail.requestBody.truncated, true); - assert.equal(detail.responseBody.truncated, true); + assert.ok(detail.requestBody.bodyRef); + assert.ok(detail.responseBody.bodyRef); + assert.equal(detail.requestBody.sizeBytes, Buffer.byteLength(requestBody)); + assert.equal(detail.responseBody.sizeBytes, Buffer.byteLength(responseBody)); + assert.equal(detail.requestBody.truncated, false); + assert.equal(detail.responseBody.truncated, false); + assert.equal(await readStoredBodyText(store, page.items[0].id, "request"), requestBody); + assert.equal(await readStoredBodyText(store, page.items[0].id, "response"), responseBody); } finally { await store.close(); rmSync(dir, { force: true, recursive: true }); @@ -423,7 +430,9 @@ test("RequestLogStore enforces a total byte budget for unmatched raw trace rows" const oldest = await store.getDetail({ id: oldestEntry.id }); const newest = await store.getDetail({ id: newestEntry.id }); assert.match(oldest.requestBody.text, /worker-model/); - assert.match(newest.requestBody.text, /^(?:17)+$/); + assert.ok(newest.requestBody.bodyRef); + assert.match(newest.requestBody.text, /bytes omitted from preview/); + assert.match(await readStoredBodyText(store, newestEntry.id, "request"), /^(?:17)+$/); } finally { await store.close(); rmSync(dir, { force: true, recursive: true }); @@ -460,7 +469,8 @@ test("RequestLogRuntime preserves original response sizes when normal body captu assert.equal(metadataEntry.resolvedModel, "resolved-model"); assert.equal(metadataEntry.responseModel, "response-model"); assert.equal(truncated.responseBody.sizeBytes, Buffer.byteLength(fullResponse)); - assert.equal(Buffer.byteLength(truncated.responseBody.text), 512 * 1024); + assert.ok(truncated.responseBody.bodyRef); + assert.equal(Buffer.byteLength(await readStoredBodyText(runtime, truncatedEntry.id, "response")), 512 * 1024); assert.equal(truncated.responseBody.truncated, true); assert.equal(metadata.responseBody.sizeBytes, Buffer.byteLength("exists")); assert.equal(metadata.responseBody.text, ""); @@ -586,7 +596,7 @@ test("RequestLogRuntime keeps the record body-removal policy for later file-back } }); -test("RequestLogRuntime retains inline raw trace bodies below the default safety limit", async () => { +test("RequestLogRuntime stores large inline raw trace bodies in sidecar storage", async () => { const dir = mkdtempSync(path.join(tmpdir(), "ccr-request-log-runtime-raw-body-test-")); const runtime = createRuntime(dir); try { @@ -604,7 +614,9 @@ test("RequestLogRuntime retains inline raw trace bodies below the default safety const page = await runtime.list({ pageSize: 25 }); const detail = await runtime.getDetail({ id: page.items[0].id }); assert.equal(detail.responseBody.sizeBytes, Buffer.byteLength(body)); - assert.equal(detail.responseBody.text.length, body.length); + assert.ok(detail.responseBody.bodyRef); + assert.match(detail.responseBody.text, /bytes omitted from preview/); + assert.equal(await readStoredBodyText(runtime, page.items[0].id, "response"), body); assert.equal(detail.responseBody.truncated, false); } finally { await runtime.close({ timeoutMs: 5_000 }); @@ -612,7 +624,7 @@ test("RequestLogRuntime retains inline raw trace bodies below the default safety } }); -test("RequestLogRuntime bounds file-backed raw bodies and cleans bundles only after ACK", async () => { +test("RequestLogRuntime stores file-backed raw bodies and cleans bundles only after ACK", async () => { const dir = mkdtempSync(path.join(tmpdir(), "ccr-request-log-runtime-raw-file-test-")); const spoolDir = path.join(dir, "raw-trace-spool"); const bundleDir = path.join(spoolDir, "bundle"); @@ -643,8 +655,10 @@ test("RequestLogRuntime bounds file-backed raw bodies and cleans bundles only af const page = await runtime.list({ pageSize: 25 }); const detail = await runtime.getDetail({ id: page.items[0].id }); assert.equal(detail.responseBody.sizeBytes, Buffer.byteLength(body)); - assert.equal(detail.responseBody.text.length, 256 * 1024); - assert.equal(detail.responseBody.truncated, true); + assert.ok(detail.responseBody.bodyRef); + assert.match(detail.responseBody.text, /bytes omitted from preview/); + assert.equal(await readStoredBodyText(runtime, page.items[0].id, "response"), body); + assert.equal(detail.responseBody.truncated, false); } finally { await runtime.close({ timeoutMs: 5_000 }); rmSync(dir, { force: true, recursive: true }); @@ -685,8 +699,11 @@ test("RequestLogRuntime compacts Base64 images while reading file-backed raw tra const detail = await runtime.getDetail({ id: page.items[0].id }); const captured = JSON.parse(detail.requestBody.text); assert.equal(detail.requestBody.sizeBytes, Buffer.byteLength(body)); - assert.equal(detail.requestBody.truncated, true); + assert.ok(detail.requestBody.bodyRef); + assert.equal(detail.requestBody.truncated, false); assert.match(captured.image_url.url, /^data:image\/jpeg;base64,\[base64 image omitted from log;/); + const fullBody = JSON.parse(await readStoredBodyText(runtime, page.items[0].id, "request")); + assert.equal(fullBody.image_url.url, `data:image/jpeg;base64,${"A".repeat(512 * 1024)}`); assert.equal(existsSync(bundleDir), false); } finally { await runtime.close({ timeoutMs: 5_000 }); @@ -1621,6 +1638,20 @@ function createRuntime(dir, options = {}) { }); } +async function readStoredBodyText(source, id, side) { + let offset = 0; + const chunks = []; + while (true) { + const chunk = await source.getBodyChunk({ id, length: 512 * 1024, offset, side }); + assert.ok(chunk, `expected ${side} body chunk at offset ${offset}`); + chunks.push(chunk.text); + if (chunk.eof) break; + assert.ok(chunk.nextOffset > offset); + offset = chunk.nextOffset; + } + return chunks.join(""); +} + function createRecord(requestId) { const now = new Date().toISOString(); return { diff --git a/packages/core/test/integration/observability/request-log-store.test.mjs b/packages/core/test/integration/observability/request-log-store.test.mjs index fb8a9f70..34c56705 100644 --- a/packages/core/test/integration/observability/request-log-store.test.mjs +++ b/packages/core/test/integration/observability/request-log-store.test.mjs @@ -316,6 +316,104 @@ test("RequestLogStore keeps list rows lightweight and detail rows complete", asy } }); +test("RequestLogStore keeps large request bodies in sidecar storage and reads them by chunk", async () => { + const dir = mkdtempSync(path.join(tmpdir(), "ccr-request-log-sidecar-test-")); + let store; + try { + store = new RequestLogStore(path.join(dir, "request-logs.sqlite")); + const body = JSON.stringify({ + messages: [{ content: "hello", role: "user" }], + model: "request-model", + padding: "中🙂".repeat(40 * 1024), + tail: "request-tail" + }); + const response = JSON.stringify({ + model: "response-model", + padding: "y".repeat(220 * 1024), + tail: "response-tail", + usage: { + input_tokens: 3, + output_tokens: 4, + total_tokens: 7 + } + }); + + await store.record({ + completedAt: new Date().toISOString(), + durationMs: 42, + method: "POST", + path: "/v1/messages", + providerName: "test-provider", + requestBody: Buffer.from(body, "utf8"), + requestHeaders: { "content-type": "application/json" }, + requestId: "request-log-sidecar-test", + responseBodyText: response, + responseHeaders: { "content-type": "application/json" }, + startedAt: new Date().toISOString(), + statusCode: 200, + url: "http://127.0.0.1:3456/v1/messages" + }); + + const page = await store.list({ pageSize: 25 }); + assert.equal(page.items.length, 1); + assert.equal(page.items[0].requestBody.text, ""); + assert.equal(page.items[0].requestBody.preview, true); + assert.ok(page.items[0].requestBody.bodyRef); + + const detail = await store.getDetail({ id: page.items[0].id }); + assert.ok(detail); + assert.equal(detail.requestBody.preview, true); + assert.equal(detail.requestBody.truncated, false); + assert.match(detail.requestBody.text, /bytes omitted from preview/); + assert.match(detail.requestBody.text, /request-tail/); + assert.equal(detail.responseBody?.preview, true); + assert.equal(detail.responseBody?.truncated, false); + assert.match(detail.responseBody?.text ?? "", /response-tail/); + + const requestChunk = await store.getBodyChunk({ + id: detail.id, + length: 512 * 1024, + offset: 0, + side: "request" + }); + assert.ok(requestChunk); + assert.equal(requestChunk.eof, true); + assert.equal(requestChunk.truncated, false); + assert.equal(requestChunk.text, body); + + let unicodeOffset = 0; + const unicodeChunks = []; + while (true) { + const unicodeChunk = await store.getBodyChunk({ + id: detail.id, + length: 4097, + offset: unicodeOffset, + side: "request" + }); + assert.ok(unicodeChunk); + unicodeChunks.push(unicodeChunk.text); + if (unicodeChunk.eof) break; + assert.ok(unicodeChunk.nextOffset > unicodeOffset); + unicodeOffset = unicodeChunk.nextOffset; + } + assert.equal(unicodeChunks.join(""), body); + + const responseChunk = await store.getBodyChunk({ + id: detail.id, + length: 512 * 1024, + offset: 0, + side: "response" + }); + assert.ok(responseChunk); + assert.equal(responseChunk.eof, true); + assert.equal(responseChunk.truncated, false); + assert.equal(responseChunk.text, response); + } finally { + await store?.close(); + rmSync(dir, { force: true, recursive: true }); + } +}); + test("RequestLogStore stores decoded Claude App route models for observability", async () => { const dir = mkdtempSync(path.join(tmpdir(), "ccr-request-log-claude-app-model-test-")); try { diff --git a/packages/core/test/unit/observability/raw-trace-sync.test.mjs b/packages/core/test/unit/observability/raw-trace-sync.test.mjs index 0ff38e1b..beb98857 100644 --- a/packages/core/test/unit/observability/raw-trace-sync.test.mjs +++ b/packages/core/test/unit/observability/raw-trace-sync.test.mjs @@ -6,7 +6,7 @@ import { Readable } from "node:stream"; import test from "node:test"; import { createDefaultAppConfig } from "@ccr/core/config/default-config.ts"; import { rawTraceSyncHeader } from "@ccr/core/gateway/internal/shared.ts"; -import { rawTraceHardMaxBodyBytes } from "@ccr/core/observability/request-log-limits.ts"; +import { rawTraceHardMaxBodyBytes, rawTraceMaxPartBytes } from "@ccr/core/observability/request-log-limits.ts"; import { applyRawTraceRequestLogPolicy, buildRawTraceConfig, @@ -117,15 +117,16 @@ test("raw trace defers body persistence when the upstream status is unknown", () assert.equal(policy.update.requestBodyText, "private request body"); }); -test("raw trace source defaults to the 50 MB hard body ceiling", () => { +test("raw trace source keeps raw body parts unbounded for sidecar storage", () => { const config = createConfig(); const previous = process.env.CCR_RAW_TRACE_ENABLED; process.env.CCR_RAW_TRACE_ENABLED = "1"; try { const rawTrace = buildRawTraceConfig(config, "sync-token"); - assert.equal(rawTrace.maxPartBytes, rawTraceHardMaxBodyBytes); + assert.equal(rawTrace.maxPartBytes, rawTraceMaxPartBytes); + assert.ok(rawTrace.maxPartBytes > rawTraceHardMaxBodyBytes); config.observability.requestLogMaxBodyBytes = Number.MAX_SAFE_INTEGER; - assert.equal(buildRawTraceConfig(config, "sync-token").maxPartBytes, rawTraceHardMaxBodyBytes); + assert.equal(buildRawTraceConfig(config, "sync-token").maxPartBytes, rawTraceMaxPartBytes); } finally { if (previous === undefined) delete process.env.CCR_RAW_TRACE_ENABLED; else process.env.CCR_RAW_TRACE_ENABLED = previous; diff --git a/packages/electron/src/main/ipc.ts b/packages/electron/src/main/ipc.ts index 0b2e6abe..ce64a740 100644 --- a/packages/electron/src/main/ipc.ts +++ b/packages/electron/src/main/ipc.ts @@ -47,7 +47,7 @@ import { getPluginMarketplace } from "@ccr/core/plugins/marketplace"; import { ensureProxyCertificateAuthority } from "@ccr/core/proxy/certificates"; import { proxyService } from "@ccr/core/proxy/service"; import { listMcpServerTools } from "@ccr/core/mcp/tool-discovery"; -import { getAgentAnalysis, getAgentTracePayload, getRequestLogDetail, getRequestLogs } from "@ccr/core/observability/request-log-store"; +import { getAgentAnalysis, getAgentTracePayload, getRequestLogBodyChunk, getRequestLogDetail, getRequestLogs } from "@ccr/core/observability/request-log-store"; import trayController from "./tray-controller"; import { appUpdateService } from "./update-service"; import { getUsageStats } from "@ccr/core/usage/store"; @@ -124,6 +124,7 @@ ipcMain.handle(IPC_CHANNELS.appGetProxyNetworkCaptures, () => proxyService.getNe ipcMain.handle(IPC_CHANNELS.appGetProxyStatus, () => proxyService.getStatus()); ipcMain.handle(IPC_CHANNELS.appGetPluginMarketplace, () => getPluginMarketplace()); ipcMain.handle(IPC_CHANNELS.appGetRequestLogDetail, (_event, request) => getRequestLogDetail(request)); +ipcMain.handle(IPC_CHANNELS.appGetRequestLogBodyChunk, (_event, request) => getRequestLogBodyChunk(request)); ipcMain.handle(IPC_CHANNELS.appGetRequestLogs, (_event, filter?: RequestLogListFilter) => getRequestLogs(filter)); ipcMain.handle(IPC_CHANNELS.appGetUpdateStatus, () => appUpdateService.getStatus()); ipcMain.handle(IPC_CHANNELS.appGetUsageStats, (_event, range?: UsageStatsRange, filter?: UsageStatsFilter) => getUsageStats(range, filter)); diff --git a/packages/electron/src/main/preload.ts b/packages/electron/src/main/preload.ts index 1df40f43..3d713478 100644 --- a/packages/electron/src/main/preload.ts +++ b/packages/electron/src/main/preload.ts @@ -71,6 +71,8 @@ import type { ProxyNetworkSnapshot, ProxyStatus, RequestLogDetailRequest, + RequestLogBodyChunk, + RequestLogBodyChunkRequest, RequestLogEntry, RequestLogListFilter, RequestLogPage, @@ -129,6 +131,7 @@ contextBridge.exposeInMainWorld("ccr", { getProxyNetworkCaptures: () => invoke(IPC_CHANNELS.appGetProxyNetworkCaptures) as Promise, getProxyStatus: () => invoke(IPC_CHANNELS.appGetProxyStatus) as Promise, getRequestLogDetail: (request: RequestLogDetailRequest) => invoke(IPC_CHANNELS.appGetRequestLogDetail, request) as Promise, + getRequestLogBodyChunk: (request: RequestLogBodyChunkRequest) => invoke(IPC_CHANNELS.appGetRequestLogBodyChunk, request) as Promise, getRequestLogs: (filter?: RequestLogListFilter) => invoke(IPC_CHANNELS.appGetRequestLogs, filter) as Promise, getUpdateStatus: () => invoke(IPC_CHANNELS.appGetUpdateStatus) as Promise, getUsageStats: (range?: UsageStatsRange, filter?: UsageStatsFilter) => invoke(IPC_CHANNELS.appGetUsageStats, range, filter) as Promise, diff --git a/packages/ui/src/pages/home/components/network-logs.tsx b/packages/ui/src/pages/home/components/network-logs.tsx index 4f41907a..e7c13a44 100644 --- a/packages/ui/src/pages/home/components/network-logs.tsx +++ b/packages/ui/src/pages/home/components/network-logs.tsx @@ -12,7 +12,7 @@ import { logResolvedRouteModel, logSelectOptions, motion, MoveRight, Network, networkCodeLabel, networkExchangeMatchesQuery, networkHeaderRows, networkLifecycleLabel, networkQueryRows, networkRowId, networkSummaryRows, Pause, Play, ProxyNetworkBody, ProxyNetworkExchange, ProxyNetworkSnapshot, ProxyStatus, - ReactNode, ReactPointerEvent, RefreshCw, RequestLogBody, RequestLogEntry, RequestLogListFilter, + ReactNode, ReactPointerEvent, RefreshCw, RequestLogBody, RequestLogBodyChunk, RequestLogEntry, RequestLogListFilter, RequestLogPage, requestLogPageSizeOptions, RequestLogStatusFilter, requestLogStatusOptions, Search, Select, translateOptions, Trash2, useAppNumberLocale, useAppText, useCallback, useEffect, useMemo, useRef, useState @@ -24,6 +24,7 @@ type NetworkResponseTab = "body" | "header" | "raw"; const logJsonAutoExpandEntryLimit = 60; const logJsonContainerPreviewLimit = 80; const logJsonAutoExpandTextLimit = 160 * 1024; +const logBodyAutoLoadJsonBytes = 2 * 1024 * 1024; const logBodyWorkerFilterDebounceMs = 180; type LogTableColumnId = "time" | "status" | "stream" | "model" | "credential" | "tokens" | "duration"; type LogTableColumn = { @@ -992,12 +993,14 @@ function LogExpandedDetails({ ) : null}
- + @@ -1510,6 +1513,14 @@ type LogBodyPanelView = FormattedLogBody & { visible: string; }; +type LogBodyChunkPanelView = { + bodyKey: string; + chunk?: RequestLogBodyChunk; + error: string; + loading: boolean; + previousOffsets: number[]; +}; + function useLogBodyWorkerView( body: RequestLogBody | undefined, bodyKey: string, @@ -1743,6 +1754,8 @@ function LogJsonPanel({ className, headerEmptyLabel = "No values", headers, + requestLogId, + side, subtitle, title }: { @@ -1750,6 +1763,8 @@ function LogJsonPanel({ className?: string; headerEmptyLabel?: string; headers?: Record; + requestLogId: number; + side: "request" | "response"; subtitle?: string; title: string; }) { @@ -1758,23 +1773,55 @@ function LogJsonPanel({ const [preferTextBody, setPreferTextBody] = useState(false); const [bodyMode, setBodyMode] = useState("preview"); const [fullscreenOpen, setFullscreenOpen] = useState(false); + const [loadedBody, setLoadedBody] = useState(); + const [chunkView, setChunkView] = useState(); + const [fullBodyLoading, setFullBodyLoading] = useState(false); + const [fullBodyError, setFullBodyError] = useState(""); const [query, setQuery] = useState(""); - const bodyKey = logBodyCacheKey(body); - const bodyView = useLogBodyWorkerView(body, bodyKey, bodyMode, query); + const fullBodyLoadIdRef = useRef(0); + const sourceBodyKey = logBodyCacheKey(body); + const effectiveBody = loadedBody && loadedBody.bodyRef && loadedBody.bodyRef === body?.bodyRef + ? loadedBody + : body; + const bodyKey = logBodyCacheKey(effectiveBody); + const bodyView = useLogBodyWorkerView(effectiveBody, bodyKey, bodyMode, query); + const chunkViewActive = chunkView?.bodyKey === sourceBodyKey; + const toolbarBodyView = fullBodyLoading || fullBodyError + ? { + ...bodyView, + error: fullBodyError || bodyView.error, + loading: fullBodyLoading || bodyView.loading, + mode: fullBodyLoading ? "full" as const : bodyView.mode + } + : bodyView; + const displayedToolbarBodyView = chunkViewActive + ? { + ...toolbarBodyView, + error: chunkView.error || toolbarBodyView.error, + loading: chunkView.loading, + mode: "full" as const, + preview: false + } + : toolbarBodyView; const formatted = bodyView.text; const visible = bodyView.visible; const headerRows = useMemo(() => networkHeaderRows(headers ?? {}), [headers]); - const [expandedJsonPaths, setExpandedJsonPaths] = useState>(() => createInitialVisibleJsonPaths(bodyView)); - const showJsonTree = bodyView.json !== undefined && query.trim() === "" && !preferTextBody && !bodyView.preview; + const [expandedJsonPaths, setExpandedJsonPaths] = useState>(() => createInitialVisibleJsonPaths(bodyView, side)); + const showJsonTree = bodyView.json !== undefined && query.trim() === "" && !preferTextBody; useEffect(() => { + fullBodyLoadIdRef.current += 1; + setLoadedBody(undefined); + setChunkView(undefined); + setFullBodyLoading(false); + setFullBodyError(""); setPreferTextBody(false); setBodyMode("preview"); - }, [bodyKey]); + }, [sourceBodyKey]); useEffect(() => { - setExpandedJsonPaths(createInitialVisibleJsonPaths(bodyView)); - }, [bodyView.bodyKey, bodyView.json, bodyView.text]); + setExpandedJsonPaths(createInitialVisibleJsonPaths(bodyView, side)); + }, [bodyView.bodyKey, bodyView.json, bodyView.text, side]); useEffect(() => { if (!fullscreenOpen) { @@ -1801,6 +1848,134 @@ function LogJsonPanel({ }); } + async function loadBodyChunk(offset: number) { + if (!effectiveBody?.bodyRef || !window.ccr?.getRequestLogBodyChunk) { + setBodyMode("full"); + return; + } + const loadId = fullBodyLoadIdRef.current + 1; + fullBodyLoadIdRef.current = loadId; + setBodyMode("full"); + setChunkView((current) => ({ + bodyKey: sourceBodyKey, + chunk: current?.bodyKey === sourceBodyKey ? current.chunk : undefined, + error: "", + loading: true, + previousOffsets: nextChunkPreviousOffsets(current, sourceBodyKey, offset) + })); + try { + const chunk = await window.ccr.getRequestLogBodyChunk({ + id: requestLogId, + length: 1024 * 1024, + offset, + side + }); + if (fullBodyLoadIdRef.current !== loadId) { + return; + } + if (!chunk) { + throw new Error(t("Request log body is not available.")); + } + setChunkView({ + bodyKey: sourceBodyKey, + chunk, + error: "", + loading: false, + previousOffsets: nextChunkPreviousOffsets(chunkView, sourceBodyKey, offset) + }); + } catch (error) { + if (fullBodyLoadIdRef.current === loadId) { + setChunkView((current) => ({ + bodyKey: sourceBodyKey, + chunk: current?.bodyKey === sourceBodyKey ? current.chunk : undefined, + error: error instanceof Error ? error.message : String(error), + loading: false, + previousOffsets: current?.bodyKey === sourceBodyKey ? current.previousOffsets : [] + })); + } + } + } + + async function loadInlineFullBody() { + if (!effectiveBody?.bodyRef || !window.ccr?.getRequestLogBodyChunk) { + return; + } + const loadId = fullBodyLoadIdRef.current + 1; + fullBodyLoadIdRef.current = loadId; + setFullBodyLoading(true); + setFullBodyError(""); + try { + const chunks: string[] = []; + let offset = 0; + let lastChunk: RequestLogBodyChunk | undefined; + while (true) { + const chunk = await window.ccr.getRequestLogBodyChunk({ + id: requestLogId, + length: 1024 * 1024, + offset, + side + }); + if (fullBodyLoadIdRef.current !== loadId) { + return; + } + if (!chunk) { + throw new Error(t("Request log body is not available.")); + } + chunks.push(chunk.text); + lastChunk = chunk; + if (chunk.eof) { + break; + } + offset = chunk.nextOffset ?? offset + chunk.length; + await new Promise((resolve) => window.requestAnimationFrame(() => resolve())); + } + setLoadedBody({ + ...effectiveBody, + ...(lastChunk?.bodyRef ? { bodyRef: lastChunk.bodyRef } : {}), + contentType: lastChunk?.contentType ?? effectiveBody.contentType, + encoding: lastChunk?.encoding ?? effectiveBody.encoding, + preview: false, + sizeBytes: lastChunk?.sizeBytes ?? effectiveBody.sizeBytes, + text: chunks.join(""), + truncated: Boolean(lastChunk?.truncated) + }); + setBodyMode("full"); + } catch (error) { + if (fullBodyLoadIdRef.current === loadId) { + setFullBodyError(error instanceof Error ? error.message : String(error)); + } + } finally { + if (fullBodyLoadIdRef.current === loadId) setFullBodyLoading(false); + } + } + + useEffect(() => { + if ( + !effectiveBody?.bodyRef || + !effectiveBody.preview || + loadedBody || + fullBodyLoading || + fullBodyError || + bodyMode !== "preview" + ) { + return; + } + if (effectiveBody.sizeBytes <= logBodyAutoLoadJsonBytes && isJsonLikeLogBody(effectiveBody)) { + void loadInlineFullBody(); + return; + } + if (!chunkViewActive && !chunkView?.loading) { + void loadBodyChunk(0); + } + }, [bodyMode, chunkView, chunkViewActive, effectiveBody, fullBodyError, fullBodyLoading, loadedBody]); + + const displayedCopyText = chunkViewActive + ? chunkView.chunk?.text ?? "" + : formatted; + const displayedVisible = chunkViewActive + ? filterStaticLogBodyText(chunkView.chunk?.text ?? "", query) + : visible; + return (
@@ -1827,9 +2002,8 @@ function LogJsonPanel({ {selectedTab === "body" ? ( <> setBodyMode("full")} + body={effectiveBody} + bodyView={displayedToolbarBodyView} onQueryChange={setQuery} onToggleTextBody={() => setPreferTextBody((current) => !current)} preferTextBody={preferTextBody} @@ -1838,27 +2012,32 @@ function LogJsonPanel({ /> setFullscreenOpen(true)} > - + {chunkViewActive ? ( + + ) : ( + + )} {fullscreenOpen ? ( setFullscreenOpen(false)} - onLoadFullBody={() => setBodyMode("full")} + onLoadChunk={loadBodyChunk} onQueryChange={setQuery} onToggleJsonPath={toggleJsonPath} onToggleTextBody={() => setPreferTextBody((current) => !current)} @@ -1867,7 +2046,7 @@ function LogJsonPanel({ showJsonTree={showJsonTree} subtitle={subtitle} title={title} - visible={visible} + visible={displayedVisible} value={bodyView.json} /> ) : null} @@ -1885,7 +2064,6 @@ function LogJsonPanel({ function LogJsonBodyToolbar({ body, bodyView, - onLoadFullBody, onQueryChange, onToggleTextBody, preferTextBody, @@ -1894,7 +2072,6 @@ function LogJsonBodyToolbar({ }: { body?: RequestLogBody; bodyView: LogBodyPanelView; - onLoadFullBody: () => void; onQueryChange: (value: string) => void; onToggleTextBody: () => void; preferTextBody: boolean; @@ -1902,12 +2079,8 @@ function LogJsonBodyToolbar({ title: string; }) { const t = useAppText(); - const canLoadFullBody = bodyView.preview && bodyView.large && query.trim() === ""; const canToggleJsonText = bodyView.json !== undefined && query.trim() === ""; - const showToggleButton = canLoadFullBody || canToggleJsonText; - const toggleLabel = canLoadFullBody - ? bodyView.loading && bodyView.mode === "full" ? t("Loading full payload...") : t("Show full content") - : preferTextBody ? "JSON" : t("Show full content"); + const toggleLabel = preferTextBody ? "JSON" : t("Text"); return (
@@ -1921,11 +2094,10 @@ function LogJsonBodyToolbar({ value={query} />
- {showToggleButton ? ( + {canToggleJsonText ? ( + + {rangeLabel ? {rangeLabel} : null} + {chunkView.loading ? {t("Loading full payload...")} : null} + {chunkView.error ? {chunkView.error} : null} +
+ 
{content}
+
+ ); +} + function LogJsonFullscreenViewer({ body, bodyView, + chunkView, copyLabel, copyText, expandedJsonPaths, onClose, - onLoadFullBody, + onLoadChunk, onQueryChange, onToggleJsonPath, onToggleTextBody, @@ -1982,11 +2211,12 @@ function LogJsonFullscreenViewer({ }: { body?: RequestLogBody; bodyView: LogBodyPanelView; + chunkView?: LogBodyChunkPanelView; copyLabel: string; copyText: string; expandedJsonPaths: Set; onClose: () => void; - onLoadFullBody: () => void; + onLoadChunk: (offset: number) => void | Promise; onQueryChange: (value: string) => void; onToggleJsonPath: (path: string) => void; onToggleTextBody: () => void; @@ -2024,7 +2254,6 @@ function LogJsonFullscreenViewer({
- + {chunkView ? ( + + ) : ( + + )}
@@ -2053,9 +2286,11 @@ function logBodyCacheKey(body: RequestLogBody | undefined): string { } const text = body.text ?? ""; return [ + body.bodyRef ?? "", body.encoding ?? "", body.contentType ?? "", body.sizeBytes, + body.preview ? "preview" : "full", body.truncated ? "truncated" : "complete", text.length, text.slice(0, 96), @@ -2063,10 +2298,42 @@ function logBodyCacheKey(body: RequestLogBody | undefined): string { ].join("\u001f"); } -function createInitialVisibleJsonPaths(bodyView: FormattedLogBody): Set { +function isJsonLikeLogBody(body: RequestLogBody | undefined): boolean { + if (!body || body.encoding === "base64") { + return false; + } + const contentType = body.contentType?.split(";", 1)[0]?.trim().toLowerCase() ?? ""; + if (contentType === "application/json" || contentType.endsWith("+json")) { + return true; + } + const first = body.text.trimStart().charAt(0); + return first === "{" || first === "["; +} + +function nextChunkPreviousOffsets( + current: LogBodyChunkPanelView | undefined, + bodyKey: string, + nextOffset: number +): number[] { + if (!current || current.bodyKey !== bodyKey || !current.chunk) { + return []; + } + if (nextOffset > current.chunk.offset) { + return [...current.previousOffsets, current.chunk.offset]; + } + if (nextOffset < current.chunk.offset) { + return current.previousOffsets.slice(0, -1); + } + return current.previousOffsets; +} + +function createInitialVisibleJsonPaths(bodyView: FormattedLogBody, side?: "request" | "response"): Set { if (!isJsonContainer(bodyView.json)) { return new Set(); } + if (side === "request") { + return new Set(["$"]); + } if ( bodyView.text.length > logJsonAutoExpandTextLimit || jsonContainerEntryCount(bodyView.json, logJsonAutoExpandEntryLimit + 1) > logJsonAutoExpandEntryLimit diff --git a/packages/ui/src/pages/home/shared/external.tsx b/packages/ui/src/pages/home/shared/external.tsx index 8eac742e..f5599ffd 100644 --- a/packages/ui/src/pages/home/shared/external.tsx +++ b/packages/ui/src/pages/home/shared/external.tsx @@ -206,6 +206,7 @@ import type { ProxyNetworkSnapshot, ProxyStatus, RequestLogBody, + RequestLogBodyChunk, RequestLogEntry, RequestLogListFilter, RequestLogPage, @@ -569,7 +570,7 @@ export type { ProviderAccountMeter, ProviderAccountStandardConnectorConfig, ProviderAccountSnapshot, ProviderAccountTestPath, ProviderAccountTestResult, ProviderDeepLinkPayload, ProviderDeepLinkRequest, ProviderCredentialConfig, ProfileConfig, ProfileOpenSurface, ProfileRuntimeStatus, CodexProfileConfigFormat, ProfileScope, ProfileSurface, ProxyCertificateInstallResult, ProxyCertificateStatus, ProxyNetworkBody, - ProxyNetworkExchange, ProxyNetworkSnapshot, ProxyStatus, RequestLogBody, RequestLogEntry, RequestLogListFilter, RequestLogPage, + ProxyNetworkExchange, ProxyNetworkSnapshot, ProxyStatus, RequestLogBody, RequestLogBodyChunk, RequestLogEntry, RequestLogListFilter, RequestLogPage, RequestLogStatusFilter, RouterConfig, RouterFallbackConfig, RouterFallbackMode, RouterRule, RouterRuleType, TrayComponentVariants, TrayBalanceProgressConfig, TrayWidgetConfig, TrayWidgetType, TrayWidgetVariant, TrayWindowModuleId, UsageComparisonRow, UsageSeriesPoint, UsageStatsFilter, UsageStatsRange, UsageStatsSnapshot, UsageTotals, VirtualModelBaseModelMode, VirtualModelExecutionMode, VirtualModelFusionCustomToolConfig, VirtualModelFusionVisionConfig, VirtualModelFusionWebSearchConfig, VirtualModelFusionWebSearchProvider, VirtualModelProfileConfig, VirtualModelToolVisibility, ProviderIdentitySafetyIssue, ProviderPreset, ProviderPresetEndpoint diff --git a/packages/ui/src/pages/home/shared/i18n.tsx b/packages/ui/src/pages/home/shared/i18n.tsx index ca02bb95..42e0d0c5 100644 --- a/packages/ui/src/pages/home/shared/i18n.tsx +++ b/packages/ui/src/pages/home/shared/i18n.tsx @@ -249,6 +249,7 @@ export const appCopy: Record = { "No request headers": "No request headers", "No response headers": "No response headers", "筛选 JSON...": "Filter JSON...", + "Text": "Text", "body": "Body", "header": "Headers", "入": "in", @@ -1593,7 +1594,7 @@ export const appCopy: Record = { "Yes": "是", "Set as default provider": "设为默认供应商", "Show credential settings": "显示凭据配置", - "Show full content": "查看完整内容", + "Text": "文本", "Session": "会话", "Session Detail": "会话详情", "Session Requests": "会话请求", diff --git a/packages/ui/src/pages/home/shared/log-body-worker-protocol.ts b/packages/ui/src/pages/home/shared/log-body-worker-protocol.ts index 0c905ed9..6bbbebfc 100644 --- a/packages/ui/src/pages/home/shared/log-body-worker-protocol.ts +++ b/packages/ui/src/pages/home/shared/log-body-worker-protocol.ts @@ -1,5 +1,5 @@ import type { RequestLogBody } from "@ccr/core/contracts/app"; -import type { FormattedLogBody } from "./logs"; +import { formatLogBodyView, type FormattedLogBody } from "./logs"; export const logBodyLargeTextThreshold = 256 * 1024; export const logBodyPreviewTextLimit = 160 * 1024; @@ -69,7 +69,7 @@ export function isLargeLogBody( if (!body) { return false; } - return Math.max(body.sizeBytes, body.text.length) > threshold; + return Boolean(body.preview) || Math.max(body.sizeBytes, body.text.length) > threshold; } export function createLogBodyPreviewText( @@ -101,3 +101,27 @@ export function createLogBodyPreviewText( tail ].join("\n"); } + +export function formatLogBodyForWorker( + body: RequestLogBody | undefined, + mode: LogBodyFormatMode, + largeTextThreshold = logBodyLargeTextThreshold, + previewTextLimit = logBodyPreviewTextLimit +): FormattedLogBody & { + large: boolean; + preview: boolean; + sourceSizeBytes: number; +} { + const large = isLargeLogBody(body, largeTextThreshold); + const preview = large && mode !== "full"; + const formattedBodyView = formatLogBodyView(body); + const bodyView = preview && formattedBodyView.json === undefined + ? { text: createLogBodyPreviewText(body, previewTextLimit) } + : formattedBodyView; + return { + ...bodyView, + large, + preview, + sourceSizeBytes: body?.sizeBytes ?? 0 + }; +} diff --git a/packages/ui/src/pages/home/shared/log-body.worker.ts b/packages/ui/src/pages/home/shared/log-body.worker.ts index 3dace470..375ab82b 100644 --- a/packages/ui/src/pages/home/shared/log-body.worker.ts +++ b/packages/ui/src/pages/home/shared/log-body.worker.ts @@ -1,6 +1,5 @@ import { - createLogBodyPreviewText, - isLargeLogBody, + formatLogBodyForWorker, logBodyLargeTextThreshold, logBodyPreviewTextLimit, type LogBodyFilterRequest, @@ -9,7 +8,7 @@ import { type LogBodyWorkerRequest, type LogBodyWorkerResponse } from "./log-body-worker-protocol"; -import { filterLogText, formatLogBodyView, type FormattedLogBody } from "./logs"; +import { filterLogText, type FormattedLogBody } from "./logs"; type CachedFormattedBody = FormattedLogBody & { bodyKey: string; @@ -50,12 +49,12 @@ worker.onmessage = (event: MessageEvent) => { function formatBody(request: LogBodyFormatRequest): LogBodyFormatResult { const threshold = request.largeTextThreshold ?? logBodyLargeTextThreshold; const previewLimit = request.previewTextLimit ?? logBodyPreviewTextLimit; - const large = isLargeLogBody(request.body, threshold); - const preview = large && request.mode !== "full"; - const bodyView = preview - ? { text: createLogBodyPreviewText(request.body, previewLimit) } - : formatLogBodyView(request.body); - const sourceSizeBytes = request.body?.sizeBytes ?? 0; + const { large, preview, sourceSizeBytes, ...bodyView } = formatLogBodyForWorker( + request.body, + request.mode, + threshold, + previewLimit + ); const visible = filterLogText(bodyView.text, request.query); cachedBody = { diff --git a/packages/ui/src/pages/home/shared/logs.ts b/packages/ui/src/pages/home/shared/logs.ts index 56e9574f..d15ced2c 100644 --- a/packages/ui/src/pages/home/shared/logs.ts +++ b/packages/ui/src/pages/home/shared/logs.ts @@ -126,7 +126,9 @@ export function logBodyKey(body: RequestLogBody | undefined): string { return "missing"; } return JSON.stringify([ + body.bodyRef ?? "", body.encoding ?? "", + body.preview ? "preview" : "full", body.sizeBytes, body.text ?? "" ]); diff --git a/packages/ui/src/types/electron.d.ts b/packages/ui/src/types/electron.d.ts index a38c1084..a747d727 100644 --- a/packages/ui/src/types/electron.d.ts +++ b/packages/ui/src/types/electron.d.ts @@ -70,6 +70,8 @@ import type { ProxyNetworkSnapshot, ProxyStatus, RequestLogDetailRequest, + RequestLogBodyChunk, + RequestLogBodyChunkRequest, RequestLogEntry, RequestLogListFilter, RequestLogPage, @@ -117,6 +119,7 @@ declare global { getProxyNetworkCaptures: () => Promise; getProxyStatus: () => Promise; getRequestLogDetail: (request: RequestLogDetailRequest) => Promise; + getRequestLogBodyChunk: (request: RequestLogBodyChunkRequest) => Promise; getRequestLogs: (filter?: RequestLogListFilter) => Promise; getUpdateStatus: () => Promise; getUsageStats: (range?: UsageStatsRange, filter?: UsageStatsFilter) => Promise; diff --git a/packages/ui/src/web-client-bridge.ts b/packages/ui/src/web-client-bridge.ts index f1fef70e..df32c637 100644 --- a/packages/ui/src/web-client-bridge.ts +++ b/packages/ui/src/web-client-bridge.ts @@ -133,6 +133,7 @@ const webClientBridge: CcrApi = { getProxyNetworkCaptures: () => rpc("getProxyNetworkCaptures") as ReturnType, getProxyStatus: () => rpc("getProxyStatus") as ReturnType, getRequestLogDetail: (request) => rpc("getRequestLogDetail", [request]) as ReturnType, + getRequestLogBodyChunk: (request) => rpc("getRequestLogBodyChunk", [request]) as ReturnType, getRequestLogs: (filter) => rpc("getRequestLogs", [filter]) as ReturnType, getUpdateStatus: () => rpc("getUpdateStatus") as ReturnType, getUsageStats: (range, filter) => rpc("getUsageStats", [range, filter]) as ReturnType, diff --git a/packages/ui/test/unit/network-format.test.ts b/packages/ui/test/unit/network-format.test.ts index 0b2d9581..4d6ab59b 100644 --- a/packages/ui/test/unit/network-format.test.ts +++ b/packages/ui/test/unit/network-format.test.ts @@ -1,5 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; +import type { RequestLogBody } from "@ccr/core/contracts/app"; +import { formatLogBodyForWorker } from "@ccr/ui/pages/home/shared/log-body-worker-protocol.ts"; import { formatRouteTracePath } from "@ccr/ui/pages/home/shared/network.ts"; test("route trace paths use request and response dotted notation", () => { @@ -28,3 +30,23 @@ test("route trace paths use request and response dotted notation", () => { "request.url" ); }); + +test("request log preview bodies still format parseable JSON as JSON", () => { + const body: RequestLogBody = { + bodyRef: "preview-json-body", + contentType: "application/json", + encoding: "utf8", + preview: true, + sizeBytes: 512 * 1024, + text: JSON.stringify({ messages: [{ role: "user", content: "hello" }], model: "test-model" }), + truncated: false + }; + + const view = formatLogBodyForWorker(body, "preview", 256 * 1024, 160 * 1024); + assert.equal(view.preview, true); + assert.deepEqual(view.json, { + messages: [{ role: "user", content: "hello" }], + model: "test-model" + }); + assert.match(view.text, /"model": "test-model"/); +});