mirror of
https://github.com/aiclientproxy/proxycast.git
synced 2026-09-24 23:10:56 +08:00
feat: finalize latest v0.97.0 release payload
This commit is contained in:
+1
-1
@@ -131,7 +131,7 @@ npm run harness:eval:trend
|
||||
### 记录 Harness eval 历史窗口
|
||||
|
||||
```bash
|
||||
node scripts/harness-eval-runner.mjs --record-history-dir "./artifacts/history" --history-retain 30
|
||||
npm run harness:eval:history:record
|
||||
```
|
||||
|
||||
### 运行 Harness cleanup / slop 报告
|
||||
|
||||
@@ -88,6 +88,7 @@ Lime 当前不直接把“真实模型重放平台”一次做完,而是先固
|
||||
2. 解析固定 fixture 与工作区自动发现 case
|
||||
3. 校验 replay case 最小四件套与关键 JSON 字段
|
||||
4. 输出统一 JSON / Markdown 摘要,并聚合 `suite tag / failure mode / review decision status / risk level` 分布
|
||||
5. 继续聚合 `observabilitySignals`,把 `requestTelemetry / artifactValidator / browser/gui smoke` 的证据缺口也纳入 trend 与 cleanup
|
||||
|
||||
当前它**不直接执行真实模型重放**,而是先把“样本是否可评估、摘要是否可归档”工程化。
|
||||
|
||||
@@ -172,6 +173,13 @@ node scripts/harness-replay-promote.mjs \
|
||||
|
||||
从 `2026-03-27` 起,这个历史窗口不再依赖 workflow 里的 `cp / ls / xargs` 拼接,而是直接由 `scripts/harness-eval-runner.mjs --record-history-dir` 负责写入和裁剪,保证本地与 nightly 走同一条跨平台主链。
|
||||
|
||||
同一天新增的 `scripts/harness-eval-history-record.mjs` 又把这条主链向前推了一步:
|
||||
|
||||
- 本地可以一次命令完成 `summary -> history -> trend -> cleanup`
|
||||
- 本地默认把 history window 写到 `./.lime/harness/history`,并把 trend / cleanup 报表写到 `./.lime/harness/reports`
|
||||
- `cleanup report` 在未显式传入 `--trend-input / --trend-history-dir` 时,会优先自动发现 `./.lime/harness/history`,再回退到 `./artifacts/history`
|
||||
- 修复后的趋势积累不再只属于 nightly,开发者本地也能拿到同口径的历史窗口
|
||||
|
||||
## 常用命令
|
||||
|
||||
```bash
|
||||
@@ -195,14 +203,17 @@ node scripts/harness-eval-runner.mjs \
|
||||
--output-json "./tmp/harness-eval-summary.json" \
|
||||
--output-markdown "./tmp/harness-eval-summary.md"
|
||||
|
||||
# 记录本地 history window,供 trend / cleanup 复用
|
||||
# 推荐的本地一体化入口:记录 history,并同时刷新 trend / cleanup
|
||||
npm run harness:eval:history:record
|
||||
|
||||
# 如需显式指定本地 history window
|
||||
node scripts/harness-eval-runner.mjs \
|
||||
--record-history-dir "./artifacts/history" \
|
||||
--record-history-dir "./.lime/harness/history" \
|
||||
--history-retain 30
|
||||
|
||||
# 从历史 summary 目录生成趋势报告
|
||||
node scripts/harness-eval-trend-report.mjs \
|
||||
--history-dir "./artifacts/history" \
|
||||
--history-dir "./.lime/harness/history" \
|
||||
--output-json "./tmp/harness-eval-trend.json" \
|
||||
--output-markdown "./tmp/harness-eval-trend.md"
|
||||
```
|
||||
@@ -218,6 +229,7 @@ Runner 摘要至少回答下面这些问题:
|
||||
- 哪些 case 属于什么 suite tag / failure mode
|
||||
- 哪些 case 默认需要人工复核
|
||||
- 哪些 case 已经记录人工审核状态与风险等级
|
||||
- 哪些 case 还缺 `observabilitySummary` 或仍处于 `requestTelemetry:unlinked`、`artifactValidator:known_gap` 等证据缺口
|
||||
- 工作区 replay 是否已经开始形成增量样本
|
||||
|
||||
如果摘要回答不了这些问题,就说明 runner 还不算进入 current 主链。
|
||||
@@ -228,6 +240,7 @@ Trend 报告至少还要回答:
|
||||
- 哪些 suite 在 latest 里变差了
|
||||
- 哪些 failure mode / suite tag 在 latest 里增长或退化了
|
||||
- 哪些人工审核状态 / 风险等级在 latest 里新增、减少或发生迁移
|
||||
- 哪些 observability signal gap 在 latest 里增加、减少或仍然停留在 current 样本中
|
||||
- 当前只有 trend seed,还是已经开始形成真正的历史窗口
|
||||
|
||||
## 与其他事实源的关系
|
||||
@@ -250,9 +263,9 @@ Trend 报告至少还要回答:
|
||||
|
||||
`P3-6` 做完之后,下一刀优先级建议固定为:
|
||||
|
||||
1. 把 cleanup report 接入更稳定的历史窗口,避免长期停留在 trend seed
|
||||
2. 继续补 observability 证据字段,让 grader 和 cleanup report 都能消费更多 request / timeline / artifact 关联
|
||||
3. 逐步提高 repo current 样本质量,而不是只增加数量
|
||||
1. 继续补 observability 证据字段,让 grader、analysis handoff 和 cleanup report 都能消费更多 `request / timeline / artifact` 关联
|
||||
2. 逐步提高 repo current 样本质量,而不是只增加数量
|
||||
3. 让 review-decision / 风险等级 / replay promotion 继续回挂到同一条趋势主线
|
||||
4. 再考虑是否引入真实模型执行或 transcript grading
|
||||
|
||||
## 非目标
|
||||
|
||||
@@ -21,7 +21,9 @@
|
||||
"relativeRoot": ".lime/harness/sessions/fixture-session-minimal-pending-request/evidence",
|
||||
"absoluteRoot": "/workspace/lime/.lime/harness/sessions/fixture-session-minimal-pending-request/evidence",
|
||||
"knownGaps": [
|
||||
"当前 fixture 不包含真实浏览器快照,只验证 replay 样本结构与 grader 约定。"
|
||||
"当前 Lime 虽然已有全局 RequestLog / token / retry / duration 遥测,但 RequestLog 尚未携带 session/thread/turn 关联键,Evidence Pack 只能导出 request telemetry coverage gap,不能给出精确的会话级摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 ArtifactDocument validator outcome 与修复问题摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 GUI smoke / browser 验证结果。"
|
||||
],
|
||||
"artifacts": [
|
||||
{
|
||||
@@ -36,6 +38,90 @@
|
||||
}
|
||||
]
|
||||
},
|
||||
"observabilitySummary": {
|
||||
"schemaVersion": "v1",
|
||||
"correlation": {
|
||||
"correlationKeys": [
|
||||
"session_id",
|
||||
"thread_id",
|
||||
"turn_id",
|
||||
"pending_request_id",
|
||||
"queued_turn_id",
|
||||
"subagent_session_id"
|
||||
],
|
||||
"sessionId": "fixture-session-minimal-pending-request",
|
||||
"threadId": "fixture-thread-minimal-pending-request",
|
||||
"activeTurnId": "turn-fixture-001",
|
||||
"pendingRequestIds": ["approval-fixture-001"],
|
||||
"queuedTurnIds": [],
|
||||
"subagentSessionIds": []
|
||||
},
|
||||
"counts": {
|
||||
"turnCount": 1,
|
||||
"itemCount": 2,
|
||||
"pendingRequestCount": 1,
|
||||
"queuedTurnCount": 0,
|
||||
"warningCount": 0,
|
||||
"failedToolCallCount": 0,
|
||||
"failedCommandCount": 0,
|
||||
"subagentCount": 0,
|
||||
"recentArtifactCount": 2
|
||||
},
|
||||
"latest": {
|
||||
"warning": null,
|
||||
"failedTool": null,
|
||||
"failedCommand": null
|
||||
},
|
||||
"signalCoverage": [
|
||||
{
|
||||
"signal": "correlation",
|
||||
"status": "exported",
|
||||
"source": "runtime thread identity",
|
||||
"detail": "当前证据包已导出 session/thread/turn/pending request/subagent 关联键。"
|
||||
},
|
||||
{
|
||||
"signal": "timeline",
|
||||
"status": "exported",
|
||||
"source": "timeline.json",
|
||||
"detail": "当前证据包已导出最近 turn 与 item 时间线。"
|
||||
},
|
||||
{
|
||||
"signal": "warnings",
|
||||
"status": "exported",
|
||||
"source": "thread.diagnostics",
|
||||
"detail": "当前线程没有 diagnostics,但 warning 通道已保留在导出结构中。"
|
||||
},
|
||||
{
|
||||
"signal": "requestTelemetry",
|
||||
"status": "unlinked",
|
||||
"source": "lime_infra.telemetry",
|
||||
"detail": "Lime 已有 workspace 级 request telemetry,但当前 RequestLog 还未携带 session/thread/turn 元数据。"
|
||||
},
|
||||
{
|
||||
"signal": "artifactValidator",
|
||||
"status": "known_gap",
|
||||
"source": "artifact_document_validator",
|
||||
"detail": "Artifact validator outcome 尚未回挂到当前 evidence pack。"
|
||||
},
|
||||
{
|
||||
"signal": "browserVerification",
|
||||
"status": "known_gap",
|
||||
"source": "browser runtime",
|
||||
"detail": "浏览器截图、DOM 快照和交互验证结果尚未接入 evidence pack。"
|
||||
},
|
||||
{
|
||||
"signal": "guiSmoke",
|
||||
"status": "known_gap",
|
||||
"source": "verify:gui-smoke",
|
||||
"detail": "GUI smoke 结果尚未作为结构化证据回挂当前会话。"
|
||||
}
|
||||
],
|
||||
"knownGaps": [
|
||||
"当前 Lime 虽然已有全局 RequestLog / token / retry / duration 遥测,但 RequestLog 尚未携带 session/thread/turn 关联键,Evidence Pack 只能导出 request telemetry coverage gap,不能给出精确的会话级摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 ArtifactDocument validator outcome 与修复问题摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 GUI smoke / browser 验证结果。"
|
||||
]
|
||||
},
|
||||
"recentArtifacts": [
|
||||
"docs/tech/harness/implementation-blueprint.md",
|
||||
".lime/harness/sessions/fixture-session-minimal-pending-request/handoff/handoff.md"
|
||||
|
||||
@@ -86,6 +86,90 @@
|
||||
},
|
||||
"incidents": []
|
||||
},
|
||||
"observability": {
|
||||
"schemaVersion": "v1",
|
||||
"correlation": {
|
||||
"correlationKeys": [
|
||||
"session_id",
|
||||
"thread_id",
|
||||
"turn_id",
|
||||
"pending_request_id",
|
||||
"queued_turn_id",
|
||||
"subagent_session_id"
|
||||
],
|
||||
"sessionId": "fixture-session-minimal-pending-request",
|
||||
"threadId": "fixture-thread-minimal-pending-request",
|
||||
"activeTurnId": "turn-fixture-001",
|
||||
"pendingRequestIds": ["approval-fixture-001"],
|
||||
"queuedTurnIds": [],
|
||||
"subagentSessionIds": []
|
||||
},
|
||||
"counts": {
|
||||
"turnCount": 1,
|
||||
"itemCount": 2,
|
||||
"pendingRequestCount": 1,
|
||||
"queuedTurnCount": 0,
|
||||
"warningCount": 0,
|
||||
"failedToolCallCount": 0,
|
||||
"failedCommandCount": 0,
|
||||
"subagentCount": 0,
|
||||
"recentArtifactCount": 2
|
||||
},
|
||||
"latest": {
|
||||
"warning": null,
|
||||
"failedTool": null,
|
||||
"failedCommand": null
|
||||
},
|
||||
"signalCoverage": [
|
||||
{
|
||||
"signal": "correlation",
|
||||
"status": "exported",
|
||||
"source": "runtime thread identity",
|
||||
"detail": "当前证据包已导出 session/thread/turn/pending request/subagent 关联键。"
|
||||
},
|
||||
{
|
||||
"signal": "timeline",
|
||||
"status": "exported",
|
||||
"source": "timeline.json",
|
||||
"detail": "当前证据包已导出最近 turn 与 item 时间线。"
|
||||
},
|
||||
{
|
||||
"signal": "warnings",
|
||||
"status": "exported",
|
||||
"source": "thread.diagnostics",
|
||||
"detail": "当前线程没有 diagnostics,但 warning 通道已保留在导出结构中。"
|
||||
},
|
||||
{
|
||||
"signal": "requestTelemetry",
|
||||
"status": "unlinked",
|
||||
"source": "lime_infra.telemetry",
|
||||
"detail": "Lime 已有 workspace 级 request telemetry,但当前 RequestLog 还未携带 session/thread/turn 元数据。"
|
||||
},
|
||||
{
|
||||
"signal": "artifactValidator",
|
||||
"status": "known_gap",
|
||||
"source": "artifact_document_validator",
|
||||
"detail": "Artifact validator outcome 尚未回挂到当前 evidence pack。"
|
||||
},
|
||||
{
|
||||
"signal": "browserVerification",
|
||||
"status": "known_gap",
|
||||
"source": "browser runtime",
|
||||
"detail": "浏览器截图、DOM 快照和交互验证结果尚未接入 evidence pack。"
|
||||
},
|
||||
{
|
||||
"signal": "guiSmoke",
|
||||
"status": "known_gap",
|
||||
"source": "verify:gui-smoke",
|
||||
"detail": "GUI smoke 结果尚未作为结构化证据回挂当前会话。"
|
||||
}
|
||||
],
|
||||
"knownGaps": [
|
||||
"当前 Lime 虽然已有全局 RequestLog / token / retry / duration 遥测,但 RequestLog 尚未携带 session/thread/turn 关联键,Evidence Pack 只能导出 request telemetry coverage gap,不能给出精确的会话级摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 ArtifactDocument validator outcome 与修复问题摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 GUI smoke / browser 验证结果。"
|
||||
]
|
||||
},
|
||||
"linkedArtifacts": {
|
||||
"handoffBundle": {
|
||||
"relativeRoot": ".lime/harness/sessions/fixture-session-minimal-pending-request/handoff",
|
||||
@@ -95,7 +179,9 @@
|
||||
"relativeRoot": ".lime/harness/sessions/fixture-session-minimal-pending-request/evidence",
|
||||
"artifactCount": 4,
|
||||
"knownGaps": [
|
||||
"当前 fixture 不包含真实浏览器快照,只验证 replay 样本结构与 grader 约定。"
|
||||
"当前 Lime 虽然已有全局 RequestLog / token / retry / duration 遥测,但 RequestLog 尚未携带 session/thread/turn 关联键,Evidence Pack 只能导出 request telemetry coverage gap,不能给出精确的会话级摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 ArtifactDocument validator outcome 与修复问题摘要。",
|
||||
"当前 Evidence Pack 尚未纳入 GUI smoke / browser 验证结果。"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user