mirror of
https://github.com/dataelement/bisheng.git
synced 2026-08-30 17:58:00 +08:00
Merge branch 'feat/3.0.0-beta2' into 3.0-vibe
This commit is contained in:
@@ -37,6 +37,9 @@ from bisheng.linsight.domain.services.binary_content_guard import (
|
||||
CODE_INTERPRETER_TOOL,
|
||||
build_binary_guards,
|
||||
)
|
||||
from bisheng.linsight.domain.services.invalid_tool_call_middleware import (
|
||||
build_invalid_tool_call_repair_middleware,
|
||||
)
|
||||
from bisheng.linsight.domain.services.resilience_middleware import build_resilience_middleware
|
||||
from bisheng.linsight.domain.services.tool_loop_middleware import build_tool_loop_breaker_middleware
|
||||
from bisheng.llm.domain.services import LLMService
|
||||
@@ -138,6 +141,7 @@ __SKILL_DELIVERABLE_LINE__ - 3a(始终):write_file 写 output/<name>.md
|
||||
- 每个问题**必须给 2-4 个具体预设选项**(options),每项是一个简短的选项文案(纯字符串),优先让用户点选;只有该信息天然无法预设选项(如“请输入你的身高”)时,才把该问题的 options 留空走开放输入——但问题本身仍要写出来,不能整个 questions 留空。
|
||||
- 多选问题(multiple=true,如输出格式):用户可勾选多项。
|
||||
- 收集“输出格式”用一个多选问题,选项含 markdown / html / docx / pdf。
|
||||
- reason / question / options 文本里**不要出现英文双引号 `"`**(它会破坏工具调用的 JSON 参数,整次澄清会直接失效);需要引用词语时用中文引号「」或“”。每个键只写一次(例如 multiple 不要重复)。
|
||||
- 一次性把所有要问的问完。不要罗列工具或能力限制,也不要预先解释工作流。
|
||||
- 【正确示例】questions 必须是这样的 JSON 数组(照此结构直接填——切勿把问题写进 reason,也切勿把数组序列化成字符串):
|
||||
questions=[
|
||||
@@ -653,6 +657,7 @@ async def ask_user(
|
||||
{"question": "问题标题", "options": ["选项1", "选项2"], "multiple": false}。
|
||||
options 为空表示开放式自由输入;multiple=true 表示多选。仅当确实没有任何
|
||||
结构化问题、只需给一句总体说明时,才省略 questions。
|
||||
文本内不要使用英文双引号(会破坏 JSON 参数),需要引用时用中文引号「」/“”。
|
||||
|
||||
Returns:
|
||||
用户的回答文本。
|
||||
@@ -865,6 +870,14 @@ async def create_linsight_agent(
|
||||
from bisheng.tool.domain.langchain.linsight_export import init_linsight_export_tools
|
||||
|
||||
export_tools = init_linsight_export_tools(backend)
|
||||
# Invalid tool-call repair (after_model). Appended LAST on purpose: after_model
|
||||
# hooks run in REVERSE middleware order, so this one sees the model output
|
||||
# first and the tool-loop breaker / TodoList hooks then see the repaired call.
|
||||
# It appends no system prompt, so the language tail above stays the tail.
|
||||
# ``tools`` = the exact list bound below, for the repaired-key sanity check.
|
||||
middlewares.append(
|
||||
build_invalid_tool_call_repair_middleware(tools=[*tools, ask_user, *export_tools], is_subagent=False)
|
||||
)
|
||||
# The researcher subagent is a separate subgraph: the main-graph middleware
|
||||
# above does NOT wrap its internal model calls, so it carries its OWN
|
||||
# resilience instance (is_subagent=True) which DEGRADES a content-filter /
|
||||
@@ -886,6 +899,9 @@ async def create_linsight_agent(
|
||||
# its TodoList/Filesystem framework prompts), so the researcher also
|
||||
# reasons in the user's language.
|
||||
_LanguageTailMiddleware(_LINSIGHT_LANGUAGE_DIRECTIVE_ZH),
|
||||
# Same invalid-call repair on the subagent's own graph (see the main-graph
|
||||
# note above for why it goes last).
|
||||
build_invalid_tool_call_repair_middleware(tools=researcher.get("tools"), is_subagent=True),
|
||||
]
|
||||
# Advertise search_knowledge_base in the system prompt IFF it is actually in
|
||||
# `tools` (init_linsight_tools injects it only when the user selected a KB /
|
||||
|
||||
@@ -0,0 +1,271 @@
|
||||
"""Invalid tool-call repair middleware for Linsight (``after_model``).
|
||||
|
||||
Incident (2026-08-27, 114, session ea5d2cee… / version 6049532c…): the model
|
||||
decided — correctly — to ``ask_user`` for a clarification, but the ``arguments``
|
||||
string it emitted was not valid JSON (an unescaped ``"`` inside the ``question``
|
||||
text: ``"question": "您说的"按照各市要求"具体是指什么?"``). langchain parks such a
|
||||
call in ``AIMessage.invalid_tool_calls``; the agent loop's model→tools edge only
|
||||
looks at ``tool_calls`` (``factory._make_model_to_tools_edge``: ``len(tool_calls)
|
||||
== 0`` → END), so the graph ended on an empty-content AIMessage, no ToolMessage
|
||||
ever reached the model, and the task closed as ``Task produced no result``
|
||||
(11090). The three ``ask_user`` self-heal layers in ``agent_factory``
|
||||
(``_coerce_questions`` / options salvage / empty-questions nudge) never ran —
|
||||
they live INSIDE the tool body, one layer below the JSON parse that failed.
|
||||
|
||||
This middleware sits right after the model node and handles the AIMessage
|
||||
before the routing edge sees it:
|
||||
|
||||
1. **Repair** — ``json_repair`` recovers the intended dict for every invalid
|
||||
call (on the live payload it restores the question text byte-for-byte). A
|
||||
repaired call is written back as a real ``tool_calls`` entry on a copy of the
|
||||
AIMessage with the SAME id, so the ``messages`` reducer replaces the message
|
||||
in place, the edge routes it to the tools node, and the stream mapper — which
|
||||
already emitted the ``start`` frame from the streamed chunks — closes it with
|
||||
the normal end frame (langgraph dedupes node-output messages by id, so no
|
||||
duplicate start frame is emitted).
|
||||
2. **Nudge** — a call that cannot be repaired (or whose repaired keys are not a
|
||||
subset of the tool's schema, i.e. the repair produced debris) gets an error
|
||||
``ToolMessage`` explaining the JSON error and how to fix it. If the turn has no
|
||||
valid call left at all, the middleware jumps straight back to the model so it
|
||||
can re-issue the call — ONCE per user turn (counted via the marker in the
|
||||
ToolMessage), mirroring the empty-questions nudge's no-infinite-retry rule.
|
||||
On the second miss the ToolMessage is still appended (keeps the transcript
|
||||
consistent for the next request: langchain_openai serialises
|
||||
``invalid_tool_calls`` into the outgoing ``tool_calls``, so an unanswered one
|
||||
would 400 on strict OpenAI-compatible endpoints) but the loop ends as before.
|
||||
|
||||
Runs FIRST among the ``after_model`` hooks (they execute in reverse middleware
|
||||
order and this one is appended last), so the tool-loop breaker and deepagents'
|
||||
TodoList hook see the repaired call, not the invalid one.
|
||||
|
||||
One instance per graph (main + researcher subagent) — the subagent's graph is
|
||||
outside the main-graph middleware, exactly like the tool-loop breaker.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from collections.abc import Iterable, Sequence
|
||||
from typing import Any
|
||||
|
||||
from json_repair import json_repair
|
||||
from langchain.agents.middleware.types import AgentMiddleware, hook_config
|
||||
from langchain_core.messages import AIMessage, ToolMessage
|
||||
from loguru import logger
|
||||
|
||||
# Substring of the corrective ToolMessage so a later pass can count "already
|
||||
# nudged this turn" from the message history alone (no per-instance state — the
|
||||
# agent is rebuilt on resume and a middleware attribute would not survive).
|
||||
INVALID_ARGS_MARKER = "参数不是合法的 JSON"
|
||||
|
||||
_INVALID_ARGS_HINT = (
|
||||
"⚠️ 你上一次对 {name} 的调用没有执行:{marker}({error})。"
|
||||
"最常见的原因是字符串值内部出现了未转义的英文双引号。"
|
||||
"请只重新调用一次 {name},并保证参数是合法 JSON:"
|
||||
'字符串内需要引号时改用中文引号「」/“”,或写成 \\" 转义;'
|
||||
"不要把参数整体或嵌套数组序列化成字符串;不要重复同一个键。"
|
||||
)
|
||||
|
||||
# How much of the raw argument string to echo back in the error (a truncated
|
||||
# ``write_file`` content can be tens of KB — the model does not need it back).
|
||||
_RAW_PREVIEW_CAP = 160
|
||||
|
||||
|
||||
def _state_messages(state: object) -> list:
|
||||
if isinstance(state, dict):
|
||||
return state.get("messages") or []
|
||||
return getattr(state, "messages", None) or []
|
||||
|
||||
|
||||
def _tool_arg_keys(tools: Iterable[Any] | None) -> dict[str, frozenset[str]]:
|
||||
"""``{tool_name: arg keys}`` for the tools whose schema is known up front.
|
||||
|
||||
deepagents' framework tools (write_file / write_todos / task …) are registered
|
||||
by its own middleware and are not in this list — an unknown tool is accepted
|
||||
on repair and left to the tool node's own validation.
|
||||
"""
|
||||
out: dict[str, frozenset[str]] = {}
|
||||
for t in tools or []:
|
||||
name = getattr(t, "name", None)
|
||||
if not name:
|
||||
continue
|
||||
try:
|
||||
args = getattr(t, "args", None) or {}
|
||||
keys = frozenset(str(k) for k in args.keys())
|
||||
except Exception: # pragma: no cover - defensive: a tool with a weird schema
|
||||
continue
|
||||
if keys:
|
||||
out[str(name)] = keys
|
||||
return out
|
||||
|
||||
|
||||
def _json_error(raw: str) -> str:
|
||||
"""The stdlib decode error for ``raw`` — the actionable part of the hint."""
|
||||
try:
|
||||
json.loads(raw)
|
||||
except Exception as exc: # JSONDecodeError / TypeError
|
||||
return str(exc)
|
||||
return "arguments did not decode to a JSON object"
|
||||
|
||||
|
||||
def _repair_args(raw: object) -> dict[str, Any] | None:
|
||||
"""Best-effort recovery of a tool-call argument dict from a malformed string.
|
||||
|
||||
Order: strict ``json.loads`` (covers the rare invalid_tool_call whose text is
|
||||
valid JSON but not an object) → ``json_repair`` → one unwrap of a
|
||||
double-encoded JSON string. Anything that is not a non-empty dict is ``None``.
|
||||
"""
|
||||
if isinstance(raw, dict):
|
||||
return raw or None
|
||||
if raw is None:
|
||||
return None
|
||||
text = str(raw).strip()
|
||||
if not text:
|
||||
return None
|
||||
obj: Any = None
|
||||
try:
|
||||
obj = json.loads(text)
|
||||
except Exception:
|
||||
try:
|
||||
obj = json_repair.loads(text)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(obj, str) and obj.strip().startswith("{"):
|
||||
# Double-encoded: the whole object was serialised as one JSON string.
|
||||
try:
|
||||
obj = json_repair.loads(obj)
|
||||
except Exception:
|
||||
return None
|
||||
if isinstance(obj, dict) and obj:
|
||||
return obj
|
||||
return None
|
||||
|
||||
|
||||
def _nudge_count_this_turn(messages: list) -> int:
|
||||
"""How many corrective ToolMessages were already issued since the last human
|
||||
turn. A fresh human message re-arms the (single) nudge — same contract as
|
||||
``agent_factory._empty_retry_count``."""
|
||||
count = 0
|
||||
for m in messages:
|
||||
if isinstance(m, dict):
|
||||
role = m.get("type") or m.get("role")
|
||||
content = m.get("content")
|
||||
else:
|
||||
role = getattr(m, "type", None)
|
||||
content = getattr(m, "content", None)
|
||||
if role in ("human", "user"):
|
||||
count = 0
|
||||
elif role == "tool" and isinstance(content, str) and INVALID_ARGS_MARKER in content:
|
||||
count += 1
|
||||
return count
|
||||
|
||||
|
||||
class LinsightInvalidToolCallRepairMiddleware(AgentMiddleware):
|
||||
"""Turn ``AIMessage.invalid_tool_calls`` into executable calls (or a retry)."""
|
||||
|
||||
def __init__(self, *, tools: Sequence[Any] | None = None, is_subagent: bool = False) -> None:
|
||||
super().__init__()
|
||||
self.tools = [] # registers no extra tools
|
||||
self.is_subagent = is_subagent
|
||||
self._arg_keys = _tool_arg_keys(tools)
|
||||
|
||||
@property
|
||||
def name(self) -> str:
|
||||
# Role-distinct, stable name so the two instances (main vs subagent) never
|
||||
# collide in a shared middleware list.
|
||||
return f"LinsightInvalidToolCallRepair{'Sub' if self.is_subagent else 'Main'}"
|
||||
|
||||
# ------------------------------------------------------------------ core
|
||||
|
||||
def _plausible(self, name: str | None, args: dict[str, Any]) -> bool:
|
||||
"""Reject a "repair" whose keys are not in the tool's schema — that is
|
||||
json_repair turning a broken string into key/value debris, and executing
|
||||
it would only trade one error for a stranger one."""
|
||||
known = self._arg_keys.get(name or "")
|
||||
if not known:
|
||||
return True
|
||||
return set(args.keys()) <= known
|
||||
|
||||
def _process(self, state: Any) -> dict[str, Any] | None:
|
||||
messages = _state_messages(state)
|
||||
if not messages:
|
||||
return None
|
||||
ai = messages[-1]
|
||||
if not isinstance(ai, AIMessage):
|
||||
return None
|
||||
invalid = list(getattr(ai, "invalid_tool_calls", None) or [])
|
||||
if not invalid:
|
||||
return None
|
||||
|
||||
repaired: list[dict[str, Any]] = []
|
||||
still_invalid: list[dict[str, Any]] = []
|
||||
tool_msgs: list[ToolMessage] = []
|
||||
for itc in invalid:
|
||||
name = itc.get("name") or "unknown"
|
||||
call_id = itc.get("id")
|
||||
raw = itc.get("args")
|
||||
args = _repair_args(raw)
|
||||
if args is not None and self._plausible(name, args) and call_id:
|
||||
repaired.append({"name": name, "args": args, "id": call_id, "type": "tool_call"})
|
||||
continue
|
||||
still_invalid.append(itc)
|
||||
if not call_id:
|
||||
# No id → nothing for a ToolMessage to answer; the request
|
||||
# serialiser drops id-less calls too, so there is nothing to pair.
|
||||
continue
|
||||
raw_text = "" if raw is None else str(raw)
|
||||
preview = raw_text[:_RAW_PREVIEW_CAP] + ("…" if len(raw_text) > _RAW_PREVIEW_CAP else "")
|
||||
error = _json_error(raw_text) if raw_text else "arguments were empty"
|
||||
content = _INVALID_ARGS_HINT.format(name=name, marker=INVALID_ARGS_MARKER, error=error)
|
||||
if preview:
|
||||
content += f"\n收到的参数开头:{preview}"
|
||||
tool_msgs.append(ToolMessage(content=content, name=name, tool_call_id=call_id, status="error"))
|
||||
|
||||
update: dict[str, Any] = {}
|
||||
new_messages: list[Any] = []
|
||||
valid_calls = list(getattr(ai, "tool_calls", None) or [])
|
||||
if repaired:
|
||||
# Same id → the add_messages reducer replaces the AIMessage in place.
|
||||
new_ai = ai.model_copy(
|
||||
update={"tool_calls": [*valid_calls, *repaired], "invalid_tool_calls": still_invalid}
|
||||
)
|
||||
new_messages.append(new_ai)
|
||||
valid_calls = [*valid_calls, *repaired]
|
||||
new_messages.extend(tool_msgs)
|
||||
|
||||
nudged = False
|
||||
if not valid_calls and tool_msgs:
|
||||
# Nothing executable is left this turn: hand the error back to the
|
||||
# model right away — but only once per user turn.
|
||||
if _nudge_count_this_turn(messages) == 0:
|
||||
update["jump_to"] = "model"
|
||||
nudged = True
|
||||
|
||||
if new_messages:
|
||||
update["messages"] = new_messages
|
||||
logger.info(
|
||||
"BS_LINSIGHT_INVALID_TOOLCALL graph={} repaired={} unrepaired={} nudged={}",
|
||||
"sub" if self.is_subagent else "main",
|
||||
[c["name"] for c in repaired],
|
||||
[c.get("name") for c in still_invalid],
|
||||
nudged,
|
||||
)
|
||||
return update or None
|
||||
|
||||
# ------------------------------------------------------------------ hooks
|
||||
|
||||
@hook_config(can_jump_to=["model"])
|
||||
def after_model(self, state, runtime):
|
||||
return self._process(state)
|
||||
|
||||
@hook_config(can_jump_to=["model"])
|
||||
async def aafter_model(self, state, runtime):
|
||||
return self._process(state)
|
||||
|
||||
|
||||
def build_invalid_tool_call_repair_middleware(
|
||||
*, tools: Sequence[Any] | None = None, is_subagent: bool
|
||||
) -> LinsightInvalidToolCallRepairMiddleware:
|
||||
"""Construct a middleware instance (one per graph); ``tools`` is the graph's
|
||||
bound tool list, used only to sanity-check repaired argument keys."""
|
||||
return LinsightInvalidToolCallRepairMiddleware(tools=tools, is_subagent=is_subagent)
|
||||
@@ -0,0 +1,372 @@
|
||||
"""Unit tests for LinsightInvalidToolCallRepairMiddleware.
|
||||
|
||||
Regression for the 2026-08-27 incident (114, version 6049532c…): the model's
|
||||
``ask_user`` arguments were not valid JSON (unescaped ``"`` inside a question),
|
||||
langchain parked the call in ``invalid_tool_calls``, the agent loop ended on an
|
||||
empty AIMessage and the task closed as ``Task produced no result``.
|
||||
|
||||
Covers:
|
||||
- the LIVE payload is repaired in place (same message id, question text intact),
|
||||
- unrepairable args → error ToolMessage + one ``jump_to: model`` per user turn,
|
||||
- mixed valid/invalid turns keep the valid calls and never jump,
|
||||
- schema-key sanity check rejects json_repair debris for a known tool,
|
||||
- pass-through when there is nothing to do,
|
||||
- the prompt/tool docstring carry the no-English-double-quotes rule.
|
||||
|
||||
``asyncio_mode = auto`` — async tests need no decorator.
|
||||
"""
|
||||
|
||||
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
|
||||
|
||||
from bisheng.linsight.domain.services import agent_factory
|
||||
from bisheng.linsight.domain.services.invalid_tool_call_middleware import (
|
||||
INVALID_ARGS_MARKER,
|
||||
LinsightInvalidToolCallRepairMiddleware,
|
||||
_repair_args,
|
||||
build_invalid_tool_call_repair_middleware,
|
||||
)
|
||||
|
||||
# The exact arguments string deepseek-v4-flash emitted in the live session: the
|
||||
# ``question`` value contains unescaped double quotes (``reason`` is escaped
|
||||
# correctly) and the second question repeats the ``multiple`` key.
|
||||
_LIVE_RAW_ARGS = (
|
||||
'{"reason": "您提到\\"按照各市要求重新生成\\",但当前映射表说明没有涉及\\"各市\\"的维度。", '
|
||||
'"questions": [{"question": "您说的"按照各市要求"具体是指什么?", '
|
||||
'"options": ["按城市/地区分类重新组织映射表", "按照各市的格式规范要求调整文档", "其他(请说明)"], '
|
||||
'"multiple": false}, {"question": "重新生成的文档输出格式?", '
|
||||
'"options": ["Word(docx)", "Markdown(md)"], "multiple": true, "multiple": false}]}'
|
||||
)
|
||||
_CALL_ID = "call_30d0f438439a467da532c3d1"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- helpers
|
||||
|
||||
|
||||
def _invalid_ai(raw=_LIVE_RAW_ARGS, name="ask_user", call_id=_CALL_ID, msg_id="lc_run--live", valid_calls=None):
|
||||
return AIMessage(
|
||||
content="",
|
||||
id=msg_id,
|
||||
tool_calls=list(valid_calls or []),
|
||||
invalid_tool_calls=[{"name": name, "args": raw, "id": call_id, "error": None, "type": "invalid_tool_call"}],
|
||||
response_metadata={"finish_reason": "tool_calls"},
|
||||
)
|
||||
|
||||
|
||||
def _mw(tools=None, is_subagent=False):
|
||||
return build_invalid_tool_call_repair_middleware(tools=tools, is_subagent=is_subagent)
|
||||
|
||||
|
||||
def _state(*messages):
|
||||
return {"messages": [HumanMessage(content="上述说明按照各市要求重新生成"), *messages]}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- repair
|
||||
|
||||
|
||||
def test_repair_args_recovers_live_payload_intact():
|
||||
args = _repair_args(_LIVE_RAW_ARGS)
|
||||
assert isinstance(args, dict)
|
||||
assert set(args) == {"reason", "questions"}
|
||||
# The question text survives byte-for-byte, quotes included.
|
||||
assert args["questions"][0]["question"] == '您说的"按照各市要求"具体是指什么?'
|
||||
assert args["questions"][0]["options"][0] == "按城市/地区分类重新组织映射表"
|
||||
assert args["questions"][1]["question"] == "重新生成的文档输出格式?"
|
||||
|
||||
|
||||
def test_repair_args_rejects_non_objects_and_garbage():
|
||||
assert _repair_args("") is None
|
||||
assert _repair_args(None) is None
|
||||
assert _repair_args("[1, 2, 3]") is None # valid JSON, not an object
|
||||
assert _repair_args('"just a string"') is None
|
||||
assert _repair_args("{}") is None
|
||||
|
||||
|
||||
def test_repair_args_unwraps_double_encoded_object():
|
||||
assert _repair_args('"{\\"reason\\": \\"why\\", \\"questions\\": []}"') == {"reason": "why", "questions": []}
|
||||
|
||||
|
||||
async def test_live_payload_is_repaired_in_place_and_routed_to_tools():
|
||||
mw = _mw(tools=[agent_factory.ask_user])
|
||||
ai = _invalid_ai()
|
||||
update = await mw.aafter_model(_state(ai), runtime=None)
|
||||
|
||||
assert update is not None
|
||||
assert "jump_to" not in update # the routing edge now sees tool_calls → tools node
|
||||
(new_ai,) = update["messages"]
|
||||
assert isinstance(new_ai, AIMessage)
|
||||
assert new_ai.id == ai.id # same id → add_messages replaces, no duplicate start frame
|
||||
assert new_ai.invalid_tool_calls == []
|
||||
assert len(new_ai.tool_calls) == 1
|
||||
tc = new_ai.tool_calls[0]
|
||||
assert tc["name"] == "ask_user" and tc["id"] == _CALL_ID and tc["type"] == "tool_call"
|
||||
assert tc["args"]["questions"][0]["question"] == '您说的"按照各市要求"具体是指什么?'
|
||||
# The original message object is not mutated (model_copy).
|
||||
assert ai.tool_calls == [] and len(ai.invalid_tool_calls) == 1
|
||||
|
||||
|
||||
def test_sync_hook_matches_async_hook():
|
||||
mw = _mw(tools=[agent_factory.ask_user])
|
||||
update = mw.after_model(_state(_invalid_ai()), runtime=None)
|
||||
assert update and update["messages"][0].tool_calls[0]["name"] == "ask_user"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- nudge
|
||||
|
||||
|
||||
async def test_unrepairable_args_get_error_tool_message_and_one_jump_back_to_model():
|
||||
mw = _mw()
|
||||
ai = _invalid_ai(raw="reason: 需要确认 questions: [", call_id="call_x")
|
||||
update = await mw.aafter_model(_state(ai), runtime=None)
|
||||
|
||||
assert update["jump_to"] == "model"
|
||||
(tm,) = update["messages"]
|
||||
assert isinstance(tm, ToolMessage)
|
||||
assert tm.tool_call_id == "call_x" and tm.name == "ask_user" and tm.status == "error"
|
||||
assert INVALID_ARGS_MARKER in tm.content
|
||||
assert "ask_user" in tm.content and "中文引号" in tm.content
|
||||
# The raw text is echoed back (capped) so the model can see what went wrong.
|
||||
assert "reason: 需要确认" in tm.content
|
||||
|
||||
|
||||
async def test_second_unrepairable_call_in_same_turn_does_not_jump_again():
|
||||
mw = _mw()
|
||||
first_ai = _invalid_ai(raw="{{{", call_id="call_1", msg_id="m1")
|
||||
first_update = await mw.aafter_model(_state(first_ai), runtime=None)
|
||||
assert first_update["jump_to"] == "model"
|
||||
(first_tm,) = first_update["messages"]
|
||||
|
||||
second_ai = _invalid_ai(raw="{{{", call_id="call_2", msg_id="m2")
|
||||
second_update = await mw.aafter_model(_state(first_ai, first_tm, second_ai), runtime=None)
|
||||
# Still answers the dangling call (transcript stays consistent) but no retry:
|
||||
# the no-infinite-loop guarantee — the run ends as it did before this middleware.
|
||||
assert "jump_to" not in second_update
|
||||
(second_tm,) = second_update["messages"]
|
||||
assert second_tm.tool_call_id == "call_2" and INVALID_ARGS_MARKER in second_tm.content
|
||||
|
||||
|
||||
async def test_new_human_turn_re_arms_the_nudge():
|
||||
mw = _mw()
|
||||
stale_tm = ToolMessage(content=f"old {INVALID_ARGS_MARKER}", tool_call_id="old", name="ask_user", status="error")
|
||||
ai = _invalid_ai(raw="{{{", call_id="call_new")
|
||||
state = {"messages": [HumanMessage(content="q1"), stale_tm, HumanMessage(content="q2"), ai]}
|
||||
update = await mw.aafter_model(state, runtime=None)
|
||||
assert update["jump_to"] == "model"
|
||||
|
||||
|
||||
async def test_mixed_valid_and_invalid_calls_keep_valid_and_never_jump():
|
||||
mw = _mw()
|
||||
valid = {"name": "ls", "args": {"path": "/"}, "id": "call_ok", "type": "tool_call"}
|
||||
ai = _invalid_ai(raw="{{{", name="write_file", call_id="call_bad", valid_calls=[valid])
|
||||
update = await mw.aafter_model(_state(ai), runtime=None)
|
||||
|
||||
assert "jump_to" not in update # the edge will run the valid call
|
||||
(tm,) = update["messages"]
|
||||
assert isinstance(tm, ToolMessage) and tm.tool_call_id == "call_bad" and tm.name == "write_file"
|
||||
|
||||
|
||||
async def test_repaired_call_is_appended_after_existing_valid_calls():
|
||||
mw = _mw()
|
||||
valid = {"name": "ls", "args": {"path": "/"}, "id": "call_ok", "type": "tool_call"}
|
||||
ai = _invalid_ai(raw='{"file_path": "/output/a.md", "content": "x"', name="write_file", valid_calls=[valid])
|
||||
update = await mw.aafter_model(_state(ai), runtime=None)
|
||||
(new_ai,) = update["messages"]
|
||||
assert [tc["id"] for tc in new_ai.tool_calls] == ["call_ok", _CALL_ID]
|
||||
assert new_ai.tool_calls[1]["args"] == {"file_path": "/output/a.md", "content": "x"}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- schema sanity
|
||||
|
||||
|
||||
async def test_repair_debris_for_known_tool_is_rejected_and_nudged():
|
||||
mw = _mw(tools=[agent_factory.ask_user])
|
||||
# json_repair turns this into a dict, but its keys are not ask_user's.
|
||||
ai = _invalid_ai(raw='{"why": "x", "items": []}', call_id="call_d")
|
||||
update = await mw.aafter_model(_state(ai), runtime=None)
|
||||
assert update["jump_to"] == "model"
|
||||
assert isinstance(update["messages"][0], ToolMessage)
|
||||
|
||||
|
||||
async def test_unknown_tool_is_repaired_without_schema_check():
|
||||
mw = _mw(tools=[agent_factory.ask_user])
|
||||
ai = _invalid_ai(raw='{"todos": [{"content": "a", "status": "pending"}', name="write_todos", call_id="call_t")
|
||||
update = await mw.aafter_model(_state(ai), runtime=None)
|
||||
assert "jump_to" not in update
|
||||
assert update["messages"][0].tool_calls[0]["args"]["todos"][0]["content"] == "a"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- pass-through
|
||||
|
||||
|
||||
async def test_no_invalid_calls_is_a_noop():
|
||||
mw = _mw()
|
||||
ai = AIMessage(content="", tool_calls=[{"name": "ls", "args": {}, "id": "c", "type": "tool_call"}])
|
||||
assert await mw.aafter_model(_state(ai), runtime=None) is None
|
||||
assert await mw.aafter_model(_state(AIMessage(content="done")), runtime=None) is None
|
||||
assert await mw.aafter_model(_state(ToolMessage(content="x", tool_call_id="c")), runtime=None) is None
|
||||
assert await mw.aafter_model({"messages": []}, runtime=None) is None
|
||||
|
||||
|
||||
async def test_invalid_call_without_id_is_ignored():
|
||||
mw = _mw()
|
||||
ai = AIMessage(
|
||||
content="",
|
||||
invalid_tool_calls=[
|
||||
{"name": "ask_user", "args": "{{{", "id": None, "error": None, "type": "invalid_tool_call"}
|
||||
],
|
||||
)
|
||||
assert await mw.aafter_model(_state(ai), runtime=None) is None
|
||||
|
||||
|
||||
def test_instance_names_are_role_distinct_and_register_no_tools():
|
||||
main, sub = _mw(is_subagent=False), _mw(is_subagent=True)
|
||||
assert main.name == "LinsightInvalidToolCallRepairMain"
|
||||
assert sub.name == "LinsightInvalidToolCallRepairSub"
|
||||
assert main.tools == [] and sub.tools == []
|
||||
assert isinstance(main, LinsightInvalidToolCallRepairMiddleware)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- prompt
|
||||
|
||||
|
||||
def test_prompt_and_tool_doc_forbid_english_double_quotes_in_ask_user_text():
|
||||
prompt = agent_factory._build_linsight_system_prompt(False)
|
||||
assert "不要出现英文双引号" in prompt
|
||||
assert "中文引号「」" in prompt
|
||||
assert "不要使用英文双引号" in agent_factory.ask_user.description
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- graph wiring
|
||||
|
||||
|
||||
def _scripted_model(script):
|
||||
"""A minimal chat model that replays ``script`` (AIMessages) call by call —
|
||||
same shape as _e2e_llm_resilience_runner's ScriptedFaultModel, trimmed."""
|
||||
from langchain_core.language_models.chat_models import BaseChatModel
|
||||
from langchain_core.outputs import ChatGeneration, ChatResult
|
||||
|
||||
class _Scripted(BaseChatModel):
|
||||
_script: list = []
|
||||
_calls: list = [0]
|
||||
_seen: list = []
|
||||
|
||||
@property
|
||||
def _llm_type(self) -> str:
|
||||
return "scripted"
|
||||
|
||||
def _next(self, messages):
|
||||
i = self._calls[0]
|
||||
self._calls[0] += 1
|
||||
self._seen.append(list(messages))
|
||||
return self._script[min(i, len(self._script) - 1)]
|
||||
|
||||
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
return ChatResult(generations=[ChatGeneration(message=self._next(messages))])
|
||||
|
||||
async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs):
|
||||
return ChatResult(generations=[ChatGeneration(message=self._next(messages))])
|
||||
|
||||
def bind_tools(self, tools, **kwargs):
|
||||
return self.bind(**kwargs)
|
||||
|
||||
m = _Scripted()
|
||||
m._script = list(script)
|
||||
m._calls = [0]
|
||||
m._seen = []
|
||||
return m
|
||||
|
||||
|
||||
def _echo_tool():
|
||||
from langchain_core.tools import tool
|
||||
|
||||
@tool
|
||||
def echo(text: str) -> str:
|
||||
"""Echo ``text`` back."""
|
||||
return f"echo:{text}"
|
||||
|
||||
return echo
|
||||
|
||||
|
||||
async def test_graph_executes_repaired_call_through_the_real_agent_loop():
|
||||
from langchain.agents import create_agent
|
||||
|
||||
echo = _echo_tool()
|
||||
broken = AIMessage(
|
||||
content="",
|
||||
id="ai-1",
|
||||
invalid_tool_calls=[
|
||||
{
|
||||
"name": "echo",
|
||||
"args": '{"text": "he said "hi" to me"}',
|
||||
"id": "c1",
|
||||
"error": None,
|
||||
"type": "invalid_tool_call",
|
||||
}
|
||||
],
|
||||
)
|
||||
model = _scripted_model([broken, AIMessage(content="final", id="ai-2")])
|
||||
agent = create_agent(model=model, tools=[echo], middleware=[_mw(tools=[echo])])
|
||||
|
||||
result = await agent.ainvoke({"messages": [HumanMessage(content="go")]})
|
||||
|
||||
tool_msgs = [m for m in result["messages"] if isinstance(m, ToolMessage)]
|
||||
assert len(tool_msgs) == 1 and tool_msgs[0].tool_call_id == "c1"
|
||||
assert tool_msgs[0].content == 'echo:he said "hi" to me' # the tool really ran on the repaired args
|
||||
ai_msgs = [m for m in result["messages"] if isinstance(m, AIMessage)]
|
||||
assert ai_msgs[0].id == "ai-1" and ai_msgs[0].invalid_tool_calls == [] and ai_msgs[0].tool_calls[0]["id"] == "c1"
|
||||
assert result["messages"][-1].content == "final"
|
||||
assert model._calls[0] == 2
|
||||
|
||||
|
||||
async def test_graph_nudges_model_once_when_args_cannot_be_repaired():
|
||||
from langchain.agents import create_agent
|
||||
|
||||
echo = _echo_tool()
|
||||
broken = AIMessage(
|
||||
content="",
|
||||
id="ai-1",
|
||||
invalid_tool_calls=[{"name": "echo", "args": "{{{", "id": "c1", "error": None, "type": "invalid_tool_call"}],
|
||||
)
|
||||
fixed = AIMessage(
|
||||
content="", id="ai-2", tool_calls=[{"name": "echo", "args": {"text": "ok"}, "id": "c2", "type": "tool_call"}]
|
||||
)
|
||||
model = _scripted_model([broken, fixed, AIMessage(content="final", id="ai-3")])
|
||||
agent = create_agent(model=model, tools=[echo], middleware=[_mw(tools=[echo])])
|
||||
|
||||
result = await agent.ainvoke({"messages": [HumanMessage(content="go")]})
|
||||
|
||||
# Call 2 was the retry: it saw the corrective ToolMessage for c1 in its prompt.
|
||||
assert model._calls[0] == 3
|
||||
second_prompt = model._seen[1]
|
||||
assert any(
|
||||
isinstance(m, ToolMessage) and m.tool_call_id == "c1" and INVALID_ARGS_MARKER in m.content
|
||||
for m in second_prompt
|
||||
)
|
||||
tool_msgs = [m for m in result["messages"] if isinstance(m, ToolMessage)]
|
||||
assert [t.tool_call_id for t in tool_msgs] == ["c1", "c2"]
|
||||
assert tool_msgs[1].content == "echo:ok"
|
||||
assert result["messages"][-1].content == "final"
|
||||
|
||||
|
||||
async def test_graph_ends_after_second_unrepairable_call_without_looping():
|
||||
from langchain.agents import create_agent
|
||||
|
||||
echo = _echo_tool()
|
||||
|
||||
def _broken(i):
|
||||
return AIMessage(
|
||||
content="",
|
||||
id=f"ai-{i}",
|
||||
invalid_tool_calls=[
|
||||
{"name": "echo", "args": "{{{", "id": f"c{i}", "error": None, "type": "invalid_tool_call"}
|
||||
],
|
||||
)
|
||||
|
||||
model = _scripted_model([_broken(1), _broken(2), _broken(3)])
|
||||
agent = create_agent(model=model, tools=[echo], middleware=[_mw(tools=[echo])])
|
||||
|
||||
result = await agent.ainvoke({"messages": [HumanMessage(content="go")]})
|
||||
|
||||
# One nudge, then the loop ends (no third model call) — no infinite retry.
|
||||
assert model._calls[0] == 2
|
||||
tool_msgs = [m for m in result["messages"] if isinstance(m, ToolMessage)]
|
||||
assert [t.tool_call_id for t in tool_msgs] == ["c1", "c2"]
|
||||
Reference in New Issue
Block a user