Merge branch 'feat/3.0.0-beta2' into 3.0-vibe

This commit is contained in:
LineWalker
2026-08-27 15:27:35 +08:00
3 changed files with 659 additions and 0 deletions
@@ -37,6 +37,9 @@ from bisheng.linsight.domain.services.binary_content_guard import (
CODE_INTERPRETER_TOOL,
build_binary_guards,
)
from bisheng.linsight.domain.services.invalid_tool_call_middleware import (
build_invalid_tool_call_repair_middleware,
)
from bisheng.linsight.domain.services.resilience_middleware import build_resilience_middleware
from bisheng.linsight.domain.services.tool_loop_middleware import build_tool_loop_breaker_middleware
from bisheng.llm.domain.services import LLMService
@@ -138,6 +141,7 @@ __SKILL_DELIVERABLE_LINE__ - 3a(始终):write_file 写 output/<name>.md
- 每个问题**必须给 2-4 个具体预设选项**(options),每项是一个简短的选项文案(纯字符串),优先让用户点选;只有该信息天然无法预设选项(如“请输入你的身高”)时,才把该问题的 options 留空走开放输入——但问题本身仍要写出来,不能整个 questions 留空。
- 多选问题(multiple=true,如输出格式):用户可勾选多项。
- 收集“输出格式”用一个多选问题,选项含 markdown / html / docx / pdf。
- reason / question / options 文本里**不要出现英文双引号 `"`**(它会破坏工具调用的 JSON 参数,整次澄清会直接失效);需要引用词语时用中文引号「」或“”。每个键只写一次(例如 multiple 不要重复)。
- 一次性把所有要问的问完。不要罗列工具或能力限制,也不要预先解释工作流。
- 【正确示例】questions 必须是这样的 JSON 数组(照此结构直接填——切勿把问题写进 reason,也切勿把数组序列化成字符串):
questions=[
@@ -653,6 +657,7 @@ async def ask_user(
{"question": "问题标题", "options": ["选项1", "选项2"], "multiple": false}。
options 为空表示开放式自由输入;multiple=true 表示多选。仅当确实没有任何
结构化问题、只需给一句总体说明时,才省略 questions。
文本内不要使用英文双引号(会破坏 JSON 参数),需要引用时用中文引号「」/“”。
Returns:
用户的回答文本。
@@ -865,6 +870,14 @@ async def create_linsight_agent(
from bisheng.tool.domain.langchain.linsight_export import init_linsight_export_tools
export_tools = init_linsight_export_tools(backend)
# Invalid tool-call repair (after_model). Appended LAST on purpose: after_model
# hooks run in REVERSE middleware order, so this one sees the model output
# first and the tool-loop breaker / TodoList hooks then see the repaired call.
# It appends no system prompt, so the language tail above stays the tail.
# ``tools`` = the exact list bound below, for the repaired-key sanity check.
middlewares.append(
build_invalid_tool_call_repair_middleware(tools=[*tools, ask_user, *export_tools], is_subagent=False)
)
# The researcher subagent is a separate subgraph: the main-graph middleware
# above does NOT wrap its internal model calls, so it carries its OWN
# resilience instance (is_subagent=True) which DEGRADES a content-filter /
@@ -886,6 +899,9 @@ async def create_linsight_agent(
# its TodoList/Filesystem framework prompts), so the researcher also
# reasons in the user's language.
_LanguageTailMiddleware(_LINSIGHT_LANGUAGE_DIRECTIVE_ZH),
# Same invalid-call repair on the subagent's own graph (see the main-graph
# note above for why it goes last).
build_invalid_tool_call_repair_middleware(tools=researcher.get("tools"), is_subagent=True),
]
# Advertise search_knowledge_base in the system prompt IFF it is actually in
# `tools` (init_linsight_tools injects it only when the user selected a KB /
@@ -0,0 +1,271 @@
"""Invalid tool-call repair middleware for Linsight (``after_model``).
Incident (2026-08-27, 114, session ea5d2cee… / version 6049532c…): the model
decided — correctly — to ``ask_user`` for a clarification, but the ``arguments``
string it emitted was not valid JSON (an unescaped ``"`` inside the ``question``
text: ``"question": "您说的"按照各市要求"具体是指什么?"``). langchain parks such a
call in ``AIMessage.invalid_tool_calls``; the agent loop's model→tools edge only
looks at ``tool_calls`` (``factory._make_model_to_tools_edge``: ``len(tool_calls)
== 0`` → END), so the graph ended on an empty-content AIMessage, no ToolMessage
ever reached the model, and the task closed as ``Task produced no result``
(11090). The three ``ask_user`` self-heal layers in ``agent_factory``
(``_coerce_questions`` / options salvage / empty-questions nudge) never ran —
they live INSIDE the tool body, one layer below the JSON parse that failed.
This middleware sits right after the model node and handles the AIMessage
before the routing edge sees it:
1. **Repair** — ``json_repair`` recovers the intended dict for every invalid
call (on the live payload it restores the question text byte-for-byte). A
repaired call is written back as a real ``tool_calls`` entry on a copy of the
AIMessage with the SAME id, so the ``messages`` reducer replaces the message
in place, the edge routes it to the tools node, and the stream mapper — which
already emitted the ``start`` frame from the streamed chunks — closes it with
the normal end frame (langgraph dedupes node-output messages by id, so no
duplicate start frame is emitted).
2. **Nudge** — a call that cannot be repaired (or whose repaired keys are not a
subset of the tool's schema, i.e. the repair produced debris) gets an error
``ToolMessage`` explaining the JSON error and how to fix it. If the turn has no
valid call left at all, the middleware jumps straight back to the model so it
can re-issue the call — ONCE per user turn (counted via the marker in the
ToolMessage), mirroring the empty-questions nudge's no-infinite-retry rule.
On the second miss the ToolMessage is still appended (keeps the transcript
consistent for the next request: langchain_openai serialises
``invalid_tool_calls`` into the outgoing ``tool_calls``, so an unanswered one
would 400 on strict OpenAI-compatible endpoints) but the loop ends as before.
Runs FIRST among the ``after_model`` hooks (they execute in reverse middleware
order and this one is appended last), so the tool-loop breaker and deepagents'
TodoList hook see the repaired call, not the invalid one.
One instance per graph (main + researcher subagent) — the subagent's graph is
outside the main-graph middleware, exactly like the tool-loop breaker.
"""
from __future__ import annotations
import json
from collections.abc import Iterable, Sequence
from typing import Any
from json_repair import json_repair
from langchain.agents.middleware.types import AgentMiddleware, hook_config
from langchain_core.messages import AIMessage, ToolMessage
from loguru import logger
# Substring of the corrective ToolMessage so a later pass can count "already
# nudged this turn" from the message history alone (no per-instance state — the
# agent is rebuilt on resume and a middleware attribute would not survive).
INVALID_ARGS_MARKER = "参数不是合法的 JSON"
_INVALID_ARGS_HINT = (
"⚠️ 你上一次对 {name} 的调用没有执行:{marker}{error})。"
"最常见的原因是字符串值内部出现了未转义的英文双引号。"
"请只重新调用一次 {name},并保证参数是合法 JSON"
'字符串内需要引号时改用中文引号「」/“”,或写成 \\" 转义;'
"不要把参数整体或嵌套数组序列化成字符串;不要重复同一个键。"
)
# How much of the raw argument string to echo back in the error (a truncated
# ``write_file`` content can be tens of KB — the model does not need it back).
_RAW_PREVIEW_CAP = 160
def _state_messages(state: object) -> list:
if isinstance(state, dict):
return state.get("messages") or []
return getattr(state, "messages", None) or []
def _tool_arg_keys(tools: Iterable[Any] | None) -> dict[str, frozenset[str]]:
"""``{tool_name: arg keys}`` for the tools whose schema is known up front.
deepagents' framework tools (write_file / write_todos / task …) are registered
by its own middleware and are not in this list — an unknown tool is accepted
on repair and left to the tool node's own validation.
"""
out: dict[str, frozenset[str]] = {}
for t in tools or []:
name = getattr(t, "name", None)
if not name:
continue
try:
args = getattr(t, "args", None) or {}
keys = frozenset(str(k) for k in args.keys())
except Exception: # pragma: no cover - defensive: a tool with a weird schema
continue
if keys:
out[str(name)] = keys
return out
def _json_error(raw: str) -> str:
"""The stdlib decode error for ``raw`` — the actionable part of the hint."""
try:
json.loads(raw)
except Exception as exc: # JSONDecodeError / TypeError
return str(exc)
return "arguments did not decode to a JSON object"
def _repair_args(raw: object) -> dict[str, Any] | None:
"""Best-effort recovery of a tool-call argument dict from a malformed string.
Order: strict ``json.loads`` (covers the rare invalid_tool_call whose text is
valid JSON but not an object) → ``json_repair`` → one unwrap of a
double-encoded JSON string. Anything that is not a non-empty dict is ``None``.
"""
if isinstance(raw, dict):
return raw or None
if raw is None:
return None
text = str(raw).strip()
if not text:
return None
obj: Any = None
try:
obj = json.loads(text)
except Exception:
try:
obj = json_repair.loads(text)
except Exception:
return None
if isinstance(obj, str) and obj.strip().startswith("{"):
# Double-encoded: the whole object was serialised as one JSON string.
try:
obj = json_repair.loads(obj)
except Exception:
return None
if isinstance(obj, dict) and obj:
return obj
return None
def _nudge_count_this_turn(messages: list) -> int:
"""How many corrective ToolMessages were already issued since the last human
turn. A fresh human message re-arms the (single) nudge — same contract as
``agent_factory._empty_retry_count``."""
count = 0
for m in messages:
if isinstance(m, dict):
role = m.get("type") or m.get("role")
content = m.get("content")
else:
role = getattr(m, "type", None)
content = getattr(m, "content", None)
if role in ("human", "user"):
count = 0
elif role == "tool" and isinstance(content, str) and INVALID_ARGS_MARKER in content:
count += 1
return count
class LinsightInvalidToolCallRepairMiddleware(AgentMiddleware):
"""Turn ``AIMessage.invalid_tool_calls`` into executable calls (or a retry)."""
def __init__(self, *, tools: Sequence[Any] | None = None, is_subagent: bool = False) -> None:
super().__init__()
self.tools = [] # registers no extra tools
self.is_subagent = is_subagent
self._arg_keys = _tool_arg_keys(tools)
@property
def name(self) -> str:
# Role-distinct, stable name so the two instances (main vs subagent) never
# collide in a shared middleware list.
return f"LinsightInvalidToolCallRepair{'Sub' if self.is_subagent else 'Main'}"
# ------------------------------------------------------------------ core
def _plausible(self, name: str | None, args: dict[str, Any]) -> bool:
"""Reject a "repair" whose keys are not in the tool's schema — that is
json_repair turning a broken string into key/value debris, and executing
it would only trade one error for a stranger one."""
known = self._arg_keys.get(name or "")
if not known:
return True
return set(args.keys()) <= known
def _process(self, state: Any) -> dict[str, Any] | None:
messages = _state_messages(state)
if not messages:
return None
ai = messages[-1]
if not isinstance(ai, AIMessage):
return None
invalid = list(getattr(ai, "invalid_tool_calls", None) or [])
if not invalid:
return None
repaired: list[dict[str, Any]] = []
still_invalid: list[dict[str, Any]] = []
tool_msgs: list[ToolMessage] = []
for itc in invalid:
name = itc.get("name") or "unknown"
call_id = itc.get("id")
raw = itc.get("args")
args = _repair_args(raw)
if args is not None and self._plausible(name, args) and call_id:
repaired.append({"name": name, "args": args, "id": call_id, "type": "tool_call"})
continue
still_invalid.append(itc)
if not call_id:
# No id → nothing for a ToolMessage to answer; the request
# serialiser drops id-less calls too, so there is nothing to pair.
continue
raw_text = "" if raw is None else str(raw)
preview = raw_text[:_RAW_PREVIEW_CAP] + ("" if len(raw_text) > _RAW_PREVIEW_CAP else "")
error = _json_error(raw_text) if raw_text else "arguments were empty"
content = _INVALID_ARGS_HINT.format(name=name, marker=INVALID_ARGS_MARKER, error=error)
if preview:
content += f"\n收到的参数开头:{preview}"
tool_msgs.append(ToolMessage(content=content, name=name, tool_call_id=call_id, status="error"))
update: dict[str, Any] = {}
new_messages: list[Any] = []
valid_calls = list(getattr(ai, "tool_calls", None) or [])
if repaired:
# Same id → the add_messages reducer replaces the AIMessage in place.
new_ai = ai.model_copy(
update={"tool_calls": [*valid_calls, *repaired], "invalid_tool_calls": still_invalid}
)
new_messages.append(new_ai)
valid_calls = [*valid_calls, *repaired]
new_messages.extend(tool_msgs)
nudged = False
if not valid_calls and tool_msgs:
# Nothing executable is left this turn: hand the error back to the
# model right away — but only once per user turn.
if _nudge_count_this_turn(messages) == 0:
update["jump_to"] = "model"
nudged = True
if new_messages:
update["messages"] = new_messages
logger.info(
"BS_LINSIGHT_INVALID_TOOLCALL graph={} repaired={} unrepaired={} nudged={}",
"sub" if self.is_subagent else "main",
[c["name"] for c in repaired],
[c.get("name") for c in still_invalid],
nudged,
)
return update or None
# ------------------------------------------------------------------ hooks
@hook_config(can_jump_to=["model"])
def after_model(self, state, runtime):
return self._process(state)
@hook_config(can_jump_to=["model"])
async def aafter_model(self, state, runtime):
return self._process(state)
def build_invalid_tool_call_repair_middleware(
*, tools: Sequence[Any] | None = None, is_subagent: bool
) -> LinsightInvalidToolCallRepairMiddleware:
"""Construct a middleware instance (one per graph); ``tools`` is the graph's
bound tool list, used only to sanity-check repaired argument keys."""
return LinsightInvalidToolCallRepairMiddleware(tools=tools, is_subagent=is_subagent)
@@ -0,0 +1,372 @@
"""Unit tests for LinsightInvalidToolCallRepairMiddleware.
Regression for the 2026-08-27 incident (114, version 6049532c…): the model's
``ask_user`` arguments were not valid JSON (unescaped ``"`` inside a question),
langchain parked the call in ``invalid_tool_calls``, the agent loop ended on an
empty AIMessage and the task closed as ``Task produced no result``.
Covers:
- the LIVE payload is repaired in place (same message id, question text intact),
- unrepairable args → error ToolMessage + one ``jump_to: model`` per user turn,
- mixed valid/invalid turns keep the valid calls and never jump,
- schema-key sanity check rejects json_repair debris for a known tool,
- pass-through when there is nothing to do,
- the prompt/tool docstring carry the no-English-double-quotes rule.
``asyncio_mode = auto`` — async tests need no decorator.
"""
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage
from bisheng.linsight.domain.services import agent_factory
from bisheng.linsight.domain.services.invalid_tool_call_middleware import (
INVALID_ARGS_MARKER,
LinsightInvalidToolCallRepairMiddleware,
_repair_args,
build_invalid_tool_call_repair_middleware,
)
# The exact arguments string deepseek-v4-flash emitted in the live session: the
# ``question`` value contains unescaped double quotes (``reason`` is escaped
# correctly) and the second question repeats the ``multiple`` key.
_LIVE_RAW_ARGS = (
'{"reason": "您提到\\"按照各市要求重新生成\\",但当前映射表说明没有涉及\\"各市\\"的维度。", '
'"questions": [{"question": "您说的"按照各市要求"具体是指什么?", '
'"options": ["按城市/地区分类重新组织映射表", "按照各市的格式规范要求调整文档", "其他(请说明)"], '
'"multiple": false}, {"question": "重新生成的文档输出格式?", '
'"options": ["Worddocx", "Markdownmd"], "multiple": true, "multiple": false}]}'
)
_CALL_ID = "call_30d0f438439a467da532c3d1"
# --------------------------------------------------------------------------- helpers
def _invalid_ai(raw=_LIVE_RAW_ARGS, name="ask_user", call_id=_CALL_ID, msg_id="lc_run--live", valid_calls=None):
return AIMessage(
content="",
id=msg_id,
tool_calls=list(valid_calls or []),
invalid_tool_calls=[{"name": name, "args": raw, "id": call_id, "error": None, "type": "invalid_tool_call"}],
response_metadata={"finish_reason": "tool_calls"},
)
def _mw(tools=None, is_subagent=False):
return build_invalid_tool_call_repair_middleware(tools=tools, is_subagent=is_subagent)
def _state(*messages):
return {"messages": [HumanMessage(content="上述说明按照各市要求重新生成"), *messages]}
# --------------------------------------------------------------------------- repair
def test_repair_args_recovers_live_payload_intact():
args = _repair_args(_LIVE_RAW_ARGS)
assert isinstance(args, dict)
assert set(args) == {"reason", "questions"}
# The question text survives byte-for-byte, quotes included.
assert args["questions"][0]["question"] == '您说的"按照各市要求"具体是指什么?'
assert args["questions"][0]["options"][0] == "按城市/地区分类重新组织映射表"
assert args["questions"][1]["question"] == "重新生成的文档输出格式?"
def test_repair_args_rejects_non_objects_and_garbage():
assert _repair_args("") is None
assert _repair_args(None) is None
assert _repair_args("[1, 2, 3]") is None # valid JSON, not an object
assert _repair_args('"just a string"') is None
assert _repair_args("{}") is None
def test_repair_args_unwraps_double_encoded_object():
assert _repair_args('"{\\"reason\\": \\"why\\", \\"questions\\": []}"') == {"reason": "why", "questions": []}
async def test_live_payload_is_repaired_in_place_and_routed_to_tools():
mw = _mw(tools=[agent_factory.ask_user])
ai = _invalid_ai()
update = await mw.aafter_model(_state(ai), runtime=None)
assert update is not None
assert "jump_to" not in update # the routing edge now sees tool_calls → tools node
(new_ai,) = update["messages"]
assert isinstance(new_ai, AIMessage)
assert new_ai.id == ai.id # same id → add_messages replaces, no duplicate start frame
assert new_ai.invalid_tool_calls == []
assert len(new_ai.tool_calls) == 1
tc = new_ai.tool_calls[0]
assert tc["name"] == "ask_user" and tc["id"] == _CALL_ID and tc["type"] == "tool_call"
assert tc["args"]["questions"][0]["question"] == '您说的"按照各市要求"具体是指什么?'
# The original message object is not mutated (model_copy).
assert ai.tool_calls == [] and len(ai.invalid_tool_calls) == 1
def test_sync_hook_matches_async_hook():
mw = _mw(tools=[agent_factory.ask_user])
update = mw.after_model(_state(_invalid_ai()), runtime=None)
assert update and update["messages"][0].tool_calls[0]["name"] == "ask_user"
# --------------------------------------------------------------------------- nudge
async def test_unrepairable_args_get_error_tool_message_and_one_jump_back_to_model():
mw = _mw()
ai = _invalid_ai(raw="reason: 需要确认 questions: [", call_id="call_x")
update = await mw.aafter_model(_state(ai), runtime=None)
assert update["jump_to"] == "model"
(tm,) = update["messages"]
assert isinstance(tm, ToolMessage)
assert tm.tool_call_id == "call_x" and tm.name == "ask_user" and tm.status == "error"
assert INVALID_ARGS_MARKER in tm.content
assert "ask_user" in tm.content and "中文引号" in tm.content
# The raw text is echoed back (capped) so the model can see what went wrong.
assert "reason: 需要确认" in tm.content
async def test_second_unrepairable_call_in_same_turn_does_not_jump_again():
mw = _mw()
first_ai = _invalid_ai(raw="{{{", call_id="call_1", msg_id="m1")
first_update = await mw.aafter_model(_state(first_ai), runtime=None)
assert first_update["jump_to"] == "model"
(first_tm,) = first_update["messages"]
second_ai = _invalid_ai(raw="{{{", call_id="call_2", msg_id="m2")
second_update = await mw.aafter_model(_state(first_ai, first_tm, second_ai), runtime=None)
# Still answers the dangling call (transcript stays consistent) but no retry:
# the no-infinite-loop guarantee — the run ends as it did before this middleware.
assert "jump_to" not in second_update
(second_tm,) = second_update["messages"]
assert second_tm.tool_call_id == "call_2" and INVALID_ARGS_MARKER in second_tm.content
async def test_new_human_turn_re_arms_the_nudge():
mw = _mw()
stale_tm = ToolMessage(content=f"old {INVALID_ARGS_MARKER}", tool_call_id="old", name="ask_user", status="error")
ai = _invalid_ai(raw="{{{", call_id="call_new")
state = {"messages": [HumanMessage(content="q1"), stale_tm, HumanMessage(content="q2"), ai]}
update = await mw.aafter_model(state, runtime=None)
assert update["jump_to"] == "model"
async def test_mixed_valid_and_invalid_calls_keep_valid_and_never_jump():
mw = _mw()
valid = {"name": "ls", "args": {"path": "/"}, "id": "call_ok", "type": "tool_call"}
ai = _invalid_ai(raw="{{{", name="write_file", call_id="call_bad", valid_calls=[valid])
update = await mw.aafter_model(_state(ai), runtime=None)
assert "jump_to" not in update # the edge will run the valid call
(tm,) = update["messages"]
assert isinstance(tm, ToolMessage) and tm.tool_call_id == "call_bad" and tm.name == "write_file"
async def test_repaired_call_is_appended_after_existing_valid_calls():
mw = _mw()
valid = {"name": "ls", "args": {"path": "/"}, "id": "call_ok", "type": "tool_call"}
ai = _invalid_ai(raw='{"file_path": "/output/a.md", "content": "x"', name="write_file", valid_calls=[valid])
update = await mw.aafter_model(_state(ai), runtime=None)
(new_ai,) = update["messages"]
assert [tc["id"] for tc in new_ai.tool_calls] == ["call_ok", _CALL_ID]
assert new_ai.tool_calls[1]["args"] == {"file_path": "/output/a.md", "content": "x"}
# --------------------------------------------------------------------------- schema sanity
async def test_repair_debris_for_known_tool_is_rejected_and_nudged():
mw = _mw(tools=[agent_factory.ask_user])
# json_repair turns this into a dict, but its keys are not ask_user's.
ai = _invalid_ai(raw='{"why": "x", "items": []}', call_id="call_d")
update = await mw.aafter_model(_state(ai), runtime=None)
assert update["jump_to"] == "model"
assert isinstance(update["messages"][0], ToolMessage)
async def test_unknown_tool_is_repaired_without_schema_check():
mw = _mw(tools=[agent_factory.ask_user])
ai = _invalid_ai(raw='{"todos": [{"content": "a", "status": "pending"}', name="write_todos", call_id="call_t")
update = await mw.aafter_model(_state(ai), runtime=None)
assert "jump_to" not in update
assert update["messages"][0].tool_calls[0]["args"]["todos"][0]["content"] == "a"
# --------------------------------------------------------------------------- pass-through
async def test_no_invalid_calls_is_a_noop():
mw = _mw()
ai = AIMessage(content="", tool_calls=[{"name": "ls", "args": {}, "id": "c", "type": "tool_call"}])
assert await mw.aafter_model(_state(ai), runtime=None) is None
assert await mw.aafter_model(_state(AIMessage(content="done")), runtime=None) is None
assert await mw.aafter_model(_state(ToolMessage(content="x", tool_call_id="c")), runtime=None) is None
assert await mw.aafter_model({"messages": []}, runtime=None) is None
async def test_invalid_call_without_id_is_ignored():
mw = _mw()
ai = AIMessage(
content="",
invalid_tool_calls=[
{"name": "ask_user", "args": "{{{", "id": None, "error": None, "type": "invalid_tool_call"}
],
)
assert await mw.aafter_model(_state(ai), runtime=None) is None
def test_instance_names_are_role_distinct_and_register_no_tools():
main, sub = _mw(is_subagent=False), _mw(is_subagent=True)
assert main.name == "LinsightInvalidToolCallRepairMain"
assert sub.name == "LinsightInvalidToolCallRepairSub"
assert main.tools == [] and sub.tools == []
assert isinstance(main, LinsightInvalidToolCallRepairMiddleware)
# --------------------------------------------------------------------------- prompt
def test_prompt_and_tool_doc_forbid_english_double_quotes_in_ask_user_text():
prompt = agent_factory._build_linsight_system_prompt(False)
assert "不要出现英文双引号" in prompt
assert "中文引号「」" in prompt
assert "不要使用英文双引号" in agent_factory.ask_user.description
# --------------------------------------------------------------------------- graph wiring
def _scripted_model(script):
"""A minimal chat model that replays ``script`` (AIMessages) call by call —
same shape as _e2e_llm_resilience_runner's ScriptedFaultModel, trimmed."""
from langchain_core.language_models.chat_models import BaseChatModel
from langchain_core.outputs import ChatGeneration, ChatResult
class _Scripted(BaseChatModel):
_script: list = []
_calls: list = [0]
_seen: list = []
@property
def _llm_type(self) -> str:
return "scripted"
def _next(self, messages):
i = self._calls[0]
self._calls[0] += 1
self._seen.append(list(messages))
return self._script[min(i, len(self._script) - 1)]
def _generate(self, messages, stop=None, run_manager=None, **kwargs):
return ChatResult(generations=[ChatGeneration(message=self._next(messages))])
async def _agenerate(self, messages, stop=None, run_manager=None, **kwargs):
return ChatResult(generations=[ChatGeneration(message=self._next(messages))])
def bind_tools(self, tools, **kwargs):
return self.bind(**kwargs)
m = _Scripted()
m._script = list(script)
m._calls = [0]
m._seen = []
return m
def _echo_tool():
from langchain_core.tools import tool
@tool
def echo(text: str) -> str:
"""Echo ``text`` back."""
return f"echo:{text}"
return echo
async def test_graph_executes_repaired_call_through_the_real_agent_loop():
from langchain.agents import create_agent
echo = _echo_tool()
broken = AIMessage(
content="",
id="ai-1",
invalid_tool_calls=[
{
"name": "echo",
"args": '{"text": "he said "hi" to me"}',
"id": "c1",
"error": None,
"type": "invalid_tool_call",
}
],
)
model = _scripted_model([broken, AIMessage(content="final", id="ai-2")])
agent = create_agent(model=model, tools=[echo], middleware=[_mw(tools=[echo])])
result = await agent.ainvoke({"messages": [HumanMessage(content="go")]})
tool_msgs = [m for m in result["messages"] if isinstance(m, ToolMessage)]
assert len(tool_msgs) == 1 and tool_msgs[0].tool_call_id == "c1"
assert tool_msgs[0].content == 'echo:he said "hi" to me' # the tool really ran on the repaired args
ai_msgs = [m for m in result["messages"] if isinstance(m, AIMessage)]
assert ai_msgs[0].id == "ai-1" and ai_msgs[0].invalid_tool_calls == [] and ai_msgs[0].tool_calls[0]["id"] == "c1"
assert result["messages"][-1].content == "final"
assert model._calls[0] == 2
async def test_graph_nudges_model_once_when_args_cannot_be_repaired():
from langchain.agents import create_agent
echo = _echo_tool()
broken = AIMessage(
content="",
id="ai-1",
invalid_tool_calls=[{"name": "echo", "args": "{{{", "id": "c1", "error": None, "type": "invalid_tool_call"}],
)
fixed = AIMessage(
content="", id="ai-2", tool_calls=[{"name": "echo", "args": {"text": "ok"}, "id": "c2", "type": "tool_call"}]
)
model = _scripted_model([broken, fixed, AIMessage(content="final", id="ai-3")])
agent = create_agent(model=model, tools=[echo], middleware=[_mw(tools=[echo])])
result = await agent.ainvoke({"messages": [HumanMessage(content="go")]})
# Call 2 was the retry: it saw the corrective ToolMessage for c1 in its prompt.
assert model._calls[0] == 3
second_prompt = model._seen[1]
assert any(
isinstance(m, ToolMessage) and m.tool_call_id == "c1" and INVALID_ARGS_MARKER in m.content
for m in second_prompt
)
tool_msgs = [m for m in result["messages"] if isinstance(m, ToolMessage)]
assert [t.tool_call_id for t in tool_msgs] == ["c1", "c2"]
assert tool_msgs[1].content == "echo:ok"
assert result["messages"][-1].content == "final"
async def test_graph_ends_after_second_unrepairable_call_without_looping():
from langchain.agents import create_agent
echo = _echo_tool()
def _broken(i):
return AIMessage(
content="",
id=f"ai-{i}",
invalid_tool_calls=[
{"name": "echo", "args": "{{{", "id": f"c{i}", "error": None, "type": "invalid_tool_call"}
],
)
model = _scripted_model([_broken(1), _broken(2), _broken(3)])
agent = create_agent(model=model, tools=[echo], middleware=[_mw(tools=[echo])])
result = await agent.ainvoke({"messages": [HumanMessage(content="go")]})
# One nudge, then the loop ends (no third model call) — no infinite retry.
assert model._calls[0] == 2
tool_msgs = [m for m in result["messages"] if isinstance(m, ToolMessage)]
assert [t.tool_call_id for t in tool_msgs] == ["c1", "c2"]