mirror of
https://github.com/AstrBotDevs/AstrBot.git
synced 2026-08-31 01:40:25 +08:00
perf: detect repeated tool calls by name and arguments (#9311)
* fix: detect repeated tool calls by name and arguments * Update tests/test_tool_loop_agent_runner.py Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com> --------- Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
@@ -148,21 +148,23 @@ class ToolLoopAgentRunner(BaseAgentRunner[TContext]):
|
||||
MALFORMED_TOOL_NAME_PLACEHOLDER = "__malformed_tool_name__"
|
||||
REPEATED_TOOL_NOTICE_L1_TEMPLATE = (
|
||||
"\n\n[SYSTEM NOTICE] By the way, you have executed the same tool "
|
||||
"`{tool_name}` {streak} times consecutively. Double-check whether another "
|
||||
"tool, different arguments, or a summary would move the task forward better."
|
||||
"`{tool_name}` with the same arguments {streak} times consecutively. "
|
||||
"Double-check whether another tool, different arguments, or a summary would "
|
||||
"move the task forward better."
|
||||
)
|
||||
REPEATED_TOOL_NOTICE_L2_TEMPLATE = (
|
||||
"\n\n[SYSTEM NOTICE] Important: you have executed the same tool "
|
||||
"`{tool_name}` {streak} times consecutively. Unless this repetition is "
|
||||
"clearly necessary, stop repeating the same action and either switch "
|
||||
"tools, refine parameters, or summarize what is still missing."
|
||||
"`{tool_name}` with the same arguments {streak} times consecutively. "
|
||||
"Unless this repetition is clearly necessary, stop repeating the same action "
|
||||
"and either switch tools, refine parameters, or summarize what is still "
|
||||
"missing."
|
||||
)
|
||||
REPEATED_TOOL_NOTICE_L3_TEMPLATE = (
|
||||
"\n\n[SYSTEM NOTICE] Important: you have executed the same tool "
|
||||
"`{tool_name}` {streak} times consecutively. Repetition is now very "
|
||||
"high. Continue only if each call is clearly producing new information. "
|
||||
"Otherwise, change strategy, adjust arguments, or explain the limitation "
|
||||
"to the user."
|
||||
"`{tool_name}` with the same arguments {streak} times consecutively. "
|
||||
"Repetition is now very high. Continue only if each call is clearly producing "
|
||||
"new information. Otherwise, change strategy, adjust arguments, or explain "
|
||||
"the limitation to the user."
|
||||
)
|
||||
TOOL_RESULT_OVERFLOW_NOTICE_TEMPLATE = (
|
||||
"Truncated tool output preview shown above. "
|
||||
@@ -281,6 +283,7 @@ class ToolLoopAgentRunner(BaseAgentRunner[TContext]):
|
||||
self._pending_follow_ups: list[FollowUpTicket] = []
|
||||
self._follow_up_seq = 0
|
||||
self._last_tool_name: str | None = None
|
||||
self._last_tool_args: dict[str, T.Any] | None = None
|
||||
self._same_tool_streak = 0
|
||||
|
||||
# These two are used for tool schema mode handling
|
||||
@@ -662,11 +665,29 @@ class ToolLoopAgentRunner(BaseAgentRunner[TContext]):
|
||||
return content
|
||||
return f"{content}{notice}"
|
||||
|
||||
def _track_tool_call_streak(self, tool_name: str) -> int:
|
||||
if tool_name == self._last_tool_name:
|
||||
def _track_tool_call_streak(
|
||||
self,
|
||||
tool_name: str,
|
||||
tool_args: dict[str, T.Any] | None,
|
||||
) -> int:
|
||||
"""Track consecutive tool calls with the same name and arguments.
|
||||
|
||||
Args:
|
||||
tool_name: Name of the called tool.
|
||||
tool_args: Arguments passed to the tool.
|
||||
|
||||
Returns:
|
||||
Number of consecutive calls with the same name and arguments.
|
||||
"""
|
||||
normalized_args = {} if tool_args is None else tool_args
|
||||
if (
|
||||
tool_name == self._last_tool_name
|
||||
and normalized_args == self._last_tool_args
|
||||
):
|
||||
self._same_tool_streak += 1
|
||||
else:
|
||||
self._last_tool_name = tool_name
|
||||
self._last_tool_args = copy.deepcopy(normalized_args)
|
||||
self._same_tool_streak = 1
|
||||
return self._same_tool_streak
|
||||
|
||||
@@ -1032,7 +1053,10 @@ class ToolLoopAgentRunner(BaseAgentRunner[TContext]):
|
||||
llm_response.tools_call_ids,
|
||||
):
|
||||
tool_result_blocks_start = len(tool_call_result_blocks)
|
||||
tool_call_streak = self._track_tool_call_streak(func_tool_name)
|
||||
tool_call_streak = self._track_tool_call_streak(
|
||||
func_tool_name,
|
||||
func_tool_args,
|
||||
)
|
||||
yield _HandleFunctionToolsResult.from_message_chain(
|
||||
MessageChain(
|
||||
type="tool_call",
|
||||
|
||||
@@ -323,9 +323,14 @@ class CapturingToolLoopProvider(MockProvider):
|
||||
|
||||
|
||||
class SequentialToolProvider(MockProvider):
|
||||
def __init__(self, tool_sequence: list[str]):
|
||||
def __init__(
|
||||
self,
|
||||
tool_sequence: list[str],
|
||||
tool_args_sequence: list[dict[str, Any]] | None = None,
|
||||
):
|
||||
super().__init__()
|
||||
self.tool_sequence = tool_sequence
|
||||
self.tool_args_sequence = tool_args_sequence
|
||||
|
||||
async def text_chat(self, **kwargs) -> LLMResponse:
|
||||
self.call_count += 1
|
||||
@@ -338,11 +343,16 @@ class SequentialToolProvider(MockProvider):
|
||||
)
|
||||
|
||||
tool_name = self.tool_sequence[self.call_count - 1]
|
||||
tool_args = (
|
||||
self.tool_args_sequence[self.call_count - 1]
|
||||
if self.tool_args_sequence is not None
|
||||
else {"query": f"step-{self.call_count}"}
|
||||
)
|
||||
return LLMResponse(
|
||||
role="assistant",
|
||||
completion_text="",
|
||||
tools_call_name=[tool_name],
|
||||
tools_call_args=[{"query": f"step-{self.call_count}"}],
|
||||
tools_call_args=[tool_args],
|
||||
tools_call_ids=[f"call_{self.call_count}"],
|
||||
usage=TokenUsage(input_other=10, output=5),
|
||||
)
|
||||
@@ -968,7 +978,10 @@ async def test_same_tool_consecutive_results_include_escalating_guidance(
|
||||
):
|
||||
runner_cls = type(runner)
|
||||
total_calls = runner_cls.REPEATED_TOOL_NOTICE_L3_THRESHOLD
|
||||
provider = SequentialToolProvider(["test_tool"] * total_calls)
|
||||
provider = SequentialToolProvider(
|
||||
["test_tool"] * total_calls,
|
||||
[{"query": "same"}] * total_calls,
|
||||
)
|
||||
tool = FunctionTool(
|
||||
name="test_tool",
|
||||
description="测试工具",
|
||||
@@ -1031,6 +1044,46 @@ async def test_same_tool_consecutive_results_include_escalating_guidance(
|
||||
assert level_3_notice in content
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_tool_with_different_args_does_not_include_repeated_guidance(
|
||||
runner, mock_tool_executor, mock_hooks
|
||||
):
|
||||
runner_cls = type(runner)
|
||||
total_calls = runner_cls.REPEATED_TOOL_NOTICE_L3_THRESHOLD
|
||||
provider = SequentialToolProvider(["test_tool"] * total_calls)
|
||||
tool = FunctionTool(
|
||||
name="test_tool",
|
||||
description="测试工具",
|
||||
parameters={"type": "object", "properties": {"query": {"type": "string"}}},
|
||||
handler=AsyncMock(),
|
||||
)
|
||||
request = ProviderRequest(
|
||||
prompt="使用不同参数连续执行工具",
|
||||
func_tool=ToolSet(tools=[tool]),
|
||||
contexts=[],
|
||||
)
|
||||
|
||||
await runner.reset(
|
||||
provider=provider,
|
||||
request=request,
|
||||
run_context=ContextWrapper(context=None),
|
||||
tool_executor=mock_tool_executor,
|
||||
agent_hooks=mock_hooks,
|
||||
streaming=False,
|
||||
)
|
||||
|
||||
async for _ in runner.step_until_done(total_calls + 1):
|
||||
pass
|
||||
|
||||
tool_messages = [
|
||||
m for m in runner.run_context.messages if getattr(m, "role", None) == "tool"
|
||||
]
|
||||
assert len(tool_messages) == total_calls
|
||||
assert all(
|
||||
"[SYSTEM NOTICE]" not in str(message.content) for message in tool_messages
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_same_tool_streak_resets_after_switching_tools(
|
||||
runner, mock_tool_executor, mock_hooks
|
||||
@@ -1038,7 +1091,8 @@ async def test_same_tool_streak_resets_after_switching_tools(
|
||||
runner_cls = type(runner)
|
||||
repeated_after_reset = runner_cls.REPEATED_TOOL_NOTICE_L1_THRESHOLD
|
||||
provider = SequentialToolProvider(
|
||||
["test_tool", "other_tool", *(["test_tool"] * repeated_after_reset)]
|
||||
["test_tool", "other_tool", *(["test_tool"] * repeated_after_reset)],
|
||||
[{"query": "same"}] * (repeated_after_reset + 2),
|
||||
)
|
||||
tool_a = FunctionTool(
|
||||
name="test_tool",
|
||||
|
||||
Reference in New Issue
Block a user