From 994eb89d1c114d5a8b70682840aec76a1f66d168 Mon Sep 17 00:00:00 2001 From: t Date: Thu, 15 Oct 2026 01:53:43 -0400 Subject: [PATCH] fix(prompt_agent): a conversation's tool_call_count sums its turns Each turn's terminal message reports only that turn's tool calls, and the PromptResponse recorded the last one, so a three-turn user-sim run with 3 + 39 + 19 calls reported 19. The count is now the sum over turns, and stays None when no turn reports one. Co-Authored-By: Claude Opus 5.5 --- .../task_step/task_steps/prompt_agent.py | 7 +- ...st_prompt_agent_conversation_tool_calls.py | 87 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 tst/unit/task_step/test_prompt_agent_conversation_tool_calls.py diff --git a/src/agent_env/task_step/task_steps/prompt_agent.py b/src/agent_env/task_step/task_steps/prompt_agent.py index eb2ddb37..d9d83d40 100644 --- a/src/agent_env/task_step/task_steps/prompt_agent.py +++ b/src/agent_env/task_step/task_steps/prompt_agent.py @@ -490,6 +490,9 @@ async def _execute_conversation( current_user_parts = initial_parts final_terminal = protocol.TerminalResponse(response_text="") + # Each turn's terminal reports only that turn's tool calls, so the conversation's count is + # their sum. None until a turn reports one, so agents that don't report it stay None. + conversation_tool_call_count: Optional[int] = None final_task_id = "" per_turn_trajectory_urls: list[str | None] = [] source_agent_per_turn_prompt_parts: list[list[dict] | None] = [] @@ -544,6 +547,8 @@ def record_turn() -> None: live_turn.ended(target_state) status_msg = (result.get("status") or {}).get("message") or {} final_terminal = protocol.TerminalResponse.from_message(status_msg) + if final_terminal.tool_call_count is not None: + conversation_tool_call_count = (conversation_tool_call_count or 0) + final_terminal.tool_call_count agent_response_parts = status_msg.get("parts") or [{"kind": "text", "text": final_terminal.response_text}] conversation_store.complete_a2a_task( @@ -713,7 +718,7 @@ def record_turn() -> None: agent_trajectory_object_prefix=trajectory_output_prefix, target_agent_per_turn_trajectory_object_urls=per_turn_trajectory_urls or None, source_agent_per_turn_prompt_parts=source_agent_per_turn_prompt_parts or None, - tool_call_count=final_terminal.tool_call_count, + tool_call_count=conversation_tool_call_count, model=model, error_type=final_terminal.error_type, error_code=final_terminal.error_code, diff --git a/tst/unit/task_step/test_prompt_agent_conversation_tool_calls.py b/tst/unit/task_step/test_prompt_agent_conversation_tool_calls.py new file mode 100644 index 00000000..8d1f3af6 --- /dev/null +++ b/tst/unit/task_step/test_prompt_agent_conversation_tool_calls.py @@ -0,0 +1,87 @@ +"""A multi-turn conversation's tool_call_count is the sum of its turns, not the last turn's. + +Each turn's terminal message reports only that turn's tool calls, so recording the last one +made a usersim run with 3 + 39 + 19 calls report 19. +""" + +from unittest.mock import MagicMock + +import pytest +from a2a.types import TaskState + +from agent_env.task_step.context import DeployedAgent, TaskStepContext +from agent_env.task_step.task_steps import prompt_agent as pa_mod +from agent_env.task_step.task_steps.prompt_agent import PromptAgentTaskStep + + +def _wire(monkeypatch, solver_tool_calls: list): + """Mock the conversation's HTTP and store boundary. The solver's n-th reply reports + solver_tool_calls[n] (omitted when None); the user-sim never says done.""" + cfg = MagicMock() + cfg.get_model_params.return_value = {} + cfg.get_default_human_a2a_url.return_value = "http://hub/a2a/human" + monkeypatch.setattr(pa_mod, "get_config", lambda: cfg) + for fn in ("create_conversation", "add_a2a_task", "complete_a2a_task", "mark_closed", "get_conversation"): + monkeypatch.setattr(pa_mod.conversation_store, fn, lambda *a, **kw: None) + solver_turns = iter(solver_tool_calls) + + async def post_agent_config(url, fields): + return None + + async def send(url, parts, message_id, context_id, timeout): + return f"task-{message_id}", None + + async def poll(url, task_id, timeout, interval, **_): + if "user-sim" in url: + parts = [{"kind": "text", "text": '{"message": "keep going", "done": false}'}] + else: + count = next(solver_turns) + parts = [{"kind": "text", "text": "done"}] + if count is not None: + parts.append({"kind": "data", "data": {"tool_call_count": count}}) + return {"status": {"state": TaskState.completed, "message": {"parts": parts}}} + + monkeypatch.setattr(pa_mod.protocol, "post_agent_config", post_agent_config) + monkeypatch.setattr(pa_mod.protocol, "send_a2a_message", send) + monkeypatch.setattr(pa_mod.protocol, "poll_a2a_task", poll) + + +def _context() -> TaskStepContext: + ctx = TaskStepContext(instance_id="ti-1") + ctx.deployed_agents.append(DeployedAgent(agent_name="solver", api_url="http://solver", a2a_url="http://solver", + a2a_card={})) + ctx.deployed_agents.append(DeployedAgent(agent_name="usersim", api_url="http://user-sim", + a2a_url="http://user-sim", a2a_card={})) + return ctx + + +def _step(turns: int) -> PromptAgentTaskStep: + return PromptAgentTaskStep(id="solve", version=None, prompt="hi", agent_name="solver", + max_conversation_turns=turns, user_agent_name="usersim") + + +@pytest.mark.asyncio +async def test_the_count_is_the_sum_of_every_turn(monkeypatch): + _wire(monkeypatch, [3, 39, 19]) + + ctx = await _step(3).execute(_context()) + + assert ctx.prompt_responses[-1].tool_call_count == 61 + + +@pytest.mark.asyncio +async def test_a_turn_without_a_count_adds_nothing(monkeypatch): + _wire(monkeypatch, [4, None, 6]) + + ctx = await _step(3).execute(_context()) + + assert ctx.prompt_responses[-1].tool_call_count == 10 + + +@pytest.mark.asyncio +async def test_it_stays_none_when_no_turn_reports_one(monkeypatch): + _wire(monkeypatch, [None, None]) + + ctx = await _step(2).execute(_context()) + + assert ctx.prompt_responses[-1].tool_call_count is None