Skip to content
14 changes: 10 additions & 4 deletions astrbot/core/agent/runners/tool_loop_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -1107,10 +1107,16 @@ async def step(self):
# If there are cached images and the model supports image input,
# append a user message with images so LLM can see them
if cached_images:
modalities = self.provider.provider_config.get("modalities", [])
supports_image = (
not modalities or "image" in modalities
) # Empty list is treated as unconfigured for backward compatibility
modalities = self.provider.provider_config.get("modalities") or []
# Unlike the request-assembly path above, tool-result images
# are new multimodal content the provider never asked for.
# Be conservative: only append them when the provider config
# explicitly declares image support. An empty, null, or missing
# modalities value means capabilities are unknown, and sending
# images to a text-only model produces a 400 that aborts the
# entire turn. The text description in result_parts already
# tells the LLM the image exists and where it is cached.
supports_image = "image" in modalities
if supports_image:
# Build user message with images for LLM to review
image_parts = []
Expand Down
62 changes: 62 additions & 0 deletions tests/test_tool_loop_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -974,6 +974,68 @@ def fake_save_image(
]


@pytest.mark.asyncio
@pytest.mark.parametrize(
("modalities", "expect_image"),
[
([], False),
(None, False),
(["text", "tool_use"], False),
(["image", "tool_use"], True),
],
)
async def test_tool_result_images_require_explicit_image_modality(
runner, provider_request, mock_hooks, monkeypatch, modalities, expect_image
):
from astrbot.core.agent.tool_image_cache import tool_image_cache

provider = SingleToolThenFinalProvider("test_tool")
provider.provider_config["modalities"] = modalities
image_reads = []

def fake_save_image(
base64_data, tool_call_id, tool_name, index=0, mime_type="image/png"
):
return SimpleNamespace(
file_path="call_123_0.png", mime_type=mime_type, tool_name=tool_name
)

def fake_read_image(file_path, mime_type):
image_reads.append((file_path, mime_type))
return "dGVzdA==", mime_type

monkeypatch.setattr(tool_image_cache, "save_image", fake_save_image)
monkeypatch.setattr(tool_image_cache, "get_image_base64_by_path", fake_read_image)

await runner.reset(
provider=provider,
request=provider_request,
run_context=ContextWrapper(context=None),
tool_executor=MockMixedContentToolExecutor,
agent_hooks=mock_hooks,
streaming=False,
)

async for _ in runner.step_until_done(3):
pass

image_messages = [
message
for message in runner.run_context.messages
if isinstance(message.content, list)
and any(isinstance(part, ImageURLPart) for part in message.content)
]
assert len(image_messages) == int(expect_image)
assert image_reads == ([("call_123_0.png", "image/png")] if expect_image else [])

tool_messages = [
message for message in runner.run_context.messages if message.role == "tool"
]
assert "Image returned and cached at path='call_123_0.png'." in str(
tool_messages[0].content
)


@pytest.mark.asyncio
async def test_runner_replaces_runtime_image_context_before_provider_call(
runner, provider_request, mock_hooks
Expand Down
Loading