Pass structured_content to LLM even when content is set (#62)

The Splunk MCP Server App returns both the content and structured_content
field set, but we were not taking use of structured_content in case
content was set.

This change fixes that.

Author: mateusz834

splunklib/ai/engines/langchain.py

@@ -15,7 +15,7 @@
 import logging
 import uuid
 from collections.abc import Sequence
-from dataclasses import dataclass
+from dataclasses import asdict, dataclass
 from functools import partial
 from time import monotonic
 from typing import Any, Awaitable, Callable, cast, override
@@ -77,7 +77,7 @@
     ToolMessage,
 )
 from splunklib.ai.model import OpenAIModel, PredefinedModel
-from splunklib.ai.tools import Tool, ToolException
+from splunklib.ai.tools import Tool, ToolException, ToolResult
 
 # RESERVED_LC_TOOL_PREFIX represents a prefix that is reserved for internal use
 # and no user-visible tool or subagent name can contain it (as a prefix).
@@ -295,7 +295,7 @@ def _debug_before_model(state: AgentState) -> None:
 def _create_langchain_tool(tool: Tool) -> BaseTool:
     async def _tool_call(
         **kwargs: dict[str, Any],
-    ) -> tuple[list[str], dict[str, Any] | None]:
+    ) -> dict[str, Any] | list[str]:
         try:
             result = await tool.func(**kwargs)
         except ToolException as e:
@@ -305,14 +305,26 @@ async def _tool_call(
                 "ToolException from langchain should not be raised in tool.func"
             )
 
-        return result.content, result.structured_content
+        if result.structured_content:
+            # For both local tools and remote tools (Splunk MCP Server App),
+            # the primary payload is returned in structured_content.
+            # The content field is typically minimal for remote tools and empty for local tools.
+            #
+            # FastMCP behaves slightly differently: when structured_content is returned,
+            # it also includes json.dumps(structured_content) in the content field.
+            #
+            # If we introduce support for additional MCP implementations in the future,
+            # this assumption may need to be revisited. For now, this approach is fine.
+            # The worst-case scenario is that the same information is provided to the LLM twice.
+            return asdict(result)  # both content + structured_content
+        return result.content
 
     return StructuredTool(
         name=_normalize_tool_name(tool.name),
         description=tool.description,
         args_schema=tool.input_schema,
         coroutine=_tool_call,
-        response_format="content_and_artifact",
+        response_format="content",
         handle_tool_error=True,
         tags=tool.tags,
     )

splunklib/ai/tools.py

@@ -302,10 +302,6 @@ def _convert_tool_result(
         if isinstance(content, TextContent):
             text_contents.append(content.text)
 
-    # If there is no text content, use the structuredContent as text content.
-    if len(text_contents) == 0:
-        text_contents.append(json.dumps(result.structuredContent))
-
     return ToolResult(
         content=text_contents, structured_content=result.structuredContent
     )

tests/integration/ai/test_agent_mcp_tools.py

@@ -1,9 +1,12 @@
 import asyncio
 import contextlib
+from dataclasses import asdict, dataclass
 import os
 import socket
+from typing import Annotated
 from unittest.mock import patch
 
+from mcp.types import CallToolResult, TextContent
 import pytest
 from starlette.middleware import Middleware
 import uvicorn
@@ -448,6 +451,106 @@ async def lifespan(app: Starlette):
                 response = result.messages[-1].content
                 assert "31.5" in response, "Invalid LLM response"
 
+    @patch(
+        "splunklib.ai.agent._testing_local_tools_path",
+        os.path.join(
+            os.path.dirname(__file__),
+            "testdata",
+            "non_existent.py",
+        ),
+    )
+    @patch("splunklib.ai.agent._testing_app_id", "app_id")
+    @pytest.mark.asyncio
+    async def test_tool_call_text_content_with_structured_output(self) -> None:
+        pytest.importorskip("langchain_openai")
+
+        mcp = FastMCP("MCP Server", streamable_http_path="/")
+
+        @dataclass
+        class Result:
+            celsius_degrees: str
+
+        @mcp.tool(description="Returns the current temperature in the city")
+        def temperature(city: str) -> Annotated[CallToolResult, Result]:
+            if city == "Krakow":
+                temperature = "31.5C"
+            else:
+                temperature = "22.1C"
+
+            # The Splunk MCP Server App returns a succeeded message in the content
+            # and a proper output in the structured_content field.
+            return CallToolResult(
+                content=[
+                    TextContent(
+                        type="text",
+                        text=f"Tool call succeeded, temperature in {city} found",
+                    )
+                ],
+                structuredContent=asdict(Result(temperature)),
+            )
+
+        @contextlib.asynccontextmanager
+        async def lifespan(app: Starlette):
+            async with mcp.session_manager.run():
+                yield
+
+        async with run_http_server(
+            Starlette(
+                routes=[
+                    Mount("/services/mcp", app=mcp.streamable_http_app()),
+                    Route(
+                        "/services/authorization/tokens",
+                        tokens_handler,
+                        methods=["POST"],
+                    ),
+                ],
+                lifespan=lifespan,
+            )
+        ) as (host, port):
+            service = await asyncio.to_thread(
+                lambda: connect(
+                    scheme="http",
+                    host=host,
+                    port=port,
+                    splunkToken=AUTH_TOKEN,
+                    autologin=True,
+                    username="admin",  # not required, but set to avoid mocking the authentication/current-context endpoint
+                ),
+            )
+
+            async with Agent(
+                model=(await self.model()),
+                system_prompt="You must use the available tools to perform requested operations",
+                service=service,
+                use_mcp_tools=True,
+            ) as agent:
+                result = await agent.invoke(
+                    [
+                        HumanMessage(
+                            content=(
+                                "What is the weather like today in Krakow? Use the provided tools to check the temperature."
+                                "Return a short response, containing the tool response."
+                            ),
+                        )
+                    ]
+                )
+
+                found_tool_message = False
+                for msg in result.messages:
+                    if isinstance(msg, ToolMessage):
+                        found_tool_message = True
+                        # Both text content and structured_content should be in the
+                        # content of a tool response.
+                        assert (
+                            "Tool call succeeded, temperature in Krakow found"
+                            in msg.content
+                        )
+                        assert '"celsius_degrees": "31.5C"' in msg.content
+                assert found_tool_message, "missing ToolMessage in agent response"
+
+                response = result.messages[-1].content
+                assert "31.5" in response, "Invalid LLM response"
+
 
 @contextlib.asynccontextmanager
 async def run_http_server(app: Starlette):